ml_meta_perf.figures

  1"""Generate the study's figure set from a finished report.
  2
  3Figures carry no titles: each is written for a LaTeX ``figure`` environment where the
  4caption supplies the description. ``CAPTIONS`` holds a suggested caption per file,
  5including the disclosures that would otherwise have to be drawn into the image.
  6"""
  7
  8from __future__ import annotations
  9
 10from pathlib import Path
 11
 12import polars as pl
 13
 14from ml_meta_perf.data import DATASET_FEATURES, MODEL_FEATURES, columns_as_arrays, load, target
 15from ml_meta_perf.experiment import Report
 16from ml_meta_perf.plots import (
 17    count_below_floor,
 18    decision_quality,
 19    equation_comparison,
 20    practice_effects,
 21    predicted_versus_actual,
 22    ranking_quality,
 23    term_count_curve,
 24    term_effects,
 25)
 26
 27ADDITIVE_REFERENCE_ROW = "reference: additive mean-based reference"
 28
 29#: The figure set in the order the study presents it. **The order is the identity**: files
 30#: are written as ``01_equation_comparison.png`` and so on, so a figure can be named by its
 31#: number in a review comment or a caption without anyone having to agree on which
 32#: "comparison" plot is meant. Position in this tuple is the only place a number is written
 33#: down -- `figure_name` derives it -- so re-ordering the set renumbers the files and the
 34#: captions together and cannot leave the two disagreeing.
 35FIGURE_ORDER: tuple[str, ...] = (
 36    "equation_comparison",
 37    "term_count_curve",
 38    "predicted_vs_actual",
 39    "term_effects",
 40    "practice_effects",
 41    "ranking_quality",
 42    "decision_quality",
 43)
 44
 45
 46def figure_name(stem: str) -> str:
 47    """The published filename for a figure, ``NN_stem.png``, numbered from `FIGURE_ORDER`."""
 48    return f"{FIGURE_ORDER.index(stem) + 1:02d}_{stem}.png"
 49
 50
 51def _additive_reference(report: Report) -> float | None:
 52    """The additive mean-based reference as this run computed it.
 53
 54    Read from the report rather than hardcoded: the reference is a property of the data,
 55    so a literal here would silently drift out of step with the comparison table the
 56    moment the meta-dataset changed.
 57    """
 58    matched = report.comparison.filter(pl.col("equation") == ADDITIVE_REFERENCE_ROW)
 59    return float(matched["r2"][0]) if matched.height else None
 60
 61
 62def _published_length(report: Report) -> int | None:
 63    """How many terms the reported E3-Valid equation actually has."""
 64    return len(report.e3.equation.terms) or None
 65
 66
 67def generate(report: Report, destination: str | Path, data: str | Path | None = None) -> list[Path]:
 68    """Write every figure and return the paths, in the order they appear in the README.
 69
 70    ``data`` must be the meta-dataset the report was fitted on. Defaulting it to the
 71    packaged one is only safe because that is the usual case; a caller who fitted on
 72    another file and does not pass it here would get figures drawn from the wrong rows.
 73    """
 74    folder = Path(destination)
 75    folder.mkdir(parents=True, exist_ok=True)
 76    frame = load(data)
 77    columns = columns_as_arrays(frame, DATASET_FEATURES + MODEL_FEATURES)
 78    truth = target(frame)
 79    predicted = report.e3.equation.predict(columns)
 80
 81    written = [
 82        equation_comparison(report.comparison, folder / figure_name("equation_comparison")),
 83        term_count_curve(
 84            report.e3.curve,
 85            folder / figure_name("term_count_curve"),
 86            reference=_additive_reference(report),
 87            marker=_published_length(report),
 88            marker_label="selected term count",
 89        ),
 90        predicted_versus_actual(truth, predicted, folder / figure_name("predicted_vs_actual")),
 91        term_effects(report.effects, folder / figure_name("term_effects")),
 92        practice_effects(report.practices, folder / figure_name("practice_effects")),
 93        ranking_quality(report.selection, folder / figure_name("ranking_quality")),
 94        decision_quality(report.decision_baselines, folder / figure_name("decision_quality")),
 95    ]
 96    # The list is what the README and the chapters index against, so it has to come back in
 97    # `FIGURE_ORDER`. Asserting it here means a call added out of order fails the run rather
 98    # than shipping a figure numbered one thing and referenced as another.
 99    expected = [folder / figure_name(stem) for stem in FIGURE_ORDER]
100    if written != expected:
101        raise AssertionError(f"figures written out of FIGURE_ORDER: {[path.name for path in written]}")
102    return written
103
104
105def captions(report: Report, data: str | Path | None = None) -> dict[str, str]:
106    """Suggested LaTeX captions, carrying the disclosures kept out of the images."""
107    frame = load(data)
108    columns = columns_as_arrays(frame, DATASET_FEATURES + MODEL_FEATURES)
109    truth = target(frame)
110    hidden = count_below_floor(truth, report.e3.equation.predict(columns))
111
112    return {
113        figure_name("equation_comparison"): (
114            "Each fitted equation against the level it is read against, all scored on the "
115            f"same {frame.height} rows. Dataset-only and model-only equations are bounded by what "
116            "their group identity can explain. The additive mean-based reference bounds only an equation "
117            "additive in dataset and model effects, which E3 is not: its mixed terms can "
118            "carry interactions beyond that reference, and the current E3 passes it. "
119            "All bars use in-sample (IS) estimates."
120        ),
121        figure_name("term_count_curve"): (
122            "Accuracy against equation length for E3 under in-sample (IS), "
123            "leave-one-dataset-out (LODO), and leave-one-model-out (LOMO) evaluation. "
124            "The vertical line is the published length, chosen as the first knee of the "
125            "smoothed worst-protocol curve followed by a sustained plateau. "
126            "The additive mean-based reference is the best score reachable by an equation additive in "
127            "dataset and model effects; an equation passes it only by representing the "
128            "dataset-by-model interaction the additive reference cannot."
129        ),
130        figure_name("predicted_vs_actual"): (
131            "Predicted against actual MCC for E3. Axes start at 0; "
132            f"{hidden} point below that is not shown. The equation compresses toward the "
133            "middle of the range, as a shrunk linear fit will."
134        ),
135        figure_name("term_effects"): (
136            "Per-term effect on predicted MCC, measured as the swing between the term's "
137            "10th and 90th percentile. Sign follows the fitted weight."
138        ),
139        figure_name("practice_effects"): (
140            "Per-feature effect on predicted MCC between the feature's lowest and highest "
141            "decile, shaded by the confidence its practice was rated at. Only the confidence "
142            "levels present in the table appear in the legend."
143        ),
144        figure_name("decision_quality"): (
145            "F1 of the above-or-below-threshold decision against the threshold, one line per "
146            "evaluation protocol: in-sample (IS), leave-one-dataset-out (LODO), "
147            "leave-one-model-out (LOMO), and doubly held-out (DHO). The four differ only in "
148            "what the equation was allowed to see, so the "
149            "spread between them is the cost of generalisation on this task. F1 rather than "
150            "accuracy because the classes are unbalanced at the outer thresholds, where "
151            "always answering with the larger class reaches 0.51 accuracy at an F1 of zero."
152        ),
153        figure_name("ranking_quality"): (
154            "Head-of-list ranking quality per dataset under doubly held-out (DHO) evaluation, "
155            "beside what a bad ranking "
156            "costs. Left: average precision and reciprocal rank, with a star where the "
157            "top-ranked choice is within 0.01 MCC of the dataset's best. Right: the MCC given "
158            "up by taking the top-ranked model. Both are "
159            "scored with the dataset and the model of every cell held out of the fit."
160        ),
161    }