ml_meta_perf.guidance

  1"""Best practices from the literature, weighed against what this study measured.
  2
  3A best practice is not a property of a fitted equation. It is a piece of general,
  4transferable advice -- short enough to remember, cheap enough to apply -- that already
  5circulates in the field and that evidence can *support*, *qualify* or *challenge*. The
  6equation analysis in `ml_meta_perf.report` is evidence. It is not itself the advice, and a
  7sentence of the form "higher `nr_norm` went with lower MCC" is a measurement rather than
  8something anyone can act on.
  9
 10So this module runs the other way round from `ml_meta_perf.practices`. Instead of reading
 11guidance out of the weights, it starts from practices the tabular machine-learning
 12literature already recommends, and asks what this corpus says about each one. That makes
 13the study's contribution a *test of received advice* rather than a source of new advice,
 14which is the honest position for 20 datasets from one domain.
 15
 16Each `Practice` is written by hand, because a general recommendation and its citation are
 17not things a fitting procedure produces. Each `Verdict` is computed: the module measures
 18the relevant quantity on the meta-dataset and picks the verdict from a stated threshold,
 19so re-running on other data can overturn any of them.
 20
 21Verdicts mean:
 22
 23* **supported** -- this corpus shows the effect the practice predicts, at a size worth acting on;
 24* **qualified** -- the effect is present but smaller, or holds only under a condition worth stating;
 25* **challenged** -- this corpus shows the opposite of what the practice predicts;
 26* **not tested** -- the study adopts or assumes the practice and cannot be evidence for it.
 27
 28That last verdict matters. A study that reports only the practices it happens to confirm
 29is not evidence about practice, so a practice the study merely assumes is marked as such
 30rather than counted as a win.
 31
 32Study chapter: [6. Best practices against the equation][study-chapter] -- the rationale, in
 33prose, with the figures.
 34
 35[study-chapter]: https://github.com/mariolpantunes/ml-meta-perf/blob/main/assets/docs/06-practices.md
 36"""
 37
 38from __future__ import annotations
 39
 40from collections.abc import Callable
 41from dataclasses import dataclass
 42
 43import numpy as np
 44import polars as pl
 45
 46from ml_meta_perf.attribution import contributions as attribution_contributions
 47from ml_meta_perf.data import (
 48    DATASET_COLUMN,
 49    DATASET_FEATURES,
 50    MODEL_COLUMN,
 51    MODEL_FAMILY,
 52    NEURAL_FAMILIES,
 53    TARGET_COLUMN,
 54    TREE_FAMILIES,
 55)
 56from ml_meta_perf.experiment import Report
 57from ml_meta_perf.stats import spearman
 58from ml_meta_perf.terms import Term
 59from ml_meta_perf.validate import paired_comparison
 60
 61SUPPORTED = "supported"
 62QUALIFIED = "qualified"
 63CHALLENGED = "challenged"
 64NOT_TESTED = "not tested"
 65
 66
 67#: What a practice predicts a feature does to MCC as that feature rises.
 68RAISES, LOWERS = "raises", "lowers"
 69
 70
 71@dataclass(frozen=True)
 72class Practice:
 73    """One general recommendation, and where it comes from."""
 74
 75    id: str
 76    statement: str
 77    source: str
 78    #: Why the field believes it, in one line -- so a reader can judge whether this
 79    #: corpus is even the right place to test it.
 80    rationale: str
 81    #: Raw features the practice makes a claim about, each with the direction it predicts
 82    #: MCC moves as that feature rises. **This is what lets a practice be checked against
 83    #: the published equation rather than only against corpus averages.** A verdict drawn
 84    #: from family means says the advice holds on this data; a term-level agreement says the
 85    #: *equation* encodes it, which is a stronger and more falsifiable claim -- and the one
 86    #: an interpretability-first study is actually in a position to make.
 87    #:
 88    #: Empty for a practice that makes no claim about a single feature. Most do not: a
 89    #: family-level recommendation is a claim about *rows*, not about a coefficient, and
 90    #: `group_claims` is how those are checked.
 91    expectations: tuple[tuple[str, str], ...] = ()
 92
 93
 94@dataclass(frozen=True)
 95class Verdict:
 96    """What this study says about one practice."""
 97
 98    practice: Practice
 99    verdict: str
100    evidence: str
101    magnitude: float
102
103
104#: Measured evidence a verdict can draw on, assembled once and passed to every check.
105@dataclass(frozen=True)
106class Evidence:
107    """Everything the checks need, measured from the meta-dataset and the study."""
108
109    frame: pl.DataFrame
110    family_means: pl.DataFrame
111    complete_family_means: pl.DataFrame
112    complete_model_means: pl.DataFrame
113    n_complete_datasets: int
114    ranking_agreement: float
115    missing_cells: int
116    total_cells: int
117    decomposition: pl.DataFrame
118    comparison: pl.DataFrame
119    leakage: pl.DataFrame
120    practices: pl.DataFrame
121    #: Mean ranking quality of the equation and of the per-model-mean baseline, over the
122    #: held-out datasets. **Read `ap`, `mrr`, `hit_at_1` and `regret`.** Spearman is carried
123    #: for continuity and must not decide anything: on this corpus it sits between 0.63 and
124    #: 0.73 for every predictor *and* every baseline, including a constant, so a verdict
125    #: resting on it is resting on a quantity that cannot separate what it is comparing.
126    #: See `validate.ranking_report`.
127    selection_ranking: dict[str, float]
128    baseline_ranking: dict[str, float]
129    #: The same two, per held-out dataset and in one group order, so a check can run a
130    #: paired test rather than compare two means over twenty folds. `PairedResult` explains
131    #: why the means alone are not enough here.
132    selection_per_group: pl.DataFrame
133    baseline_per_group: pl.DataFrame
134
135    def family(self, name: str, complete: bool = True) -> float:
136        """Mean MCC of one learner family, on the complete-grid subset by default."""
137        table = self.complete_family_means if complete else self.family_means
138        matched = table.filter(pl.col("family") == name)
139        return float(matched["mcc"][0]) if matched.height else float("nan")
140
141    def families(self, names: tuple[str, ...], complete: bool = True) -> float:
142        """Mean MCC across several families, weighted by their row counts."""
143        table = self.complete_family_means if complete else self.family_means
144        matched = table.filter(pl.col("family").is_in(list(names)))
145        if not matched.height:
146            return float("nan")
147        weights = matched["n"].to_numpy().astype(np.float64)
148        return float(np.average(matched["mcc"].to_numpy(), weights=weights))
149
150    def scored(self, equation: str) -> float:
151        """The R2 of a row of `comparison`, matched by prefix.
152
153        By prefix because the labels carry their term count -- for example, "E3, dataset +
154        model" -- and that count moves whenever the configuration does. Matching the whole
155        string would make every caller here break on a change that is not about them.
156        """
157        matched = self.comparison.filter(pl.col("equation").str.starts_with(equation))
158        return float(matched["r2"][0]) if matched.height else float("nan")
159
160    def protocol(self, name: str) -> float:
161        matched = self.leakage.filter(pl.col("protocol") == name)
162        return float(matched["r2"][0]) if matched.height else float("nan")
163
164    def practice_effect(self, feature: str) -> float:
165        matched = self.practices.filter(pl.col("feature") == feature)
166        return float(matched["effect"][0]) if matched.height else float("nan")
167
168    def practice_terms(self, feature: str) -> int:
169        """How many of the equation's terms this feature appears in, 0 if none do."""
170        matched = self.practices.filter(pl.col("feature") == feature)
171        if not matched.height or "n_terms" not in matched.columns:
172            return 0
173        return int(matched["n_terms"][0])
174
175
176#: The ranking columns a verdict may read, in the order `ranking_report` documents them.
177RANKING_METRICS: tuple[str, ...] = ("ap", "mrr", "hit_at_1", "regret")
178
179
180def _mean_ranking(table: pl.DataFrame) -> dict[str, float]:
181    """Mean of each ranking metric over the held-out groups."""
182    return {
183        name: float(np.mean(table[name].to_numpy())) for name in (*RANKING_METRICS, "spearman") if name in table.columns
184    }
185
186
187def _family_means(frame: pl.DataFrame) -> pl.DataFrame:
188    return (
189        frame.with_columns(pl.col(MODEL_COLUMN).replace(MODEL_FAMILY).alias("family"))
190        .group_by("family")
191        .agg(
192            pl.col(TARGET_COLUMN).mean().alias("mcc"),
193            pl.col(MODEL_COLUMN).n_unique().alias("models"),
194            pl.len().alias("n"),
195        )
196        .sort("mcc", descending=True)
197    )
198
199
200def _model_means(frame: pl.DataFrame) -> pl.DataFrame:
201    return (
202        frame.group_by(MODEL_COLUMN)
203        .agg(pl.col(TARGET_COLUMN).mean().alias("mcc"), pl.len().alias("n"))
204        .sort("mcc", descending=True)
205    )
206
207
208def gather(frame: pl.DataFrame, report: Report) -> Evidence:
209    """Measure everything the practice checks need.
210
211    The *complete-grid* tables are the ones the family comparisons use. Sixteen of the
212    500 (dataset, model) cells are absent and they are not absent at random -- eight
213    models are missing from the same two datasets -- so a mean taken over all rows
214    compares those eight models on a different set of problems from the other seventeen.
215    Restricting to the datasets where every model ran removes that.
216    """
217    from ml_meta_perf.stats import spearman
218    from ml_meta_perf.validate import baseline_group_mean, ranking_report
219
220    n_models = frame[MODEL_COLUMN].n_unique()
221    counts = frame.select(DATASET_COLUMN, MODEL_COLUMN).unique().group_by(DATASET_COLUMN).len()
222    complete = counts.filter(pl.col("len") == n_models)[DATASET_COLUMN].to_list()
223    # No dataset ran every model: there is no complete grid to restrict to, so the family
224    # comparisons fall back to every row and the agreement figure is undefined rather than
225    # silently computed over an empty subset.
226    subset = frame.filter(pl.col(DATASET_COLUMN).is_in(complete)) if complete else frame
227
228    every = _model_means(frame).sort(MODEL_COLUMN)
229    restricted = _model_means(subset).sort(MODEL_COLUMN)
230    agreement = (
231        spearman(every["mcc"].to_numpy(), restricted["mcc"].to_numpy())
232        if complete and every.height == restricted.height
233        else float("nan")
234    )
235
236    truth = frame[TARGET_COLUMN].to_numpy().astype(np.float64)
237    datasets = frame[DATASET_COLUMN].to_numpy().astype(str)
238    models = frame[MODEL_COLUMN].to_numpy().astype(str)
239    baseline = ranking_report(truth, baseline_group_mean(truth, datasets, models), datasets)
240
241    selection = report.selection
242    return Evidence(
243        frame=frame,
244        family_means=_family_means(frame),
245        complete_family_means=_family_means(subset),
246        complete_model_means=_model_means(subset),
247        n_complete_datasets=len(complete),
248        ranking_agreement=agreement,
249        # Distinct pairs, not rows: a meta-dataset with a repeated (dataset, model) cell
250        # would otherwise report a negative number of missing ones.
251        missing_cells=n_models * frame[DATASET_COLUMN].n_unique()
252        - frame.select(DATASET_COLUMN, MODEL_COLUMN).unique().height,
253        total_cells=n_models * frame[DATASET_COLUMN].n_unique(),
254        decomposition=report.decomposition,
255        comparison=report.comparison,
256        leakage=report.leakage,
257        practices=report.practices,
258        selection_ranking=_mean_ranking(selection),
259        baseline_ranking=_mean_ranking(baseline),
260        selection_per_group=selection.sort("group") if "group" in selection.columns else selection,
261        baseline_per_group=baseline.sort("group") if "group" in baseline.columns else baseline,
262    )
263
264
265# --------------------------------------------------------------------------------------
266# The catalogue. Statements and citations are written by hand; every verdict is computed.
267# --------------------------------------------------------------------------------------
268
269CATALOGUE: tuple[Practice, ...] = (
270    Practice(
271        id="profile-the-data-first",
272        statement=(
273            "Characterise the dataset before choosing a model. What the data is like bounds "
274            "what any model can reach, and that bound is usually the larger effect."
275        ),
276        source="Zha et al., 'Data-centric Artificial Intelligence: A Survey', ACM Computing Surveys 57(5) (2025)",
277        rationale=(
278            "The data-centric position holds that returns from improving data exceed returns "
279            "from swapping architectures. It is an argument about where to spend effort."
280        ),
281        # No single feature: the claim is that the *dataset half* of the meta-data matters
282        # more than the model half, which is a claim about the two blocks of the equation
283        # rather than about any one column. `group_shares` is where it is read.
284    ),
285    Practice(
286        id="tree-ensembles-first",
287        statement=(
288            "On tabular data, start from tree ensembles. Reach for a neural architecture only "
289            "when a tree ensemble has been tried and found wanting."
290        ),
291        source=(
292            "Grinsztajn, Oyallon & Varoquaux, NeurIPS 2022 Datasets and Benchmarks Track; "
293            "Shwartz-Ziv & Armon, 'Tabular Data: Deep Learning is Not All You Need', "
294            "Information Fusion 81, 84-90 (2022)"
295        ),
296        rationale=(
297            "Trees handle irregular, non-smooth target functions and uninformative features, "
298            "which is what tabular data usually contains."
299        ),
300    ),
301    Practice(
302        id="pretrained-tabular-baseline",
303        statement=(
304            "Include a pretrained tabular model (TabPFN, TabICL) in the first round of "
305            "candidates: it costs one fit and is frequently competitive with a tuned ensemble."
306        ),
307        source=(
308            "Hollmann et al., 'Accurate predictions on small data with a tabular foundation "
309            "model' (TabPFN), Nature 637, 319-326 (2025); "
310            "Qu, Holzmuller, Varoquaux & Le Morvan, 'TabICL', ICML 2025"
311        ),
312        rationale=(
313            "In-context learning on tabular data removes the tuning budget that usually "
314            "separates a quick baseline from a competitive one."
315        ),
316    ),
317    Practice(
318        id="hold-out-whole-groups",
319        statement=(
320            "When rows share a group -- a subject, a site, a dataset -- validate by holding out "
321            "whole groups. A random split reports a number that will not survive deployment."
322        ),
323        source="Walsh et al., 'Machine learning reporting standards', Nature Methods 18 (2021)",
324        rationale=(
325            "Any feature constant within a group lets the model recognise the group rather "
326            "than generalise to it, and a random split puts the group on both sides."
327        ),
328    ),
329    Practice(
330        id="clean-noise-before-adding-capacity",
331        statement=(
332            "Spend the first effort on reducing noise in the data, not on a larger model. "
333            "Noise sets a ceiling that capacity cannot lift."
334        ),
335        source=(
336            "Zha et al., 'Data-centric AI: Perspectives and Challenges', "
337            "SIAM International Conference on Data Mining (SDM) 2023, 945-948"
338        ),
339        rationale=(
340            "Irreducible error from noisy features or labels bounds every model on that data, "
341            "so capacity spent against it buys nothing."
342        ),
343        expectations=(("ns_ratio", LOWERS),),
344    ),
345    Practice(
346        id="prefer-outlier-robust-learners",
347        statement=(
348            "On real-world data that has not been carefully curated, prefer a learner with "
349            "built-in robustness to outliers."
350        ),
351        source=(
352            "Grinsztajn, Oyallon & Varoquaux, NeurIPS 2022 Datasets and Benchmarks Track, "
353            "on non-smooth targets and outliers"
354        ),
355        rationale=(
356            "Real tabular data carries outliers that a squared-error learner chases and a "
357            "split-based or margin-based one largely ignores."
358        ),
359        # `nr_outliers` counts outliers in the *data*; the practice is about resistance to
360        # them in the *learner*. The expectation is on the data side only, and the verdict
361        # says why that does not settle the practice.
362        expectations=(("nr_outliers", LOWERS),),
363    ),
364    Practice(
365        id="capacity-is-not-free",
366        statement=(
367            "Match capacity to the problem. A larger, more expensive model is not a safer "
368            "default; on small tabular problems it is usually a worse one."
369        ),
370        source="Shwartz-Ziv & Armon, Information Fusion 81, 84-90 (2022)",
371        rationale=(
372            "Capacity beyond what the sample supports fits noise, and the cost is paid twice: "
373            "in accuracy and in the tuning budget needed to recover it."
374        ),
375        # The practice says more capacity is not safer, so it predicts that raising
376        # `Processing Units Number` does not raise MCC. `Model Capability` is the opposite
377        # claim in the same sentence -- capability *matched* to the problem does help -- and
378        # both are in the equation, so both are checkable.
379        expectations=(("Processing Units Number", LOWERS), ("Model Capability", RAISES)),
380    ),
381    Practice(
382        id="beat-the-trivial-baseline",
383        statement=(
384            "Before adopting a meta-learner to choose models, check it against 'use whatever "
385            "usually works'. Ranking is an easier problem than prediction and often needs less."
386        ),
387        source="Rice, 'The Algorithm Selection Problem' (1976); standard meta-learning practice",
388        rationale=(
389            "A per-model mean over previous datasets carries most of the ranking signal at "
390            "zero modelling cost, and is the baseline any selection method has to clear."
391        ),
392    ),
393    Practice(
394        id="report-excluded-runs",
395        statement=(
396            "Report which (dataset, model) runs were excluded and why. Aggregate comparisons "
397            "over an incomplete grid compare different models on different problems."
398        ),
399        source="Walsh et al., Nature Methods 18 (2021); benchmarking reporting standards",
400        rationale=(
401            "Runs usually go missing where a model struggles or will not fit, so exclusions "
402            "are correlated with the outcome being measured."
403        ),
404    ),
405    Practice(
406        id="use-a-balanced-metric",
407        statement=(
408            "Score imbalanced classification with a metric that accounts for all four "
409            "confusion-matrix cells -- MCC rather than accuracy or F1."
410        ),
411        source="Chicco & Jurman, BMC Genomics 21:6 (2020)",
412        rationale=(
413            "Accuracy and F1 can both look strong on a classifier that has learned only the majority class; MCC cannot."
414        ),
415    ),
416)
417
418_BY_ID = {practice.id: practice for practice in CATALOGUE}
419
420
421def _profile_the_data_first(evidence: Evidence) -> Verdict:
422    rows = {
423        row["knowing only"]: float(row["variance_explained"]) for row in evidence.decomposition.iter_rows(named=True)
424    }
425    dataset, model = rows.get("dataset identity", float("nan")), rows.get("model identity", float("nan"))
426    captured_dataset = evidence.scored("E1, dataset only") / dataset if dataset else float("nan")
427    gap = dataset - model
428    return Verdict(
429        practice=_BY_ID["profile-the-data-first"],
430        verdict=SUPPORTED if gap > 0.02 else QUALIFIED,
431        evidence=(
432            f"Knowing only which dataset a row came from explains {dataset:.1%} of MCC variance; "
433            f"knowing only which model, {model:.1%}. "
434            f"{len(DATASET_FEATURES)} dataset meta-features reach {captured_dataset:.0%} of what "
435            "dataset identity explains. The model-mean reference does not bound E2, because "
436            "Processing Units Number can vary with dataset shape; its score is reported "
437            "separately without a fraction-of-ceiling claim."
438        ),
439        magnitude=gap,
440    )
441
442
443def _tree_ensembles_first(evidence: Evidence) -> Verdict:
444    trees = evidence.families(TREE_FAMILIES)
445    neural = evidence.families(NEURAL_FAMILIES)
446    tabular_nn = evidence.family("tabular NN")
447    generic_nn = evidence.family("generic NN")
448    return Verdict(
449        practice=_BY_ID["tree-ensembles-first"],
450        verdict=SUPPORTED if trees - neural > 0.05 else QUALIFIED,
451        evidence=(
452            f"On the {evidence.n_complete_datasets} datasets where every model ran, tree-based "
453            f"families average MCC {trees:.3f} against {neural:.3f} for neural ones. The neural "
454            f"side splits sharply: architectures built for tabular data reach {tabular_nn:.3f} "
455            f"while a plain MLP or DNN reaches {generic_nn:.3f}, last of the ten families. The "
456            "advice holds, and it holds most strongly against exactly the architectures that are "
457            "not designed for this kind of data."
458        ),
459        magnitude=trees - neural,
460    )
461
462
463def _pretrained_tabular_baseline(evidence: Evidence) -> Verdict:
464    foundation = evidence.family("tabular foundation")
465    best_tree = evidence.complete_family_means.filter(pl.col("family").is_in(list(TREE_FAMILIES)))
466    top = float(np.max(best_tree["mcc"].to_numpy())) if best_tree.height else float("nan")
467    order = evidence.complete_model_means[MODEL_COLUMN].to_list()
468    ranks = {name: index + 1 for index, name in enumerate(order)}
469    placings = sorted(ranks.get(name, 0) for name in ("TabICL", "TabPFN") if name in ranks)
470    return Verdict(
471        practice=_BY_ID["pretrained-tabular-baseline"],
472        verdict=SUPPORTED if foundation >= top - 0.01 else QUALIFIED,
473        evidence=(
474            f"Pretrained tabular models average MCC {foundation:.3f}, against {top:.3f} for the "
475            f"best tree family, and place {' and '.join(str(place) for place in placings)} of "
476            f"{len(order)} models. They match the strongest tree ensembles here without a tuning "
477            "budget, which is the whole of the claim."
478        ),
479        magnitude=foundation - top,
480    )
481
482
483def _hold_out_whole_groups(evidence: Evidence) -> Verdict:
484    random = evidence.protocol("random 10-fold (leaky)")
485    grouped = evidence.protocol("leave-one-dataset-out")
486    inflation = random - grouped
487    return Verdict(
488        practice=_BY_ID["hold-out-whole-groups"],
489        verdict=SUPPORTED if inflation > 0.02 else QUALIFIED,
490        evidence=(
491            f"The same equation scores R² {random:.3f} under a random 10-fold split and "
492            f"{grouped:.3f} when whole datasets are held out -- {inflation:.3f} of pure "
493            "protocol. Dataset meta-features are constant within a dataset, so a random fold "
494            "shows the equation rows from a dataset it is being scored on."
495        ),
496        magnitude=inflation,
497    )
498
499
500def _clean_noise_before_adding_capacity(evidence: Evidence) -> Verdict:
501    effect = evidence.practice_effect("ns_ratio")
502    if np.isnan(effect):
503        return Verdict(
504            practice=_BY_ID["clean-noise-before-adding-capacity"],
505            verdict=NOT_TESTED,
506            evidence="No noise-to-signal practice survived the extraction filters in this run.",
507            magnitude=float("nan"),
508        )
509    return Verdict(
510        practice=_BY_ID["clean-noise-before-adding-capacity"],
511        verdict=SUPPORTED if effect < -0.05 else QUALIFIED,
512        evidence=(
513            f"Noise-to-signal ratio moves predicted MCC by {effect:.2f} between its lowest and "
514            "highest decile, and it is one of only two features whose direction inside the "
515            "equation agrees with its own correlation against MCC -- so it is not an artefact of "
516            "conditioning. Nothing on the model side of the equation offsets it."
517        ),
518        magnitude=effect,
519    )
520
521
522def _prefer_outlier_robust_learners(evidence: Evidence) -> Verdict:
523    """Untestable on this corpus, and it will stay that way.
524
525    The practice needs a column saying whether a *learner* resists outliers.
526    The generated corpus does not contain such a learner descriptor.
527
528    The nearby column is not a substitute. ``nr_outliers`` counts outliers in the *data*,
529    which is a property of the problem rather than of the learner, so a term over it
530    answers "do outliers hurt?" and not "does robustness help?". Re-pointing the check at
531    it would produce a verdict that reads as though the practice had been tested.
532
533    Nor can it be recovered: no model may be re-run and no descriptor measured under this
534    study's standing constraints, so the only route left is asserting a robustness ordinal
535    from the literature, which
536    would test the assertion rather than the practice. This returns `NOT_TESTED`
537    unconditionally and says why, which is the honest report of a practice this study
538    cannot weigh.
539    """
540    return Verdict(
541        practice=_BY_ID["prefer-outlier-robust-learners"],
542        verdict=NOT_TESTED,
543        evidence=(
544            "This corpus cannot weigh it. The practice is about a property of the learner, "
545            "and the generated meta-dataset has no learner-robustness descriptor. "
546            "`nr_outliers` counts outliers in the data, not resistance to them in the model, "
547            "so it answers a different question. Reported as untested rather than answered "
548            "with the nearest available number."
549        ),
550        magnitude=float("nan"),
551    )
552
553
554def _capacity_is_not_free(evidence: Evidence) -> Verdict:
555    generic = evidence.family("generic NN")
556    trees = evidence.families(TREE_FAMILIES)
557    capacity_terms = evidence.practice_terms("Processing Units Number")
558    equation_says = (
559        f" The equation says it conditionally rather than flatly: `Processing Units Number` "
560        f"appears in {capacity_terms} of its terms alongside other properties; "
561        "the observed association depends on other recorded properties. It does not isolate "
562        "a capacity intervention."
563        if capacity_terms
564        else " Capacity does not survive into the equation's terms in this run, so the family "
565        "means are the whole of the evidence here."
566    )
567    return Verdict(
568        practice=_BY_ID["capacity-is-not-free"],
569        verdict=SUPPORTED if generic < trees - 0.05 else QUALIFIED,
570        evidence=(
571            f"Generic neural networks average MCC {generic:.3f} against {trees:.3f} for tree ensembles.{equation_says}"
572        ),
573        magnitude=generic - trees,
574    )
575
576
577def _beat_the_trivial_baseline(evidence: Evidence) -> Verdict:
578    """Did the meta-learner actually beat "use whatever usually works"?
579
580    Two things this check must not do, both of which it did.
581
582    It must not decide on Spearman. That column sits between 0.63 and 0.73 for every
583    predictor and every baseline on this corpus, including a constant, so the two sides
584    were once separated by 0.0006 -- a margin that chose a published verdict while
585    measuring nothing. `validate.ranking_report` says so where it is computed.
586
587    It must not decide on a difference of means over twenty folds. A mean difference does
588    not say whether the advantage is broad or driven by a few large wins. The verdict
589    therefore runs `validate.paired_comparison` on the per-dataset scores and reads the
590    interval, not the mean alone.
591
592    The practice claims the trivial baseline is *competitive*, so an inconclusive paired
593    test is the practice being right rather than a failure to measure. Only a baseline that
594    loses on a majority of metrics with an interval excluding zero can challenge it.
595    """
596    selection, baseline = evidence.selection_per_group, evidence.baseline_per_group
597    usable = [
598        name
599        for name in RANKING_METRICS
600        if name in selection.columns
601        and name in baseline.columns
602        and selection.height == baseline.height
603        and selection.height > 1
604    ]
605    if not usable:
606        return Verdict(
607            practice=_BY_ID["beat-the-trivial-baseline"],
608            verdict=NOT_TESTED,
609            evidence="No per-dataset ranking table was available to pair in this run.",
610            magnitude=float("nan"),
611        )
612
613    results = {
614        name: paired_comparison(
615            selection[name].to_numpy().astype(np.float64),
616            baseline[name].to_numpy().astype(np.float64),
617            lower_is_better=name == "regret",
618        )
619        for name in usable
620    }
621    decisive = [name for name, result in results.items() if result.significant]
622    equation_ahead = [name for name in decisive if results[name].mean > 0.0]
623
624    if decisive and not equation_ahead:
625        verdict = SUPPORTED
626        summary = "and the equation loses on a margin this corpus can actually resolve"
627    elif len(equation_ahead) > len(usable) / 2:
628        verdict = CHALLENGED
629        summary = "and the equation clears it by a margin that survives a paired test"
630    else:
631        verdict = QUALIFIED
632        summary = (
633            "and the two cannot be separated -- which is the practice being right, since it "
634            "claims the trivial baseline is competitive rather than that it wins"
635        )
636
637    detail = "; ".join(
638        f"{name} {evidence.selection_ranking[name]:.3f} against "
639        f"{evidence.baseline_ranking[name]:.3f}, equation better on {results[name].wins} of "
640        f"{results[name].n} datasets that differ, 95% CI "
641        f"[{results[name].low:+.3f}, {results[name].high:+.3f}]"
642        for name in usable
643    )
644    return Verdict(
645        practice=_BY_ID["beat-the-trivial-baseline"],
646        verdict=verdict,
647        evidence=(
648            f"Tested against this study's own equation {summary}. Ranking models within a "
649            f"held-out dataset, against the per-model-mean baseline -- {detail}. Every "
650            "interval is a paired bootstrap over the twenty held-out datasets, because a "
651            "difference of two means over twenty folds is not yet a measurement."
652        ),
653        magnitude=results[usable[0]].mean,
654    )
655
656
657def _report_excluded_runs(evidence: Evidence) -> Verdict:
658    share = evidence.missing_cells / evidence.total_cells if evidence.total_cells else 0.0
659    return Verdict(
660        practice=_BY_ID["report-excluded-runs"],
661        verdict=SUPPORTED,
662        evidence=(
663            f"{evidence.missing_cells} of {evidence.total_cells} (dataset, model) cells are "
664            f"absent -- {share:.0%} -- and they are not absent at random: eight models are "
665            f"missing from datasets excluded by the minimum-size policy. Measuring the bias rather than assuming it "
666            f"is small: restricting to the {evidence.n_complete_datasets} complete datasets moves "
667            f"the model ranking by Spearman {evidence.ranking_agreement:.3f}, so the ordering "
668            "survives, but a mean over all rows still compares eight of the models on a "
669            "different set of problems from the rest. Every family figure quoted here uses the "
670            "complete subset for that reason."
671        ),
672        magnitude=share,
673    )
674
675
676def _use_a_balanced_metric(evidence: Evidence) -> Verdict:
677    truth = evidence.frame[TARGET_COLUMN].to_numpy().astype(np.float64)
678    zeros = int(np.sum(truth == 0.0))
679    return Verdict(
680        practice=_BY_ID["use-a-balanced-metric"],
681        verdict=NOT_TESTED,
682        evidence=(
683            "This study adopts MCC as its target and never measures an alternative, so it is not "
684            f"evidence for the practice. What it does show is the shape MCC has: {zeros} of "
685            f"{truth.shape[0]} rows sit at exactly 0, which is a classifier that has learned "
686            "nothing being scored as having learned nothing. Accuracy would not have said that."
687        ),
688        magnitude=float(zeros) / float(truth.shape[0]),
689    )
690
691
692CHECKS: dict[str, Callable[[Evidence], Verdict]] = {
693    "profile-the-data-first": _profile_the_data_first,
694    "tree-ensembles-first": _tree_ensembles_first,
695    "pretrained-tabular-baseline": _pretrained_tabular_baseline,
696    "hold-out-whole-groups": _hold_out_whole_groups,
697    "clean-noise-before-adding-capacity": _clean_noise_before_adding_capacity,
698    "prefer-outlier-robust-learners": _prefer_outlier_robust_learners,
699    "capacity-is-not-free": _capacity_is_not_free,
700    "beat-the-trivial-baseline": _beat_the_trivial_baseline,
701    "report-excluded-runs": _report_excluded_runs,
702    "use-a-balanced-metric": _use_a_balanced_metric,
703}
704
705
706def assess(frame: pl.DataFrame, report: Report) -> list[Verdict]:
707    """Weigh every catalogued practice against this study, in catalogue order."""
708    evidence = gather(frame, report)
709    return [CHECKS[practice.id](evidence) for practice in CATALOGUE]
710
711
712#: Below this rank correlation a (term, feature) pair has no direction worth stating. The same
713#: floor `practices.MIN_DIRECTION` applies to the per-feature statements, and for the same
714#: reason: a correlation of 0.05 between a feature and a term's contribution is the other
715#: features that term contains moving, not the feature being asked about. Without it the table
716#: reports a sign for a pairing that has none, which reads as a disagreement.
717MIN_TERM_DIRECTION = 0.15
718
719
720def feature_position(term: Term, feature: str) -> str:
721    """Where a feature sits inside a term: ``numerator``, ``denominator``, or ``factor``.
722
723    **This is what makes a sign readable**, and without it the table looks self-contradictory.
724    A feature in a denominator enters the term inverted, so a negatively-weighted ratio
725    contributes *more* as that feature rises -- which is arithmetic, not a disagreement with
726    whatever the practice claims. Printing the position next to the direction is what lets a
727    reader see that ``eq_num_attr / PUN`` and ``PUN / nr_class`` carrying opposite signs is one
728    coherent statement about a ratio rather than two conflicting ones about a quantity.
729    """
730    match term.operation:
731        case "ratio":
732            below = 1
733        case "sum_ratio":
734            below = 2
735        case "ratio_of_sums":
736            below = 2
737        case _:
738            return "factor"
739    for index, operand in enumerate(term.operands):
740        if feature in set(operand.features if isinstance(operand, Term) else (operand.feature,)):
741            return "denominator" if index >= below else "numerator"
742    return "factor"
743
744
745def equation_evidence(report: Report, columns: dict[str, np.ndarray]) -> pl.DataFrame:
746    """Each practice paired with the **terms** of the published equation that carry it.
747
748    **The equation is a sum of terms, so a term is the unit a practice has to be checked
749    against.** A raw feature is not: `Processing Units Number` can appear in several terms,
750    in numerators and denominators and under different transforms, and collapsing that
751    into one per-feature direction throws away exactly what a reader wants -- which part of
752    the equation encodes the advice, how strongly, and with what sign.
753
754    One row per (practice, term) pair. A term is paired with a practice when it contains a
755    feature the practice makes a claim about.
756
757    ``beta`` is the **strength**: the standardised weight, already in MCC units because the
758    target is centred but never scaled, so it is comparable across terms whose raw units have
759    nothing to do with each other. ``effect`` is what the term is worth on this data -- the
760    swing in its contribution across the middle 80% of its range -- because a large weight on
761    a term that barely varies is not important.
762
763    ``direction`` is the **sign**, and it is *measured* rather than derived. Reading it off the
764    weight would be wrong as soon as the feature sits in a denominator or under a reciprocal,
765    which several of these do: the rank correlation between the feature and the contribution
766    that term actually makes is the only thing that answers "as this rises, what does this
767    term do to predicted MCC". ``agrees`` compares that with what the practice predicts.
768
769    A practice can therefore be **encoded by several terms that disagree with each other**,
770    and that is a finding rather than a defect: it means the equation says the effect is
771    conditional on which other quantity the feature is measured against.
772    """
773    equation = report.e3.equation
774    # `EquationReport.stability` is optional, and an absent one means "the folds were never
775    # re-run" rather than "no term was ever reselected". The column comes back as NaN in that
776    # case, which reads as unknown; a zero would read as a term the folds rejected.
777    fold_choices = report.e3.stability
778    stability = (
779        {row["term"]: float(row["frequency"]) for row in fold_choices.to_dicts()}
780        if fold_choices is not None and fold_choices.height
781        else {}
782    )
783    contributions = attribution_contributions(equation, columns)
784    betas = dict(zip((term.name for term in equation.terms), equation.standardized_weights, strict=True))
785
786    rows: list[dict[str, object]] = []
787    for practice in CATALOGUE:
788        for feature, expected in practice.expectations:
789            carrying = [(index, term) for index, term in enumerate(equation.terms) if feature in set(term.features)]
790            if not carrying:
791                rows.append(
792                    {
793                        "practice": practice.id,
794                        "feature": feature,
795                        "expected": expected,
796                        "term": "",
797                        "position": "",
798                        "beta": float("nan"),
799                        "effect": float("nan"),
800                        "stability": float("nan"),
801                        "rho": float("nan"),
802                        "direction": "",
803                        "agrees": "not selected",
804                    }
805                )
806                continue
807            for index, term in carrying:
808                share = contributions[:, index]
809                low, high = np.percentile(share, [10.0, 90.0])
810                slope = spearman(columns[feature], share)
811                weak = abs(slope) < MIN_TERM_DIRECTION
812                observed = "" if weak else (RAISES if slope > 0.0 else LOWERS)
813                rows.append(
814                    {
815                        "practice": practice.id,
816                        "feature": feature,
817                        "expected": expected,
818                        "term": term.name,
819                        "position": feature_position(term, feature),
820                        "beta": float(betas[term.name]),
821                        "effect": float(high - low),
822                        "stability": stability.get(term.name, float("nan")),
823                        "rho": float(slope),
824                        "direction": observed,
825                        "agrees": "no direction" if weak else ("yes" if observed == expected else "no"),
826                    }
827                )
828    return pl.DataFrame(
829        rows,
830        schema={
831            "practice": pl.String,
832            "feature": pl.String,
833            "expected": pl.String,
834            "term": pl.String,
835            "position": pl.String,
836            "beta": pl.Float64,
837            "effect": pl.Float64,
838            "stability": pl.Float64,
839            "rho": pl.Float64,
840            "direction": pl.String,
841            "agrees": pl.String,
842        },
843    )
844
845
846def equation_coverage(report: Report, columns: dict[str, np.ndarray]) -> dict[str, int]:
847    """How much of the catalogue the equation's *terms* can be held against, and what they say.
848
849    Kept separate from the verdict tally, and the distinction is the point. A verdict is drawn
850    from corpus statistics -- family means, variance shares, paired tests -- so "5 supported"
851    says the advice holds on these twenty datasets, which any study with this corpus could
852    establish and which the fitted equation plays no part in. This counts the narrower and
853    harder thing: how many of the equation's terms carry a practice at all, and how many of those
854    pairings come out the way the practice predicts.
855
856    Counted over (practice, term) pairs rather than over practices, because a practice carried
857    by five terms that disagree with each other is not one verdict -- it is five readings, and
858    collapsing them would hide the disagreement that makes the equation worth reading.
859    """
860    evidence = equation_evidence(report, columns)
861    verdicts = evidence["agrees"].to_list()
862    return {
863        "practices": len(CATALOGUE),
864        "with_feature_claims": sum(1 for practice in CATALOGUE if practice.expectations),
865        "pairs": sum(1 for verdict in verdicts if verdict in ("yes", "no")),
866        "agree": verdicts.count("yes"),
867        "disagree": verdicts.count("no"),
868        "undirected": verdicts.count("no direction"),
869        "unselected": verdicts.count("not selected"),
870        # How many of the disagreements are the same feature entering as a denominator. If
871        # every one of them is, the "disagreement" is arithmetic rather than a conflict: a
872        # negatively-weighted ratio contributes more as its denominator grows, so a marginal
873        # expectation about the feature can never match a term that only speaks about a ratio.
874        "disagree_in_denominator": sum(
875            1 for row in evidence.to_dicts() if row["agrees"] == "no" and row["position"] == "denominator"
876        ),
877        "terms": len(report.e3.equation.terms),
878        "carrying_terms": len({term for term in evidence["term"].to_list() if term}),
879    }
880
881
882def as_table(verdicts: list[Verdict]) -> pl.DataFrame:
883    """The verdicts as a table, for the report and for a paper."""
884    if not verdicts:
885        return pl.DataFrame(
886            schema={
887                "id": pl.String,
888                "practice": pl.String,
889                "verdict": pl.String,
890                "magnitude": pl.Float64,
891                "source": pl.String,
892            }
893        )
894    return pl.DataFrame(
895        [
896            {
897                "id": verdict.practice.id,
898                "practice": verdict.practice.statement,
899                "verdict": verdict.verdict,
900                "magnitude": verdict.magnitude,
901                "source": verdict.practice.source,
902            }
903            for verdict in verdicts
904        ]
905    )
906
907
908def render(verdicts: list[Verdict]) -> str:
909    """The verdicts as readable markdown, one section per practice."""
910    if not verdicts:
911        return "_No practices assessed._"
912    counts: dict[str, int] = {}
913    for verdict in verdicts:
914        counts[verdict.verdict] = counts.get(verdict.verdict, 0) + 1
915    summary = ", ".join(f"{count} {name}" for name, count in sorted(counts.items()))
916
917    lines = [f"{len(verdicts)} practices assessed: {summary}.\n"]
918    for index, verdict in enumerate(verdicts, start=1):
919        lines.append(f"### {index}. {verdict.practice.statement}\n")
920        lines.append(f"**Verdict: {verdict.verdict}.** {verdict.evidence}\n")
921        lines.append(f"*Practice from:* {verdict.practice.source}. {verdict.practice.rationale}\n")
922    return "\n".join(lines)