ml_meta_perf.practices

  1"""Turning a fitted equation into written guidance.
  2
  3This is the payoff the whole project trades accuracy for, so it is worth being exact
  4about what it is and is not. A practice here is a statement of the form "across this
  5meta-dataset, higher X went with higher/lower MCC, worth about D MCC points". That is an
  6**association measured on 20 datasets**, not a causal claim and not a guarantee; the
  7`confidence` column carries the evidence needed to discount it.
  8
  9The derivation is deliberately empirical rather than symbolic. Reading a sign off a
 10weight is wrong as soon as a feature appears in more than one term, or appears inside a
 11ratio's denominator, or appears under a transform that flips its monotonicity. Instead
 12every term's contribution is evaluated over the real data, contributions are summed per
 13feature, and the direction is the rank correlation between the feature and the total it
 14drives. That answers "what does this equation actually do as this feature rises", which
 15is the question a practitioner is asking.
 16
 17Study chapter: [6. Best practices against the equation][study-chapter] -- the rationale, in
 18prose, with the figures.
 19
 20[study-chapter]: https://github.com/mariolpantunes/ml-meta-perf/blob/main/assets/docs/06-practices.md
 21"""
 22
 23from __future__ import annotations
 24
 25import numpy as np
 26import polars as pl
 27
 28from ml_meta_perf.attribution import contributions
 29from ml_meta_perf.data import FEATURE_GLOSSARY
 30from ml_meta_perf.model import Equation
 31from ml_meta_perf.stats import spearman
 32
 33#: A feature has to move predicted MCC by at least this much, between the bottom and top
 34#: decile of its own range, before it is worth writing down as guidance.
 35MIN_EFFECT = 0.02
 36
 37#: And it has to have been selected in at least this fraction of cross-validation folds.
 38#: A term picked in 4 of 20 folds is an artefact of which datasets landed in the training
 39#: split; publishing it as advice would be the most misleading thing this module could do.
 40MIN_STABILITY = 0.5
 41
 42#: And its relationship has to be monotone enough for "higher X" to mean anything. A
 43#: feature can carry a large effect through a non-monotone term -- a product with another
 44#: feature that changes sign, say -- and no directional sentence would be true of it.
 45MIN_DIRECTION = 0.15
 46
 47#: Fraction of rows taken as the low and high end when measuring an effect.
 48DECILE = 0.1
 49
 50
 51def feature_practices(
 52    equation: Equation,
 53    columns: dict[str, np.ndarray],
 54    stability: pl.DataFrame | None = None,
 55) -> pl.DataFrame:
 56    """One row per raw feature the equation uses, strongest effect first.
 57
 58    ``effect`` is signed: the mean MCC this feature's terms contribute in its top decile
 59    minus the mean they contribute in its bottom decile. Ordering by the feature is the
 60    whole point -- a quantile of the contributions alone would be order-invariant and
 61    would describe the term's spread rather than its response to the feature.
 62
 63    ``direction`` is the rank correlation between the feature and the MCC it drives, so
 64    it survives transforms and multiple appearances. ``stability`` is the mean selection
 65    frequency of the terms involved, when a cross-validated stability table is supplied.
 66    A published-equation term absent from a non-empty stability table was selected in zero
 67    folds and therefore contributes zero to that mean. ``NaN`` is reserved for the case in
 68    which no stability evaluation was supplied.
 69
 70    A feature appearing in a term shared with another feature is credited the whole of
 71    that term, so two features sharing one dominant term will show the same effect. That
 72    is honest -- the term cannot be split between them -- but it means ``effect`` reads
 73    as "how much MCC moves across this feature's range", not "how much this feature owns".
 74    """
 75    matrix = contributions(equation, columns)
 76    frequency = _frequency_lookup(stability)
 77
 78    features = sorted({feature for term in equation.terms for feature in term.features})
 79    if not features:
 80        # An empty frame built from an empty row list carries no schema, so every later
 81        # column reference fails. Declare it instead.
 82        return pl.DataFrame(
 83            schema={
 84                "feature": pl.String,
 85                "meaning": pl.String,
 86                "n_terms": pl.Int64,
 87                "direction": pl.Float64,
 88                "effect": pl.Float64,
 89                "stability": pl.Float64,
 90            }
 91        )
 92    rows: list[dict[str, object]] = []
 93    for feature in features:
 94        indices = [index for index, term in enumerate(equation.terms) if feature in term.features]
 95        total = matrix[:, indices].sum(axis=1)
 96        values = columns[feature]
 97        order = np.argsort(values)
 98        edge = max(1, round(DECILE * values.shape[0]))
 99        effect = float(total[order[-edge:]].mean() - total[order[:edge]].mean())
100        rows.append(
101            {
102                "feature": feature,
103                "meaning": FEATURE_GLOSSARY.get(feature, feature),
104                "n_terms": len(indices),
105                "direction": spearman(values, total),
106                "effect": effect,
107                "stability": float(
108                    np.mean([frequency.get(equation.terms[index].name, 0.0) for index in indices])
109                )
110                if frequency
111                else float("nan"),
112            }
113        )
114    return pl.DataFrame(rows).sort(pl.col("effect").abs(), descending=True)
115
116
117def _frequency_lookup(stability: pl.DataFrame | None) -> dict[str, float]:
118    if stability is None or stability.height == 0:
119        return {}
120    return dict(zip(stability["term"].to_list(), stability["frequency"].to_list(), strict=True))
121
122
123def _confidence(effect: float, stability: float) -> str:
124    if np.isnan(stability):
125        return "unrated"
126    if stability >= 0.85 and abs(effect) >= 0.10:
127        return "strong"
128    if stability >= MIN_STABILITY and abs(effect) >= 0.05:
129        return "moderate"
130    return "weak"
131
132
133def best_practices(
134    equation: Equation,
135    columns: dict[str, np.ndarray],
136    stability: pl.DataFrame | None = None,
137    *,
138    min_effect: float = MIN_EFFECT,
139    min_stability: float = MIN_STABILITY,
140    min_direction: float = MIN_DIRECTION,
141) -> pl.DataFrame:
142    """Written practices, filtered to those the evidence actually supports.
143
144    A feature is reported when it moves predicted MCC by at least ``min_effect``, its
145    relationship is monotone enough to describe in a sentence (``min_direction``), and
146    its terms survived at least ``min_stability`` of the cross-validation folds. Features
147    failing any test are dropped rather than reported with a caveat -- a practice nobody
148    should act on is better left unwritten.
149    """
150    table = feature_practices(equation, columns, stability)
151    if table.height == 0:
152        return table.with_columns(pl.lit("").alias("practice"), pl.lit("").alias("confidence"))
153
154    kept = table.filter(
155        (pl.col("effect").abs() >= min_effect)
156        & (pl.col("direction").abs() >= min_direction)
157        # The two measurements have to agree on the sign, and occasionally they do not.
158        # ``direction`` is a rank correlation over the whole range; ``effect`` compares
159        # the top decile against the bottom. A feature can be monotonically negative
160        # overall and still contribute more MCC in its top decile than its bottom -- and
161        # the sentence would then say "went with lower MCC" while quoting a positive
162        # decile difference as its evidence, which is worse than saying nothing. On the
163        # published equation this drops exactly one feature, inference cost, whose
164        # direction is -0.17 against a decile difference of +0.03.
165        & (pl.col("direction").sign() == pl.col("effect").sign())
166    )
167    if stability is not None:
168        kept = kept.filter(pl.col("stability").is_nan() | (pl.col("stability") >= min_stability))
169
170    statements: list[str] = []
171    confidences: list[str] = []
172    for row in kept.iter_rows(named=True):
173        rises = float(row["direction"]) > 0.0
174        statements.append(
175            f"Higher {row['meaning']} went with {'higher' if rises else 'lower'} MCC "
176            f"(about {abs(float(row['effect'])):.2f} MCC between its lowest and highest decile)."
177        )
178        confidences.append(_confidence(float(row["effect"]), float(row["stability"])))
179
180    return kept.with_columns(
181        pl.Series("practice", statements),
182        pl.Series("confidence", confidences),
183    )
184
185
186def concordance(
187    first: Equation,
188    second: Equation,
189    columns: dict[str, np.ndarray],
190    *,
191    names: tuple[str, str] = ("first", "second"),
192) -> pl.DataFrame:
193    """Do two independently-discovered equations give the same advice?
194
195    This is the question that decides whether the extracted practices describe *the data*
196    or merely describe *one search*. Fit two equations by methods that share no machinery
197    -- an enumerated library with beam selection, and a dendrogram cut with no selection
198    at all -- then compare the guidance rather than the R2.
199
200    Agreement on **direction** is the claim that matters. If a short, less accurate
201    equation and a long, more accurate one both say that higher class-centre separation
202    goes with lower MCC, that statement is a property of the meta-dataset, and the
203    accuracy gap between the two equations is beside the point.
204
205    Agreement on **effect magnitude** is a separate and weaker claim, reported alongside
206    so the two are not conflated. Two equations can agree on every direction while
207    disagreeing on which effect is largest.
208    """
209    left = {row["feature"]: row for row in feature_practices(first, columns).iter_rows(named=True)}
210    right = {row["feature"]: row for row in feature_practices(second, columns).iter_rows(named=True)}
211
212    rows: list[dict[str, object]] = []
213    for feature in sorted(set(left) & set(right)):
214        a, b = left[feature], right[feature]
215        rows.append(
216            {
217                "feature": feature,
218                f"direction_{names[0]}": a["direction"],
219                f"direction_{names[1]}": b["direction"],
220                "agrees": bool(np.sign(a["direction"]) == np.sign(b["direction"])),
221                f"effect_{names[0]}": a["effect"],
222                f"effect_{names[1]}": b["effect"],
223            }
224        )
225    if not rows:
226        return pl.DataFrame(schema={"feature": pl.String, "agrees": pl.Boolean})
227    return pl.DataFrame(rows)
228
229
230def concordance_summary(table: pl.DataFrame, names: tuple[str, str] = ("first", "second")) -> dict[str, float]:
231    """Headline numbers from a concordance table: agreement rate and effect-rank correlation."""
232    if table.height == 0 or "agrees" not in table.columns:
233        return {"shared": 0.0, "direction_agreement": float("nan"), "effect_rank_correlation": float("nan")}
234    agreement = float(table["agrees"].sum()) / table.height
235    left = np.abs(table[f"effect_{names[0]}"].to_numpy())
236    right = np.abs(table[f"effect_{names[1]}"].to_numpy())
237    return {
238        "shared": float(table.height),
239        "direction_agreement": agreement,
240        "effect_rank_correlation": spearman(left, right),
241    }
242
243
244def render(practices: pl.DataFrame) -> str:
245    """The practices as a numbered, readable list."""
246    if practices.height == 0:
247        return "No practice met the effect and stability thresholds."
248    lines: list[str] = []
249    for index, row in enumerate(practices.iter_rows(named=True), start=1):
250        lines.append(f"{index:2d}. [{row['confidence']:8s}] {row['practice']}")
251    return "\n".join(lines)