ml_meta_perf.cli

  1"""The one entry point that runs every phase of the study: ``python -m ml_meta_perf``.
  2
  3Given a meta-dataset it screens the term library, fits E1, E2, E3-Valid and E3-MAX,
  4cross-validates them under the retained grouped protocols, extracts the
  5practices, writes the figures and generates the written report -- in one command, from
  6one set of parameters, so a result can be reproduced by repeating the command line rather
  7than by rerunning a notebook in the right order.
  8
  9Every hyperparameter comes from ``config/study.json`` -- the command line holds no tuned
 10default of its own -- so a bare ``python -m ml_meta_perf`` reproduces the reported run. The
 11file is read with `jsonargparse`: ``--config other.json`` replaces it, any field can be
 12overridden as a dotted flag (``--search.penalty 3``, ``--selection.delta 0.02``), and
 13``--print_config`` shows the effective configuration.
 14
 15Study chapter: [4. The equation][study-chapter] -- the rationale, in
 16prose, with the figures.
 17
 18[study-chapter]: https://github.com/mariolpantunes/ml-meta-perf/blob/main/assets/docs/04-equation.md
 19"""
 20
 21from __future__ import annotations
 22
 23import sys
 24import time
 25from pathlib import Path
 26
 27import numpy as np
 28import polars as pl
 29from jsonargparse import ActionConfigFile, ArgumentParser, Namespace  # pyright: ignore[reportPrivateImportUsage]
 30
 31from ml_meta_perf.config import (
 32    DEFAULT_CONFIG_PATH,
 33    Configuration,
 34    OpaqueConfig,
 35    SelectionConfig,
 36    SweepConfig,
 37)
 38from ml_meta_perf.data import DATASET_FEATURES, DEFAULT_PATH, MODEL_FEATURES, columns_as_arrays, load, target
 39from ml_meta_perf.experiment import Report, run
 40from ml_meta_perf.practices import render as render_practices
 41from ml_meta_perf.report import term_importance, write_into_chapters
 42
 43#: What ``--phase`` accepts. ``all`` is the default and is what the study runs.
 44PHASES = ("screen", "equations", "validation", "practices", "figures", "report")
 45PROTOCOL_LABELS = {"loo_dataset": "LODO", "loo_model": "LOMO", "loo_cell": "DHO"}
 46
 47
 48def _section(title: str) -> None:
 49    print(f"\n{'=' * 78}\n{title}\n{'=' * 78}")
 50
 51
 52def _show(frame: pl.DataFrame) -> None:
 53    with pl.Config(tbl_rows=60, tbl_cols=20, tbl_width_chars=200, float_precision=3):
 54        print(frame)
 55
 56
 57def _configure_windows_output() -> None:
 58    """Keep Polars' Unicode table borders printable under a legacy Windows code page."""
 59    if sys.platform != "win32":
 60        return
 61    reconfigure = getattr(sys.stdout, "reconfigure", None)
 62    if callable(reconfigure):
 63        reconfigure(encoding="utf-8", errors="replace")
 64
 65
 66def render(
 67    report: Report,
 68    phases: frozenset[str] = frozenset(PHASES),
 69    columns: dict[str, np.ndarray] | None = None,
 70) -> None:
 71    """Print the full study to stdout, restricted to the requested phases.
 72
 73    ``columns`` is the feature matrix the report was fitted on. It is passed in rather
 74    than reloaded because reloading would use the default meta-dataset, which is the
 75    wrong one whenever ``--data`` was given.
 76    """
 77    if columns is None:
 78        columns = columns_as_arrays(load(), DATASET_FEATURES + MODEL_FEATURES)
 79
 80    if "screen" in phases:
 81        _section("Correlation screening -- linear vs monotone association with MCC")
 82        _show(report.correlations)
 83
 84    if "equations" in phases:
 85        print(
 86            "Evaluation protocols: in-sample (IS), leave-one-dataset-out (LODO), "
 87            "leave-one-model-out (LOMO), and doubly held out (DHO)."
 88        )
 89        _section("E1 -- dataset features only (fitted on all rows)")
 90        print(report.e1.equation)
 91        print(f"\nIS: {report.e1.in_sample}")
 92        for label, scores in report.e1.cross_validated.items():
 93            print(f"{PROTOCOL_LABELS.get(label, label)}: {scores}")
 94        print("\naccuracy vs number of terms:")
 95        _show(report.e1.curve)
 96
 97        _section("E2 -- model features only (the control for 'model choice dominates')")
 98        print(report.e2.equation)
 99        print(f"\nIS: {report.e2.in_sample}")
100        for label, scores in report.e2.cross_validated.items():
101            print(f"{PROTOCOL_LABELS.get(label, label)}: {scores}")
102
103        _section("E3 -- dataset + model features (fitted on all rows)")
104        print(report.e3.equation)
105        print(f"\nIS: {report.e3.in_sample}")
106        for label, scores in report.e3.cross_validated.items():
107            print(f"{PROTOCOL_LABELS.get(label, label)}: {scores}")
108        print("\naccuracy vs number of terms:")
109        _show(report.e3.curve)
110        if report.e3.stability is not None:
111            print("\nterm stability across LODO folds:")
112            _show(report.e3.stability.head(20))
113
114    if "practices" in phases:
115        _section("Where the signal lives")
116        print("variance of MCC explained by identity alone (no equation involved):")
117        _show(report.decomposition)
118        print("\nshare of E3's output variance, by which features its terms use:")
119        _show(report.shares)
120
121        _section("Term importance -- ranked by standardised weight, major terms flagged")
122        _show(term_importance(report.e3.equation, columns, DATASET_FEATURES, MODEL_FEATURES, report.e3.stability))
123
124        _section("Extracted practices")
125        print(render_practices(report.practices))
126        print("\nevidence:")
127        _show(report.practices.select("feature", "n_terms", "direction", "effect", "stability", "confidence"))
128
129        _section("What each term is worth (MCC units)")
130        _show(report.effects)
131
132    if "validation" in phases:
133        _section("E3-Valid and E3-MAX selection")
134        _show(report.grammars)
135
136        _section("E1 vs E2 vs E3 on the common scale (all rows)")
137        _show(report.comparison)
138
139        _section("Oracle ladder -- what each interaction component would be worth")
140        _show(report.oracles)
141        _section("How much of the leading interaction the equation reaches")
142        _show(report.interaction)
143
144        _section("Baselines")
145        _show(report.baselines)
146
147        _section("Validation protocol matters (same equation, different splits)")
148        _show(report.leakage)
149
150        _section("Go/no-go decision quality on held-out datasets")
151        _show(report.decision)
152
153        _section("Model selection on held-out datasets")
154        _show(report.selection)
155        print(
156            f"\nmean spearman = {report.selection['spearman'].mean():.3f}   "
157            f"mean top-1 regret = {report.selection['regret'].mean():.3f}"
158        )
159
160
161def _save_tables(report: Report, folder: Path) -> list[Path]:
162    """Every table the study produced, as CSV, for a paper's tables and plots."""
163    tables: dict[str, pl.DataFrame] = {
164        "correlations": report.correlations,
165        "reach": report.reach,
166        "ceiling": report.ceiling,
167        "curve_e1": report.e1.curve,
168        "curve_e2": report.e2.curve,
169        "curve_e3": report.e3.curve,
170        "comparison": report.comparison,
171        "baselines": report.baselines,
172        "ranking_baselines": report.ranking_baselines,
173        "decision_baselines": report.decision_baselines,
174        "leakage": report.leakage,
175        "oracles": report.oracles,
176        "interaction": report.interaction,
177        "decision": report.decision,
178        "selection": report.selection,
179        "decomposition": report.decomposition,
180        "shares": report.shares,
181        "term_effects": report.effects,
182        "feature_practices": report.feature_practices,
183        "practices": report.practices,
184        "length_choice": report.length_choice,
185        "grammars": report.grammars,
186        "optimism": report.optimism,
187    }
188    if report.e3.stability is not None:
189        tables["stability"] = report.e3.stability
190    written: list[Path] = []
191    for name, frame in tables.items():
192        path = folder / f"{name}.csv"
193        frame.write_csv(path)
194        written.append(path)
195    return written
196
197
198def build_parser() -> ArgumentParser:
199    parser = ArgumentParser(
200        prog="ml-meta-perf",
201        description=(
202            "Fit interpretable equations predicting MCC from dataset and model meta-features, "
203            "validate them under leave-one-group-out, and write the figures and the report."
204        ),
205        default_config_files=[str(DEFAULT_CONFIG_PATH)],
206    )
207    parser.add_argument(
208        "--config",
209        action=ActionConfigFile,
210        help=f"study configuration JSON (default: {DEFAULT_CONFIG_PATH.name} in config/)",
211    )
212    # The hyperparameters. Their values live only in the configuration file; each field can be
213    # overridden as a dotted flag, e.g. `--search.penalty 3`.
214    parser.add_argument("--search", type=Configuration, help="search hyperparameters shared by E1, E2 and E3")
215    parser.add_argument("--selection", type=SelectionConfig, help="the equation-length rule")
216    parser.add_argument("--opaque", type=OpaqueConfig, help="the opaque regressors' sizes")
217    parser.add_argument("--sweep", type=SweepConfig, help="the grid `ml-meta-perf-search` explores (unused here)")
218
219    parser.add_argument(
220        "--data", type=str | None, default=None, help="meta-dataset CSV (default: dataset/meta_dataset.csv)"
221    )
222    parser.add_argument("--output", type=str, default="results", help="directory for the equations and CSV tables")
223    parser.add_argument("--figures", type=str, default="assets/figures", help="directory for the generated figures")
224    # The generated results go *into* the chapters that discuss them, between markers, rather
225    # than into a report of their own. Everything outside the markers is hand-written and never
226    # touched; everything inside is rewritten on every run, so a chapter cannot carry a stale table.
227    parser.add_argument(
228        "--docs", type=str, default="assets/docs", help="chapters whose generated sections are rewritten"
229    )
230    parser.add_argument("--readme", type=str, default="README.md", help="README whose generated headline is rewritten")
231    parser.add_argument("--no-figures", action="store_true", help="skip figure generation")
232    parser.add_argument("--no-report", action="store_true", help="skip rewriting the generated chapter sections")
233    parser.add_argument("--no-tables", action="store_true", help="skip the CSV tables")
234    parser.add_argument("--quiet", action="store_true", help="write files without printing the study")
235    parser.add_argument(
236        "--phase",
237        type=str | None,
238        default=None,
239        help=f"comma-separated phases to run (default: all): {', '.join(PHASES)}",
240    )
241    return parser
242
243
244def phases_of(arguments: Namespace) -> frozenset[str]:
245    """The phases to run; every phase when none, or ``all``, was asked for."""
246    requested = [name.strip() for name in (arguments.phase or "").split(",") if name.strip()]
247    if not requested or "all" in requested:
248        return frozenset(PHASES)
249    unknown = sorted(set(requested) - set(PHASES))
250    if unknown:
251        raise SystemExit(f"ml-meta-perf: unknown phase(s): {', '.join(unknown)}")
252    return frozenset(requested)
253
254
255def main(argv: list[str] | None = None) -> int:
256    _configure_windows_output()
257    parser = build_parser()
258    arguments = parser.instantiate(parser.parse_args(argv))
259    phases = phases_of(arguments)
260
261    config: Configuration = arguments.search
262    started = time.perf_counter()
263    report = run(arguments.data, config=config, selection=arguments.selection, opaque=arguments.opaque)
264    elapsed = time.perf_counter() - started
265
266    frame = load(arguments.data)
267    columns = columns_as_arrays(frame, DATASET_FEATURES + MODEL_FEATURES)
268    source = arguments.data or str(DEFAULT_PATH)
269
270    if not arguments.quiet:
271        render(report, phases, columns)
272
273    if arguments.output:
274        destination = Path(arguments.output)
275        destination.mkdir(parents=True, exist_ok=True)
276        report.e1.equation.save(destination / "e1.json")
277        report.e3.equation.save(destination / "e3.json")
278        report.e2.equation.save(destination / "e2.json")
279        print(f"\nequations written to {destination}")
280
281        if not arguments.no_tables:
282            written = _save_tables(report, destination)
283            print(f"{len(written)} tables written to {destination}")
284
285    if arguments.docs and not arguments.no_report and "report" in phases:
286        pages = write_into_chapters(
287            report,
288            columns,
289            target(frame),
290            DATASET_FEATURES,
291            MODEL_FEATURES,
292            arguments.docs,
293            frame=frame,
294            config=config,
295            selection=arguments.selection,
296            source=source,
297            readme=arguments.readme,
298        )
299        print(f"{len(pages)} pages regenerated ({arguments.docs}, {arguments.readme})")
300
301    if arguments.figures and not arguments.no_figures and "figures" in phases:
302        from ml_meta_perf.figures import generate
303
304        written = generate(report, arguments.figures, arguments.data)
305        print(f"{len(written)} figures written to {arguments.figures}")
306
307    print(f"\ncompleted in {elapsed:.1f}s")
308    return 0
309
310
311if __name__ == "__main__":
312    sys.exit(main())