ml_meta_perf.config

  1"""The study's hyperparameters, read from ``config/study.json``.
  2
  3**Every tuned value lives in that one file**, and nowhere in the code: the search settings all
  4three equations share, the rule that chooses an equation's length, the opaque regressors'
  5sizes, and the grid the configuration sweep explores. The dataclasses below are its schema and
  6carry no defaults for anything that was tuned, so a value can only come from the file (or from
  7a command-line override of it) and cannot drift between a constant and a chapter.
  8
  9The file holds **hyperparameters only**. Which features an equation may use is not one: E1
 10searches every dataset feature, E2 every model feature, E3 both, and the search decides what
 11each equation keeps. Restricting the pool would be a second, hidden search over feature
 12subsets.
 13
 14`ml-meta-perf-search` (`ml_meta_perf.sweep`) proposes a replacement file; adopting it is a
 15reviewed change to ``config/study.json`` rather than something a run does to itself.
 16
 17Study chapter: [3. Term generation and selection][study-chapter] -- the rationale, in prose.
 18
 19[study-chapter]: https://github.com/mariolpantunes/ml-meta-perf/blob/main/assets/docs/03-term-selection.md
 20"""
 21
 22from __future__ import annotations
 23
 24import json
 25from dataclasses import dataclass
 26from functools import cache
 27from pathlib import Path
 28from typing import Any
 29
 30#: The configuration file, relative to the repository root. Resolved like the corpus
 31#: (`data.DEFAULT_PATH`): the source checkout this module runs from, else the working directory.
 32CONFIG_RELATIVE_PATH = Path("config") / "study.json"
 33
 34
 35def _default_config_path() -> Path:
 36    checkout = Path(__file__).resolve().parents[2] / CONFIG_RELATIVE_PATH
 37    working = Path.cwd() / CONFIG_RELATIVE_PATH
 38    return next((path for path in (checkout, working) if path.is_file()), checkout)
 39
 40
 41DEFAULT_CONFIG_PATH = _default_config_path()
 42
 43
 44@dataclass(frozen=True)
 45class Configuration:
 46    """The search hyperparameters, **shared by E1, E2 and E3**.
 47
 48    One configuration for all three equations is what makes the gaps between them evidence
 49    about the meta-data rather than about three tuning runs.
 50    """
 51
 52    #: The largest standard score a term may reach before it is rejected as a spike.
 53    max_abs_zscore: float
 54    #: The ridge penalty on standardised terms.
 55    penalty: float
 56    #: Terms surviving the correlation screen into the beam.
 57    pool_size: int
 58    #: The search horizon: the longest equation the search explores and the range every curve
 59    #: covers. **Not the published length** -- `selection.plateau_knee` derives that.
 60    max_terms: int
 61    #: Beam width. Eight times wider converges to the fourth decimal, so this is a cost
 62    #: control rather than a tuned knob.
 63    beam_width: int
 64    #: Raw features per term: 2 allows products and ratios of two features, 3 adds
 65    #: ratio-of-sums ``(a + b) / c``.
 66    max_arity: int
 67
 68
 69@dataclass(frozen=True)
 70class SelectionConfig:
 71    """How an equation's length is chosen from its cross-validated curve.
 72
 73    See `selection.plateau_knee`: the curve is the worst R2 over the four protocols, smoothed
 74    by a running median of ``smoothing`` lengths; the length is the first knee of its Pareto
 75    front after which the smoothed curve gains at most ``delta`` over the next ``window``
 76    lengths.
 77    """
 78
 79    delta: float
 80    window: int
 81    smoothing: int
 82    #: The capability bound's grammar: E3-MAX is fitted at this arity under the same
 83    #: configuration and takes the raw maximum of its curve.
 84    capability_arity: int
 85
 86
 87@dataclass(frozen=True)
 88class OpaqueConfig:
 89    """The opaque regressors the equation is compared against (`ml_meta_perf.opaque`).
 90
 91    Chosen on a grid scored under all four protocols: tuning on the held-out protocols can only
 92    favour the opaque side, which is the conservative direction for this study's claim.
 93    """
 94
 95    forest_trees: int
 96    #: Share of the features considered at each split.
 97    forest_max_features: float
 98    boosting_stages: int
 99    boosting_learning_rate: float
100    boosting_max_depth: int
101
102
103@dataclass(frozen=True)
104class SweepConfig:
105    """The grid `ml_meta_perf.sweep` explores, and how it chooses among close candidates."""
106
107    zscores: tuple[float, ...]
108    penalties: tuple[float, ...]
109    arities: tuple[int, ...]
110    #: Only configurations whose chosen equation has at most this many terms are proposed:
111    #: the study's readability limit, stated rather than implied.
112    readable_terms: int
113    #: Among those, candidates whose smoothed floor is within this of the best are treated as
114    #: indistinguishable on R2, and the one with the best doubly-held-out ranking average
115    #: precision is proposed.
116    band: float
117
118
119@dataclass(frozen=True)
120class StudyConfig:
121    """The whole of ``config/study.json``."""
122
123    search: Configuration
124    selection: SelectionConfig
125    opaque: OpaqueConfig
126    sweep: SweepConfig
127
128
129def parse_config(payload: dict[str, Any]) -> StudyConfig:
130    """Build the typed configuration from the file's JSON object; unknown keys are an error."""
131    sweep = payload["sweep"]
132    return StudyConfig(
133        search=Configuration(**payload["search"]),
134        selection=SelectionConfig(**payload["selection"]),
135        opaque=OpaqueConfig(**payload["opaque"]),
136        sweep=SweepConfig(
137            zscores=tuple(float(value) for value in sweep["zscores"]),
138            penalties=tuple(float(value) for value in sweep["penalties"]),
139            arities=tuple(int(value) for value in sweep["arities"]),
140            readable_terms=int(sweep["readable_terms"]),
141            band=float(sweep["band"]),
142        ),
143    )
144
145
146@cache
147def load_config(path: str | Path | None = None) -> StudyConfig:
148    """Read and validate ``config/study.json`` (or ``path``). Cached: the file is read once."""
149    resolved = Path(path) if path is not None else DEFAULT_CONFIG_PATH
150    if not resolved.is_file():
151        raise FileNotFoundError(f"study configuration not found: {resolved} (pass --config)")
152    return parse_config(json.loads(resolved.read_text(encoding="utf-8")))
153
154
155def default_configuration() -> Configuration:
156    """The shared search configuration from the study file -- what a bare run fits under."""
157    return load_config().search