Source code for passengersim.callbacks.combine

"""Combination of tracer results from several trials, workers, or saved runs.

A finalized tracer table declares how it combines with another table of the
same kind through ``df.attrs["aggregation_process"]``. That string is the only
contract between the tracer that wrote the table and the code that combines
it, because it is all that survives once a result is saved. Combination is
looked up in a registry keyed by that string.

Process identifiers are part of the saved-result format: once published, an
identifier never changes meaning. A change in meaning takes a new identifier,
so results saved before and after the change are never silently mixed.
"""

from __future__ import annotations

from collections.abc import Callable
from typing import Any

import numpy as np
import pandas as pd

Combiner = Callable[[pd.DataFrame, pd.DataFrame], pd.DataFrame]
"""Combine two tables declaring the same process into a new table.

A combiner must not modify either operand, and its result must declare the
same process (and whatever weights it needs) so that it can be combined again.
"""

PROCESS_ATTR = "aggregation_process"
"""The ``DataFrame.attrs`` key naming a table's aggregation process."""

_COMBINERS: dict[str, Combiner] = {}


[docs] def register_combiner(process: str, combiner: Combiner) -> None: """Register how tables declaring an aggregation process are combined. A tracer declares its process by setting ``df.attrs["aggregation_process"]`` on the table returned by its ``finalize`` method. Register the combiner in the module that defines the tracer, so that it is available whenever that module is imported, including before loading saved results that use it. Parameters ---------- process : str The process identifier. It is stored with saved results, so it must never change meaning once results using it exist. combiner : Callable[[pd.DataFrame, pd.DataFrame], pd.DataFrame] Function combining two tables that both declare `process` into a new table, without modifying either. The result must also declare `process`, so that it can be combined again. Raises ------ ValueError If `process` is empty or already registered. TypeError If `combiner` is not callable. """ if not isinstance(process, str) or not process: raise ValueError(f"An aggregation process must be a nonempty string, not {process!r}") if process in _COMBINERS: raise ValueError(f"A combiner for aggregation process {process!r} is already registered") if not callable(combiner): raise TypeError("A combiner must be callable") _COMBINERS[process] = combiner
[docs] def registered_processes() -> tuple[str, ...]: """Get the aggregation processes that have a registered combiner. Returns ------- tuple[str, ...] """ return tuple(_COMBINERS)
def _process_of(value: Any) -> str | None: """The aggregation process a value declares, or None.""" attrs = getattr(value, "attrs", None) return attrs.get(PROCESS_ATTR) if isinstance(attrs, dict) else None
[docs] def combine_values(key: str, a: Any, b: Any) -> Any: """Combine two values stored under the same ``callback_data`` key. Parameters ---------- key : str The ``callback_data`` key, used in error messages. a, b : Any The two values. Neither is modified. Returns ------- Any For tables declaring an aggregation process, the result of that process's combiner. Otherwise ``a + b``, which concatenates the lists of rows collected by callback functions, and adds undeclared tables. Raises ------ ValueError If the two values declare different processes (other than the legacy fallback for ``pooled_moments``), or a process with no registered combiner. """ p1, p2 = _process_of(a), _process_of(b) if p1 is None and p2 is None: # Binary `+` (never `+=`) so neither operand is mutated: for lists `+=` # extends the left list in place, which would leak every later # trial's rows into the first trial's data. return a + b if p1 != p2: if {p1, p2} == {"pooled_moments", "wgt_by_trials"}: return _pooled_with_legacy(a, b) raise ValueError( f"cannot combine callback_data[{key!r}]: one result declares aggregation process {p1!r} " f"and the other {p2!r}; results are combined only with others of the same kind" ) try: combiner = _COMBINERS[p1] except KeyError: raise ValueError( f"cannot combine callback_data[{key!r}]: no combiner is registered for aggregation process " f"{p1!r}. Import the module that defines the tracer which produced it (it registers the " "combiner with passengersim.callbacks.register_combiner) before combining these results" ) from None return combiner(a, b)
# -------------------------------------------------------------------------- # Built-in processes def _weighted_mean(a: pd.DataFrame, n1: float, b: pd.DataFrame, n2: float) -> pd.DataFrame: """Combine two weighted-mean frames, ignoring a side that carries no weight. Arithmetic between frames aligns on the union of their indexes, so an empty zero-weight frame (e.g. a tracer that recorded no post-burn samples) would otherwise turn every value of the other frame into NaN. """ if n1 == 0 and n2 != 0: return b.copy() if n2 == 0: return a.copy() return ((a * n1) + (b * n2)) / (n1 + n2) def _merge_coverage(a: pd.DataFrame, b: pd.DataFrame, n1: float, n2: float) -> dict | None: """Merge the ``segmentation_coverage`` of two sample-weighted frames. A pooled mean is complete only where every side recorded the requested scope, so the missing scopes are united. Unknown coverage (None) stays unknown, except that a side with no samples contributed no data and is ignored. """ c1 = a.attrs.get("segmentation_coverage") c2 = b.attrs.get("segmentation_coverage") if n1 == 0 and n2 != 0: c1 = c2 elif n2 == 0 and n1 != 0: c2 = c1 if c1 is None or c2 is None: return None return {field: tuple(sorted(set(c1[field]) | set(c2[field]))) for field in ("missing_markets", "missing_segments")}
[docs] def combine_wgt_by_samples(a: pd.DataFrame, b: pd.DataFrame) -> pd.DataFrame: """Combine two per-sample means, weighted by ``attrs["n_samples"]``. If either frame carries a ``segmentation_coverage`` attribute, the coverage of the two is merged as well. """ n1 = a.attrs.get("n_samples", 1) n2 = b.attrs.get("n_samples", 1) result = _weighted_mean(a, n1, b, n2) result.attrs = {"n_samples": n1 + n2, PROCESS_ATTR: "wgt_by_samples"} if "segmentation_coverage" in a.attrs or "segmentation_coverage" in b.attrs: result.attrs["segmentation_coverage"] = _merge_coverage(a, b, n1, n2) return result
[docs] def combine_wgt_by_trials(a: pd.DataFrame, b: pd.DataFrame) -> pd.DataFrame: """Combine two per-trial means, weighted by ``attrs["n_trials"]``.""" n1 = a.attrs.get("n_trials", 1) n2 = b.attrs.get("n_trials", 1) result = _weighted_mean(a, n1, b, n2) # Record the pooled weight under the same key it is read from, so chained # sums over 3+ trials stay correctly weighted. result.attrs = {"n_trials": n1 + n2, PROCESS_ATTR: "wgt_by_trials"} return result
def _moments(df: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: """Split a ``pooled_moments`` table into its count, mean, and std_dev frames.""" return ( df.xs("n", level="statistic"), df.xs("mean", level="statistic"), df.xs("std_dev", level="statistic"), )
[docs] def combine_pooled_moments(a: pd.DataFrame, b: pd.DataFrame) -> pd.DataFrame: """Pool two tables of per-cell count, mean, and sample standard deviation exactly. Each table is indexed by ``(statistic, ...)``, where ``statistic`` is ``"mean"``, ``"std_dev"`` (the sample standard deviation, ``ddof=1``), or ``"n"`` (the number of observations), aligned cell by cell. The pooled result is what a single accumulator would have reported had it observed every observation of both: counts add, means are count-weighted, and sums of squared deviations add with the usual correction for the difference between the means. A cell observed once has no standard deviation but contributes zero to the pooled sum of squared deviations. A cell with no observations contributes nothing. A cell present in only one table is passed through unchanged. """ n_trials = a.attrs.get("n_trials", 1) + b.attrs.get("n_trials", 1) if a.empty or b.empty: # A table with no rows (a tracer that observed nothing) contributes no # observations; pass the other through, keeping its labels and types. result = (b if a.empty else a).copy() result.attrs = {"n_trials": n_trials, PROCESS_ATTR: "pooled_moments"} return result n1, m1, s1 = _moments(a) n2, m2, s2 = _moments(b) index = n1.index.union(n2.index) columns = n1.columns.union(n2.columns) def aligned(frame: pd.DataFrame) -> np.ndarray: return frame.reindex(index=index, columns=columns).to_numpy(dtype=np.float64, na_value=np.nan) # A cell missing from one side (another entity, or a day never observed) # has a count of zero there. c1 = np.nan_to_num(aligned(n1), nan=0.0) c2 = np.nan_to_num(aligned(n2), nan=0.0) mean1, mean2 = aligned(m1), aligned(m2) sd1, sd2 = aligned(s1), aligned(s2) # Sum of squared deviations of each side; zero for a single observation. ss1 = np.where(c1 >= 2, np.nan_to_num(sd1) ** 2 * (c1 - 1), 0.0) ss2 = np.where(c2 >= 2, np.nan_to_num(sd2) ** 2 * (c2 - 1), 0.0) count = c1 + c2 both = (c1 > 0) & (c2 > 0) with np.errstate(invalid="ignore", divide="ignore"): delta = np.where(both, mean2 - mean1, 0.0) mean = np.where(both, (c1 * mean1 + c2 * mean2) / count, np.nan) ss = ss1 + ss2 + np.where(both, delta**2 * c1 * c2 / count, 0.0) sd = np.where(count >= 2, np.sqrt(ss / (count - 1)), np.nan) # A cell observed on one side only passes through exactly. only1 = (c1 > 0) & (c2 == 0) only2 = (c2 > 0) & (c1 == 0) mean = np.where(only1, mean1, np.where(only2, mean2, mean)) sd = np.where(only1, sd1, np.where(only2, sd2, sd)) def frame(values: np.ndarray) -> pd.DataFrame: return pd.DataFrame(values, index=index, columns=columns) result = pd.concat([frame(mean), frame(sd), frame(count)], keys=["mean", "std_dev", "n"], names=["statistic"]) result.attrs = {"n_trials": n_trials, PROCESS_ATTR: "pooled_moments"} return result
def _pooled_with_legacy(a: pd.DataFrame, b: pd.DataFrame) -> pd.DataFrame: """Combine a ``pooled_moments`` table with a legacy ``wgt_by_trials`` one. Results saved before counts were recorded hold trial-averaged standard deviations, which cannot be pooled exactly. The pooled table's counts are dropped and the two are combined as trial-weighted means, so the result declares ``wgt_by_trials`` and never claims to support exact pooling. """ def as_legacy(df: pd.DataFrame) -> pd.DataFrame: if _process_of(df) != "pooled_moments": return df legacy = df.drop(index="n", level="statistic") legacy.attrs = {"n_trials": df.attrs.get("n_trials", 1), PROCESS_ATTR: "wgt_by_trials"} return legacy return combine_wgt_by_trials(as_legacy(a), as_legacy(b))
[docs] def combine_concat_rows(a: pd.DataFrame, b: pd.DataFrame) -> pd.DataFrame: """Stack two tables of per-sample rows, such as those of separate trials. Used by tracers that keep one row per observation (for example per trial, sample, and day) rather than summary statistics. A table with no rows contributes nothing. """ if len(a.index) == 0 and len(b.index) > 0: result = b.copy() elif len(b.index) == 0: result = a.copy() else: result = pd.concat([a, b]) result.attrs = {PROCESS_ATTR: "concat_rows"} return result
register_combiner("wgt_by_samples", combine_wgt_by_samples) register_combiner("wgt_by_trials", combine_wgt_by_trials) register_combiner("pooled_moments", combine_pooled_moments) register_combiner("concat_rows", combine_concat_rows)