"""Combination of tracer results from several trials, workers, or saved runs.
A finalized tracer table declares how it combines with another table of the
same kind through ``df.attrs["aggregation_process"]``. That string is the only
contract between the tracer that wrote the table and the code that combines
it, because it is all that survives once a result is saved. Combination is
looked up in a registry keyed by that string.
Process identifiers are part of the saved-result format: once published, an
identifier never changes meaning. A change in meaning takes a new identifier,
so results saved before and after the change are never silently mixed.
"""
from __future__ import annotations
from collections.abc import Callable
from typing import Any
import numpy as np
import pandas as pd
Combiner = Callable[[pd.DataFrame, pd.DataFrame], pd.DataFrame]
"""Combine two tables declaring the same process into a new table.
A combiner must not modify either operand, and its result must declare the
same process (and whatever weights it needs) so that it can be combined again.
"""
PROCESS_ATTR = "aggregation_process"
"""The ``DataFrame.attrs`` key naming a table's aggregation process."""
_COMBINERS: dict[str, Combiner] = {}
[docs]
def register_combiner(process: str, combiner: Combiner) -> None:
"""Register how tables declaring an aggregation process are combined.
A tracer declares its process by setting
``df.attrs["aggregation_process"]`` on the table returned by its
``finalize`` method. Register the combiner in the module that defines the
tracer, so that it is available whenever that module is imported,
including before loading saved results that use it.
Parameters
----------
process : str
The process identifier. It is stored with saved results, so it must
never change meaning once results using it exist.
combiner : Callable[[pd.DataFrame, pd.DataFrame], pd.DataFrame]
Function combining two tables that both declare `process` into a new
table, without modifying either. The result must also declare
`process`, so that it can be combined again.
Raises
------
ValueError
If `process` is empty or already registered.
TypeError
If `combiner` is not callable.
"""
if not isinstance(process, str) or not process:
raise ValueError(f"An aggregation process must be a nonempty string, not {process!r}")
if process in _COMBINERS:
raise ValueError(f"A combiner for aggregation process {process!r} is already registered")
if not callable(combiner):
raise TypeError("A combiner must be callable")
_COMBINERS[process] = combiner
[docs]
def registered_processes() -> tuple[str, ...]:
"""Get the aggregation processes that have a registered combiner.
Returns
-------
tuple[str, ...]
"""
return tuple(_COMBINERS)
def _process_of(value: Any) -> str | None:
"""The aggregation process a value declares, or None."""
attrs = getattr(value, "attrs", None)
return attrs.get(PROCESS_ATTR) if isinstance(attrs, dict) else None
[docs]
def combine_values(key: str, a: Any, b: Any) -> Any:
"""Combine two values stored under the same ``callback_data`` key.
Parameters
----------
key : str
The ``callback_data`` key, used in error messages.
a, b : Any
The two values. Neither is modified.
Returns
-------
Any
For tables declaring an aggregation process, the result of that
process's combiner. Otherwise ``a + b``, which concatenates the lists
of rows collected by callback functions, and adds undeclared tables.
Raises
------
ValueError
If the two values declare different processes (other than the legacy
fallback for ``pooled_moments``), or a process with no registered
combiner.
"""
p1, p2 = _process_of(a), _process_of(b)
if p1 is None and p2 is None:
# Binary `+` (never `+=`) so neither operand is mutated: for lists `+=`
# extends the left list in place, which would leak every later
# trial's rows into the first trial's data.
return a + b
if p1 != p2:
if {p1, p2} == {"pooled_moments", "wgt_by_trials"}:
return _pooled_with_legacy(a, b)
raise ValueError(
f"cannot combine callback_data[{key!r}]: one result declares aggregation process {p1!r} "
f"and the other {p2!r}; results are combined only with others of the same kind"
)
try:
combiner = _COMBINERS[p1]
except KeyError:
raise ValueError(
f"cannot combine callback_data[{key!r}]: no combiner is registered for aggregation process "
f"{p1!r}. Import the module that defines the tracer which produced it (it registers the "
"combiner with passengersim.callbacks.register_combiner) before combining these results"
) from None
return combiner(a, b)
# --------------------------------------------------------------------------
# Built-in processes
def _weighted_mean(a: pd.DataFrame, n1: float, b: pd.DataFrame, n2: float) -> pd.DataFrame:
"""Combine two weighted-mean frames, ignoring a side that carries no weight.
Arithmetic between frames aligns on the union of their indexes, so an
empty zero-weight frame (e.g. a tracer that recorded no post-burn samples)
would otherwise turn every value of the other frame into NaN.
"""
if n1 == 0 and n2 != 0:
return b.copy()
if n2 == 0:
return a.copy()
return ((a * n1) + (b * n2)) / (n1 + n2)
def _merge_coverage(a: pd.DataFrame, b: pd.DataFrame, n1: float, n2: float) -> dict | None:
"""Merge the ``segmentation_coverage`` of two sample-weighted frames.
A pooled mean is complete only where every side recorded the requested
scope, so the missing scopes are united. Unknown coverage (None) stays
unknown, except that a side with no samples contributed no data and is
ignored.
"""
c1 = a.attrs.get("segmentation_coverage")
c2 = b.attrs.get("segmentation_coverage")
if n1 == 0 and n2 != 0:
c1 = c2
elif n2 == 0 and n1 != 0:
c2 = c1
if c1 is None or c2 is None:
return None
return {field: tuple(sorted(set(c1[field]) | set(c2[field]))) for field in ("missing_markets", "missing_segments")}
[docs]
def combine_wgt_by_samples(a: pd.DataFrame, b: pd.DataFrame) -> pd.DataFrame:
"""Combine two per-sample means, weighted by ``attrs["n_samples"]``.
If either frame carries a ``segmentation_coverage`` attribute, the
coverage of the two is merged as well.
"""
n1 = a.attrs.get("n_samples", 1)
n2 = b.attrs.get("n_samples", 1)
result = _weighted_mean(a, n1, b, n2)
result.attrs = {"n_samples": n1 + n2, PROCESS_ATTR: "wgt_by_samples"}
if "segmentation_coverage" in a.attrs or "segmentation_coverage" in b.attrs:
result.attrs["segmentation_coverage"] = _merge_coverage(a, b, n1, n2)
return result
[docs]
def combine_wgt_by_trials(a: pd.DataFrame, b: pd.DataFrame) -> pd.DataFrame:
"""Combine two per-trial means, weighted by ``attrs["n_trials"]``."""
n1 = a.attrs.get("n_trials", 1)
n2 = b.attrs.get("n_trials", 1)
result = _weighted_mean(a, n1, b, n2)
# Record the pooled weight under the same key it is read from, so chained
# sums over 3+ trials stay correctly weighted.
result.attrs = {"n_trials": n1 + n2, PROCESS_ATTR: "wgt_by_trials"}
return result
def _moments(df: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
"""Split a ``pooled_moments`` table into its count, mean, and std_dev frames."""
return (
df.xs("n", level="statistic"),
df.xs("mean", level="statistic"),
df.xs("std_dev", level="statistic"),
)
[docs]
def combine_pooled_moments(a: pd.DataFrame, b: pd.DataFrame) -> pd.DataFrame:
"""Pool two tables of per-cell count, mean, and sample standard deviation exactly.
Each table is indexed by ``(statistic, ...)``, where ``statistic`` is
``"mean"``, ``"std_dev"`` (the sample standard deviation, ``ddof=1``), or
``"n"`` (the number of observations), aligned cell by cell. The pooled
result is what a single accumulator would have reported had it observed
every observation of both: counts add, means are count-weighted, and sums
of squared deviations add with the usual correction for the difference
between the means.
A cell observed once has no standard deviation but contributes zero to the
pooled sum of squared deviations. A cell with no observations contributes
nothing. A cell present in only one table is passed through unchanged.
"""
n_trials = a.attrs.get("n_trials", 1) + b.attrs.get("n_trials", 1)
if a.empty or b.empty:
# A table with no rows (a tracer that observed nothing) contributes no
# observations; pass the other through, keeping its labels and types.
result = (b if a.empty else a).copy()
result.attrs = {"n_trials": n_trials, PROCESS_ATTR: "pooled_moments"}
return result
n1, m1, s1 = _moments(a)
n2, m2, s2 = _moments(b)
index = n1.index.union(n2.index)
columns = n1.columns.union(n2.columns)
def aligned(frame: pd.DataFrame) -> np.ndarray:
return frame.reindex(index=index, columns=columns).to_numpy(dtype=np.float64, na_value=np.nan)
# A cell missing from one side (another entity, or a day never observed)
# has a count of zero there.
c1 = np.nan_to_num(aligned(n1), nan=0.0)
c2 = np.nan_to_num(aligned(n2), nan=0.0)
mean1, mean2 = aligned(m1), aligned(m2)
sd1, sd2 = aligned(s1), aligned(s2)
# Sum of squared deviations of each side; zero for a single observation.
ss1 = np.where(c1 >= 2, np.nan_to_num(sd1) ** 2 * (c1 - 1), 0.0)
ss2 = np.where(c2 >= 2, np.nan_to_num(sd2) ** 2 * (c2 - 1), 0.0)
count = c1 + c2
both = (c1 > 0) & (c2 > 0)
with np.errstate(invalid="ignore", divide="ignore"):
delta = np.where(both, mean2 - mean1, 0.0)
mean = np.where(both, (c1 * mean1 + c2 * mean2) / count, np.nan)
ss = ss1 + ss2 + np.where(both, delta**2 * c1 * c2 / count, 0.0)
sd = np.where(count >= 2, np.sqrt(ss / (count - 1)), np.nan)
# A cell observed on one side only passes through exactly.
only1 = (c1 > 0) & (c2 == 0)
only2 = (c2 > 0) & (c1 == 0)
mean = np.where(only1, mean1, np.where(only2, mean2, mean))
sd = np.where(only1, sd1, np.where(only2, sd2, sd))
def frame(values: np.ndarray) -> pd.DataFrame:
return pd.DataFrame(values, index=index, columns=columns)
result = pd.concat([frame(mean), frame(sd), frame(count)], keys=["mean", "std_dev", "n"], names=["statistic"])
result.attrs = {"n_trials": n_trials, PROCESS_ATTR: "pooled_moments"}
return result
def _pooled_with_legacy(a: pd.DataFrame, b: pd.DataFrame) -> pd.DataFrame:
"""Combine a ``pooled_moments`` table with a legacy ``wgt_by_trials`` one.
Results saved before counts were recorded hold trial-averaged standard
deviations, which cannot be pooled exactly. The pooled table's counts are
dropped and the two are combined as trial-weighted means, so the result
declares ``wgt_by_trials`` and never claims to support exact pooling.
"""
def as_legacy(df: pd.DataFrame) -> pd.DataFrame:
if _process_of(df) != "pooled_moments":
return df
legacy = df.drop(index="n", level="statistic")
legacy.attrs = {"n_trials": df.attrs.get("n_trials", 1), PROCESS_ATTR: "wgt_by_trials"}
return legacy
return combine_wgt_by_trials(as_legacy(a), as_legacy(b))
[docs]
def combine_concat_rows(a: pd.DataFrame, b: pd.DataFrame) -> pd.DataFrame:
"""Stack two tables of per-sample rows, such as those of separate trials.
Used by tracers that keep one row per observation (for example per trial,
sample, and day) rather than summary statistics. A table with no rows
contributes nothing.
"""
if len(a.index) == 0 and len(b.index) > 0:
result = b.copy()
elif len(b.index) == 0:
result = a.copy()
else:
result = pd.concat([a, b])
result.attrs = {PROCESS_ATTR: "concat_rows"}
return result
register_combiner("wgt_by_samples", combine_wgt_by_samples)
register_combiner("wgt_by_trials", combine_wgt_by_trials)
register_combiner("pooled_moments", combine_pooled_moments)
register_combiner("concat_rows", combine_concat_rows)