Source code for hybrid_aif.reference.raif.run

"""The resumable multi-seed Mountain Car run, and its report and benchmark modes.

One chunk is :data:`CHUNK_STEPS` pooled agent steps per seed, jitted and vmapped over
seeds: ``n_blocks`` × [``train_every`` pooled collect steps → one :func:`train_step`].
Each seed collects from :data:`LANES` independent environment streams that share its
parameters and pool its step count, so a block is ``train_every // lanes`` steps per lane.
Collection auto-resets in the scan and records the fixture format.

Between chunks, on the host: episodes are split out per lane at ``is_last`` (a chunk
boundary's partial episode is carried), labelled by
:func:`~hybrid_aif.reference.raif.revise.label` and routed (success → both buffers, failure
→ ``M``). Every world-model batch comes from ``M`` unless ``--alternate``. Both buffers are
:class:`~hybrid_aif.train.buffers.StreamBuffer` rings of :data:`CAPACITY` steps, and each
seed's are seeded once from its oracle fixture through the same routing.

A checkpoint (every ``--checkpoint-every`` minutes and at shutdown, SIGINT/SIGTERM
included) holds every numeric carrier, the buffers, the keys, the counters and the partial
episodes, so ``--resume`` continues the run; bitwise continuation is not asserted.

``<out>/scores.jsonl``
    Per finished episode: ``{step, seed, lane, return, length, success, stability,
    log_entropy, coverage}``. ``step`` is the pooled agent step it ended on; ``return`` the raw
    summed reward; ``stability`` R-S's ``C``; ``log_entropy`` the episode's mean act-time policy
    entropy (:func:`act_entropy`); ``coverage`` the lane's cumulative grid fraction
    (:class:`Coverage`).
``<out>/metrics.jsonl``
    Per chunk, per seed: the trainers' losses and :func:`probe`'s diagnostics.
``<out>/checkpoint/``
    The resumable state.
``<out>/run.json``
    The resolved configuration and the launching commit.

Modes: no flag = train; ``--report`` = the §7 table from ``scores.jsonl``;
``--benchmark`` = the throughput table (:func:`benchmark`)::

    python -m hybrid_aif.reference.raif.run --seeds 0,1,2,3 --steps 1000000 --out runs/mc_2d
    python -m hybrid_aif.reference.raif.run --out runs/mc_2d --resume
    python -m hybrid_aif.reference.raif.run --out runs/mc_2d --report

Design: ``docs/design/amortization.md`` P1′ chunk 2d;
``docs/design/amortization-p0-addendum.md`` §§5–7.
"""

from __future__ import annotations

import argparse
import collections
import dataclasses
import json
import pathlib
import signal
import sys
import time
from collections.abc import Mapping
from typing import NamedTuple

import equinox as eqx
import jax
import jax.numpy as jnp
import numpy as np
from jax import Array

from hybrid_aif.controls import soft_clip
from hybrid_aif.objectives.amortized import features as _amortized_features
from hybrid_aif.reference.envs.mountain_car import MountainCar
from hybrid_aif.reference.raif import metrics as ev
from hybrid_aif.reference.raif.agent import (
    ACTOR_LR,
    BATCH_LENGTH,
    BATCH_SIZE,
    ENSEMBLE_LR,
    IMAG_HORIZON,
    TRAIN_EVERY,
    WORLD_LR,
    Agent,
    raif_mountain_car,
    to_world_batch,
    train_step,
)
from hybrid_aif.reference.raif.columns import EXTRAS, IS_SUCCESS, RECORDED, raif_episode, succeeded
from hybrid_aif.reference.raif.crspp import ROW as CRSPP_ROW
from hybrid_aif.reference.raif.crspp import crspp_recognize, embeddings
from hybrid_aif.reference.raif.fixtures import load_fixture
from hybrid_aif.reference.raif.revise import label
from hybrid_aif.system import LearnState, System, SystemParams
from hybrid_aif.train.buffers import Episode, StreamBuffer, extend, sample_windows, seed_stats
from hybrid_aif.train.imagine import rollout_returns
from hybrid_aif.train.world import assemble, posterior_states

#: Pooled agent steps per device chunk: their ``batch_steps``, and 64 gradient steps.
#: Episodes finished inside a chunk reach the buffers at its end (``--chunk-steps``).
CHUNK_STEPS = 1024

#: Environment lanes per seed: their ``envs: {amount: 4}``, four streams behind one
#: agent with the step count pooled. ``--lanes 1`` collects from a single stream.
LANES = 4

#: Their ``replay_size 5e5`` steps per buffer (§5);
#: :class:`~hybrid_aif.train.buffers.StreamBuffer` counts steps.
CAPACITY = 500_000

#: Their Mountain Car budget: 1M agent steps per seed, 4 seeds (§5). One counted step
#: is one driver transition, reset and terminal rows included, so a 100-action
#: episode counts 101.
DEFAULT_STEPS = 1_000_000
DEFAULT_SEEDS = (0, 1, 2, 3)

#: The probe batch the distillation gap is measured on — smaller than a training
#: batch because it is a diagnostic paid every chunk. See :func:`probe`.
PROBE_BATCH = 4

#: Test/smoke sizes. **Not parity**: `--small` exists so the resume round trip and
#: the throughput ratio are testable in seconds. Their ``tiny`` is the default.
SMALL = dict(
    horizon=4,
    deter=16,
    num_vars=6,
    num_classes=6,
    units=8,
    mlp_units=16,
    embed=16,
    head_units=32,
    head_layers=2,
)


# ============================================================== the per-seed carry


[docs] class SeedState(NamedTuple): """One seed's entire numeric state, the thing a checkpoint holds; leaves are stacked over seeds. ``params``/``learn_state`` carry only the seed axis. The collection carriers carry a further leading lane axis (:data:`LANES`). ``key`` is per lane (env noise, action draw, reset); ``train_key`` is per seed, one gradient step per seed however many lanes fed it. The CRSPP's parameters live in ``params.scoring``, not in a field of their own. """ params: SystemParams learn_state: LearnState beliefs: dict[str, Array] # (lanes, ...) the habit's filter carry, per lane, by factor env_state: tuple[Array, ...] # (lanes, ...) MountainCar's ``(x,)``, per lane incoming_reward: Array # (lanes,) float32 — the reward that PRODUCED env_state t_in_episode: Array # (lanes,) int32, against the TimeLimit succeeded: Array # (lanes,) bool, this episode's running has_succeeded key: Array # (lanes, 2) the collection keys train_key: Array # (2,) the gradient step's own stream, per SEED
[docs] def collect_carry(self) -> tuple: """This seed's collection carry: the tuple :func:`make_collect_fn` threads.""" return ( self.beliefs, self.env_state, self.incoming_reward, self.t_in_episode, self.succeeded, self.key, )
[docs] class Stream(NamedTuple): """A chunk's recorded collection stream in the fixture format, one row per driver transition. The columns are :data:`hybrid_aif.train.buffers.EPISODE_FIELDS` minus ``can_self_imitate``, plus the telemetry column ``log_entropy`` (dropped by :func:`as_collected`). From :func:`make_chunk_fn` the leaves are ``(seeds, lanes, chunk_steps // lanes, ...)``. ``obs`` is the row's observation, ``reward`` the reward that produced it, ``action`` the raw sample drawn at it and ``is_success`` the goal predicate on it (:func:`make_collect_fn`). """ obs: Array action: Array reward: Array is_success: Array has_succeeded: Array is_first: Array is_last: Array #: The act-time differential entropy of the distribution this row's action was #: drawn from. Telemetry, not replay data, and computed without a key; its #: per-episode mean is their ``stats/mean_log_entropy``. log_entropy: Array
# ============================================================ the device-side chunk #: The collection carry an agent-step scan threads: :class:`SeedState` minus the trained #: things and the training key, leaves on a leading lane axis. CollectCarry = tuple def _action_bounds(system: System, params: SystemParams) -> tuple[Array, Array] | None: """The control box in ``params.control``, or ``None``: what the env and the transition clip to.""" layout = system._control_layout(params) return None if layout is None else layout.bounds
[docs] def episode_start_beliefs(system: System, params: SystemParams) -> dict[str, Array]: """The belief carry an episode's first filtering step reads. The episode-start state the world model's training restart uses (:meth:`~hybrid_aif.inference.rssm.RSSMCore.initial_state`: the learned carry and the prior's mode at it, a one-hot code), advanced one GRU step with a zero action, as their ``obs_step`` does at ``is_first``. Pure; the collection scan hoists it and broadcasts it across lanes. Design: ``docs/experiments/raif-mountain-car/execution-ledger-acting.md`` D6. """ core, _ = assemble(system, params) initial = system.beliefs_from_states(params, core.initial_state()) layout = system._control_layout(params) dim = 1 if layout is None else int(layout.action_dim) return system.update_empirical_prior( params, initial, system.from_positional("actions", jnp.zeros((dim,), jnp.float32), params), )
[docs] def act_entropy(system: System, params: SystemParams, beliefs: Mapping[str, Array]) -> Array: """The act-time policy entropy ``H[pi(a | s_t)]`` at ``beliefs``: a scalar. The entropy of the actor's raw ``Normal`` head at the features :func:`~hybrid_aif.planning.habit.habit_action` uses, summed over action dimensions. Takes no key and draws nothing. Its per-episode mean is their ``stats/mean_log_entropy``; the imagined :attr:`hybrid_aif.train.imagine.ImagineMetrics.policy_entropy` is a different quantity. Design: ``docs/experiments/raif-mountain-car/series-sweep.md`` §5.3. """ module = system._habit_module(params) if module is None: raise ValueError( "act_entropy needs a policy MODULE in SystemParams.policy_prior — a vector E " "over an enumerated policy set has no act-time density to take the entropy of" ) features = _amortized_features(system.to_positional("beliefs", beliefs)) return jnp.sum(module.condition_on((features,)).entropy().midpoint)
#: The tag the act-time code sample's key is folded in with: large, so it cannot coincide #: with any ``split(k_act, n)[i]`` a planner might take from the same key (``fold_in(k, i)`` #: equals ``split(k, n)[i]`` for small ``i``; see ``train.imagine.SCORING_KEY_TAG``). ACT_SAMPLE_KEY_TAG = 0x5A11CE5
[docs] def make_collect_fn(system: System, env: MountainCar, steps: int, *, act_sampling: bool = True): """``(params, carry) → (carry, Stream)`` over ``steps`` steps per lane. Pure. ``carry``'s leaves have a leading lane axis of any width; the lanes are vmapped with ``params`` and the episode-start belief broadcast. The returned ``Stream`` leaves are ``(steps, lanes, ...)``. Each lane-step is one transition of their driver. A row is ``(obs_t, reward_t, flags_t, a_t)`` with ``reward_t`` caused by ``a_{t−1}``, so an episode of ``T`` applied actions is ``T + 1`` rows: a reset row with a real action at zero reward, and a terminal row whose observation is the final state, whose action is zeroed and after which the env is not stepped. ``reward`` the incoming reward of the previous step's action; 0 on a reset row. ``is_success`` the goal predicate on this row's observation; ``has_succeeded`` is its running OR. ``action`` the raw actor sample, zeroed on ``is_last``. The env receives it clipped and the transition :func:`~hybrid_aif.controls.soft_clip`-ed; the recorded column is raw. ``is_last`` the goal predicate on this row, or ``t_in_episode`` reaching the time limit. On ``is_last`` a ``jnp.where`` swaps in a fresh env state, :func:`episode_start_beliefs`, zero incoming reward and ``t_in_episode = 0``, so episodes straddle chunk boundaries and the host recovers them from the flags. With ``act_sampling`` the recognized code is sampled (``infer_states(key=)``) and the actor, ``log_entropy`` and the transition read that sample; without, the lane acts on the mean-field posterior. Design: ``docs/experiments/raif-mountain-car/execution-ledger-acting.md`` D1–D6. """ time_limit = int(env.time_limit) def collect(params: SystemParams, carry: CollectCarry): # the reset targets, hoisted out of the scan: a read of params, broadcast over # the lanes by `in_axes=None` bounds = _action_bounds(system, params) initial = episode_start_beliefs(system, params) def lane(inner, initial): beliefs, env_state, incoming, t_ep, succeeded, key = inner key, k_act, k_plan, k_step, k_reset = jax.random.split(key, 5) obs = env.observe(env_state)[0] # the observation this ROW is is_first = t_ep == 0 # their MeasureSuccessRate, on this observation success = env.reached_goal(env_state) ran_on = succeeded | success # their easy_mode termination, or TimeLimit._step reaching `duration` is_last = success | (t_ep >= time_limit) # keyed, the posterior is one sample, read by the actor, the entropy and the # carry alike; its key is derived from the act key so no other stream moves k_code = jax.random.fold_in(k_act, ACT_SAMPLE_KEY_TAG) if act_sampling else None observed = system.from_positional("observations", (obs, None, None)) beliefs, _ = system.infer_states(params, observed, beliefs, key=k_code) # their `log_entropy`, off the beliefs the actor reads; draws no key h_act = act_entropy(system, params, beliefs) chosen, _, _ = system.act(k_act, params, beliefs, plan_key=k_plan, metrics=False) action = system.to_positional("actions", chosen, params) # their `ClipAction` into the env and their `action_clip` rescale into the # transition; the recorded column stays raw _, stepped, reward, _ = env.agent_step( k_step, env_state, action if bounds is None else jnp.clip(action, *bounds) ) carried = system.update_empirical_prior( params, beliefs, system.from_positional("actions", soft_clip(action, bounds), params), ) _, fresh = env.reset(k_reset) def pick(a, b): return jnp.where(is_last, a, b) row = Stream( obs=obs, action=jnp.where(is_last, jnp.zeros_like(action), action), reward=incoming, is_success=success, has_succeeded=ran_on, is_first=is_first, is_last=is_last, log_entropy=h_act, ) return ( jax.tree_util.tree_map(pick, initial, carried), jax.tree_util.tree_map(pick, fresh, stepped), jnp.where(is_last, 0.0, reward).astype(jnp.float32), jnp.where(is_last, 0, t_ep + 1).astype(jnp.int32), ran_on & ~is_last, key, ), row def step(inner, _): return jax.vmap(lane, in_axes=(0, None))(inner, initial) return jax.lax.scan(step, carry, None, length=int(steps)) return collect
[docs] def make_chunk_fn( system: System, crspp: System, env: MountainCar, *, chunk_steps: int = CHUNK_STEPS, train_every: int = TRAIN_EVERY, lanes: int = LANES, batch_size: int = BATCH_SIZE, horizon: int = IMAG_HORIZON, world_lr: float = WORLD_LR, actor_lr: float = ACTOR_LR, ensemble_lr: float = ENSEMBLE_LR, sample_scoring: bool = False, act_sampling: bool = True, ): """The jitted, seed-vmapped chunk: ``(state, positive, general, use_positive) → …``. Returns ``(state, stream, losses)``, the stream's leaves ``(seeds, lanes, chunk_steps // lanes, ...)`` with each lane's steps contiguous, and the losses' ``(seeds, n_blocks)``. The outer scan is the blocks, with ``use_positive`` (the host's schedule) as its ``xs``; the inner is the ``train_every // lanes`` lane-steps, so a gradient step follows every ``train_every`` pooled steps. ``positive``/``general`` are constant through the chunk. """ if chunk_steps % train_every: raise ValueError(f"chunk_steps {chunk_steps} must be a multiple of {train_every=}") if lanes < 1 or train_every % lanes: raise ValueError( f"lanes must be a positive divisor of train_every={train_every}, got {lanes} — " "their Ratio(64/1024) counts POOLED agent steps, so one gradient block is " "train_every // lanes steps in each lane and the division must be exact" ) per_lane = train_every // int(lanes) collect_fn = make_collect_fn(system, env, per_lane, act_sampling=act_sampling) def one_seed(state: SeedState, positive: StreamBuffer, general: StreamBuffer, ups: Array): def block(st: SeedState, use_positive: Array): collected = collect_fn(st.params, st.collect_carry()) (beliefs, env_state, incoming, t_ep, succeeded, key), rows = collected # one gradient step: both buffers are sampled and the host's `use_positive` # selects one. The keys come from the seed's `train_key`, not a lane's. train_key, k_pos, k_gen, k_train = jax.random.split(st.train_key, 4) batch = jax.tree_util.tree_map( lambda a, b: jnp.where(use_positive, a, b), sample_windows(positive, k_pos, batch_size), sample_windows(general, k_gen, batch_size), ) agent = Agent( system=system, params=st.params, crspp=crspp, crspp_params=st.params.scoring[CRSPP_ROW], learn_state=st.learn_state, ) agent, losses = train_step( agent, batch, key=k_train, horizon=horizon, world_lr=world_lr, actor_lr=actor_lr, critic_lr=actor_lr, ensemble_lr=ensemble_lr, crspp_lr=world_lr, sample_scoring=sample_scoring, ) return SeedState( params=agent.params, learn_state=agent.learn_state, beliefs=beliefs, env_state=env_state, incoming_reward=incoming, t_in_episode=t_ep, succeeded=succeeded, key=key, train_key=train_key, ), (rows, losses) state, (rows, losses) = jax.lax.scan(block, state, ups) # (n_blocks, per_lane, lanes, ...) -> (lanes, n_blocks * per_lane, ...) stream = jax.tree_util.tree_map( lambda x: jnp.reshape(jnp.moveaxis(x, 2, 0), (int(lanes), -1, *x.shape[3:])), rows ) return state, stream, losses return eqx.filter_jit(jax.vmap(one_seed))
# ================================================================= host-side splitting #: Stream columns a resumed ``pending`` may lack: telemetry, backfilled with NaN. A #: pending tail missing any other column is a corrupt checkpoint. OPTIONAL_STREAM_COLUMNS = frozenset({"log_entropy"}) def _carried(pending: dict, column: str, chunk: dict): """``pending[column]``, or NaN of the right length where an older run lacked it.""" if column in pending: return pending[column] if column not in OPTIONAL_STREAM_COLUMNS: raise KeyError( f"the resumed pending episode has no {column!r} column, and it is not telemetry " f"({sorted(OPTIONAL_STREAM_COLUMNS)} are). This checkpoint's partial episodes cannot " "be continued by this code." ) held = len(next(iter(pending.values()))) return np.full((held, *np.shape(chunk[column])[1:]), np.nan, np.asarray(chunk[column]).dtype)
[docs] def split_stream( pending: dict[str, np.ndarray] | None, chunk: dict[str, np.ndarray] ) -> tuple[list[dict[str, np.ndarray]], dict[str, np.ndarray]]: """``(finished episodes, the new pending tail)``: one lane's stream split on ``is_last``. An episode a chunk boundary cut is the tail here and the head of the next chunk's ``pending``. """ cols = ( chunk if pending is None else {k: np.concatenate([_carried(pending, k, chunk), chunk[k]]) for k in chunk} ) out, start = [], 0 for end in np.flatnonzero(cols["is_last"]).tolist(): out.append({k: v[start : end + 1] for k, v in cols.items()}) start = end + 1 return out, {k: v[start:] for k, v in cols.items()}
#: Their ``bin_size`` for the ``gym`` suite (``wrappers.py:795``): a 10 × 10 grid over #: ``(position, velocity)``, so 100 cells. COVERAGE_BINS = 10
[docs] class Coverage: """Their ``MeasureCoverage`` for Mountain Car, one instance per lane. The observation box (position ``[-1.2, 0.6]``, velocity ``[-0.07, 0.07]``) is cut into :data:`COVERAGE_BINS` per axis at the interior edges, the cell id is ``COVERAGE_BINS · bin(position) + bin(velocity)``, and coverage is the fraction of cells visited. The counter is cumulative over the lane's life, not per episode, and is carried across chunks and checkpoints. It reads recorded ``obs`` on the host only. Design: ``docs/experiments/raif-mountain-car/series-sweep.md`` §5.3. """
[docs] def __init__(self, env: MountainCar, bins: int = COVERAGE_BINS, seen: set[int] | None = None): low = np.asarray([env.min_position, -env.max_speed], np.float64) high = np.asarray([env.max_position, env.max_speed], np.float64) # their `np.linspace(lows, highs, bin_size + 1)[1:-1]`: interior edges, (bins-1, 2) self.edges = np.linspace(low, high, bins + 1)[1:-1] self.bins = int(bins) self.total = float(bins**2) self.seen: set[int] = set(seen or ())
[docs] def update(self, obs: np.ndarray) -> float: """Count one episode's rows in order, then return the coverage. ``obs`` is ``(steps, 2)``, reset and terminal rows included. """ rows = np.asarray(obs, np.float64).reshape(-1, 2) # their per-factor `np.digitize(state[i], binss[:, i])`, vectorised over rows pos = np.digitize(rows[:, 0], self.edges[:, 0]) vel = np.digitize(rows[:, 1], self.edges[:, 1]) self.seen.update((pos * self.bins + vel).tolist()) return len(self.seen) / self.total
[docs] def as_collected(cols: dict[str, np.ndarray]) -> Episode: """One host-split episode as a labelled :class:`~hybrid_aif.train.buffers.Episode`. ``can_self_imitate`` is zero as recorded; :func:`hybrid_aif.reference.raif.revise.label` then replaces it with the imitation mask. """ t = int(cols["obs"].shape[0]) return label( raif_episode( can_self_imitate=np.zeros((t,), np.float32), **{k: cols[k] for k in RECORDED if k != "can_self_imitate"}, ) )
[docs] def route(positive: StreamBuffer, general: StreamBuffer, ep: Episode, add_fn=extend): """Buffer routing: success → both buffers, failure → ``M`` only. Fixture episodes included.""" general = add_fn(general, ep) if succeeded(ep): positive = add_fn(positive, ep) return positive, general
[docs] def alternate(a: object, b: object): """``a, b, a, b, …`` — the paper's strict M/M⁺ alternation as an iterator (A10 disposition 9). Not the parity default; see :func:`alternation`.""" while True: yield a yield b
[docs] def alternation(grad_index: int, n_blocks: int, ready: np.ndarray) -> Array: """The ``--alternate`` schedule: ``(seeds, n_blocks)`` bool. Not the default. The paper's strict ``i % 2`` alternation, drawing from ``M`` while a seed's ``M⁺`` is not ``ready``. The parity default is :func:`m_only`, because the oracle's alternation is inert. Design: ``docs/design/amortization-debt-ledger.md`` entry 16. """ turn = ((np.arange(n_blocks) + int(grad_index)) % 2 == 0)[None, :] return jnp.asarray(turn & np.asarray(ready, bool)[:, None])
[docs] def m_only(n_seeds: int, n_blocks: int) -> Array: """The parity default's ``use_positive`` schedule: all ``False``, every batch from ``M``.""" return jnp.zeros((int(n_seeds), int(n_blocks)), bool)
# ======================================================================== the probe
[docs] def probe( system: System, crspp: System, params: SystemParams, batch: Episode, key: Array, *, sample_scoring: bool = False, act_sampling: bool = True, ): """Diagnostics on a fixed probe batch, per seed: ``(ret, critic, gap, value, mu_act, mu_train, sigma_act, sigma_train, mu_gap)``, batch means. ``gap`` is the distillation gap ``ret − critic``: the λ-return of the explicit rollout against the ``g_tail`` critic's value at the same states, both from :func:`~hybrid_aif.train.imagine.rollout_returns`. The actor is also read on two feature paths over the same windows: the training features, and the collection lane's filter run from a cold start at each window's first step (``episode_start_beliefs`` at ``is_first``, ``infer_states``, keyed when ``act_sampling``, then the prior pushed by the recorded action). ``mu_*`` are its mean actions, ``sigma_*`` its per-dimension scales, and ``mu_gap`` the mean absolute difference of the two means. Design: ``docs/experiments/raif-mountain-car/actor-entropy-diagnostic.md`` §8. """ world = to_world_batch(batch) k_states, k_recognize, k_roll = jax.random.split(key, 3) states = posterior_states(system, params, world, key=k_states) beliefs = system.to_positional("beliefs", system.beliefs_from_states(params, states)) n = beliefs[0].shape[0] * beliefs[0].shape[1] flat = tuple(jnp.reshape(x, (n, *x.shape[2:])) for x in beliefs) aux = crspp_recognize( crspp, params.scoring[CRSPP_ROW], embeddings(system, params, world.obs), key=k_recognize, is_first=batch.is_first, ) est = rollout_returns( system, params, system.from_positional("beliefs", flat), key=k_roll, aux_beliefs={ CRSPP_ROW: crspp.from_positional( "beliefs", tuple(jnp.reshape(x, (n, *x.shape[2:])) for x in aux) ) }, sample_scoring=sample_scoring, # the rollout the critic was fit to, not another ) ret, critic = jnp.mean(est.ret), jnp.mean(est.critic) # the act-time path: the lane's filter over the same windows (keys derived, not # split, so the columns above keep their draws) bounds = _action_bounds(system, params) initial = episode_start_beliefs(system, params) k_act = jax.random.fold_in(key, ACT_SAMPLE_KEY_TAG) def window(obs, act, first, k): def step(carry, xs): qs, kk = carry o, a, f = xs kk, k_code = jax.random.split(kk) qs = jax.tree_util.tree_map(lambda i_, b: jnp.where(f, i_, b), initial, qs) qs, _ = system.infer_states( params, system.from_positional("observations", (o, None, None)), qs, key=k_code if act_sampling else None, ) feat = _amortized_features(system.to_positional("beliefs", qs)) pushed = system.update_empirical_prior( params, qs, system.from_positional("actions", soft_clip(a, bounds), params) ) return (pushed, kk), feat return jax.lax.scan(step, (initial, k), (obs, act, first))[1] keys = jax.random.split(k_act, batch.obs.shape[0]) f_act = jnp.reshape(jax.vmap(window)(batch.obs, batch.action, batch.is_first, keys), (n, -1)) f_train = jax.vmap(lambda *b: _amortized_features(b))(*flat) module = system._habit_module(params) def read(feats): return jax.vmap(lambda f: module.diag_moments((f,)))(feats) # (mean, scale), (n, dim) mu_act, sigma_act = read(f_act) mu_train, sigma_train = read(f_train) return ( ret, critic, ret - critic, jnp.mean(est.value), jnp.mean(mu_act), jnp.mean(mu_train), jnp.mean(sigma_act), jnp.mean(sigma_train), jnp.mean(jnp.abs(mu_act - mu_train)), )
# ======================================================================= the harness def _code_provenance() -> dict[str, str | bool]: """``{sha, dirty}`` for the checkout this run is launched from, best-effort. A missing ``git`` or a tarball checkout records ``unknown`` rather than raising. """ import subprocess def git(*args: str) -> str | None: try: out = subprocess.run( ("git", "-C", str(pathlib.Path(__file__).resolve().parent), *args), capture_output=True, text=True, timeout=10, ) except (OSError, subprocess.SubprocessError): return None return out.stdout.strip() if out.returncode == 0 else None sha = git("rev-parse", "HEAD") status = git("status", "--porcelain") return {"sha": sha or "unknown", "dirty": bool(status) if status is not None else "unknown"} #: What a knob's absence from a stored ``run.json`` means: what the code did before the #: knob existed, not today's default. A knob missing from both this table and the #: stored file makes :func:`parse` refuse the resume. _ABSENT_MEANS: dict[str, object] = { "lanes": 1, # single-stream collection "winit": "default", # eqx's initializers "sample_scoring": False, # an expected (mean-field) preferred rollout "act_sampling": False, }
[docs] @dataclasses.dataclass class Config: """The resolved run configuration — written to ``run.json`` and re-read on resume.""" out: str seeds: tuple[int, ...] = DEFAULT_SEEDS steps: int = DEFAULT_STEPS chunk_steps: int = CHUNK_STEPS train_every: int = TRAIN_EVERY lanes: int = LANES batch_size: int = BATCH_SIZE seq_len: int = BATCH_LENGTH capacity: int = CAPACITY horizon: int = IMAG_HORIZON time_limit: int = 100 probe_batch: int = PROBE_BATCH checkpoint_every: float = 15.0 # minutes metrics_every: int = 1 # chunks small: bool = False seed_offset: int = 0 # the agent-init key per seed is `seed_offset + seed` fixtures: bool = True # False = every world-model batch from M (the oracle's behaviour); True = the # paper's i % 2 alternation. See `alternation`. alternate: bool = False # the world model's and the critic's initializer: "normal" is theirs, "default" # the engine's. A field so that a run's arm is recorded in `run.json`. winit: str = "normal" # the CRSPP's preferred rollout inside `imagine`: False = expected, True = sampled sample_scoring: bool = False act_sampling: bool = True # the act-time code is a sample (their obs_step), not the mean field @property def tiny(self) -> dict: return dict(SMALL) if self.small else {} @property def path(self) -> pathlib.Path: return pathlib.Path(self.out)
[docs] def build(cfg: Config): """Step 0: ``(system, crspp, env, state, buffers, probe_batch, seed_reports, horizon)``. Deterministic in the config alone, so ``--resume`` rebuilds the shapes by calling it and then overwrites the leaves. ``horizon`` is resolved once, for both the agent and :func:`make_chunk_fn`. """ env = MountainCar(time_limit=cfg.time_limit) tiny = dict(cfg.tiny) horizon = int(tiny.pop("horizon", cfg.horizon)) agents = [ raif_mountain_car( jax.random.PRNGKey(cfg.seed_offset + s), horizon=horizon, winit=cfg.winit, **tiny ) for s in cfg.seeds ] system, crspp = agents[0].system, agents[0].crspp buffers, reports = [], [] for s in cfg.seeds: positive = StreamBuffer.empty(cfg.capacity, cfg.seq_len, 2, 1, extras=EXTRAS) general = StreamBuffer.empty(cfg.capacity, cfg.seq_len, 2, 1, extras=EXTRAS) if cfg.fixtures: episodes, provenance = load_fixture(fixture_path(s)) reports.append((s, seed_stats(episodes, success=IS_SUCCESS), provenance)) for ep in episodes: # the same routing rule as collected episodes; these all succeeded positive, general = route(positive, general, label(ep)) buffers.append([positive, general]) lanes = int(cfg.lanes) keys = [jax.random.PRNGKey(1_000_000 + cfg.seed_offset + s) for s in cfg.seeds] # one env reset and one key per lane, so the lanes do not start in lockstep resets = [jax.vmap(env.reset)(jax.random.split(jax.random.fold_in(k, 0), lanes)) for k in keys] state = _stack( [ SeedState( params=a.params, learn_state=a.learn_state, beliefs=_lanes(episode_start_beliefs(system, a.params), lanes), env_state=r[1], incoming_reward=jnp.zeros((lanes,), jnp.float32), t_in_episode=jnp.zeros((lanes,), jnp.int32), succeeded=jnp.zeros((lanes,), jnp.bool_), key=jax.random.split(jax.random.fold_in(k, 1), lanes), train_key=jax.random.fold_in(k, 2), ) for a, r, k in zip(agents, resets, keys, strict=True) ] ) batch = _probe_batch(cfg, buffers) return system, crspp, env, state, buffers, batch, reports, horizon
def _lanes(tree, lanes: int): """``tree`` with every leaf broadcast onto a leading lane axis. Lanes share the learned initial belief and diverge through their own reset and key. """ return jax.tree_util.tree_map(lambda x: jnp.broadcast_to(x, (int(lanes), *jnp.shape(x))), tree) def _probe_batch(cfg: Config, buffers: list) -> Episode: """A fixed ``(seeds, probe_batch, seq_len, ...)`` batch, drawn once, from M⁺. Under ``--no-fixtures`` M⁺ starts empty and the probe is a batch of zeros until it fills. """ return _stack( [ sample_windows(buffers[i][0], jax.random.PRNGKey(7_000_000 + i), cfg.probe_batch) for i in range(len(cfg.seeds)) ] )
[docs] def fixture_path(seed: int) -> pathlib.Path: """``reference/raif/fixtures/oracle_mc_seed<N>.npz`` — the oracle's own collect stream.""" import hybrid_aif.reference.raif.fixtures as fx return pathlib.Path(fx.__file__).parent / f"oracle_mc_seed{int(seed)}.npz"
def _stack(xs: list): return jax.tree_util.tree_map(lambda *a: jnp.stack(a), *xs) # =================================================================== checkpointing
[docs] class Counters(NamedTuple): """What the JSON sidecar carries: the loop's position, not its arrays.""" step: int chunk: int grad_index: int episodes: tuple[int, ...] wall_seconds: float
[docs] def save_checkpoint( cfg: Config, state, buffers, pending, counters: Counters, coverage=None ) -> pathlib.Path: """Serialise everything numeric, then the sidecar, whose presence means the checkpoint is valid. Files land as ``.tmp`` and are renamed, and ``checkpoint.json`` is written last, so a process killed mid-checkpoint leaves the previous one intact. """ directory = cfg.path / "checkpoint" directory.mkdir(parents=True, exist_ok=True) _serialise(directory / "state.eqx", state) _serialise(directory / "buffers.eqx", buffers) flat = {} for i, per_seed in enumerate(pending): for j, held in enumerate(per_seed): for name, value in (held or {}).items(): flat[f"{i}:{j}:{name}"] = np.asarray(value) tmp = directory / "pending.tmp" with tmp.open("wb") as handle: # a path would gain a second `.npz` suffix np.savez(handle, **flat) tmp.replace(directory / "pending.npz") record = {"counters": counters._asdict(), "config": dataclasses.asdict(cfg)} if coverage is not None: # the visited-cell sets per seed per lane: coverage is cumulative over the # lane's life, so a resume continues it record["coverage"] = [[sorted(c.seen) for c in per_seed] for per_seed in coverage] tmp = directory / "checkpoint.json.tmp" tmp.write_text(json.dumps(record, indent=2, sort_keys=True) + "\n") tmp.replace(directory / "checkpoint.json") return directory
def _serialise(path: pathlib.Path, tree) -> None: tmp = path.with_suffix(path.suffix + ".tmp") eqx.tree_serialise_leaves(tmp, tree) tmp.replace(path) #: One pending-episode slot per seed per lane: the type the host's boundary carry is. PendingGrid = list[list[dict[str, np.ndarray] | None]]
[docs] def empty_pending(cfg: Config) -> PendingGrid: """``[seeds][lanes]`` of ``None`` — nothing in flight. One place builds the grid.""" return [[None] * int(cfg.lanes) for _ in cfg.seeds]
[docs] def load_coverage(cfg: Config, env: MountainCar) -> list[list[Coverage]]: """``[seeds][lanes]`` of :class:`Coverage`, restored from the sidecar if it is there. Without a ``coverage`` key the counters start empty. """ path = cfg.path / "checkpoint" / "checkpoint.json" saved = None if path.exists(): saved = json.loads(path.read_text()).get("coverage") return [ [Coverage(env, seen=None if saved is None else saved[i][j]) for j in range(int(cfg.lanes))] for i in range(len(cfg.seeds)) ]
[docs] def load_checkpoint(cfg: Config, state, buffers): """``(state, buffers, pending, counters)`` restored onto freshly built shapes.""" directory = cfg.path / "checkpoint" record = json.loads((directory / "checkpoint.json").read_text()) state = eqx.tree_deserialise_leaves(directory / "state.eqx", state) buffers = eqx.tree_deserialise_leaves(directory / "buffers.eqx", buffers) with np.load(directory / "pending.npz") as data: held = empty_pending(cfg) for name in data.files: index, lane, column = name.split(":", 2) slot = held[int(index)][int(lane)] or {} slot[column] = np.asarray(data[name]) held[int(index)][int(lane)] = slot counters = dict(record["counters"]) counters["episodes"] = tuple(counters["episodes"]) return state, buffers, held, Counters(**counters)
# ========================================================================= the loop def _console(*args) -> None: """``print`` with a flush, so status lines reach a redirected log.""" print(*args, flush=True)
[docs] def train(cfg: Config, *, resume: bool = False, log=_console) -> Counters: """The run. Returns the final counters; writes everything else to ``cfg.out``.""" cfg.path.mkdir(parents=True, exist_ok=True) system, crspp, env, state, buffers, probe_batch, reports, horizon = build(cfg) pending: PendingGrid = empty_pending(cfg) counters = Counters( step=0, chunk=0, grad_index=0, episodes=(0,) * len(cfg.seeds), wall_seconds=0.0 ) # the last WINDOW returns per seed, for the console line only (the record is # scores.jsonl) recent: list[collections.deque] = [collections.deque(maxlen=ev.WINDOW) for _ in cfg.seeds] # their MeasureCoverage, one per lane, cumulative over the lane's life (`Coverage`) coverage = ( load_coverage(cfg, env) if resume else [[Coverage(env) for _ in range(int(cfg.lanes))] for _ in cfg.seeds] ) if resume: state, buffers, pending, counters = load_checkpoint(cfg, state, buffers) for i, s in enumerate(cfg.seeds): recent[i].extend( r["return"] for r in read_scores(cfg.path / "scores.jsonl") if r["seed"] == s ) log( f"resumed at step {counters.step:,} (chunk {counters.chunk}, " f"{sum(counters.episodes):,} episodes)" ) else: # the commit the run was launched from, and whether the tree was dirty record = dataclasses.asdict(cfg) | {"code": _code_provenance()} (cfg.path / "run.json").write_text(json.dumps(record, indent=2) + "\n") for s, stats, provenance in reports: log( f"seed {s} seeded from {provenance['commit_sha'][:12]}: " f"{stats.n_episodes} episodes, mean return {stats.mean_return:.1f}, " f"SR {stats.success_rate:.2f}, mean length {stats.mean_length:.1f}" ) chunk_fn = make_chunk_fn( system, crspp, env, chunk_steps=cfg.chunk_steps, train_every=cfg.train_every, lanes=cfg.lanes, batch_size=cfg.batch_size, horizon=horizon, sample_scoring=cfg.sample_scoring, act_sampling=cfg.act_sampling, ) probe_fn = eqx.filter_jit( jax.vmap( lambda p, b, k: probe( system, crspp, p, b, k, sample_scoring=cfg.sample_scoring, act_sampling=cfg.act_sampling, ) ) ) add_fn = eqx.filter_jit(extend) n_blocks = cfg.chunk_steps // cfg.train_every scores = (cfg.path / "scores.jsonl").open("a") metrics = (cfg.path / "metrics.jsonl").open("a") stopping = {"now": False} def _stop(signum, _frame): stopping["now"] = True log(f"signal {signum}: finishing the chunk, then checkpointing") for sig in (signal.SIGINT, signal.SIGTERM): try: signal.signal(sig, _stop) except ValueError: # not the main thread (a test, or a driver process) pass started, last_checkpoint = time.time(), time.time() try: while counters.step < cfg.steps and not stopping["now"]: if cfg.alternate: # the labeled A/B, not the parity default — see `alternation` # their fallback test is `len(positive_replay) < args.batch_steps` # (`run/train.py:160`) — windows against batch_size · batch_length ready = np.asarray([b[0].ready(cfg.batch_size * cfg.seq_len) for b in buffers]) ups = alternation(counters.grad_index, n_blocks, ready) else: ups = m_only(len(cfg.seeds), n_blocks) t0 = time.time() state, stream, losses = chunk_fn( state, _stack([b[0] for b in buffers]), _stack([b[1] for b in buffers]), ups ) jax.block_until_ready(losses.world.total) device_seconds = time.time() - t0 step = counters.step + cfg.chunk_steps counts = list(counters.episodes) columns = {name: np.asarray(getattr(stream, name)) for name in Stream._fields} for i, s in enumerate(cfg.seeds): for j in range(int(cfg.lanes)): # the pooled agent step each episode ended on: `consumed` counts # steps in this lane, starting negative by the pending head's # length, and one lane-step is `lanes` pooled steps held = pending[i][j] consumed = -len(held["reward"]) if held is not None else 0 episodes, pending[i][j] = split_stream( held, {k: v[i, j] for k, v in columns.items()} ) for ep_cols in episodes: ep = as_collected(ep_cols) buffers[i][0], buffers[i][1] = route( buffers[i][0], buffers[i][1], ep, add_fn=add_fn ) counts[i] += 1 consumed += int(ep_cols["reward"].shape[0]) total = float(np.sum(ep_cols["reward"])) recent[i].append(total) scores.write( json.dumps( { "step": counters.step + consumed * int(cfg.lanes), "seed": int(s), "lane": j, "return": total, "length": int(ep_cols["reward"].shape[0]), "success": bool(ep_cols["has_succeeded"][-1]), # R-S's accumulator, which drops the is_last # row's reward (`ev.stability_c`) "stability": ev.stability_c(ep_cols["reward"], ev.MC_OFFSET), # their `stats/mean_log_entropy`: the act-time # policy entropy averaged over the episode "log_entropy": float(np.mean(ep_cols["log_entropy"])), # their `episode/coverage_mean`, read off this # lane's cumulative counter at the episode's end "coverage": coverage[i][j].update(ep_cols["obs"]), } ) + "\n" ) scores.flush() counters = Counters( step=step, chunk=counters.chunk + 1, grad_index=counters.grad_index + n_blocks, episodes=tuple(counts), wall_seconds=counters.wall_seconds + (time.time() - t0), ) if counters.chunk % max(1, cfg.metrics_every) == 0: gaps = probe_fn( state.params, probe_batch, jax.random.split(jax.random.PRNGKey(counters.chunk), len(cfg.seeds)), ) for i, s in enumerate(cfg.seeds): # G's decomposition, keyed by term name per_term = { f"term.{name}": float(value[i, -1]) for name, value in losses.imagine.per_term.items() } metrics.write( json.dumps( { "step": step, "seed": int(s), **per_term, "world": float(losses.world.total[i, -1]), "decoder": float(losses.world.decoder[i, -1]), "dyn": float(losses.world.dyn[i, -1]), "rep": float(losses.world.rep[i, -1]), "actor": float(losses.imagine.actor_loss[i, -1]), "critic": float(losses.imagine.critic_loss[i, -1]), "ensemble": float(losses.ensemble.total[i, -1]), "crspp": float(losses.crspp.total[i, -1]), "crspp_positive": float(losses.crspp.positive[i, -1]), "ret": float(losses.imagine.ret[i, -1]), "critic_value": float(losses.imagine.critic_value[i, -1]), "bc_logprob": float(losses.imagine.bc_logprob[i, -1]), # their `train/actor_refresh_mean`, a term of the # actor loss; `bc_logprob` is the unmasked mean # and not comparable (`series-sweep.md` §6.1) "bc_term": float(losses.imagine.bc_term[i, -1]), # the actor channel: `policy_entropy` and # `bc_weight` show a collapse to the σ floor and # the imitation weight left in the batch; # `advantage` against `retnorm_low/high` shows # the return normaliser's band "advantage": float(losses.imagine.advantage[i, -1]), "policy_entropy": float(losses.imagine.policy_entropy[i, -1]), "prior_entropy": float(losses.imagine.prior_entropy[i, -1]), "bc_weight": float(losses.imagine.bc_weight[i, -1]), "retnorm_low": float(losses.imagine.retnorm_low[i, -1]), "retnorm_high": float(losses.imagine.retnorm_high[i, -1]), "value": float(losses.imagine.value[i, -1]), # how much of the imagined horizon the continuation # head is truncating — the direct read on `start_cont` "rollout_weight": float(losses.imagine.rollout_weight_mean[i, -1]), "probe_ret": float(gaps[0][i]), "probe_critic": float(gaps[1][i]), "distill_gap": float(gaps[2][i]), "probe_value": float(gaps[3][i]), # the actor on the act-time and the training feature # paths, same states: a widening gap is the actor # acting off the distribution it trains on "probe_mu_act": float(gaps[4][i]), "probe_mu_train": float(gaps[5][i]), "probe_sigma_act": float(gaps[6][i]), "probe_sigma_train": float(gaps[7][i]), "probe_mu_gap": float(gaps[8][i]), } ) + "\n" ) metrics.flush() log( f"[{counters.step:>9,}/{cfg.steps:,}] {device_seconds:5.1f}s/chunk " f"({cfg.chunk_steps / device_seconds:6.1f} steps/s) " f"eps={sum(counters.episodes):>6,} " f"return={'/'.join(f'{np.mean(r):.0f}' if r else '—' for r in recent)} " f"world={float(jnp.mean(losses.world.total[:, -1])):7.3f} " f"finite={bool(jnp.all(jnp.isfinite(losses.world.total)))}" ) if (time.time() - last_checkpoint) / 60.0 >= cfg.checkpoint_every: save_checkpoint(cfg, state, buffers, pending, counters, coverage) last_checkpoint = time.time() log(f"checkpoint at step {counters.step:,}") finally: scores.close() metrics.close() # never mask the exception that got us here: a failure to write the # shutdown checkpoint is logged, not raised over the real cause try: save_checkpoint(cfg, state, buffers, pending, counters, coverage) written = "checkpoint written" except Exception as exc: # noqa: BLE001 written = f"CHECKPOINT FAILED: {exc!r}" log( f"stopped at step {counters.step:,} after {(time.time() - started) / 3600:.2f} h; " f"{written}" ) return counters
# ============================================================================ eval def read_scores(path: pathlib.Path) -> list[dict]: if not pathlib.Path(path).exists(): return [] with pathlib.Path(path).open() as handle: return [json.loads(line) for line in handle if line.strip()]
[docs] class SeedReport(NamedTuple): """One seed's §7 row. Every field is a *window*, not an endpoint — §7's ruling.""" seed: int episodes: int steps: int best_window: float best_window_at: str final_acr: float last_500k: float sr_window: float sr_oracle: float rs: float
[docs] def seed_report( rows: list[dict], seed: int, *, window: int = ev.WINDOW, span: int = 200_000, tail: int = 500_000, offset: float = ev.MC_OFFSET, threshold: float = -100.0, ) -> SeedReport: """The §7-comparable numbers for one seed, from ``scores.jsonl`` rows. ``best_window`` The best mean over complete tiled ``span``-step windows (the log reaching within 1% of a window's end); a partial trailing window is not counted. ``final_acr`` §6's ACR: the mean over the last ``window`` episodes (an endpoint read). ``last_500k`` The mean over episodes in the final ``tail`` steps. ``sr_window`` §6's SR: the fraction of the last ``window`` episodes reaching the goal. ``sr_oracle`` The oracle table's SR: the fraction of the last ``tail`` steps' episodes returning above ``threshold``, a different statistic from §6's. ``rs`` §6's R-S at the Mountain Car offset, over each logged episode's ``C`` (:func:`_stability`). Design: ``docs/design/amortization-p0-addendum.md`` §7. """ mine = sorted((r for r in rows if r["seed"] == seed), key=lambda r: r["step"]) if not mine: raise ValueError(f"no episodes logged for seed {seed}") returns = np.asarray([r["return"] for r in mine], np.float64) steps = np.asarray([r["step"] for r in mine], np.int64) last = int(steps[-1]) best, best_at = -np.inf, "—" # a window counts once the log reaches it to within 1% of its end: the run's last # episode can end a few dozen steps short of a window that the run's budget covers for low in range(0, last + span // 100 - span + 1, span): inside = (steps >= low) & (steps < low + span) if inside.sum() >= 2 and returns[inside].mean() > best: best, best_at = float(returns[inside].mean()), f"{low // 1000}–{(low + span) // 1000}k" if not np.isfinite(best): # a run shorter than one window: the whole run IS the window best, best_at = float(returns.mean()), f"0–{last // 1000}k (partial)" in_tail = steps > last - tail return SeedReport( seed=seed, episodes=len(mine), steps=last, best_window=best, best_window_at=best_at, final_acr=ev.acr(returns[-window:]), last_500k=float(returns[in_tail].mean()), sr_window=ev.sr(np.asarray([r["success"] for r in mine[-window:]])), sr_oracle=float(np.mean(returns[in_tail] > threshold)), rs=ev.relative_stability(_stability(mine, offset), offset=offset, window=window), )
def _stability(rows: list[dict], offset: float) -> np.ndarray: """R-S's ``C`` per episode: the logged ``stability`` field, or the fallback for older logs. A log without the field falls back to ``return + offset·length``, which over-counts by the final row's reward. A log mixing both kinds raises ``ValueError``. """ old = [r for r in rows if "stability" not in r] new = [r for r in rows if "stability" in r] if old and new: raise ValueError( f"scores.jsonl mixes {len(old)} episode row(s) with no 'stability' " f"field and {len(new)} with one: R-S's C is accumulated differently in the two " "(their MeasureRStability drops the is_last row's reward), so folding them into " "one history would report a number that is neither. Re-run, or report the halves " "separately." ) if new: return np.asarray([r["stability"] for r in new], np.float64) return np.asarray([r["return"] + offset * r["length"] for r in old], np.float64)
[docs] def report(cfg: Config, log=_console) -> list[SeedReport]: """Print the §7-comparable table from ``scores.jsonl``.""" rows = read_scores(cfg.path / "scores.jsonl") if not rows: log(f"no episodes in {cfg.path / 'scores.jsonl'}") return [] present = sorted({r["seed"] for r in rows}) out = [seed_report(rows, s) for s in present] log("") log("R-AIF state-based Mountain Car — addendum §7 comparison (window-honest)") log( f"{'seed':>4} {'steps':>10} {'eps':>7} {'best 200k':>11} {'at':>12} " f"{'final-100':>10} {'last-500k':>10} {'SR(§6)':>7} {'SR(>-100)':>10} {'R-S':>6}" ) for r in out: log( f"{r.seed:>4} {r.steps:>10,} {r.episodes:>7,} {r.best_window:>11.1f} " f"{r.best_window_at:>12} {r.final_acr:>10.1f} {r.last_500k:>10.1f} " f"{r.sr_window:>7.2f} {r.sr_oracle:>10.2f} {r.rs:>6.3f}" ) log("") log( "oracle (8 seeds): best-200k −68.0…−69.5 / final-100 −67.6…−110.0 / " "last-500k −69.5…−79.4 / SR(>-100) 0.86…0.98" ) log( "target (§7): converged ACR −68 to −69.5 on the WINDOWS, plus the oracle's shape: " "first successes by ~2–4k, a plateau near −80 over 50–200k, a failure phase at " "~225–450k recovered by 375–450k, converged after" ) if out: log( f"mean best-200k over {len(out)} seed(s): " f"{np.mean([r.best_window for r in out]):.1f} ± " f"{np.std([r.best_window for r in out]):.1f}" ) return out
# ======================================================================= benchmark
[docs] def benchmark(cfg: Config, *, reps: int = 3, log=_console) -> dict: """Jitted steps/s for the full train cycle, one seed against ``len(seeds)`` vmapped. Measured at the run's shapes with the run's chunk function. Reports both rates, their ratio, the projected wall clock for ``cfg.steps`` × seeds, and the collection half's own time (``collect_seconds``) from the same :func:`make_collect_fn`. """ out = {} for n in (1, len(cfg.seeds)): sub = dataclasses.replace(cfg, seeds=tuple(cfg.seeds[:n])) system, crspp, env, state, buffers, _, _, horizon = build(sub) chunk_fn = make_chunk_fn( system, crspp, env, chunk_steps=sub.chunk_steps, train_every=sub.train_every, lanes=sub.lanes, batch_size=sub.batch_size, horizon=horizon, sample_scoring=sub.sample_scoring, act_sampling=sub.act_sampling, ) n_blocks = sub.chunk_steps // sub.train_every ups = m_only(n, n_blocks) # the parity default's schedule; timing is identical either way args = (_stack([b[0] for b in buffers]), _stack([b[1] for b in buffers]), ups) t0 = time.time() state, _, losses = chunk_fn(state, *args) jax.block_until_ready(losses.world.total) compile_seconds = time.time() - t0 t0 = time.time() for _ in range(reps): state, _, losses = chunk_fn(state, *args) jax.block_until_ready(losses.world.total) seconds = (time.time() - t0) / reps # the collection half alone, at the same shapes and the same POOLED step count # (`chunk_steps // lanes` steps in each of `lanes` lanes) only_collect = eqx.filter_jit( jax.vmap( make_collect_fn( system, env, sub.chunk_steps // int(sub.lanes), act_sampling=sub.act_sampling ) ) ) carry = state.collect_carry() jax.block_until_ready(only_collect(state.params, carry)[1].reward) t0 = time.time() for _ in range(reps): jax.block_until_ready(only_collect(state.params, carry)[1].reward) collect_seconds = (time.time() - t0) / reps out[n] = { "seeds": n, "compile_seconds": compile_seconds, "chunk_seconds": seconds, "collect_seconds": collect_seconds, "collect_fraction": collect_seconds / seconds, "agent_steps_per_second": sub.chunk_steps / seconds, "grad_steps_per_second": n_blocks / seconds, "total_agent_steps_per_second": n * sub.chunk_steps / seconds, } log( f"{n} seed(s): {seconds:7.2f} s/chunk of {sub.chunk_steps} steps " f"= {sub.chunk_steps / seconds:7.2f} agent steps/s per seed, " f"{n * sub.chunk_steps / seconds:7.2f} aggregate; " f"{seconds / n_blocks:6.3f} s/gradient step; " f"collect {collect_seconds:.2f}s = {100 * collect_seconds / seconds:.1f}% " f"(compile {compile_seconds:.1f}s)" ) one, many = out[1], out[len(cfg.seeds)] ratio = many["total_agent_steps_per_second"] / one["total_agent_steps_per_second"] hours = cfg.steps / many["agent_steps_per_second"] / 3600.0 log(f"vmap({len(cfg.seeds)}) aggregate throughput ratio vs 1 seed: {ratio:.2f}×") sequential = cfg.steps * len(cfg.seeds) / one["agent_steps_per_second"] / 3600.0 log(f"projected wall clock for {cfg.steps:,} steps × {len(cfg.seeds)} seeds: {hours:.1f} h") log(f"the same run sequentially: {sequential:.1f} h") out["sequential_hours"] = sequential out["ratio"] = ratio out["projected_hours"] = hours return out
# ============================================================================ CLI def parse(argv: list[str] | None = None) -> tuple[Config, argparse.Namespace]: p = argparse.ArgumentParser(prog="python -m hybrid_aif.reference.raif.run", description=__doc__) p.add_argument("--out", required=True, help="run directory") p.add_argument("--seeds", default=",".join(str(s) for s in DEFAULT_SEEDS)) p.add_argument( "--steps", type=int, default=None, help=f"agent steps per seed (default {DEFAULT_STEPS:,}); on --resume it EXTENDS the run", ) p.add_argument("--chunk-steps", type=int, default=CHUNK_STEPS) p.add_argument( "--lanes", type=int, default=LANES, help=( f"parallel env lanes per seed (default {LANES} = their envs.amount); must divide " "--train-every, since their train_ratio counts POOLED agent steps. 1 is a " "single-stream harness." ), ) p.add_argument("--batch-size", type=int, default=BATCH_SIZE) p.add_argument("--seq-len", type=int, default=BATCH_LENGTH) p.add_argument( "--capacity", type=int, default=CAPACITY, help="replay steps per buffer, not sequences" ) p.add_argument("--horizon", type=int, default=IMAG_HORIZON) p.add_argument("--checkpoint-every", type=float, default=15.0, help="minutes") p.add_argument("--metrics-every", type=int, default=1, help="chunks") p.add_argument("--small", action="store_true", help="test-scale sizes; NOT parity") p.add_argument("--no-fixtures", action="store_true", help="start with empty buffers") p.add_argument( "--alternate", action="store_true", help=( "alternate world-model batches M/M+ per the PAPER — a labeled A/B, not parity: " "the oracle's own alternation is inert and it trains every batch on M" ), ) p.add_argument( "--sample-scoring", action="store_true", help=( "roll the CRSPP's preferred trajectory SAMPLED inside imagine (theirs, ledger 6.5) " "instead of expected/mean-field (the default)" ), ) p.add_argument( "--no-act-sampling", action="store_true", help=("act on the mean-field posterior instead of a sampled code; kept for the ablation"), ) p.add_argument( "--winit", choices=("normal", "default"), default="normal", help=( "the world model's and the critic's initializer. 'normal' is theirs " "and the parity default; 'default' is the engine's, the isolation control's arm" ), ) p.add_argument("--resume", action="store_true") p.add_argument("--report", action="store_true") p.add_argument("--benchmark", action="store_true") args = p.parse_args(argv) cfg = Config( out=args.out, seeds=tuple(int(s) for s in args.seeds.split(",") if s.strip() != ""), steps=DEFAULT_STEPS if args.steps is None else args.steps, chunk_steps=args.chunk_steps, lanes=args.lanes, batch_size=args.batch_size, seq_len=args.seq_len, capacity=args.capacity, horizon=args.horizon, checkpoint_every=args.checkpoint_every, metrics_every=args.metrics_every, small=args.small, fixtures=not args.no_fixtures, alternate=args.alternate, winit=args.winit, sample_scoring=args.sample_scoring, act_sampling=not args.no_act_sampling, ) if args.resume and (cfg.path / "run.json").exists(): # the stored configuration wins: every knob fixes a shape, except `--steps` and # the two reporting cadences, which the checkpoint does not hold stored = json.loads((cfg.path / "run.json").read_text()) stored["seeds"] = tuple(stored["seeds"]) # `code` is provenance, not configuration; a resume from another commit is # reported, not refused launched_from = stored.pop("code", None) # a knob absent from a stored run.json takes the value it had before the knob # existed (`_ABSENT_MEANS`), never today's default; any other absence refuses for knob, before_it_existed in _ABSENT_MEANS.items(): stored.setdefault(knob, before_it_existed) missing = sorted({f.name for f in dataclasses.fields(Config)} - set(stored) - {"out"}) if missing: raise ValueError( f"{cfg.path / 'run.json'} records no {missing}, so this resume cannot know what " "the checkpoint was built with. Defaulting them would rebuild whatever today's " "code builds, against a checkpoint from whatever it built then. Add the value the " "run was actually launched with to its run.json (and to reference.raif.run's " "_ABSENT_MEANS if the knob postdates runs that are still resumable)." ) # `out` is where this invocation writes, not the stored run's old path stored["out"] = args.out if args.steps is not None: stored["steps"] = args.steps stored["checkpoint_every"] = args.checkpoint_every stored["metrics_every"] = args.metrics_every cfg = Config(**stored) now = _code_provenance() if launched_from and launched_from.get("sha") != now["sha"]: print( f"note: resuming a run launched from {str(launched_from.get('sha'))[:12]} " f"on a checkout at {str(now['sha'])[:12]}", file=sys.stderr, ) return cfg, args def main(argv: list[str] | None = None) -> int: cfg, args = parse(argv) if args.report: report(cfg) elif args.benchmark: benchmark(cfg) else: train(cfg, resume=args.resume) return 0 if __name__ == "__main__": sys.exit(main())