Source code for hybrid_aif.train.buffers

"""Replay: one episode format, three buffers for two execution regimes.

Replay is agent-coupled, so it lives beside the trainers; it reads
:mod:`hybrid_aif.data`, never the reverse.

============================  ====================  ==========================
                              :class:`ReplayBuffer`  :class:`StaticBuffer`,
                                                     :class:`StreamBuffer`
============================  ====================  ==========================
sampled                       in **Python**, between  **inside** jit /
                              jitted calls            ``lax.scan``
shapes                        dynamic; schema from    static, declared at
                              the first episode       allocation
episodes                      ragged, as collected    fitted to ``seq_len`` /
                                                      spliced into a stream
storage                       numpy ring, one array   a **pytree** of JAX
                              per column              arrays
columns                       whatever was written    the :class:`Episode`
                                                      format
extras                        ``Dataset`` windowing,  rides a ``lax.scan``
                              pluggable samplers,     carry
                              ``dump()`` to any
                              registered format
============================  ====================  ==========================

Use :class:`ReplayBuffer` unless you need in-jit sampling. ``num_steps`` ≈
``seq_len`` and ``max_steps`` ≈ ``capacity·seq_len``; :func:`as_episode` and
:func:`episode_columns` bridge the regimes. The device-side buffers differ in their
sequence contract:

============================  ======================  ==========================
                              :class:`StaticBuffer`   :class:`StreamBuffer`
============================  ======================  ==========================
a stored unit is              one whole episode,      one **step**
                              fitted to ``seq_len``
``capacity`` counts           sequences               steps
an episode shorter than       padded by freezing      nothing — it is
``seq_len``                   its final step          ``length`` steps of stream
a sampled window              starts at an episode's  starts anywhere, and may
                              first step              cross episode boundaries
``is_first``                  true only at step 0     true wherever an episode
                                                      begins inside the window
needs                         nothing of the RSSM     ``RSSMCore.observe``'s
                                                      ``is_first`` reset
============================  ======================  ==========================

Buffer policy (several buffers, routing, fallback) stays in the caller's loop.

Design: docs/design/amortization.md A12 chunk 2c §(d′)
"""

from __future__ import annotations

from collections import deque
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
from typing import Any, NamedTuple

import equinox as eqx
import jax
import jax.numpy as jnp
import numpy as np
from jax import Array

from hybrid_aif.data.dataset import Dataset
from hybrid_aif.data.format import check_schema, episode_schema, get_format, public_items

# ============================================================== the episode format


[docs] class ColumnSpec(NamedTuple): """One task-specific column of an :class:`Episode`.""" #: The column's per-step trailing shape. shape: tuple[int, ...] = () #: The column's dtype, by name. dtype: str = "float32" #: How :func:`fit_length` pads the column: ``"zero"`` or ``"hold"`` (repeat the last value). pad: str = "zero"
[docs] class Episode(NamedTuple): """One episode's per-step columns — the format the whole training side speaks. Leaves are ``(T, ...)`` for one episode, ``(B, T, ...)`` for a batch. Row ``t`` pairs ``obs[t]`` with the action then drawn at it, so ``reward[t]`` is the reward of ``action[t−1]`` and every flag is a predicate on ``obs[t]``. ``T`` applied actions make ``T + 1`` rows: a reset row (``is_first``, ``reward = 0``) and a terminal row with the action zeroed. """ #: The observations, ``(T, obs_dim)`` float32. obs: Array #: ``a_t``, the raw draw taken at ``obs[t]``, ``(T, action_dim)`` float32. action: Array #: The reward that produced ``obs[t]``, that is ``a_{t-1}``'s, ``(T,)`` float32. reward: Array #: ``1 - terminal``, a predicate on ``obs[t]`` as recorded by the collector, ``(T,)`` #: float32. cont: Array #: Whether ``obs[t]`` starts the episode (the reset row), ``(T,)`` bool. is_first: Array #: Whether ``obs[t]`` ends the episode, ``(T,)`` bool. is_last: Array #: The task's own per-step columns, ``(T, ...)`` each, carried through untouched. extras: dict[str, Array] = {} @property def length(self) -> int: return self.obs.shape[-2]
[docs] def prev_actions(self) -> Array: """The actions re-indexed so entry ``t`` is ``a_{t−1}``; step 0 gets zeros. :func:`~hybrid_aif.train.world.fit_world`'s convention. """ zero = jnp.zeros_like(self.action[..., :1, :]) return jnp.concatenate([zero, self.action[..., :-1, :]], axis=-2)
#: The engine's columns: what every episode carries, whatever the task. EPISODE_FIELDS: tuple[str, ...] = ("obs", "action", "reward", "cont", "is_first", "is_last") #: How :func:`fit_length` pads each engine column. _CORE_PAD: dict[str, str] = { "obs": "hold", "action": "zero", "reward": "zero", "cont": "hold", "is_first": "zero", } Schema = Mapping[str, ColumnSpec]
[docs] def episode( obs: Array, action: Array, reward: Array, *, is_first: Array, is_last: Array, cont: Array | None = None, **extras: Array, ) -> Episode: """An :class:`Episode` from its columns, cast to the engine dtypes and checked. ``cont`` defaults to all ones. Any further keyword is a task column, stored in ``extras`` with its own dtype and a leading ``T`` axis checked. """ t = int(jnp.asarray(obs).shape[0]) core = { "obs": jnp.asarray(obs, jnp.float32), "action": jnp.asarray(action, jnp.float32), "reward": jnp.asarray(reward, jnp.float32), "cont": jnp.ones((t,), jnp.float32) if cont is None else jnp.asarray(cont, jnp.float32), "is_first": jnp.asarray(is_first, jnp.bool_), "is_last": jnp.asarray(is_last, jnp.bool_), } for name in ("obs", "action"): if core[name].ndim != 2: raise ValueError( f"episode(): {name} has shape {core[name].shape}, expected (T, dim) — a " "per-step vector even where the dimension is 1" ) for name in ("reward", "cont", "is_first", "is_last"): if core[name].ndim != 1: raise ValueError( f"episode(): {name} has shape {core[name].shape}, expected (T,) — one number " "per step" ) cast_extras = {name: jnp.asarray(value) for name, value in extras.items()} lengths = { name: int(value.shape[0]) if value.ndim else -1 for name, value in {**core, **cast_extras}.items() } if len(set(lengths.values())) != 1: raise ValueError(f"episode(): columns disagree on T: {lengths}") return Episode(**core, extras=cast_extras)
[docs] def fit_length(ep: Episode, seq_len: int, pad: Mapping[str, str] | None = None) -> Episode: """One episode at exactly ``seq_len`` steps: truncated, or padded by freezing. Padding repeats the final observation and ``cont`` with zero action and zero reward; ``is_last`` marks the final stored step. Task columns pad by ``pad[name]`` (``"hold"`` or ``"zero"``; zeros when not given). """ if seq_len < 1: raise ValueError(f"seq_len must be positive, got {seq_len}") t = ep.length if t >= seq_len: out = jax.tree_util.tree_map(lambda x: x[:seq_len], ep) return out._replace(is_last=jnp.zeros((seq_len,), jnp.bool_).at[-1].set(True)) n = seq_len - t extra_rules = dict(pad or {}) if bad := sorted(set(extra_rules) & set(EPISODE_FIELDS)): raise ValueError( f"fit_length: pad names engine columns {bad}; only task columns take a rule" ) rules = {**_CORE_PAD, **extra_rules} def padded(name: str, x: Array) -> Array: rule = rules.get(name, "zero") if rule == "hold": return jnp.concatenate([x, jnp.repeat(x[-1:], n, axis=0)], axis=0) if rule == "zero": return jnp.concatenate([x, jnp.zeros((n, *x.shape[1:]), x.dtype)], axis=0) raise ValueError( f"fit_length: pad rule for column {name!r} is {rule!r}; the rules are 'hold' and 'zero'" ) core = {name: padded(name, getattr(ep, name)) for name in EPISODE_FIELDS if name != "is_last"} return Episode( **core, is_last=jnp.zeros((seq_len,), jnp.bool_).at[-1].set(True), extras={name: padded(name, x) for name, x in ep.extras.items()}, )
# ======================================================= host-side, dynamic schema Sampler = Callable[["ReplayBuffer", int, np.random.Generator], np.ndarray] """The type of a :class:`ReplayBuffer`'s clip sampler: ``sampler(buffer, batch_size, rng)`` returns ``batch_size`` flat clip indices in ``[0, len(buffer))``, which :meth:`ReplayBuffer.sample` turns into clips. :func:`uniform_sampler` is the default.""" def uniform_sampler(buffer: ReplayBuffer, batch_size: int, rng: np.random.Generator): """The default :data:`Sampler`: ``batch_size`` clip indices drawn uniformly, with replacement, from ``[0, len(buffer))`` by ``rng``. Raises :class:`RuntimeError` when the buffer holds no clip.""" n = len(buffer) if n == 0: raise RuntimeError("ReplayBuffer.sample: no clips available") return rng.integers(0, n, size=batch_size)
[docs] class ReplayBuffer(Dataset): """Ring-storage replay buffer with whole-episode FIFO eviction. Host-side. Both a :class:`~hybrid_aif.data.dataset.Dataset` and a :class:`~hybrid_aif.data.format.Writer`. Storage is one numpy ring per column of ``max_steps`` rows, allocated from the first episode's schema; clips never cross episode boundaries. ``dump(path, format)`` persists through any registered format. Arguments: - ``max_steps``: capacity budget in raw steps. - ``num_steps``: observation frames per clip (Dataset windowing). - ``frameskip``: stride between observation frames within a clip. - ``sampler``: pluggable :data:`Sampler` used by :meth:`sample`; uniform over valid clips by default. - ``seed``: seed for the default rng handed to the sampler. """
[docs] def __init__( self, max_steps: int, num_steps: int = 1, frameskip: int = 1, sampler: Sampler | None = None, seed: int | None = None, ) -> None: if max_steps <= 0: raise ValueError(f"max_steps must be positive, got {max_steps}") if num_steps <= 0 or frameskip <= 0: raise ValueError( f"num_steps and frameskip must be positive, got {num_steps=} {frameskip=}" ) self.max_steps = int(max_steps) self.num_steps = num_steps self.frameskip = frameskip self.sampler: Sampler = sampler if sampler is not None else uniform_sampler self.rng = np.random.default_rng(seed) self._cols: dict[str, np.ndarray] = {} self._schema: dict[str, tuple[tuple[int, ...], np.dtype]] | None = None self._episodes: deque[tuple[int, int]] = deque() # (ring_start, ep_len) self._head = 0 self._size = 0 self._clip_cum: np.ndarray | None = None self._clips: list[tuple[int, int]] | None = None
# -- Writer side ------------------------------------------------------- def __enter__(self) -> ReplayBuffer: return self def __exit__(self, *exc) -> None: return None def write_episode(self, ep_data: dict) -> None: if self._schema is None: schema = episode_schema(ep_data) else: check_schema(self._schema, ep_data, "ReplayBuffer.write_episode") schema = self._schema stored = dict(public_items(ep_data)) ep_len = next(iter(stored.values())).shape[0] if ep_len == 0: return if ep_len > self.max_steps: raise ValueError(f"episode of length {ep_len} exceeds max_steps={self.max_steps}") if self._schema is None: self._schema = schema for col, (shape, dtype) in schema.items(): self._cols[col] = np.empty((self.max_steps, *shape), dtype=dtype) while self._size + ep_len > self.max_steps: _, evicted = self._episodes.popleft() self._size -= evicted positions = (self._head + np.arange(ep_len)) % self.max_steps for col, arr in stored.items(): self._cols[col][positions] = arr self._episodes.append((self._head, ep_len)) self._head = (self._head + ep_len) % self.max_steps self._size += ep_len self._clip_cum = None self._clips = None def write_episodes(self, episodes: Iterable[dict]) -> None: for ep in episodes: self.write_episode(ep) # -- Dataset side ------------------------------------------------------ @property def column_names(self) -> list[str]: return list(self._cols) @property def num_episodes(self) -> int: return len(self._episodes) @property def num_steps_stored(self) -> int: return self._size @property def lengths(self) -> np.ndarray: return np.asarray([ln for _, ln in self._episodes], dtype=np.int64) @property def offsets(self) -> np.ndarray: lens = self.lengths return np.concatenate(([0], np.cumsum(lens[:-1]))).astype(np.int64) def _load_slice(self, ep_idx: int, start: int, end: int) -> dict[str, np.ndarray]: ring_start, ep_len = self._episodes[ep_idx] if not 0 <= start <= end <= ep_len: raise IndexError(f"slice [{start}:{end}] out of episode of length {ep_len}") positions = (ring_start + np.arange(start, end)) % self.max_steps return {col: arr[positions] for col, arr in self._cols.items()}
[docs] def sample( self, batch_size: int, rng: np.random.Generator | None = None ) -> dict[str, np.ndarray]: """Draw a batch of clips via the sampler; arrays are stacked to ``(batch_size, num_steps, ...)``.""" idx = np.asarray(self.sampler(self, batch_size, rng if rng is not None else self.rng)) eps, starts = self.locate(idx) clips = [self._clip(int(e), int(s)) for e, s in zip(eps, starts, strict=True)] return {col: np.stack([c[col] for c in clips]) for col in self.column_names}
# -- Persistence -------------------------------------------------------
[docs] def iter_episodes(self) -> Iterator[dict[str, np.ndarray]]: """Yield stored episodes oldest-first as contiguous column dicts.""" for ring_start, ep_len in list(self._episodes): positions = (ring_start + np.arange(ep_len)) % self.max_steps yield {col: arr[positions] for col, arr in self._cols.items()}
[docs] def dump(self, path, format: str, mode: str = "overwrite", **kwargs: Any) -> None: """Persist current contents through the registered writer for ``format``.""" import hybrid_aif.data.formats # noqa: F401 (registers built-in formats) with get_format(format).open_writer(path, mode=mode, **kwargs) as writer: writer.write_episodes(self.iter_episodes())
# ====================================================== device-side, static shapes
[docs] class StaticBuffer(eqx.Module): """A uniform replay buffer of fixed-length episode-aligned sequences. A pytree. One JAX array per column, written FIFO into a ring. ``size`` and ``cursor`` are traced scalars, so :func:`add` and :func:`sample` are pure and the buffer can ride a ``lax.scan`` carry. """ #: The stored sequences, one array per column, ``(capacity, seq_len, ...)``. data: Episode #: The number of live slots (traced). size: Array #: The next slot to write (traced). cursor: Array #: The number of sequences held. capacity: int = eqx.field(static=True) #: The steps per sequence. seq_len: int = eqx.field(static=True) #: The task's own columns, ``(name, ColumnSpec)`` pairs. schema: tuple[tuple[str, ColumnSpec], ...] = eqx.field(static=True, default=()) @property def pad_rules(self) -> dict[str, str]: """How :func:`fit_length` pads each task column, from the schema.""" return {name: spec.pad for name, spec in self.schema}
[docs] @classmethod def empty( cls, capacity: int, seq_len: int, obs_dim: int, action_dim: int, extras: Schema | None = None, ) -> StaticBuffer: """An allocated, empty buffer. Shapes are fixed here and never change. ``extras`` declares the task's columns (:class:`ColumnSpec` per name); every episode written must carry exactly those. """ n, t = int(capacity), int(seq_len) if capacity < 1: raise ValueError(f"capacity must be positive, got {capacity}") schema = _schema(extras) lead = (n, t) if cls is StaticBuffer else (n,) return cls( data=Episode( obs=jnp.zeros((*lead, int(obs_dim)), jnp.float32), action=jnp.zeros((*lead, int(action_dim)), jnp.float32), reward=jnp.zeros(lead, jnp.float32), cont=jnp.ones(lead, jnp.float32), is_first=jnp.zeros(lead, jnp.bool_), is_last=jnp.zeros(lead, jnp.bool_), extras={ name: jnp.zeros((*lead, *spec.shape), jnp.dtype(spec.dtype)) for name, spec in schema }, ), size=jnp.zeros((), jnp.int32), cursor=jnp.zeros((), jnp.int32), capacity=n, seq_len=t, schema=schema, )
def __len__(self) -> int: """Live sequences, as a Python int (host-side); inside jit read :attr:`size`.""" return int(self.size)
[docs] def ready(self, batch_size: int) -> bool: """Whether a full batch can be drawn without repeating the whole buffer. Host-side.""" return len(self) >= int(batch_size)
def _check_extras(buffer: StaticBuffer | StreamBuffer, ep: Episode) -> None: declared = dict(buffer.schema) if set(ep.extras) != set(declared): raise ValueError( f"the episode carries task columns {sorted(ep.extras)} but the buffer was " f"allocated for {sorted(declared)}: declare them in .empty(extras=...) — a buffer " "holds exactly the columns its schema names" ) for name, spec in declared.items(): column = ep.extras[name] if column.dtype != jnp.dtype(spec.dtype) or tuple(column.shape[1:]) != tuple(spec.shape): raise ValueError( f"task column {name!r} is {column.dtype} with per-step shape " f"{tuple(column.shape[1:])} but the buffer's schema says {spec.dtype} " f"{tuple(spec.shape)}; cast or reshape it before writing — a store would cast " "silently" ) def _schema(extras: Schema | None) -> tuple[tuple[str, ColumnSpec], ...]: """A buffer's static schema from ``extras``: names to :class:`ColumnSpec`, validated.""" out = [] for name, spec in dict(extras or {}).items(): spec = ColumnSpec(*spec) if name in EPISODE_FIELDS: raise ValueError(f"{name!r} is an engine column, not a task column") if spec.pad not in ("hold", "zero"): raise ValueError(f"column {name!r}: pad must be 'hold' or 'zero', got {spec.pad!r}") out.append((name, ColumnSpec(tuple(int(d) for d in spec.shape), str(spec.dtype), spec.pad))) return tuple(out)
[docs] def add(buffer: StaticBuffer, ep: Episode) -> StaticBuffer: """One episode written into a :class:`StaticBuffer` ring: a new buffer, nothing mutated. ``ep`` is fitted to ``seq_len`` first. Eviction is FIFO by slot. Compute any column derived from the whole episode before adding. """ _check_extras(buffer, ep) fitted = fit_length(ep, buffer.seq_len, pad=buffer.pad_rules) slot = buffer.cursor % buffer.capacity data = jax.tree_util.tree_map(lambda store, row: store.at[slot].set(row), buffer.data, fitted) return StaticBuffer( data=data, size=jnp.minimum(buffer.size + 1, buffer.capacity), cursor=slot + 1, capacity=buffer.capacity, seq_len=buffer.seq_len, schema=buffer.schema, )
[docs] def sample(buffer: StaticBuffer, key: Array, batch_size: int) -> Episode: """A uniform batch of whole sequences: leaves ``(batch_size, seq_len, ...)``. Samples with replacement over the live slots. An empty buffer draws slot 0, so guard with :meth:`StaticBuffer.ready`. """ index = jax.random.randint(key, (int(batch_size),), 0, jnp.maximum(buffer.size, 1)) return jax.tree_util.tree_map(lambda store: store[index], buffer.data)
[docs] class StreamBuffer(eqx.Module): """A uniform replay buffer over a flat step stream. Device-side, a pytree. Static shapes and traced counters like :class:`StaticBuffer`, but ``capacity`` counts steps: :func:`extend` splices each episode onto one ring in completion order, and :func:`sample_windows` draws ``seq_len`` contiguous steps starting anywhere, so a window may cross episode boundaries; its ``is_first`` column marks the joins for :meth:`~hybrid_aif.inference.rssm.RSSMCore.observe`. Nothing is padded or truncated; eviction is FIFO by step. Design: docs/design/amortization.md A12 chunk 2c §(d′) """ #: The step stream, one array per column, ``(capacity, ...)``. data: Episode #: The number of live steps (traced). size: Array #: The next step slot to write (traced). cursor: Array #: The number of steps held. capacity: int = eqx.field(static=True) #: The steps per sampled window. seq_len: int = eqx.field(static=True) #: The task's own columns, ``(name, ColumnSpec)`` pairs. schema: tuple[tuple[str, ColumnSpec], ...] = eqx.field(static=True, default=()) @property def pad_rules(self) -> dict[str, str]: """How :func:`fit_length` pads each task column, from the schema.""" return {name: spec.pad for name, spec in self.schema}
[docs] @classmethod def empty( cls, capacity: int, seq_len: int, obs_dim: int, action_dim: int, extras: Schema | None = None, ) -> StreamBuffer: """An allocated, empty buffer. Shapes are fixed here and never change. ``extras`` declares the task's columns (:class:`ColumnSpec` per name); every episode written must carry exactly those. """ n, t = int(capacity), int(seq_len) if t < 1: raise ValueError(f"seq_len must be positive, got {seq_len}") if n < t: raise ValueError( f"capacity must be at least seq_len ({t} steps), got {capacity} — capacity is " "in STEPS here, not sequences, and a stream shorter than one window can never " "be sampled" ) schema = _schema(extras) lead = (n, t) if cls is StaticBuffer else (n,) return cls( data=Episode( obs=jnp.zeros((*lead, int(obs_dim)), jnp.float32), action=jnp.zeros((*lead, int(action_dim)), jnp.float32), reward=jnp.zeros(lead, jnp.float32), cont=jnp.ones(lead, jnp.float32), is_first=jnp.zeros(lead, jnp.bool_), is_last=jnp.zeros(lead, jnp.bool_), extras={ name: jnp.zeros((*lead, *spec.shape), jnp.dtype(spec.dtype)) for name, spec in schema }, ), size=jnp.zeros((), jnp.int32), cursor=jnp.zeros((), jnp.int32), capacity=n, seq_len=t, schema=schema, )
def __len__(self) -> int: """Live steps, as a Python int (host-side); inside jit read :attr:`size`.""" return int(self.size) @property def windows(self) -> int: """How many distinct ``seq_len``-step windows the live stream offers. ``size − seq_len + 1``, floored at 0: one window per valid start offset. """ return max(int(self.size) - self.seq_len + 1, 0)
[docs] def ready(self, batch_steps: int) -> bool: """Whether the stream offers at least ``batch_steps`` windows yet. Host-side.""" return self.windows >= int(batch_steps)
[docs] def extend(buffer: StreamBuffer, ep: Episode) -> StreamBuffer: """One episode's steps spliced onto the stream: a new buffer, nothing mutated. Every column, ``extras`` included, is written at the same ring positions; the episode keeps its own ``is_first``/``is_last``. Compute any column derived from the whole episode before extending. Jit-able, with ``ep.length`` static per trace. """ if jnp.ndim(ep.obs) != 2: raise ValueError( f"extend takes ONE episode: obs (T, obs_dim), got {jnp.shape(ep.obs)} — a batch of " "episodes is a loop over this call, since each has its own length" ) _check_extras(buffer, ep) t = ep.length if t > buffer.capacity: raise ValueError( f"episode of length {t} exceeds the stream's capacity of {buffer.capacity} steps" ) positions = (buffer.cursor + jnp.arange(t, dtype=jnp.int32)) % buffer.capacity data = jax.tree_util.tree_map( lambda store, column: store.at[positions].set(column), buffer.data, ep ) return StreamBuffer( data=data, size=jnp.minimum(buffer.size + t, buffer.capacity), cursor=(buffer.cursor + t) % buffer.capacity, capacity=buffer.capacity, seq_len=buffer.seq_len, schema=buffer.schema, )
[docs] def sample_windows(buffer: StreamBuffer, key: Array, batch_size: int) -> Episode: """A uniform batch of ``seq_len``-step windows: leaves ``(batch_size, seq_len, ...)``. Start offsets are uniform over valid starts, with replacement; ``is_first`` marks every episode boundary in the window, and step 0's flag is forced true. A stream shorter than one window floors the offset range at one slot, so guard with :meth:`StreamBuffer.ready`. """ oldest = (buffer.cursor - buffer.size) % buffer.capacity starts = jnp.maximum(buffer.size - buffer.seq_len + 1, 1) offset = jax.random.randint(key, (int(batch_size), 1), 0, starts) index = (oldest + offset + jnp.arange(buffer.seq_len, dtype=jnp.int32)) % buffer.capacity window = jax.tree_util.tree_map(lambda store: store[index], buffer.data) return window._replace(is_first=window.is_first.at[:, 0].set(True))
# ================================================================== the two, bridged
[docs] def as_episode(columns: Mapping[str, np.ndarray]) -> Episode: """A :class:`ReplayBuffer` episode dict → an :class:`Episode`. ``obs`` and ``action`` are required; ``reward`` defaults to zeros, ``cont`` to ones, ``is_first``/``is_last`` to the first and last row. Every other column becomes a task column in ``extras``, with the dtype it arrived in. """ if "obs" not in columns or "action" not in columns: raise ValueError( f"as_episode: no 'obs'/'action' column (got {sorted(columns)}); those two carry " "the episode's shape, so the format cannot be filled in without them" ) t = int(np.asarray(columns["obs"]).shape[0]) core = { "obs": columns["obs"], "action": columns["action"], "reward": columns.get("reward", jnp.zeros((t,), jnp.float32)), "cont": columns.get("cont", jnp.ones((t,), jnp.float32)), "is_first": columns.get("is_first", jnp.zeros((t,), jnp.bool_).at[0].set(True)), "is_last": columns.get("is_last", jnp.zeros((t,), jnp.bool_).at[-1].set(True)), } extras = {name: value for name, value in columns.items() if name not in EPISODE_FIELDS} return episode( core["obs"], core["action"], core["reward"], is_first=core["is_first"], is_last=core["is_last"], cont=core["cont"], **extras, )
[docs] def episode_columns(ep: Episode) -> dict[str, np.ndarray]: """An :class:`Episode` → a column dict for :meth:`ReplayBuffer.write_episode`: the engine columns and the task columns, flat, as numpy arrays.""" out = {name: np.asarray(getattr(ep, name)) for name in EPISODE_FIELDS} out.update({name: np.asarray(value) for name, value in ep.extras.items()}) return out
[docs] def stack_episodes( episodes: Sequence[Episode], seq_len: int, pad: Mapping[str, str] | None = None ) -> Episode: """A list of episodes as one batched :class:`Episode`, each fitted to ``seq_len`` (:func:`fit_length`, with ``pad`` the task columns' rules).""" if not episodes: raise ValueError("stack_episodes: no episodes") fitted = [fit_length(ep, seq_len, pad=pad) for ep in episodes] return jax.tree_util.tree_map(lambda *xs: jnp.stack(xs), *fitted)
__all__ = [ "EPISODE_FIELDS", "ColumnSpec", "Episode", "ReplayBuffer", "Sampler", "Schema", "SeedStats", "StaticBuffer", "StreamBuffer", "add", "as_episode", "episode", "episode_columns", "extend", "fit_length", "sample", "sample_windows", "seed_stats", "stack_episodes", "uniform_sampler", ] # ------------------------------------------------------------- seed statistics class SeedStats(NamedTuple): """Seed-policy quality over a set of episodes. ``mean_return`` is the summed reward per episode; ``mean_length`` the steps to the first success when a success column is named, else the whole episode; ``success_rate`` the fraction with a success step, or ``None`` without a column. """ n_episodes: int mean_return: float success_rate: float | None mean_length: float def seed_stats(episodes: Sequence[Episode], success: str | None = None) -> SeedStats: """Mean return / success rate / mean length over a list of episodes. ``success`` names the task column in ``extras`` holding the per-step success flag; without it the success rate is ``None`` and lengths are whole episodes. """ if not episodes: raise ValueError("seed_stats: no episodes") returns = [float(jnp.sum(ep.reward)) for ep in episodes] if success is None: return SeedStats( len(episodes), float(np.mean(returns)), None, float(np.mean([ep.length for ep in episodes])), ) flags = [np.asarray(ep.extras[success]).astype(bool) for ep in episodes] lengths = [ int(np.flatnonzero(f)[0]) + 1 if f.any() else int(ep.length) for f, ep in zip(flags, episodes, strict=True) ] return SeedStats( n_episodes=len(episodes), mean_return=float(np.mean(returns)), success_rate=float(np.mean([bool(f.any()) for f in flags])), mean_length=float(np.mean(lengths)), )