Source code for hybrid_aif.reference.raif.columns

"""R-AIF's episode columns, as task columns of the engine's :class:`Episode`.

``is_success`` is the goal predicate on the row's observation, ``has_succeeded`` its running
OR, ``can_self_imitate`` the expert flag that :mod:`.revise` overwrites with the imitation
mask, and ``pref_label`` is ρ. In Mountain Car the goal ends the episode, so
:func:`raif_episode` writes ``cont = 1 − is_success``.

Design: ``docs/design/amortization-p0-addendum.md`` §4.
"""

from __future__ import annotations

import jax.numpy as jnp
import numpy as np
from jax import Array

from hybrid_aif.train.buffers import ColumnSpec, Episode, episode

IS_SUCCESS = "is_success"
HAS_SUCCEEDED = "has_succeeded"
CAN_SELF_IMITATE = "can_self_imitate"
PREF_LABEL = "pref_label"

#: The task columns and how a buffer pads them: the two flags hold their last
#: value (a padded step is still "after the goal"), the weight and the label are
#: zero where nothing was done or earned.
EXTRAS: dict[str, ColumnSpec] = {
    IS_SUCCESS: ColumnSpec((), "bool", "hold"),
    HAS_SUCCEEDED: ColumnSpec((), "bool", "hold"),
    CAN_SELF_IMITATE: ColumnSpec((), "float32", "zero"),
    PREF_LABEL: ColumnSpec((), "float32", "zero"),
}

#: The **recorded** columns — the on-disk fixture format and the collect stream's
#: arrays; ``pref_label`` is derived and never stored.
RECORDED: tuple[str, ...] = (
    "obs",
    "action",
    "reward",
    IS_SUCCESS,
    HAS_SUCCEEDED,
    CAN_SELF_IMITATE,
    "is_first",
    "is_last",
)


[docs] def raif_episode( *, obs: Array, action: Array, reward: Array, is_success: Array, has_succeeded: Array, can_self_imitate: Array, is_first: Array, is_last: Array, pref_label: Array | None = None, ) -> Episode: """An :class:`Episode` from their recorded columns: ``cont = 1 − is_success``, the four task columns in ``extras`` (ρ zero until :func:`.revise.label`).""" success = jnp.asarray(is_success, jnp.bool_) t = int(success.shape[0]) return episode( obs, action, reward, is_first=is_first, is_last=is_last, cont=1.0 - success.astype(jnp.float32), **{ IS_SUCCESS: success, HAS_SUCCEEDED: jnp.asarray(has_succeeded, jnp.bool_), CAN_SELF_IMITATE: jnp.asarray(can_self_imitate, jnp.float32), PREF_LABEL: jnp.zeros((t,), jnp.float32) if pref_label is None else jnp.asarray(pref_label, jnp.float32), }, )
[docs] def recorded_columns(ep: Episode) -> dict[str, np.ndarray]: """The recorded columns of ``ep`` as numpy arrays, in :data:`RECORDED` order.""" out = {} for name in RECORDED: out[name] = np.asarray(ep.extras[name] if name in ep.extras else getattr(ep, name)) return out
[docs] def succeeded(ep: Episode) -> bool: """Whether the episode reached the goal: its final ``has_succeeded``.""" return bool(ep.extras[HAS_SUCCEEDED][-1])