Source code for hybrid_aif.reference.raif.revise

"""Self-revision: the per-episode backward pass that produces ρ and the imitation mask.

A port of their ``callbacks.py::lookback_on_episode`` with their ``configs.yaml:109``
hyperparameters. It runs once per finished episode, before the episode is written to the
buffers (:func:`label`). Pure, unbatched and jit-able: one episode in, two ``(T,)`` arrays
out.

Design: ``docs/design/amortization-p0-addendum.md`` §4.
"""

from typing import NamedTuple

import jax
import jax.numpy as jnp
from jax import Array

from hybrid_aif.reference.raif.columns import (
    CAN_SELF_IMITATE,
    HAS_SUCCEEDED,
    IS_SUCCESS,
    PREF_LABEL,
)
from hybrid_aif.train.buffers import Episode

#: Their ``configs.yaml:109`` triple: the backward discount, the quantization
#: floor below which the carry is zeroed, and the failure branch's decay.
ALPHA = 0.4
EPSILON = 0.001
BETA = 0.3


[docs] class PrefLabels(NamedTuple): """What ``lookback_on_episode`` writes back into the episode. Both ``(T,)``. ``pref_label`` is ρ ∈ [−1, 1]. ``can_self_imitate`` is the propagated imitation mask, a float weight in [0, 1] that replaces the recorded flag and gates the actor's BC term. """ pref_label: Array can_self_imitate: Array
[docs] def pref_labels( ep: Episode, *, alpha: float = ALPHA, epsilon: float = EPSILON, beta: float = BETA, ) -> PrefLabels: """``lookback_on_episode``, ported: ``(pref_label, can_self_imitate)``. ``alpha``, ``epsilon`` and ``beta`` are their ``disc``, ``thr`` and ``fail``. The steps: - the expert flags are gated by episode success, ``can_self_imitate *= has_succeeded[-1]``; - a backward carry seeded from the last step's gated flag credits step ``t−1`` from ``is_success[t]`` geometrically at ``alpha``, zeroed below ``epsilon``; ``mask[T−1]`` is the seed and is not thresholded; - the mask is the clipped sum of the gated flags and the carry, shifted one step forward into ``positive_mask`` (slot 0 from ``has_succeeded[0]``); - in a failed episode each step gets ``−beta**(T−1−t)``, so the last step is at −1; the expert exemption on this branch is a no-op after the success gate and is kept as their code has it; - ``pref_label = clip(positive_mask + neg, −1, 1)`` with ``pref_label[0] = 0``. Raises ``ValueError`` if ``alpha`` is outside [0, 1), ``epsilon`` is negative or ``beta`` is outside (0, 1). """ if not 0.0 <= alpha < 1.0: raise ValueError(f"alpha (their disc) must be in [0, 1), got {alpha}") if epsilon < 0.0: raise ValueError(f"epsilon (their thr) must be non-negative, got {epsilon}") if not 0.0 < beta < 1.0: # strictly positive: their failure_decays construction divides by `fail` raise ValueError(f"beta (their fail) must be in (0, 1), got {beta}") t = ep.length succeeded = ep.extras[HAS_SUCCEEDED][-1].astype(jnp.float32) gated = ep.extras[CAN_SELF_IMITATE].astype(jnp.float32) * succeeded success = ep.extras[IS_SUCCESS].astype(jnp.float32) def backward(carry: Array, s: Array) -> tuple[Array, Array]: # their `carry = max(carry * disc, is_success[t]); carry *= (carry >= thr)` c = jnp.maximum(carry * alpha, s) c = c * (c >= epsilon).astype(jnp.float32) return c, c if t > 1: # the scan reads is_success[1:] in reverse; the step at index j credits mask[j] _, credited = jax.lax.scan(backward, jnp.maximum(gated[-1], 0.0), success[1:], reverse=True) mask = jnp.concatenate([credited, jnp.maximum(gated[-1:], 0.0)]) else: mask = jnp.maximum(gated[-1:], 0.0) final = jnp.clip(gated + mask, 0.0, 1.0) positive = jnp.concatenate([ep.extras[HAS_SUCCEEDED][:1].astype(jnp.float32), final[:-1]]) decays = jnp.cumprod(jnp.full((t,), beta, jnp.float32))[::-1] / beta decays = decays * (1.0 - succeeded) neg = -decays * (1.0 - gated) rho = jnp.clip((positive + neg).at[0].set(0.0), -1.0, 1.0) return PrefLabels(pref_label=rho, can_self_imitate=final)
[docs] def label(ep: Episode, **kwargs) -> Episode: """``ep`` with both columns ``lookback_on_episode`` writes: ρ and the imitation mask. Call it on a finished episode, before :func:`hybrid_aif.train.buffers.add`; on a stored window the last step is not the episode's last step. Afterwards ``can_self_imitate`` is the float mask, not the recorded expert flag. """ out = pref_labels(ep, **kwargs) return ep._replace( extras={**ep.extras, PREF_LABEL: out.pref_label, CAN_SELF_IMITATE: out.can_self_imitate} )
[docs] def positive_gate(rho: Array) -> Array: """``max(0, sgn ρ)``: the CRSPP KL's hard mask to positive states (paper Eq. 5).""" return jnp.maximum(0.0, jnp.sign(rho))