Source code for hybrid_aif.reference.raif.world
"""Reference world models as declared Systems: R-AIF's ``tiny`` RSSM and its CRSPP.
The placement of :func:`tiny_core`'s parts into slots::
state factors deter = the Deterministic GRU carry (controlled: the action)
stoch = the 32x32 categorical code (parent: deter, same step)
transition transition:deter GRU(deter, flat stoch_{t-1}, action)
transition:stoch p(stoch | deter_t)
initial priors initial_state_prior:deter LearnedVector
initial_state_prior:stoch DerivedInitial()
modalities likelihood:state symlog_mse over [deter, flat stoch]
likelihood:reward symlog_disc, 255 bins over [-20, 20]
likelihood:cont binary (continuation, not termination)
recognition recognition:embed MLP over the state observation (symlog inputs)
recognition:stoch q(stoch | deter_t, embed)
``deter`` is declared first because ``stoch``'s transition parent is the carry at the same
step. The G side is not part of the world model; the agent mounts it
(:func:`hybrid_aif.reference.raif.agent.raif_mountain_car`).
Design: ``docs/design/amortization-p0-addendum.md`` §§1–3.
"""
import dataclasses
import jax
import jax.numpy as jnp
from jax import Array
from hybrid_aif.controls import Box, control_name
from hybrid_aif.distributions import MLPCategorical
from hybrid_aif.distributions.neural import LearnedVector, MLPPredictor, PredictorHead
from hybrid_aif.factors import DerivedInitial, Deterministic, Recognition
from hybrid_aif.inference.rssm import RSSMCore
from hybrid_aif.reference.raif import crspp as _crspp # noqa: F401 (registers the CRSPP row)
from hybrid_aif.system import System, SystemParams
#: Their ``winit`` values mapped onto the engine's initializer names.
_INIT = {"default": "default", "normal": "trunc_normal_avg"}
[docs]
def init_of(winit: str) -> str:
"""The engine's ``init=`` for their ``winit`` (``"default"`` | ``"normal"``)."""
if winit not in _INIT:
raise ValueError(f"unknown winit {winit!r}; theirs are {sorted(_INIT)}")
return _INIT[winit]
CARRY_NAME = "deter"
CODE_NAME = "stoch"
STATE_MODALITY = "state"
REWARD_MODALITY = "reward"
CONT_MODALITY = "cont"
EMBED_VARIABLE = "embed"
[docs]
def tiny_core(
obs_dim: int,
action_dim: int,
*,
key: Array,
deter: int = 128,
num_vars: int = 32,
num_classes: int = 32,
units: int = 32,
mlp_units: int = 64,
mlp_layers: int = 2,
embed: int = 64,
unimix: float = 0.01,
num_bins: int = 255,
bin_low: float = -20.0,
bin_high: float = 20.0,
activation: str = "silu",
norm: bool = True,
winit: str = "default",
) -> RSSMCore:
"""R-AIF's ``tiny`` world model as an :class:`RSSMCore`, state-based.
``rssm {deter 128, stoch 32, classes 32}``, ``units 32`` inside the RSSM and on the
reward/continuation heads, ``mlp_units 64`` / 2 layers on the encoder and decoder,
``unimix 0.01``, a 255-bin ``symlog_disc`` reward over [-20, 20]. ``winit`` reaches every
block: ``"normal"`` is their ``winit: normal, fan: avg`` with no bias where a layer is
normed; ``"default"`` is eqx's initializer with every bias kept.
Design: ``docs/experiments/raif-mountain-car/divergence-register.md`` (divergences 17, 26).
"""
keys = jax.random.split(key, 7)
flat = int(num_vars) * int(num_classes)
return RSSMCore(
gru=Deterministic.gru(
deter,
(flat, action_dim),
key=keys[0],
hidden=units,
activation=activation,
norm=norm,
init=init_of(winit),
),
prior=MLPCategorical.from_shapes(
(deter,),
num_vars,
num_classes,
key=keys[1],
width=units,
depth=1,
activation=activation,
norm=norm,
unimix=unimix,
init=init_of(winit),
),
posterior=MLPCategorical.from_shapes(
(deter, embed),
num_vars,
num_classes,
key=keys[2],
width=units,
depth=1,
activation=activation,
norm=norm,
unimix=unimix,
init=init_of(winit),
),
# their MLP encoder branch: the last normed, activated layer is the embedding,
# hence out_activation and depth = mlp_layers - 1
encoder=MLPPredictor.from_shapes(
(obs_dim,),
embed,
key=keys[3],
width=mlp_units,
depth=max(mlp_layers - 1, 0),
activation=activation,
norm=norm,
out_activation=True,
symlog_inputs=True,
init=init_of(winit),
),
decoder=PredictorHead.from_shapes(
(deter, flat),
loss="symlog_mse",
out_dim=obs_dim,
key=keys[4],
width=mlp_units,
depth=mlp_layers,
activation=activation,
norm=norm,
init=init_of(winit),
),
reward=PredictorHead.from_shapes(
(deter, flat),
loss="symlog_disc",
num_bins=num_bins,
bin_low=bin_low,
bin_high=bin_high,
key=keys[5],
width=units,
depth=2,
activation=activation,
norm=norm,
out_scale=0.0, # their reward outscale 0.0
init=init_of(winit),
),
cont=PredictorHead.from_shapes(
(deter, flat),
loss="binary",
key=keys[6],
width=units,
depth=2,
activation=activation,
norm=norm,
init=init_of(winit),
),
initial=LearnedVector.zeros(deter),
obs_dim=int(obs_dim),
action_dim=int(action_dim),
)
[docs]
def tiny_rssm(
obs_dim: int,
action_dim: int,
*,
key: Array,
action_low: float = -1.0,
action_high: float = 1.0,
policy_len: int = 8,
learned_initial: bool = True,
**tiny: object,
) -> tuple[System, SystemParams]:
"""R-AIF's ``tiny`` world model as a ``(System, SystemParams)`` pair.
``obs_dim``/``action_dim`` are the environment's (Mountain Car: 2 and 1). ``key`` and the
extra keyword arguments (``deter``, ``num_vars``, ``units``, ``unimix``, …) go to
:func:`tiny_core`. ``learned_initial=False`` freezes the initial carry at zero (their
``initial: 'zeros'``). ``policy_len`` is their ``imag_horizon``: the explicit rollout
steps before the ``g_tail`` critic closes the horizon. The carry reads a continuous control
node, its box ``[action_low, action_high]`` in ``params.control``; its action node emits the
value to the env. The G side is
mounted on the returned pair by the agent.
"""
core = tiny_core(obs_dim, action_dim, key=key, **tiny) # type: ignore[arg-type]
system = System(
A_deps=((0, 1), (0, 1), (0, 1)), # every head reads [deter, flat stoch]
B_deps=((1,), (0,)), # deter <- stoch_{t-1} (+ action); stoch <- deter_t
controlled=(True, False), # the action enters the carry, their img_step
policy_len=policy_len,
planner="cem", # continuous actions (A9); "habit" once an actor is declared
epistemic=False, # no epistemic row over these beliefs yet (2b)
state_names=(CARRY_NAME, CODE_NAME),
modality_names=(STATE_MODALITY, REWARD_MODALITY, CONT_MODALITY),
recognition=(
Recognition(variable=EMBED_VARIABLE, parents=(f"obs:{STATE_MODALITY}",)),
Recognition(variable=CODE_NAME, parents=(CARRY_NAME, f"recognition:{EMBED_VARIABLE}")),
),
)
# their `initial: 'zeros'` is the same declaration, marked untrainable
initial = core.initial if learned_initial else dataclasses.replace(core.initial, frozen=True)
params = SystemParams(
transition={CARRY_NAME: core.gru, CODE_NAME: core.prior},
initial_state_prior={CARRY_NAME: initial, CODE_NAME: DerivedInitial()},
likelihood={
STATE_MODALITY: core.decoder,
REWARD_MODALITY: core.reward,
CONT_MODALITY: core.cont,
},
recognition={EMBED_VARIABLE: core.encoder, CODE_NAME: core.posterior},
control={
control_name(CARRY_NAME): Box(
jnp.full((action_dim,), action_low, jnp.float32),
jnp.full((action_dim,), action_high, jnp.float32),
)
},
)
system._validate(params)
return system, params
[docs]
def crspp_rssm(
*,
key: Array,
policy_len: int = 8,
deter: int = 128,
num_vars: int = 32,
num_classes: int = 32,
units: int = 32,
embed: int = 64,
unimix: float = 0.01,
activation: str = "silu",
norm: bool = True,
winit: str = "default",
) -> tuple[System, SystemParams]:
"""R-AIF's CRSPP prior-preference model: ``tiny_rssm`` without the action or the heads.
A separate parameter set with its own GRU and prior and no modality, so it is only rolled
forward, never observed. Its ``recognition:stoch`` conditions on
``(deter, external:embed)``: ``embed``'s width is declared in ``System.external_inputs``
and must match the agent's encoder (``tiny_rssm``'s ``embed``, 64 by default); its value
is supplied per call by :func:`hybrid_aif.reference.raif.crspp.crspp_recognize`. The
``cosine_sim_to_pref_rollout`` row owns it through ``System.scoring`` and
``SystemParams.scoring``. ``policy_len`` is carried but unused: the row rolls the agent's
horizon.
Design: ``docs/design/amortization-debt-ledger.md`` entry 5.
"""
keys = jax.random.split(key, 3)
flat = int(num_vars) * int(num_classes)
system = System(
A_deps=(), # no decoder of its own
B_deps=((1,), (0,)), # deter <- stoch_{t-1} (no action); stoch <- deter_t
controlled=(False, False), # the whole point: no action in the transition prior
policy_len=policy_len,
epistemic=False,
state_names=(CARRY_NAME, CODE_NAME),
modality_names=(),
external_inputs=((EMBED_VARIABLE, int(embed)),),
recognition=(
Recognition(variable=CODE_NAME, parents=(CARRY_NAME, f"external:{EMBED_VARIABLE}")),
),
)
params = SystemParams(
transition=(
Deterministic.gru(
deter,
(flat,),
key=keys[0],
hidden=units,
activation=activation,
norm=norm,
init=init_of(winit),
),
MLPCategorical.from_shapes(
(deter,),
num_vars,
num_classes,
key=keys[1],
width=units,
depth=1,
activation=activation,
norm=norm,
unimix=unimix,
init=init_of(winit),
),
),
initial_state_prior=(LearnedVector.zeros(deter), DerivedInitial()),
recognition={
CODE_NAME: MLPCategorical.from_shapes(
(deter, int(embed)),
num_vars,
num_classes,
key=keys[2],
width=units,
depth=1,
activation=activation,
norm=norm,
unimix=unimix,
init=init_of(winit),
)
},
)
params = system.from_positional("params", params)
system._validate(params)
return system, params