"""``fit_world``: replay-driven world-model learning.
A pure function over explicit state: ``(System, params, data, LearnState)`` in,
the stepped triple out. The objective is DreamerV3's world-model loss, averaged
over batch and time::
L = 1.0·decoder_nll + 1.0·reward_nll + 1.0·cont_nll
+ 0.5·max(KL(sg q ‖ p), 1) + 0.1·max(KL(q ‖ sg p), 1)
built from the heads' own ``nll`` and
:func:`~hybrid_aif.distributions.neural.kl_balance`. Sequences go through
:meth:`~hybrid_aif.inference.rssm.RSSMCore.observe` (one ``lax.scan`` each) under
one ``vmap``. The loss is stated over the assembly read out of the System's slots
(:func:`assemble`), and the stepped parts are written back by slot key.
Design: docs/design/amortization.md A2, A6
"""
import dataclasses
from collections.abc import Mapping
from typing import NamedTuple
import equinox as eqx
import jax
import jax.numpy as jnp
import optax
from jax import Array
from hybrid_aif.distributions.neural import (
kl_balance,
)
from hybrid_aif.inference import amortized as _amortized
from hybrid_aif.inference.rssm import PART_FIELDS, RSSMCore, RSSMState
from hybrid_aif.system import LearnState, System, SystemParams
#: DreamerV3's world-model optimizer defaults: adam at ``lr 1e-4`` with a global-norm
#: ``clip 1000`` over the whole model. The learning rate is applied by :func:`fit_world`
#: after the optax step — the transformation stays static and the rate stays
#: *traceable*, as ``distributions.neural.OPTIMIZER`` documents for a neural slot.
DEFAULT_LR = 1e-4
DEFAULT_CLIP = 1000.0
#: The world-model adam's ``eps``: optax's default, hence ``distributions.neural.OPTIMIZER``'s,
#: so each part steps through its own ``absorb`` and no per-slot optimizer is built.
DEFAULT_EPS = 1e-8
[docs]
class WorldBatch(NamedTuple):
"""A batch of replay sequences, all shapes static."""
#: The observations, ``(B, T, obs_dim)``.
obs: Array
#: ``action[b, t]`` is ``a_{t-1}``, the action that produced ``obs[b, t]``,
#: ``(B, T, action_dim)``.
action: Array
#: The rewards, ``(B, T)``.
reward: Array
#: ``1 - done``, ``(B, T)``.
cont: Array
#: Where an episode begins inside a sequence, ``(B, T)``; the recurrent carry restarts there.
#: Episode-aligned batches may leave it ``None``.
is_first: Array | None = None
[docs]
def check(self, core: RSSMCore) -> tuple[int, int]:
"""Validate against a core and return ``(B, T)``."""
if self.obs.ndim != 3 or self.obs.shape[-1] != core.obs_dim:
raise ValueError(f"obs has shape {self.obs.shape}, expected (B, T, {core.obs_dim})")
b, t = self.obs.shape[0], self.obs.shape[1]
expected = {
"action": (self.action.shape, (b, t, core.action_dim)),
"reward": (self.reward.shape, (b, t)),
"cont": (self.cont.shape, (b, t)),
}
if self.is_first is not None:
expected["is_first"] = (self.is_first.shape, (b, t))
bad = {k: v for k, v in expected.items() if tuple(v[0]) != v[1]}
if bad:
raise ValueError(
"WorldBatch shapes disagree with obs "
f"{self.obs.shape}: "
+ "; ".join(f"{k} is {got} but should be {want}" for k, (got, want) in bad.items())
)
return b, t
[docs]
class WorldLosses(NamedTuple):
"""The objective, decomposed: every field a scalar mean over batch and time.
``dyn > dyn_raw`` means the free-bits floor binds.
"""
#: The differentiated loss (scales applied).
total: Array
#: The decoder's unscaled negative log-likelihood.
decoder: Array
#: The reward head's unscaled negative log-likelihood.
reward: Array
#: The continuation head's unscaled negative log-likelihood.
cont: Array
#: The dynamics KL ``KL[sg(posterior) || prior]``, after the free-bits floor and before its
#: scale.
dyn: Array
#: The representation KL ``KL[posterior || sg(prior)]``, after the free-bits floor and before
#: its scale.
rep: Array
#: The dynamics KL before the floor.
dyn_raw: Array
#: The representation KL before the floor.
rep_raw: Array
[docs]
def world_loss(
core: RSSMCore,
batch: WorldBatch,
key: Array,
*,
dyn_scale: float = 0.5,
rep_scale: float = 0.1,
free_nats: float = 1.0,
decoder_scale: float = 1.0,
reward_scale: float = 1.0,
cont_scale: float = 1.0,
return_states: bool = False,
) -> tuple[Array, WorldLosses] | tuple[Array, tuple[WorldLosses, RSSMState]]:
"""The world-model loss on one batch: ``(total, WorldLosses)``.
With ``return_states=True`` the auxiliary is ``(WorldLosses, RSSMState)``, the
posterior states of this forward pass, leaves ``(B, T, ...)``. Pure and
differentiable in ``core``; ``key`` draws the straight-through posterior
samples. Defaults are DreamerV3's loss scales and one-nat free-bits floor.
"""
b, _ = batch.check(core)
def one_sequence(
k: Array,
obs: Array,
action: Array,
reward: Array,
cont: Array,
is_first: Array | None = None,
):
states, post_logits, prior_logits = core.observe(k, action, obs, is_first=is_first)
# each head's own loss, per step, on the state vector it reads
parents = jax.vmap(core._features)(states)
decoder_nll = jax.vmap(core.decoder.nll)(obs, parents)
reward_nll = jax.vmap(core.reward.nll)(reward, parents)
cont_nll = jax.vmap(core.cont.nll)(cont, parents)
kl = kl_balance(
post_logits,
prior_logits,
dyn_scale=dyn_scale,
rep_scale=rep_scale,
free_nats=free_nats,
)
total = (
decoder_scale * decoder_nll
+ reward_scale * reward_nll
+ cont_scale * cont_nll
+ kl.total
)
losses = WorldLosses(
total=total,
decoder=decoder_nll,
reward=reward_nll,
cont=cont_nll,
dyn=kl.dyn,
rep=kl.rep,
dyn_raw=kl.dyn_raw,
rep_raw=kl.rep_raw,
)
return losses, states
columns = (jax.random.split(key, b), batch.obs, batch.action, batch.reward, batch.cont)
if batch.is_first is None:
per_step, states = jax.vmap(one_sequence)(*columns)
else:
per_step, states = jax.vmap(one_sequence)(*columns, batch.is_first)
losses = WorldLosses(*(jnp.mean(term) for term in per_step))
if return_states:
return losses.total, (losses, states)
return losses.total, losses
[docs]
def assemble(system: System, params: SystemParams, roles: dict[str, str] | None = None):
"""The world-model assembly behind a declared System: ``(core, slot addresses)``.
:meth:`~hybrid_aif.inference.rssm.RSSMCore.from_declarations` over the
validated amortized structure; ``roles`` names the reward and continuation
modalities when names and shapes do not decide.
"""
params = system._p(params)
structure = system._amortized_structure(params)
mods, facs = (system.modality_names, system._facs)
return RSSMCore.from_declarations(
structure,
params.likelihood,
params.transition,
params.initial_state_prior,
params.recognition,
system.A_deps,
mods,
facs,
roles=roles,
)
[docs]
def fit_world(
system: System,
params: SystemParams,
learn_state: LearnState | None,
batch: WorldBatch,
*,
key: Array,
lr: float | Array = DEFAULT_LR,
lr_by_slot: Mapping[str, float | Array] | None = None,
clip: float = DEFAULT_CLIP,
roles: dict[str, str] | None = None,
return_states: bool = False,
**loss_kwargs,
) -> (
tuple[SystemParams, LearnState, WorldLosses]
| tuple[SystemParams, LearnState, WorldLosses, RSSMState]
):
"""One gradient step of world-model learning: ``(params, learn_state, losses)``.
With ``return_states=True`` a fourth element follows: the posterior
:class:`~hybrid_aif.inference.rssm.RSSMState` of the forward pass (pre-step
parameters, gradient stopped). ``roles`` goes to :func:`assemble`,
``**loss_kwargs`` to :func:`world_loss`. Pure, jit-able, resumable from its
own return value; ``learn_state`` starts as ``system.init_learn_state(params)``
and ``None`` raises.
``lr`` may be traced. ``clip`` is one global-norm clip over the whole model
(``None`` disables it), summed in :data:`~hybrid_aif.inference.rssm.PART_FIELDS`
order; each slot then steps through its own ``LearnState`` entry.
``lr_by_slot`` is ``System.learn``'s rate grammar: a static ``0.0`` skips that
slot's step at trace time, a traced zero multiplies it. A frozen slot is not
differentiated, holds no optimizer state, and comes back bitwise.
"""
core, slots = assemble(system, params, roles=roles)
batch.check(core)
opt = System._resolve_learn_state(learn_state, system._gradient_slots(params))
rates = system._resolve_rates(lr_by_slot, "lr_by_slot", params)
fields = tuple(name for name in PART_FIELDS if getattr(core, name).learnable)
addresses = tuple(getattr(slots, name) for name in fields)
def loss_fn(parts: tuple):
model = dataclasses.replace(core, **dict(zip(fields, parts, strict=True)))
return world_loss(model, batch, key, return_states=True, **loss_kwargs)
trainable = tuple(getattr(core, name) for name in fields)
grads, (losses, states) = eqx.filter_grad(loss_fn, has_aux=True)(trainable)
if clip is not None:
clipper = optax.clip_by_global_norm(clip)
grads, _ = clipper.update(grads, clipper.init(grads))
updated: dict[str, object] = {}
for address, part, grad in zip(addresses, trainable, grads, strict=True):
rate = lr if rates is None else rates.get(address, lr)
if isinstance(rate, int | float) and float(rate) == 0.0:
continue # static zero: this slot takes no step, and its carry is untouched
updated[address], opt[address] = part.absorb(grad, opt[address], lr=rate)
# `moments` is a cross-trainer carry (the return normalizer's EMA, the slow
# critic's counter): this trainer owns none of it and passes it through unchanged.
carried = {} if learn_state is None else dict(learn_state.moments)
new_params = system._write_slots(params, updated)
new_state = LearnState(opt_states=opt, moments=carried)
if return_states:
return new_params, new_state, losses, jax.lax.stop_gradient(states)
return new_params, new_state, losses
[docs]
def posterior_states(
system: System,
params: SystemParams,
batch: WorldBatch,
*,
key: Array,
roles: dict[str, str] | None = None,
) -> RSSMState:
"""The recognition model's posterior states over a replay batch: ``(B, T, ...)``.
:meth:`~hybrid_aif.inference.rssm.RSSMCore.observe` under one ``vmap``; the
codes are straight-through samples. The start states for
:func:`~hybrid_aif.train.imagine.imagine` and the targets for
:func:`~hybrid_aif.train.ensemble.fit_ensemble`. ``roles`` goes to :func:`assemble`.
"""
core, _ = assemble(system, params, roles=roles)
b, _ = batch.check(core)
def one(k: Array, obs: Array, action: Array, is_first: Array | None = None) -> RSSMState:
states, _, _ = core.observe(k, action, obs, is_first=is_first)
return states
columns = (jax.random.split(key, b), batch.obs, batch.action)
if batch.is_first is None:
return jax.vmap(one)(*columns)
return jax.vmap(one)(*columns, batch.is_first)
def belief_tuple(structure, state: RSSMState) -> tuple[Array, ...]:
"""An :class:`~hybrid_aif.inference.rssm.RSSMState` as the positional belief tuple (internal).
Per-factor values in declaration order: the carry's vector for ``carry``, the code's tensor
for ``code``. Leading batch axes pass through. The faces take beliefs keyed by factor name:
:meth:`System.beliefs_from_states <hybrid_aif.system.System.beliefs_from_states>`.
"""
values = {_amortized.CARRY: state.deter, _amortized.CODE: state.stoch}
missing = sorted(set(structure.families) - set(values))
if missing:
raise ValueError(
f"belief_tuple: this System declares factor families {missing}, which the RSSM "
"assembly does not produce (it is one carry and one code)"
)
return tuple(values[family] for family in structure.families)