"""Neural conditional families: a parametrized function in a declaration slot.
- :class:`MLPConditional`: an MLP over the concatenated parents with a diagonal-Gaussian head;
- :class:`MLPPredictor`: the trunk alone (parents → a raw output vector), with no density;
- :class:`PredictorHead`: a trunk plus the name of the loss that reads its output;
- :class:`MLPCategorical`: a trunk with a factored, unimix-mixed categorical head;
- :class:`GRUCell`: a deterministic carry update, declared through
:class:`~hybrid_aif.factors.Deterministic`;
- :class:`LearnedVector`: a trainable vector behind an activation (a parametrized constant);
- the world-model loss pieces (:func:`kl_balance`, :func:`symlog_mse_nll`,
:func:`symlog_disc_nll`, :func:`binary_nll`).
Everything here is role-free (no class knows which slot holds it) and unbatched: ``parents`` is
a tuple of unbatched vectors concatenated in declaration order, ``parent_dims`` their widths; a
discrete parent arrives as its one-hot or probability vector, a continuous action as its raw
vector. Batch by ``vmap``. ``frozen`` is a static field: a frozen declaration has no trainable
marking, allocates no optimizer state and comes back from ``learn`` bitwise.
**Gaussian head** (:class:`MLPConditional`). The final layer emits ``2·child_dim`` numbers,
mean then scale pre-activation, read one of two ways (:attr:`MLPConditional.head`):
- ``"softplus"`` (default): mean raw, scale ``min_scale + softplus(raw)``; the floor keeps the
log-density finite;
- ``"bounded"``: mean ``tanh(raw)``, scale ``minstd + (maxstd − minstd)·sigmoid(raw + 2)``;
``minstd == maxstd`` is a fixed-σ head whose scale parameters get zero gradient.
**LayerNorm.** A normed hidden layer runs ``linear → norm → activation`` (eps 1e-3) and carries
no bias; the head layer is never normed and keeps its bias.
Design: docs/design/amortization.md A1, A2, A9, A10
"""
from typing import NamedTuple
import equinox as eqx
import jax
import jax.numpy as jnp
import optax
from jax import Array
from hybrid_aif.distributions.categorical import Categorical
from hybrid_aif.distributions.gaussian import Gaussian
#: The default optimizer, at unit learning rate: ``learn`` scales the updates by the slot's
#: resolved rate, which equals ``optax.adam(lr)`` and keeps the rate traceable.
OPTIMIZER = optax.adam(1.0)
#: ``optax.adam``'s own ``eps``; :data:`OPTIMIZER` runs at this value.
DEFAULT_EPS = 1e-8
def adam(eps: float = DEFAULT_EPS) -> optax.GradientTransformation:
""":data:`OPTIMIZER` with an explicit ``eps``: a per-slot adam at unit rate.
``eps`` enters adam's update, not its ``init``, so an optimizer state built by
:meth:`~hybrid_aif.system.System.init_learn_state` (with :data:`OPTIMIZER`) is compatible with
any ``eps``.
Design: docs/design/amortization.md chunk 2d note §(a) item 5
"""
return optax.adam(1.0, eps=float(eps))
_LOG_2PI = float(jnp.log(2.0 * jnp.pi))
_ACTIVATIONS = {
"tanh": jnp.tanh,
"relu": jax.nn.relu,
"gelu": jax.nn.gelu,
"silu": jax.nn.silu, # DreamerV3's default activation
}
def _inverse_softplus(y: float) -> float:
return float(jnp.log(jnp.expm1(jnp.asarray(y))))
#: The standard deviation of a unit normal truncated to [-2, 2]. Dividing a truncated
#: draw by it makes the weights' std come out at ``sqrt(scale / denom)`` exactly rather
#: than 0.8796× it.
_TRUNC_NORMAL_STD = 0.87962566103423978
def trunc_normal_avg(key: Array, shape: tuple[int, ...], fan_in: int, fan_out: int) -> Array:
"""Truncated-normal weights with variance scaled by the average fan, at scale 1.0.
``std = sqrt(1 / mean(fan_in, fan_out)) / 0.87962566…`` times a unit normal truncated to
``[-2, 2]`` (DreamerV3's ``winit: normal, fan: avg``). Returns a float32 array of ``shape``;
biases are the caller's. The fans are arguments so a caller can draw half of a fused layer at
that half's own fan.
"""
denom = (float(fan_in) + float(fan_out)) / 2.0
std = (1.0 / denom) ** 0.5 / _TRUNC_NORMAL_STD
return std * jax.random.truncated_normal(key, -2.0, 2.0, shape, jnp.float32)
def uniform_avg(key: Array, shape: tuple[int, ...], fan_in: int, fan_out: int) -> Array:
"""Uniform weights with variance scaled by the average fan, at scale 1.0.
``U(−limit, limit)`` with ``limit = sqrt(3 / mean(fan_in, fan_out))``: the second moment of
:func:`trunc_normal_avg` in a uniform shape (DreamerV3's ``winit: uniform``). Returns a float32
array of ``shape``. Under ``init="trunc_normal_avg"`` a stack's read-out layer draws this.
"""
denom = (float(fan_in) + float(fan_out)) / 2.0
limit = (3.0 / denom) ** 0.5
return jax.random.uniform(key, shape, jnp.float32, -limit, limit)
[docs]
class MLPConditional(eqx.Module):
"""MLP over concatenated parents → diagonal-Gaussian child (see module docstring).
The family contract is :meth:`log_prob`, :meth:`sample` and :meth:`moments`, all through
:meth:`condition_on`, the plain :class:`~hybrid_aif.distributions.Gaussian` the parents select.
:meth:`stats` (one tick's gradient of the NLL) and :meth:`absorb` (the optax step) learn it. It
is not a :class:`~hybrid_aif.distributions.Learnable` (there is no posterior over the weights):
``isinstance(decl, Learnable)`` is ``False``, so the conjugate paths leave it alone.
Construct with :meth:`from_shapes`; the field constructor stays the dataclass default so
``dataclasses.replace`` and ``eqx.tree_at`` work. Freeze with
``dataclasses.replace(decl, frozen=True)``.
The final layer emits ``2 * child_dim`` numbers, the mean and then the scale
pre-activation, read as ``head`` says.
"""
#: The linear layers; the trainable leaves live here.
layers: tuple[eqx.nn.Linear, ...]
#: The width of each parent vector, in parent order.
parent_dims: tuple[int, ...] = eqx.field(static=True)
#: The width of the child, a diagonal Gaussian.
child_dim: int = eqx.field(static=True)
#: One layer norm per hidden layer, or ``()`` for none.
norms: tuple[eqx.nn.LayerNorm, ...] = ()
#: The hidden layers' activation.
activation: str = eqx.field(static=True, default="tanh")
#: The softplus head's additive floor on the scale.
min_scale: float = eqx.field(static=True, default=1e-3)
#: The trainable marking: ``True`` keeps the weights fixed.
frozen: bool = eqx.field(static=True, default=False)
#: How the final layer is read. ``"softplus"``: mean raw, scale ``min_scale + softplus(raw)``.
#: ``"bounded"``: mean ``tanh(raw)``, scale ``minstd + (maxstd - minstd) * sigmoid(raw + 2)``;
#: ``minstd == maxstd`` is a fixed-scale head whose scale parameters get zero gradient. See
#: :mod:`hybrid_aif.distributions.neural`.
head: str = eqx.field(static=True, default="softplus")
#: The bounded head's smallest scale.
minstd: float = eqx.field(static=True, default=0.1)
#: The bounded head's largest scale.
maxstd: float = eqx.field(static=True, default=1.0)
#: How a Gaussian belief on the one continuous parent is read through the network when the
#: conditional is a likelihood on the analytic path: ``"first_order"``, a tangent at the mean,
#: or ``"unscented"``, the statistical linearisation over the sigma points
#: (:func:`~hybrid_aif.inference.linearise.linearise`).
propagation: str = eqx.field(static=True, default="first_order")
def __check_init__(self) -> None:
if not self.parent_dims or any(d <= 0 for d in self.parent_dims):
raise ValueError(
f"parent_dims must be a non-empty tuple of positive widths, got "
f"{self.parent_dims!r} (one entry per declared parent; a discrete parent's "
"width is its level count — the one-hot convention)"
)
if self.child_dim <= 0:
raise ValueError(f"child_dim must be positive, got {self.child_dim}")
if self.activation not in _ACTIVATIONS:
raise ValueError(
f"unknown activation {self.activation!r}; known activations are "
f"{sorted(_ACTIVATIONS)}"
)
if self.min_scale <= 0.0:
raise ValueError(
f"min_scale must be strictly positive, got {self.min_scale} (it is the floor "
"that keeps the conditional log-density finite; see the module docstring)"
)
if not self.layers:
raise ValueError("layers must be non-empty")
if self.layers[0].in_features != self.parent_dim:
raise ValueError(
f"first layer takes {self.layers[0].in_features} inputs but the parents "
f"concatenate to {self.parent_dim} (parent_dims={self.parent_dims})"
)
if self.layers[-1].out_features != 2 * self.child_dim:
raise ValueError(
f"final layer emits {self.layers[-1].out_features} outputs but a Gaussian "
f"head over a {self.child_dim}-dimensional child needs {2 * self.child_dim} "
"(mean, then pre-softplus scale)"
)
for a, b in zip(self.layers[:-1], self.layers[1:], strict=True):
if a.out_features != b.in_features:
raise ValueError(
f"layer widths do not chain: {a.out_features} out, {b.in_features} in"
)
if self.head not in ("softplus", "bounded"):
raise ValueError(
f"unknown head {self.head!r}; the built heads are ['bounded', 'softplus'] "
"(the module docstring's two scale parameterizations)"
)
if self.head == "bounded" and not 0.0 < self.minstd <= self.maxstd:
raise ValueError(
f"a bounded head needs 0 < minstd <= maxstd, got ({self.minstd}, {self.maxstd}) "
"— equality is the fixed-sigma case, not an error"
)
if self.propagation not in ("first_order", "unscented"):
raise ValueError(
f"unknown propagation {self.propagation!r}; the rules are "
"['first_order', 'unscented'] (how a Gaussian belief on the continuous parent "
"is read through the network)"
)
if self.norms and len(self.norms) != len(self.layers) - 1:
raise ValueError(
f"got {len(self.norms)} norm(s) for {len(self.layers) - 1} hidden layer(s) "
"(the head layer is never normed; "
"pass () for an unnormalized net)"
)
# ------------------------------------------------------------------ construction
[docs]
@classmethod
def from_shapes(
cls,
parent_dims: tuple[int, ...],
child_dim: int,
*,
key: Array,
width: int = 64,
depth: int = 2,
activation: str = "tanh",
min_scale: float = 1e-3,
init_scale: float = 1.0,
frozen: bool = False,
norm: bool = False,
head: str = "softplus",
minstd: float = 0.1,
maxstd: float = 1.0,
init: str = "default",
propagation: str = "first_order",
) -> "MLPConditional":
"""A randomly-initialized net: ``depth`` hidden layers of ``width`` units.
``init_scale`` centres a softplus head's initial standard deviation (the scale-half bias)
and must exceed ``min_scale``; a bounded head ignores it. Options, all off by default:
- ``norm=True``: a LayerNorm between each hidden linear and its activation, hidden biases
dropped;
- ``head="bounded"`` with ``minstd``/``maxstd`` (``minstd == maxstd`` is the fixed-σ head);
- ``init="trunc_normal_avg"``: :func:`trunc_normal_avg` on the hidden layers,
:func:`uniform_avg` on the fused head layer (each half at its own ``(in, child_dim)``
fan), every bias zero. ``init="default"`` is ``eqx.nn.Linear``'s initializer throughout.
``propagation`` sets the static field of that name: ``"first_order"`` (default) or
``"unscented"``.
"""
parent_dims = tuple(int(d) for d in parent_dims)
child_dim = int(child_dim)
if depth < 0:
raise ValueError(f"depth must be non-negative, got {depth}")
if init not in ("default", "trunc_normal_avg"):
raise ValueError(
f"unknown init {init!r}; the initializers are ['default', 'trunc_normal_avg'] "
"(eqx.nn.Linear's own, or a truncated normal scaled by the fan average, "
"biasless under a norm)"
)
sizes = [sum(parent_dims), *([width] * depth), 2 * child_dim]
keys = jax.random.split(key, len(sizes) - 1)
layers = [
# a normed hidden layer carries no bias (module docstring);
# the head layer (i == depth) is never normed and always keeps its bias
eqx.nn.Linear(a, b, use_bias=not (norm and i < depth), key=k)
for i, (a, b, k) in enumerate(zip(sizes[:-1], sizes[1:], list(keys), strict=True))
]
if init == "trunc_normal_avg":
wkeys = jax.random.split(jax.random.fold_in(key, 0x17), len(layers) + 1)
for i, layer in enumerate(layers):
fan_in = layer.in_features
if i == len(layers) - 1:
# the fused head is two read-outs (mean, scale): each half is drawn
# uniform at its own fan (fan_out = child_dim)
weight = jnp.concatenate(
[
uniform_avg(wkeys[i], (child_dim, fan_in), fan_in, child_dim),
uniform_avg(wkeys[-1], (child_dim, fan_in), fan_in, child_dim),
],
axis=0,
)
else:
out = layer.out_features
weight = trunc_normal_avg(wkeys[i], (out, fan_in), fan_in, out)
layer = eqx.tree_at(lambda m: m.weight, layer, weight)
if layer.bias is not None: # every bias starts at zero
layer = eqx.tree_at(lambda m: m.bias, layer, jnp.zeros_like(layer.bias))
layers[i] = layer
if head == "softplus":
if init_scale <= min_scale:
raise ValueError(
f"init_scale {init_scale} must exceed min_scale {min_scale} (the head's "
"scale is min_scale + softplus(·), so it cannot start at or below the floor)"
)
last = layers[-1]
bias = last.bias.at[child_dim:].set(_inverse_softplus(init_scale - min_scale))
layers[-1] = eqx.tree_at(lambda m: m.bias, last, bias)
return cls(
layers=tuple(layers),
parent_dims=parent_dims,
child_dim=child_dim,
norms=(tuple(eqx.nn.LayerNorm(w, eps=1e-3) for w in sizes[1:-1]) if norm else ()),
activation=activation,
min_scale=min_scale,
frozen=frozen,
head=head,
minstd=float(minstd),
maxstd=float(maxstd),
propagation=propagation,
)
# -------------------------------------------------------------------- structure
@property
def parent_dim(self) -> int:
"""Width of the concatenated parent vector."""
return sum(self.parent_dims)
@property
def learnable(self) -> bool:
"""Capability: a non-frozen net's weights are a trainable block."""
return not self.frozen
# -------------------------------------------------------------- family contract
def _concat(self, parents: tuple[Array, ...]) -> Array:
if len(parents) != len(self.parent_dims):
raise ValueError(
f"got {len(parents)} parent value(s) for {len(self.parent_dims)} declared "
f"parent(s) (parent_dims={self.parent_dims})"
)
cast = [jnp.asarray(p) for p in parents]
for i, (p, d) in enumerate(zip(cast, self.parent_dims, strict=True)):
if p.shape != (d,):
raise ValueError(
f"parent {i} has shape {p.shape}, expected ({d},) — parents are "
"unbatched vectors (a discrete parent as a one-hot / probability "
"vector); batch with vmap"
)
return jnp.concatenate(cast, axis=-1)
def _head(self, parents: tuple[Array, ...]) -> tuple[Array, Array]:
"""``(mean, scale)``: the Gaussian head's two halves, per :attr:`head`."""
h = self._concat(parents)
act = _ACTIVATIONS[self.activation]
for i, layer in enumerate(self.layers[:-1]):
h = layer(h)
if self.norms: # linear -> norm -> act (module docstring)
h = self.norms[i](h)
h = act(h)
out = self.layers[-1](h)
mean, raw = out[: self.child_dim], out[self.child_dim :]
if self.head == "bounded":
# bounded head: Normal(tanh(out), lo + (hi-lo)*sigmoid(raw+2))
lo, hi = self.minstd, self.maxstd
return jnp.tanh(mean), lo + (hi - lo) * jax.nn.sigmoid(raw + 2.0)
return mean, self.min_scale + jax.nn.softplus(raw)
[docs]
def condition_on(self, parents: tuple[Array, ...]) -> Gaussian:
"""The conditional as a plain :class:`Gaussian` with a diagonal Cholesky."""
mean, scale = self._head(parents)
return Gaussian(loc=mean, chol=jnp.diag(scale))
[docs]
def log_prob(self, child: Array, parents: tuple[Array, ...]) -> Array:
"""``log p(child | parents)`` (scalar), computed on the diagonal (:meth:`diag_nll`)."""
return -self.diag_nll(child, parents)
[docs]
def sample(self, key: Array, parents: tuple[Array, ...]) -> Array:
"""One ancestral draw of the child given its parents, shape ``(child_dim,)``."""
return self.condition_on(parents).sample(key)
[docs]
def moments(self, parents: tuple[Array, ...]) -> tuple[Array, Array]:
"""``(mean, cov)`` given the parents; ``cov`` is the full, diagonal-valued
``(child_dim, child_dim)`` matrix.
"""
mean, scale = self._head(parents)
return mean, jnp.diag(scale**2)
[docs]
def diag_moments(self, parents: tuple[Array, ...]) -> tuple[Array, Array]:
"""``(mean, scale)``: the head's diagonal view, with no covariance built."""
return self._head(parents)
[docs]
def diag_nll(self, child: Array, parents: tuple[Array, ...]) -> Array:
"""Negative log-density on the diagonal: ``0.5·Σ((y−μ)/σ)² + Σ log σ + 0.5·d·log 2π``.
Exactly ``−log_prob`` for this head, without building or solving a ``(d, d)`` Cholesky.
Design: docs/design/amortization.md chunk 2c note §(e) item 1
"""
mean, scale = self._head(parents)
z = (jnp.asarray(child) - mean) / scale
return (
0.5 * jnp.sum(z**2, axis=-1)
+ jnp.sum(jnp.log(scale), axis=-1)
+ 0.5 * self.child_dim * _LOG_2PI
)
# ------------------------------------------------------------- gradient learning
[docs]
def nll(self, child: Array, parents: tuple[Array, ...]) -> Array:
"""Negative conditional log-density: the gradient loss on one evidence pair."""
return self.diag_nll(child, parents)
[docs]
def stats(self, child: Array, parents: tuple[Array, ...]) -> "MLPConditional":
"""The per-tick statistic ``∇ nll(child, parents)``, as a same-structured pytree.
``None`` sits at every non-trainable position. Raises ``ValueError`` on a frozen
declaration.
"""
if self.frozen:
raise ValueError(
"stats() on a frozen MLPConditional: a frozen declaration has no trainable "
"parameters, so there is no gradient statistic to take (design: "
"amortization.md A1 — frozen ⇔ no trainable marking). Unfreeze with "
"dataclasses.replace(decl, frozen=False)."
)
return eqx.filter_grad(lambda m: m.nll(child, parents))(self)
[docs]
def init_opt_state(self):
"""Fresh optimizer state for this declaration — one entry of ``LearnState``."""
return OPTIMIZER.init(eqx.filter(self, eqx.is_inexact_array))
[docs]
def absorb(
self,
stats: "MLPConditional",
opt_state,
lr: float | Array = 1.0,
decay: float | Array = 1.0,
) -> tuple["MLPConditional", object]:
"""One optax step: returns ``(declaration, opt_state)``.
The optimizer state is passed in and out. ``lr`` scales the step and may be traced;
``decay`` other than ``1.0`` raises ``NotImplementedError``.
"""
if not (isinstance(decay, int | float) and float(decay) == 1.0):
raise NotImplementedError(
"decay != 1.0 on an MLPConditional slot: the conjugate families' leaky-count "
"decay (α ← decay·α + (1−decay)·anchor) has no settled meaning for network "
"weights, and weight decay toward an anchor is a different regularizer with "
"a different justification, so none is supplied here (design: "
"docs/design/amortization.md A2, which fixes the rate grammar but not decay "
"semantics for neural slots). Leave decay at 1.0 on this slot."
)
params = eqx.filter(self, eqx.is_inexact_array)
updates, opt_state = OPTIMIZER.update(stats, opt_state, params)
updates = jax.tree_util.tree_map(lambda u: lr * u, updates)
return eqx.apply_updates(self, updates), opt_state
# ==================================================================== symlog / symexp
def symlog(x: Array) -> Array:
"""``sign(x)·log(1+|x|)``, DreamerV3's symmetric log squash, inverted by :func:`symexp`."""
x = jnp.asarray(x)
return jnp.sign(x) * jnp.log1p(jnp.abs(x))
def symexp(x: Array) -> Array:
"""``sign(x)·(exp|x|−1)`` — the exact inverse of :func:`symlog`."""
x = jnp.asarray(x)
return jnp.sign(x) * jnp.expm1(jnp.abs(x))
# ================================================== the optax step, module-generic
#
# The shared optax step for modules whose gradient arrives from their consumer.
# Optimizer state is passed in and out; nothing is stashed on a module.
def init_opt_state(module, optimizer: optax.GradientTransformation = OPTIMIZER):
"""Fresh optimizer state for any module's trainable leaves — one ``LearnState`` entry."""
return optimizer.init(eqx.filter(module, eqx.is_inexact_array))
def apply_gradient(
module,
grads,
opt_state,
*,
lr: float | Array = 1.0,
decay: float | Array = 1.0,
optimizer: optax.GradientTransformation = OPTIMIZER,
):
"""One optax step on ``module`` from a same-structured gradient pytree.
Returns ``(module, opt_state)``. ``lr`` scales the update and may be traced; ``decay != 1.0``
raises ``NotImplementedError``, as in :meth:`MLPConditional.absorb`.
"""
if not (isinstance(decay, int | float) and float(decay) == 1.0):
raise NotImplementedError(
f"decay={decay!r} on a gradient-trained module: the conjugate families' leaky-count "
"decay has no settled meaning for network weights, so none is supplied "
"(design: docs/design/amortization.md A2). Leave decay at 1.0."
)
params = eqx.filter(module, eqx.is_inexact_array)
updates, opt_state = optimizer.update(grads, opt_state, params)
updates = jax.tree_util.tree_map(lambda u: lr * u, updates)
return eqx.apply_updates(module, updates), opt_state
# ============================================================== the shared MLP trunk
def _concat_parents(parents: tuple[Array, ...], parent_dims: tuple[int, ...]) -> Array:
"""The parent convention: one unbatched vector per declared parent, concatenated."""
if len(parents) != len(parent_dims):
raise ValueError(
f"got {len(parents)} parent value(s) for {len(parent_dims)} declared parent(s) "
f"(parent_dims={parent_dims})"
)
cast = [jnp.asarray(p) for p in parents]
for i, (p, d) in enumerate(zip(cast, parent_dims, strict=True)):
if p.shape != (d,):
raise ValueError(
f"parent {i} has shape {p.shape}, expected ({d},) — parents are unbatched "
"vectors (a categorical parent as its flattened one-hot); batch with vmap"
)
return jnp.concatenate(cast, axis=-1)
def _linear_stack(
sizes: list[int],
key: Array,
*,
out_scale: float,
init: str = "default",
hidden: int | None = None,
norm: bool = False,
) -> tuple[eqx.nn.Linear, ...]:
"""``len(sizes) − 1`` linears chained ``sizes[0] → sizes[1] → … → sizes[-1]``.
The first ``hidden`` layers (default: all but the last) are hidden layers, which the caller
follows with a norm and an activation; the rest is the read-out. ``init="default"`` is
``eqx.nn.Linear``'s initializer, biases everywhere. ``init="trunc_normal_avg"`` draws
:func:`trunc_normal_avg` on hidden layers (biasless when ``norm``) and :func:`uniform_avg` on
the read-out, every kept bias starting at zero. ``out_scale`` multiplies the read-out weights
and zeroes its bias (``0.0`` starts at the constant zero output). ``norm`` only places biases;
the caller builds the LayerNorms. Raises ``ValueError`` on an unknown ``init``.
"""
if init not in ("default", "trunc_normal_avg"):
raise ValueError(
f"unknown init {init!r}; the initializers are ['default', 'trunc_normal_avg'] "
"(eqx.nn.Linear's own, or a truncated normal scaled by the fan average, "
"biasless under a norm)"
)
keys = jax.random.split(key, len(sizes) - 1)
n_hidden = len(sizes) - 2 if hidden is None else int(hidden)
layers = []
for i, (a, b, k) in enumerate(zip(sizes[:-1], sizes[1:], list(keys), strict=True)):
if init == "default":
layers.append(eqx.nn.Linear(a, b, key=k))
continue
is_hidden = i < n_hidden # hidden: truncated normal, and biasless under a norm
layer = eqx.nn.Linear(a, b, use_bias=not (is_hidden and norm), key=k)
draw = trunc_normal_avg if is_hidden else uniform_avg
layer = eqx.tree_at(lambda m: m.weight, layer, draw(k, (b, a), a, b))
if layer.bias is not None: # a kept bias starts at zero
layer = eqx.tree_at(lambda m: m.bias, layer, jnp.zeros_like(layer.bias))
layers.append(layer)
if out_scale != 1.0:
last = layers[-1]
layers[-1] = eqx.tree_at(lambda m: m.weight, last, out_scale * last.weight)
if last.bias is not None:
layers[-1] = eqx.tree_at(lambda m: m.bias, layers[-1], jnp.zeros_like(last.bias))
return tuple(layers)
[docs]
class MLPPredictor(eqx.Module):
"""MLP over concatenated parents → a raw output vector, with no density of its own.
The trunk the other classes here are built from; paired with a loss it is a head without a
density family (:class:`PredictorHead`). ``depth`` hidden layers of ``width`` units, each
optionally layer-normed (eps 1e-3) then activated, and a final linear to ``out_dim``.
"""
#: The linear layers (the trainable leaves).
layers: tuple[eqx.nn.Linear, ...]
#: One layer norm per activated layer, or ``()`` for none.
norms: tuple[eqx.nn.LayerNorm, ...]
#: The width of each parent vector, in parent order.
parent_dims: tuple[int, ...] = eqx.field(static=True)
#: The width of the output vector.
out_dim: int = eqx.field(static=True)
#: The activation of the hidden layers.
activation: str = eqx.field(static=True, default="silu")
#: ``True`` norms and activates the final layer too, making the trunk an encoder.
out_activation: bool = eqx.field(static=True, default=False)
#: The trainable marking: ``True`` keeps the weights fixed.
frozen: bool = eqx.field(static=True, default=False)
#: ``True`` applies :func:`symlog` to the concatenated parents before the first layer.
symlog_inputs: bool = eqx.field(static=True, default=False)
def __check_init__(self) -> None:
if not self.parent_dims or any(d <= 0 for d in self.parent_dims):
raise ValueError(
f"parent_dims must be a non-empty tuple of positive widths, got "
f"{self.parent_dims!r} (one entry per declared parent)"
)
if self.out_dim <= 0:
raise ValueError(f"out_dim must be positive, got {self.out_dim}")
if self.activation not in _ACTIVATIONS:
raise ValueError(
f"unknown activation {self.activation!r}; known activations are "
f"{sorted(_ACTIVATIONS)}"
)
if not self.layers:
raise ValueError("layers must be non-empty")
if self.layers[0].in_features != sum(self.parent_dims):
raise ValueError(
f"first layer takes {self.layers[0].in_features} inputs but the parents "
f"concatenate to {sum(self.parent_dims)} (parent_dims={self.parent_dims})"
)
if self.layers[-1].out_features != self.out_dim:
raise ValueError(
f"final layer emits {self.layers[-1].out_features} outputs, not out_dim="
f"{self.out_dim}"
)
if self.norms and len(self.norms) != self._num_activated:
raise ValueError(
f"got {len(self.norms)} norm(s) for {self._num_activated} activated layer(s) "
"(pass () for an unnormalized trunk)"
)
[docs]
@classmethod
def from_shapes(
cls,
parent_dims: tuple[int, ...],
out_dim: int,
*,
key: Array,
width: int,
depth: int = 2,
activation: str = "silu",
norm: bool = True,
out_activation: bool = False,
out_scale: float = 1.0,
frozen: bool = False,
symlog_inputs: bool = False,
init: str = "default",
) -> "MLPPredictor":
"""A randomly-initialized trunk: ``depth`` hidden layers of ``width`` units.
``width`` is required. ``norm=True`` puts a LayerNorm (eps 1e-3) before each activation.
``out_scale`` scales the final layer's weights and zeroes its bias (``0.0`` starts at the
constant zero prediction). ``init`` is as in :func:`_linear_stack`.
"""
parent_dims = tuple(int(d) for d in parent_dims)
out_dim = int(out_dim)
if depth < 0:
raise ValueError(f"depth must be non-negative, got {depth}")
sizes = [sum(parent_dims), *([width] * depth), out_dim]
activated = len(sizes) - 1 if out_activation else len(sizes) - 2
layers = _linear_stack(
sizes, key, out_scale=out_scale, init=init, hidden=activated, norm=norm
)
norms = (
tuple(eqx.nn.LayerNorm(layer.out_features, eps=1e-3) for layer in layers[:activated])
if norm
else ()
)
return cls(
layers=layers,
norms=norms,
parent_dims=parent_dims,
out_dim=out_dim,
activation=activation,
out_activation=out_activation,
frozen=frozen,
symlog_inputs=symlog_inputs,
)
@property
def _num_activated(self) -> int:
return len(self.layers) if self.out_activation else len(self.layers) - 1
@property
def parent_dim(self) -> int:
"""Width of the concatenated parent vector."""
return sum(self.parent_dims)
@property
def learnable(self) -> bool:
"""Capability: a non-frozen trunk's weights are a trainable block."""
return not self.frozen
def __call__(self, parents: tuple[Array, ...]) -> Array:
"""The raw output vector, shape ``(out_dim,)``."""
h = _concat_parents(parents, self.parent_dims)
if self.symlog_inputs:
h = symlog(h)
act = _ACTIVATIONS[self.activation]
for i, layer in enumerate(self.layers):
h = layer(h)
if i < self._num_activated:
if self.norms:
h = self.norms[i](h)
h = act(h)
return h
[docs]
def init_opt_state(self):
"""Fresh optimizer state — one entry of a ``LearnState``."""
return init_opt_state(self)
[docs]
def absorb(self, stats, opt_state, lr: float | Array = 1.0, decay: float | Array = 1.0):
"""One optax step from a gradient pytree: ``(trunk, opt_state)``."""
return apply_gradient(self, stats, opt_state, lr=lr, decay=decay)
# ==================================================== the factored categorical head
def unimix_logits(logits: Array, unimix: float) -> Array:
"""Mix ``unimix`` of the uniform into a categorical's probabilities, returned as logits.
``probs ← (1−u)·softmax(logits) + u/num_classes``: no class probability reaches zero, so
``log_prob`` of any one-hot is finite. ``unimix`` must lie in ``[0, 1)``; ``0.0`` passes the
logits through. The last axis is the class axis.
"""
if not 0.0 <= unimix < 1.0:
raise ValueError(f"unimix must lie in [0, 1), got {unimix}")
if unimix == 0.0:
return jnp.asarray(logits)
probs = jax.nn.softmax(jnp.asarray(logits), axis=-1)
uniform = 1.0 / logits.shape[-1]
return jnp.log((1.0 - unimix) * probs + unimix * uniform)
def straight_through_onehot(key: Array, logits: Array) -> Array:
"""Sample a one-hot per variable, with the probabilities' gradient attached.
``sample + probs − sg(probs)``: the forward value is the hard draw, the backward value
``∂probs`` (DreamerV3's ``OneHotDist.sample``). ``logits`` may carry leading batch axes; the
last axis is the class axis.
"""
draw = jax.random.categorical(key, logits, axis=-1)
hard = jax.nn.one_hot(draw, logits.shape[-1], dtype=logits.dtype)
probs = jax.nn.softmax(logits, axis=-1)
return jax.lax.stop_gradient(hard) + (probs - jax.lax.stop_gradient(probs))
def onehot_mode(logits: Array) -> Array:
"""The argmax one-hot per variable, with the probabilities' gradient attached."""
hard = jax.nn.one_hot(jnp.argmax(logits, axis=-1), logits.shape[-1], dtype=logits.dtype)
probs = jax.nn.softmax(logits, axis=-1)
return jax.lax.stop_gradient(hard) + (probs - jax.lax.stop_gradient(probs))
[docs]
class MLPCategorical(eqx.Module):
"""MLP over concatenated parents → a factored categorical child.
``num_vars`` independent categoricals of ``num_classes`` classes each, unimix-mixed and sampled
as a straight-through one-hot. The child value is the one-hot code, shape ``(num_vars,
num_classes)``; :attr:`flat_dim` is its width as a downstream parent. There is no ``moments``:
:meth:`probs` is this family's moment view.
"""
#: The network from the concatenated parents to the ``num_vars * num_classes`` logits.
trunk: MLPPredictor
#: The number of independent categoricals.
num_vars: int = eqx.field(static=True)
#: The number of classes of each.
num_classes: int = eqx.field(static=True)
#: The weight of the uniform distribution mixed into each categorical.
unimix: float = eqx.field(static=True, default=0.01)
def __check_init__(self) -> None:
if self.num_vars <= 0 or self.num_classes <= 0:
raise ValueError(
f"num_vars and num_classes must be positive, got {self.num_vars} and "
f"{self.num_classes}"
)
if self.trunk.out_dim != self.num_vars * self.num_classes:
raise ValueError(
f"the trunk emits {self.trunk.out_dim} numbers but a factored categorical over "
f"{self.num_vars} variable(s) of {self.num_classes} class(es) needs "
f"{self.num_vars * self.num_classes} (one logit per (variable, class))"
)
if not 0.0 <= self.unimix < 1.0:
raise ValueError(f"unimix must lie in [0, 1), got {self.unimix}")
[docs]
@classmethod
def from_shapes(
cls,
parent_dims: tuple[int, ...],
num_vars: int,
num_classes: int,
*,
key: Array,
width: int,
depth: int = 1,
activation: str = "silu",
norm: bool = True,
unimix: float = 0.01,
frozen: bool = False,
init: str = "default",
) -> "MLPCategorical":
"""A randomly-initialized head: ``depth`` normed, activated hidden layers of ``width`` units
(required), then a linear to the ``num_vars·num_classes`` logits. ``init`` is as in
:meth:`MLPPredictor.from_shapes`.
"""
trunk = MLPPredictor.from_shapes(
parent_dims,
int(num_vars) * int(num_classes),
key=key,
width=width,
depth=depth,
activation=activation,
norm=norm,
frozen=frozen,
init=init,
)
return cls(
trunk=trunk, num_vars=int(num_vars), num_classes=int(num_classes), unimix=float(unimix)
)
# -------------------------------------------------------------------- structure
@property
def parent_dims(self) -> tuple[int, ...]:
return self.trunk.parent_dims
@property
def parent_dim(self) -> int:
return self.trunk.parent_dim
@property
def flat_dim(self) -> int:
"""Width of the flattened one-hot code — what it contributes as a parent."""
return self.num_vars * self.num_classes
@property
def frozen(self) -> bool:
return self.trunk.frozen
@property
def learnable(self) -> bool:
return self.trunk.learnable
# ---------------------------------------------------------------- the contract
[docs]
def logits(self, parents: tuple[Array, ...]) -> Array:
"""Unimix-mixed logits, shape ``(num_vars, num_classes)``."""
raw = self.trunk(parents).reshape(self.num_vars, self.num_classes)
return unimix_logits(raw, self.unimix)
[docs]
def condition_on(self, parents: tuple[Array, ...]) -> Categorical:
"""The conditional as a :class:`Categorical` whose batch axis is the variable
axis, so entropies and KLs come out per variable.
"""
return Categorical(logits=self.logits(parents))
[docs]
def probs(self, parents: tuple[Array, ...]) -> Array:
"""The ``(num_vars, num_classes)`` probability tensor: this family's moment view."""
return jax.nn.softmax(self.logits(parents), axis=-1)
def _as_code(self, child: Array) -> Array:
child = jnp.asarray(child)
if child.shape == (self.num_vars, self.num_classes):
return child
if child.shape == (self.flat_dim,):
return child.reshape(self.num_vars, self.num_classes)
raise ValueError(
f"child has shape {child.shape}; a factored-categorical child is the one-hot code, "
f"either ({self.num_vars}, {self.num_classes}) or its flattening ({self.flat_dim},)"
)
[docs]
def log_prob(self, child: Array, parents: tuple[Array, ...]) -> Array:
"""``log q(child | parents)`` for a one-hot code, summed over variables.
``Σ_v Σ_c child[v,c]·log p[v,c]``: differentiable through a straight-through sample, and
finite for every code (:func:`unimix_logits`).
"""
code = self._as_code(child)
return jnp.sum(code * jax.nn.log_softmax(self.logits(parents), axis=-1))
[docs]
def sample(self, key: Array, parents: tuple[Array, ...]) -> Array:
"""One straight-through one-hot draw, shape ``(num_vars, num_classes)``."""
return straight_through_onehot(key, self.logits(parents))
[docs]
def mode(self, parents: tuple[Array, ...]) -> Array:
"""The argmax code, same shape as :meth:`sample`."""
return onehot_mode(self.logits(parents))
# ---------------------------------------------------------------- the optax step
def init_opt_state(self):
return init_opt_state(self)
def absorb(self, stats, opt_state, lr: float | Array = 1.0, decay: float | Array = 1.0):
return apply_gradient(self, stats, opt_state, lr=lr, decay=decay)
# =================================================================== the loss pieces
def factored_categorical_kl(post_logits: Array, prior_logits: Array) -> Array:
"""``KL(q ‖ p)`` between two factored categoricals, summed over variables.
Leading batch axes are preserved. Returns a plain array (a loss term), not an
:class:`~hybrid_aif.interval.Interval`.
"""
lq = jax.nn.log_softmax(jnp.asarray(post_logits), axis=-1)
lp = jax.nn.log_softmax(jnp.asarray(prior_logits), axis=-1)
return jnp.sum(jnp.exp(lq) * (lq - lp), axis=(-2, -1))
def factored_categorical_entropy(logits: Array) -> Array:
"""``H[q]`` of a factored categorical, summed over variables.
Read off the logits, not a sampled code (whose entropy is zero); leading batch axes are
preserved. ``train.imagine``'s ``prior_entropy_scale`` term reads it.
Design: docs/design/amortization-p0-addendum.md §3
"""
lq = jax.nn.log_softmax(jnp.asarray(logits), axis=-1)
return -jnp.sum(jnp.exp(lq) * lq, axis=(-2, -1))
class KLBalance(NamedTuple):
"""The complexity term of the world-model loss, decomposed.
``total = dyn_scale·dyn + rep_scale·rep``; ``dyn``/``rep`` are the two
stop-gradient directions *after* the free-bits floor and ``dyn_raw``/
``rep_raw`` the same KLs before it — reported so a caller can see whether
the floor is currently binding (``dyn > dyn_raw`` ⇔ clamped).
"""
total: Array
dyn: Array
rep: Array
dyn_raw: Array
rep_raw: Array
def kl_balance(
post_logits: Array,
prior_logits: Array,
*,
dyn_scale: float = 0.5,
rep_scale: float = 0.1,
free_nats: float = 1.0,
) -> KLBalance:
"""KL balancing with free bits: the complexity term of a world-model loss.
- ``dyn = KL(sg(q) ‖ p)`` at ``dyn_scale`` trains the prior toward the posterior;
- ``rep = KL(q ‖ sg(p))`` at ``rep_scale`` trains the posterior toward the prior.
``free_nats`` floors each direction after the sum over variables, per element (batch by
``vmap`` and average afterwards); below the floor the term is constant with zero gradient.
Raises ``ValueError`` on a negative floor.
Design: docs/design/amortization-p0-addendum.md §3
"""
if free_nats < 0.0:
raise ValueError(f"free_nats must be non-negative, got {free_nats}")
sg = jax.lax.stop_gradient
dyn_raw = factored_categorical_kl(sg(post_logits), prior_logits)
rep_raw = factored_categorical_kl(post_logits, sg(prior_logits))
dyn = jnp.maximum(dyn_raw, free_nats)
rep = jnp.maximum(rep_raw, free_nats)
return KLBalance(
total=dyn_scale * dyn + rep_scale * rep,
dyn=dyn,
rep=rep,
dyn_raw=dyn_raw,
rep_raw=rep_raw,
)
def symlog_mse_nll(prediction: Array, target: Array) -> Array:
"""The ``symlog_mse`` loss: ``Σ_i (pred_i − symlog(target)_i)²``.
The prediction lives in symlog space; read it back in the target's units with :func:`symexp`.
"""
return jnp.sum((jnp.asarray(prediction) - symlog(target)) ** 2, axis=-1)
def disc_bins(num_bins: int = 255, low: float = -20.0, high: float = 20.0) -> Array:
"""The ``symlog_disc`` head's bin centres: ``num_bins`` evenly spaced over ``[low, high]``."""
if num_bins < 2:
raise ValueError(f"num_bins must be at least 2, got {num_bins}")
if not high > low:
raise ValueError(f"need low < high, got low={low}, high={high}")
return jnp.linspace(low, high, num_bins)
def twohot(target: Array, bins: Array) -> Array:
"""Two-hot encoding of a scalar target over ``bins`` (DreamerV3's ``DiscDist``).
Mass on the two bracketing bins, split linearly, so the encoding's expectation is the target
(clipped at the ends). Vectorize over a batch with ``vmap``.
"""
target, bins = jnp.asarray(target), jnp.asarray(bins)
if target.shape != ():
raise ValueError(
f"twohot takes a scalar target, got shape {target.shape} — vmap over batch axes"
)
n = bins.shape[0]
below = jnp.clip(jnp.sum((bins <= target).astype(jnp.int32)) - 1, 0, n - 1)
above = jnp.clip(n - jnp.sum((bins > target).astype(jnp.int32)), 0, n - 1)
equal = below == above
d_below = jnp.where(equal, 1.0, jnp.abs(bins[below] - target))
d_above = jnp.where(equal, 1.0, jnp.abs(bins[above] - target))
total = d_below + d_above
return (
jax.nn.one_hot(below, n) * (d_above / total)[..., None]
+ jax.nn.one_hot(above, n) * (d_below / total)[..., None]
)
def symlog_disc_nll(logits: Array, target: Array, bins: Array) -> Array:
"""The ``symlog_disc`` NLL: cross-entropy against the two-hot of ``symlog(target)``.
The loss of a binned-scalar head: a reward head, or a critic trained on a return target.
Design: docs/design/amortization.md A10 disposition 6, A12 item 5
"""
return -jnp.sum(twohot(symlog(target), bins) * jax.nn.log_softmax(logits, axis=-1), axis=-1)
def symlog_disc_mean(logits: Array, bins: Array) -> Array:
"""Decode a ``symlog_disc`` head: ``symexp`` of the expected bin."""
return symexp(jnp.sum(jax.nn.softmax(logits, axis=-1) * bins, axis=-1))
def binary_nll(logit: Array, target: Array) -> Array:
"""The ``binary`` head's loss: the Bernoulli NLL ``softplus(logit) − target·logit``.
Stable for large logits; ``target`` may be a soft value in [0, 1].
"""
logit, target = jnp.asarray(logit), jnp.asarray(target)
return jax.nn.softplus(logit) - target * logit
# ======================================================== trunk + loss: a declaration
#: The output-head losses :class:`PredictorHead` can name, with the width its
#: trunk must emit given the width of the value it predicts (``None`` = the value's
#: own width; ``symlog_disc`` emits one logit per bin, ``binary`` a single logit).
_HEAD_LOSSES: dict[str, int | None] = {"symlog_mse": None, "symlog_disc": -1, "binary": 1}
[docs]
class PredictorHead(eqx.Module):
"""An :class:`MLPPredictor` plus the name of the loss that reads its output.
What a ``likelihood`` slot holds for a head without a density family: ``symlog_mse``
reconstructs a vector, ``symlog_disc`` predicts a scalar through ``num_bins`` two-hot bins over
``[bin_low, bin_high]`` in symlog space, ``binary`` predicts a Bernoulli from one logit.
:meth:`nll` is the training loss and :meth:`predict` the decoded prediction, in the value's own
units. Not a Distribution or a :class:`~hybrid_aif.distributions.Conditional`, and role-free.
The three loss names are a closed list; an unknown name raises.
Design: docs/design/amortization-p0-addendum.md §2
"""
#: The network whose output the loss reads.
trunk: MLPPredictor
#: The loss name: ``"symlog_mse"``, ``"symlog_disc"`` or ``"binary"``.
loss: str = eqx.field(static=True)
#: The number of two-hot bins (``symlog_disc`` only).
num_bins: int = eqx.field(static=True, default=255)
#: The lowest bin, in symlog space (``symlog_disc`` only).
bin_low: float = eqx.field(static=True, default=-20.0)
#: The highest bin, in symlog space (``symlog_disc`` only).
bin_high: float = eqx.field(static=True, default=20.0)
def __check_init__(self) -> None:
if self.loss not in _HEAD_LOSSES:
raise ValueError(
f"unknown head loss {self.loss!r}; the built losses are "
f"{sorted(_HEAD_LOSSES)} (a symlog_mse vector head, a symlog_disc binned scalar "
"head, a binary Bernoulli head)"
)
if self.loss == "symlog_disc":
if self.num_bins < 2:
raise ValueError(f"symlog_disc needs at least 2 bins, got {self.num_bins}")
if not self.bin_high > self.bin_low:
raise ValueError(
f"symlog_disc bins need bin_low < bin_high, got {self.bin_low} and "
f"{self.bin_high}"
)
if self.trunk.out_dim != self.num_bins:
raise ValueError(
f"a symlog_disc head emits one logit per bin, so its trunk must emit "
f"num_bins={self.num_bins} numbers, not {self.trunk.out_dim}"
)
elif self.loss == "binary" and self.trunk.out_dim != 1:
raise ValueError(
f"a binary head is one logit, so its trunk must emit 1 number, not "
f"{self.trunk.out_dim}"
)
[docs]
@classmethod
def from_shapes(
cls,
parent_dims: tuple[int, ...],
*,
loss: str,
key: Array,
out_dim: int | None = None,
num_bins: int = 255,
bin_low: float = -20.0,
bin_high: float = 20.0,
width: int,
depth: int = 2,
activation: str = "silu",
norm: bool = True,
out_scale: float = 1.0,
frozen: bool = False,
symlog_inputs: bool = False,
init: str = "default",
) -> "PredictorHead":
"""A randomly-initialized head. ``out_dim`` is required for ``symlog_mse`` and rejected by
the other two losses, which fix the trunk's width (one logit per bin; one logit). ``width``
is required; the trunk arguments are :meth:`MLPPredictor.from_shapes`'s.
"""
if loss not in _HEAD_LOSSES:
raise ValueError(
f"unknown head loss {loss!r}; the built losses are {sorted(_HEAD_LOSSES)}"
)
if loss == "symlog_mse":
if out_dim is None:
raise ValueError(
"a symlog_mse head predicts a vector, so out_dim (its width) is required"
)
trunk_out = int(out_dim)
else:
if out_dim is not None:
raise ValueError(
f"a {loss} head's width is fixed by the loss ("
f"{'one logit per bin' if loss == 'symlog_disc' else 'one logit'}), so "
"out_dim must be left unset"
)
trunk_out = int(num_bins) if loss == "symlog_disc" else 1
return cls(
trunk=MLPPredictor.from_shapes(
parent_dims,
trunk_out,
key=key,
width=width,
depth=depth,
activation=activation,
norm=norm,
out_scale=out_scale,
frozen=frozen,
symlog_inputs=symlog_inputs,
init=init,
),
loss=loss,
num_bins=int(num_bins),
bin_low=float(bin_low),
bin_high=float(bin_high),
)
# -------------------------------------------------------------------- structure
@property
def parent_dims(self) -> tuple[int, ...]:
return self.trunk.parent_dims
@property
def parent_dim(self) -> int:
return self.trunk.parent_dim
@property
def out_dim(self) -> int:
"""Width of the trunk's raw output (logits, for the two classifier losses)."""
return self.trunk.out_dim
@property
def target_dim(self) -> int:
"""Width of the value this head predicts: the vector's, or 1 for a scalar."""
return self.trunk.out_dim if self.loss == "symlog_mse" else 1
@property
def frozen(self) -> bool:
return self.trunk.frozen
@property
def learnable(self) -> bool:
return self.trunk.learnable
@property
def bins(self) -> Array:
"""A ``symlog_disc`` head's bin centres (a constant; XLA folds it)."""
if self.loss != "symlog_disc":
raise ValueError(f"a {self.loss} head has no bins")
return disc_bins(self.num_bins, self.bin_low, self.bin_high)
# ---------------------------------------------------------------- the contract
def __call__(self, parents: tuple[Array, ...]) -> Array:
"""The trunk's raw output — logits for the classifier losses."""
return self.trunk(parents)
[docs]
def nll(self, target: Array, parents: tuple[Array, ...]) -> Array:
"""This head's loss on one target: the training half of the contract."""
out = self.trunk(parents)
if self.loss == "symlog_mse":
return symlog_mse_nll(out, target)
if self.loss == "symlog_disc":
return symlog_disc_nll(out, target, self.bins)
return binary_nll(out[0], target)
[docs]
def predict(self, parents: tuple[Array, ...]) -> Array:
"""The decoded prediction, in the predicted value's own units.
``symexp`` of a symlog_mse output, the expected bin of a symlog_disc head, ``σ(logit)`` for
a binary head.
"""
out = self.trunk(parents)
if self.loss == "symlog_mse":
return symexp(out)
if self.loss == "symlog_disc":
return symlog_disc_mean(out, self.bins)
return jax.nn.sigmoid(out[0])
# ---------------------------------------------------------------- the optax step
def init_opt_state(self):
return init_opt_state(self)
def absorb(self, stats, opt_state, lr: float | Array = 1.0, decay: float | Array = 1.0):
return apply_gradient(self, stats, opt_state, lr=lr, decay=decay)
# =========================================================== the deterministic map
[docs]
class GRUCell(eqx.Module):
"""A GRU carry update ``(deter, *inputs) -> deter'``: a pure map, not a conditional.
Declared through :class:`~hybrid_aif.factors.Deterministic`. DreamerV3's GRU variant: project
the non-carry inputs to ``hidden`` units, normalize, activate; concatenate the carry; one linear
to ``3·deter`` (normalized) giving ``(reset, candidate, update)``; then
``deter' = u·tanh(σ(reset)·candidate) + (1−u)·deter``, ``u = σ(update − 1)``
``inputs`` are unbatched vectors of widths ``input_dims``, in declaration order; the carry is
passed separately.
"""
#: The projection of the non-carry inputs to ``hidden`` units.
input_proj: eqx.nn.Linear
#: The linear map from ``[deter, hidden]`` to ``3 * deter``: reset, candidate and update.
gate: eqx.nn.Linear
#: The ``(input, gate)`` layer norms, or ``()`` for none.
norms: tuple[eqx.nn.LayerNorm, ...]
#: The width of the carry ``deter``.
deter_dim: int = eqx.field(static=True)
#: The widths of the non-carry inputs, in declaration order.
input_dims: tuple[int, ...] = eqx.field(static=True)
#: The activation applied after the input projection.
activation: str = eqx.field(static=True, default="silu")
def __check_init__(self) -> None:
if self.deter_dim <= 0:
raise ValueError(f"deter_dim must be positive, got {self.deter_dim}")
if not self.input_dims or any(d <= 0 for d in self.input_dims):
raise ValueError(
f"input_dims must be a non-empty tuple of positive widths, got "
f"{self.input_dims!r} (the non-carry inputs, e.g. (flat code, action))"
)
if self.activation not in _ACTIVATIONS:
raise ValueError(f"unknown activation {self.activation!r}")
if self.input_proj.in_features != sum(self.input_dims):
raise ValueError(
f"input projection takes {self.input_proj.in_features} inputs but input_dims "
f"{self.input_dims} concatenate to {sum(self.input_dims)}"
)
if self.gate.in_features != self.deter_dim + self.input_proj.out_features:
raise ValueError(
f"gate linear takes {self.gate.in_features} inputs but [deter, hidden] is "
f"{self.deter_dim} + {self.input_proj.out_features}"
)
if self.gate.out_features != 3 * self.deter_dim:
raise ValueError(
f"gate linear emits {self.gate.out_features} outputs; a GRU over a "
f"{self.deter_dim}-unit carry needs {3 * self.deter_dim} (reset, candidate, update)"
)
if self.norms and len(self.norms) != 2:
raise ValueError(f"norms must be () or (input_norm, gate_norm), got {self.norms!r}")
[docs]
@classmethod
def from_shapes(
cls,
deter_dim: int,
input_dims: tuple[int, ...],
*,
key: Array,
hidden: int,
activation: str = "silu",
norm: bool = True,
init: str = "default",
) -> "GRUCell":
"""A randomly-initialized cell; ``hidden`` (required) is the width the
non-carry inputs are projected to.
``norm=True`` puts a LayerNorm (eps 1e-3) after each of the two linears. ``init="default"``
keeps both biases and ``eqx.nn.Linear``'s draw; ``init="trunc_normal_avg"`` draws
:func:`trunc_normal_avg` weights on both linears, drops their biases when normed and starts
any kept bias at zero.
"""
if init not in ("default", "trunc_normal_avg"):
raise ValueError(
f"unknown init {init!r}; the initializers are ['default', 'trunc_normal_avg']"
)
deter_dim, hidden = int(deter_dim), int(hidden)
input_dims = tuple(int(d) for d in input_dims)
k_in, k_gate = jax.random.split(key)
n_in, n_gate = sum(input_dims), deter_dim + hidden
use_bias = init == "default" or not norm
input_proj = eqx.nn.Linear(n_in, hidden, use_bias=use_bias, key=k_in)
gate = eqx.nn.Linear(n_gate, 3 * deter_dim, use_bias=use_bias, key=k_gate)
if init == "trunc_normal_avg":
def reinit(layer: eqx.nn.Linear, k: Array) -> eqx.nn.Linear:
fan_in, fan_out = layer.in_features, layer.out_features
weight = trunc_normal_avg(k, (fan_out, fan_in), fan_in, fan_out)
layer = eqx.tree_at(lambda m: m.weight, layer, weight)
if layer.bias is not None: # a kept bias starts at zero
layer = eqx.tree_at(lambda m: m.bias, layer, jnp.zeros_like(layer.bias))
return layer
input_proj, gate = reinit(input_proj, k_in), reinit(gate, k_gate)
return cls(
input_proj=input_proj,
gate=gate,
norms=(
(eqx.nn.LayerNorm(hidden, eps=1e-3), eqx.nn.LayerNorm(3 * deter_dim, eps=1e-3))
if norm
else ()
),
deter_dim=deter_dim,
input_dims=input_dims,
activation=activation,
)
def __call__(self, deter: Array, *inputs: Array) -> Array:
"""The next carry, shape ``(deter_dim,)``. Unbatched; batch with ``vmap``."""
deter = jnp.asarray(deter)
if deter.shape != (self.deter_dim,):
raise ValueError(
f"deter has shape {deter.shape}, expected ({self.deter_dim},) — the carry is an "
"unbatched vector; batch with vmap"
)
x = self.input_proj(_concat_parents(inputs, self.input_dims))
if self.norms:
x = self.norms[0](x)
x = _ACTIVATIONS[self.activation](x)
g = self.gate(jnp.concatenate([deter, x], axis=-1))
if self.norms:
g = self.norms[1](g)
reset, candidate, update = jnp.split(g, 3, axis=-1)
candidate = jnp.tanh(jax.nn.sigmoid(reset) * candidate)
update = jax.nn.sigmoid(update - 1.0)
return update * candidate + (1.0 - update) * deter
# ======================================================== a parametrized constant
[docs]
class LearnedVector(eqx.Module):
"""A trainable vector behind an activation: a parametrized constant, with no parents.
The canonical use is an episode-start carry: zeros through ``tanh``, so a fresh one starts at
the zero carry and stays in [-1, 1]. Role-free; its gradient arrives from whatever consumes its
value. Trainable, it is DreamerV3's ``initial: 'learned'``; frozen, ``initial: 'zeros'``.
"""
#: The pre-activation parameter.
value: Array
#: The activation the value passes through (static).
activation: str = eqx.field(static=True, default="tanh")
#: The trainable marking: ``True`` keeps ``value`` fixed.
frozen: bool = eqx.field(static=True, default=False)
def __check_init__(self) -> None:
if self.activation not in _ACTIVATIONS:
raise ValueError(
f"unknown activation {self.activation!r}; known activations are "
f"{sorted(_ACTIVATIONS)}"
)
if jnp.ndim(self.value) != 1:
raise ValueError(
f"LearnedVector holds one unbatched vector, got shape {jnp.shape(self.value)}"
)
[docs]
@classmethod
def zeros(cls, dim: int, *, activation: str = "tanh", frozen: bool = False) -> "LearnedVector":
"""The zero pre-activation — hence, under ``tanh``, the zero output."""
return cls(value=jnp.zeros((int(dim),), jnp.float32), activation=activation, frozen=frozen)
@property
def dim(self) -> int:
return int(jnp.shape(self.value)[0])
@property
def learnable(self) -> bool:
"""Capability: a non-frozen vector is a trainable block."""
return not self.frozen
def __call__(self) -> Array:
"""The value, activated — shape ``(dim,)``. No parents: it is a constant."""
return _ACTIVATIONS[self.activation](self.value)
def init_opt_state(self):
return init_opt_state(self)
def absorb(self, stats, opt_state, lr: float | Array = 1.0, decay: float | Array = 1.0):
return apply_gradient(self, stats, opt_state, lr=lr, decay=decay)