"""CEM planner: sampled policy search over continuous action sequences.
A diagonal-Gaussian proposal over ``(policy_len, action_dim)`` sequences, refit to
its own elites ``cem_iters`` times: draw ``num_candidates`` sequences clipped to
the declared box, score them with the context's closures
(:func:`~hybrid_aif.planning.enumerate.evaluate_policies_total`), and refit
``mean``/``std`` to the ``elite_frac`` with the lowest G. The result is the final
candidate set with the proposal mean prepended as row 0 and
``q_pi = softmax(−γ·G)`` over it: :func:`mean_action` is the receding-horizon
action, and sampling ``q_pi`` acts from the searched posterior. The loop is one
``lax.scan``; a fixed key gives a bitwise-identical plan. ``min_std`` floors the
refit standard deviation (a proposal floor; it never changes G). E as the
proposal is not built and raises.
Design: docs/design/amortization.md A9
"""
from typing import NamedTuple
import jax
import jax.numpy as jnp
from jax import Array
from hybrid_aif.distributions import Categorical
from hybrid_aif.interval import Interval
from hybrid_aif.objectives.base import TermTree, total
from hybrid_aif.planning.base import (
Planner,
PlannerContext,
UseBound,
refuse_mixed,
register_planner,
)
from hybrid_aif.planning.enumerate import (
evaluate_policies,
evaluate_policies_total,
policy_posterior,
)
#: Defaults, in units of the declared box width per action dimension: the initial
#: proposal covers half the box, and the refit std never falls below a tenth of it.
INIT_STD_FRACTION = 0.5
MIN_STD_FRACTION = 0.1
[docs]
class CEMResult(NamedTuple):
"""One CEM plan: the posterior, the metrics, and the proposal it ended on.
``G``, ``q_pi`` and every ``tree`` leaf share the leading axis of ``candidates``.
"""
#: The posterior over the candidates.
q_pi: Categorical
#: The term tree of the objective over the candidates (``None`` under ``metrics=False``).
tree: TermTree | None
#: The scored plans, ``(num_candidates + 1, policy_len, action_dim)``; entry 0 is the final
#: proposal ``mean``.
candidates: Array
#: The expected free energy of each candidate.
G: Interval
#: The final proposal's mean, ``(policy_len, action_dim)``.
mean: Array
#: The final proposal's standard deviation, ``(policy_len, action_dim)``.
std: Array
[docs]
def mean_action(candidates: Array) -> Array:
"""The receding-horizon action: the final proposal mean's first step, ``candidates[0, 0]``."""
return jnp.asarray(candidates)[0, 0]
def _point(G: Interval, use_bound: UseBound) -> Array:
return {"midpoint": G.midpoint, "lower": G.lower, "upper": G.upper}[use_bound]
def _elite_count(num_candidates: int, elite_frac: float) -> int:
if num_candidates < 2:
raise ValueError(f"num_candidates must be at least 2, got {num_candidates}")
if not 0.0 < elite_frac <= 1.0:
raise ValueError(f"elite_frac must lie in (0, 1], got {elite_frac}")
return max(1, min(num_candidates, int(round(elite_frac * num_candidates))))
def _layout(ctx: PlannerContext):
refuse_mixed(ctx, "cem")
if ctx.continuous_controls is None:
raise ValueError(
"the CEM planner plans over continuous control sequences, and this System's control "
"nodes are discrete (level indices): give a node a box in params.control and a "
'float-reading transition, or plan with the "enumeration" planner (design: '
"amortization.md A9)."
)
return ctx.continuous_controls
[docs]
def cem_plan(
ctx: PlannerContext,
key: Array,
*,
num_candidates: int = 64,
cem_iters: int = 5,
elite_frac: float = 0.125,
init_mean: Array | None = None,
init_std: float | Array | None = None,
min_std: float | Array | None = None,
use_bound: UseBound = "midpoint",
metrics: bool = True,
) -> CEMResult:
"""Run CEM against ``ctx``; see the module docstring for the algorithm.
``init_mean`` (default: the box midpoint) is the warm start, e.g. the previous
plan shifted one step. ``init_std``/``min_std`` default to
:data:`INIT_STD_FRACTION`/:data:`MIN_STD_FRACTION` of the box width; explicit
values are in action units, scalars or ``(action_dim,)`` arrays.
``metrics=False`` skips the term tree without changing the posterior.
"""
layout = _layout(ctx)
if ctx.policy_prior is not None:
raise NotImplementedError(
"policy_prior with the CEM planner: E's representation for a sampled planner is the "
"proposal distribution itself, not a vector over an enumerated policy set (design: "
"amortization.md A3 — the 'cem' planner has no e_proposal mode). Drop "
"the policy_prior designation, or plan with the enumeration "
"planner."
)
if cem_iters < 1:
raise ValueError(f"cem_iters must be at least 1, got {cem_iters}")
n_elite = _elite_count(num_candidates, elite_frac)
horizon, action_dim = ctx.policy_len, layout.action_dim
low, high = layout.bounds
width = high - low
mean = 0.5 * (low + high) + jnp.zeros((horizon, action_dim), low.dtype)
if init_mean is not None:
mean = jnp.broadcast_to(jnp.asarray(init_mean, low.dtype), (horizon, action_dim))
std = jnp.broadcast_to(
jnp.asarray(INIT_STD_FRACTION * width if init_std is None else init_std, low.dtype),
(horizon, action_dim),
)
floor = jnp.broadcast_to(
jnp.asarray(MIN_STD_FRACTION * width if min_std is None else min_std, low.dtype),
(horizon, action_dim),
)
def score(candidates: Array) -> Interval:
return evaluate_policies_total(
candidates,
ctx.init_beliefs,
ctx.transition_fn,
ctx.step_terms_fn,
ctx.terminal_terms_fn,
)
def iteration(carry, _):
k, mean, std, _, _ = carry
k, k_draw = jax.random.split(k)
eps = jax.random.normal(k_draw, (num_candidates, horizon, action_dim), low.dtype)
candidates = jnp.clip(mean + std * eps, low, high)
scored = score(candidates)
g = _point(scored, use_bound).astype(low.dtype)
elite = candidates[jax.lax.top_k(-g, n_elite)[1]]
return (
k,
elite.mean(axis=0),
jnp.maximum(elite.std(axis=0), floor),
candidates,
scored,
), None
blank = jnp.zeros((num_candidates, horizon, action_dim), low.dtype)
# the carry holds the last iteration's scores (an Interval, whose warrant flag is
# static structure): its initial value is zeros in that same structure
scored0 = jax.tree_util.tree_map(
lambda leaf: jnp.zeros(leaf.shape, leaf.dtype), jax.eval_shape(score, blank)
)
init = (key, mean, std, blank, scored0)
(_, mean, std, candidates, scored), _ = jax.lax.scan(iteration, init, None, length=cem_iters)
candidates = jnp.concatenate([mean[None], candidates], axis=0)
if metrics:
tree = evaluate_policies(
candidates,
ctx.init_beliefs,
ctx.transition_fn,
ctx.step_terms_fn,
ctx.terminal_terms_fn,
)
G = total(tree)
else:
tree = None
# the sampled candidates were scored by the last iteration; only the mean is new
G = jax.tree_util.tree_map(lambda a, b: jnp.concatenate([a, b]), score(mean[None]), scored)
return CEMResult(
q_pi=policy_posterior(G, ctx.gamma, None, use_bound=use_bound),
tree=tree,
candidates=candidates,
G=G,
mean=mean,
std=std,
)
[docs]
def make_cem_planner(**kwargs) -> Planner:
"""A CEM :class:`~hybrid_aif.planning.base.Planner` with ``kwargs`` bound.
Pass it per call, e.g. ``infer_policies(..., planner=make_cem_planner(
num_candidates=256))``. It carries the ``needs_key`` mark.
"""
def planner(
ctx: PlannerContext, use_bound: UseBound = "midpoint", metrics: bool = True
) -> tuple[Categorical, TermTree | None, Array]:
result = _run(ctx, use_bound=use_bound, metrics=metrics, **kwargs)
return result.q_pi, result.tree, result.candidates
planner.needs_key = True # type: ignore[attr-defined]
return planner
def _run(ctx: PlannerContext, **kwargs) -> CEMResult:
if ctx.key is None:
raise ValueError(
"the CEM planner is stochastic and found no PRNG key in its PlannerContext: pass "
"plan_key= to System.infer_policies / act / planner_context (it is not split off "
"the action-selection key — a plan's randomness is the caller's to control)."
)
return cem_plan(ctx, ctx.key, **kwargs)
[docs]
@register_planner("cem", needs_key=True)
def cem_planner(
ctx: PlannerContext, use_bound: UseBound = "midpoint", metrics: bool = True
) -> tuple[Categorical, TermTree | None, Array]:
"""The :class:`~hybrid_aif.planning.base.Planner` face of this module.
Registered as ``"cem"`` with :func:`cem_plan`'s defaults. Returns
``(q_pi, tree, candidates)``, the candidates as in :class:`CEMResult`.
"""
result = _run(ctx, use_bound=use_bound, metrics=metrics)
return result.q_pi, result.tree, result.candidates