Source code for hybrid_aif.distributions.truncated

"""Diagonal Gaussian restricted to a box and renormalised: the bounded family of a control node.

Stored parameterization: ``loc`` and ``scale`` (the untruncated Gaussian's mean and standard
deviation, per dimension) and the box ``low``/``high``. Event shape ``(dim,)``; every closed form
factorises over dimensions, since a diagonal Gaussian on a box is a product of one-dimensional
truncated normals. All four leaves share one shape, broadcast at construction.

With ``α = (low − loc)/scale``, ``β = (high − loc)/scale`` and the normaliser
``Z = Φ(β) − Φ(α)``, per dimension:

- density ``φ((x − loc)/scale) / (scale·Z)`` inside the box, zero outside;
- mean ``loc + scale·(φ(α) − φ(β))/Z``;
- variance ``scale²·(1 + (αφ(α) − βφ(β))/Z − ((φ(α) − φ(β))/Z)²)``;
- entropy ``log(√(2πe)·scale·Z) + (αφ(α) − βφ(β))/(2Z)``.

An infinite bound is read 40 standard deviations beyond the other bound (the mass past it is
zero in float32), so a half-bounded or unbounded member has finite moments and gradients.

``log Z`` is computed in log space from ``log_ndtr``, a box in the upper tail reflected onto the
lower one, so a box many standard deviations from ``loc`` keeps a finite normaliser and finite
moments. Sampling is reparameterised through the inverse CDF, inverted in log space.

Design: docs/design/control-states.md C4 (Bounded Gaussian).
"""

import equinox as eqx
import jax
import jax.numpy as jnp
from jax import Array
from jax.scipy.special import log_ndtr, ndtri

from hybrid_aif.distributions.base import Distribution
from hybrid_aif.interval import Interval

_LOG_2PI = float(jnp.log(2.0 * jnp.pi))
_LOG_2 = float(jnp.log(2.0))
#: Newton steps of the log-space inverse CDF (quadratic convergence from the asymptotic start).
_NEWTON_STEPS = 8
#: Beyond this many standard deviations a box's moments come from the tail forms (see
#: :func:`_tail_moments`); the continued fraction converges to float precision there.
_TAIL = 3.0
_CF_TERMS = 24
#: A box is *narrow* when its half-width ``h`` (in standard deviations) and centre ``c`` satisfy
#: ``h·(1 + |c|) < _NARROW``: its moments then come from the expansion about the midpoint
#: (:func:`_narrow_series`), summed far enough that the neglected terms are below float32
#: precision there.
_NARROW = 1.0
#: Below this half-width (in standard deviations) a draw comes from the exponential tilt
#: ``exp(−c t)`` about the lower bound: the dropped ``t²/2`` is under float32 precision there.
_TILT_HALF_WIDTH = 1e-3
_SERIES_TERMS = 12
#: An infinite bound is read this many standard deviations out (beyond the other bound): the
#: mass past it is zero in float32, and every quantity stays finite and differentiable.
_INFINITE = 40.0


def _log1mexp(x: Array) -> Array:
    """``log(1 − exp(x))`` for ``x < 0``, accurate at both ends."""
    near = jnp.log(-jnp.expm1(jnp.minimum(x, -1e-30)))
    far = jnp.log1p(-jnp.exp(jnp.minimum(x, -_LOG_2)))
    return jnp.where(x > -_LOG_2, near, far)


def log_normaliser(alpha: Array, beta: Array) -> Array:
    """``log(Φ(β) − Φ(α))`` for ``α < β``, elementwise, in log space.

    A box below zero reads the lower tail directly; a box above zero is reflected onto it
    (``Φ(β) − Φ(α) = Φ(−α) − Φ(−β)``); a box straddling zero holds at least the mass between
    its nearer bound and zero and is taken directly. Each branch is evaluated at safe stand-in
    values where it is not selected, so gradients stay finite.
    """
    low_side, high_side = beta <= 0.0, alpha >= 0.0
    mid = ~(low_side | high_side)
    a_l, b_l = jnp.where(low_side, alpha, -2.0), jnp.where(low_side, beta, -1.0)
    a_h, b_h = jnp.where(high_side, alpha, 1.0), jnp.where(high_side, beta, 2.0)
    a_m, b_m = jnp.where(mid, alpha, -1.0), jnp.where(mid, beta, 1.0)
    lower = log_ndtr(b_l) + _log1mexp(log_ndtr(a_l) - log_ndtr(b_l))
    upper = log_ndtr(-a_h) + _log1mexp(log_ndtr(-b_h) - log_ndtr(-a_h))
    middle = jnp.log(jax.scipy.special.ndtr(b_m) - jax.scipy.special.ndtr(a_m))
    return jnp.where(low_side, lower, jnp.where(high_side, upper, middle))


def _log_phi(z: Array) -> Array:
    return -0.5 * z**2 - 0.5 * _LOG_2PI


def _narrow_series(c: Array, q: Array) -> tuple[Array, Array, Array]:
    """Normalised moments of a standard normal on ``[c − h, c + h]`` about its midpoint, as
    series in ``q = h²``: with ``t = z − c`` the weight is ``exp(−c t − t²/2)``, whose Taylor
    coefficients are ``(−1)ⁿ He_n(c)/n!`` (``He`` the probabilists' Hermite polynomials), so
    ``∫ w = 2h·M₀``, ``∫ t w = 2h³·M₁``, ``∫ t² w = 2h³·M₂`` with
    ``M₀ = Σ_k He_{2k} q^k/((2k)!(2k+1))``, ``M₁ = −Σ_k He_{2k+1} q^k/((2k+1)!(2k+3))``,
    ``M₂ = Σ_k He_{2k} q^k/((2k)!(2k+3))``, summed to :data:`_SERIES_TERMS` terms.
    Returns ``(M₀, M₁/M₀, M₂/M₀)``."""
    he = [jnp.ones_like(c), c]
    for n in range(1, 2 * _SERIES_TERMS):
        he.append(c * he[n] - n * he[n - 1])
    m0 = m1 = m2 = jnp.zeros_like(c)
    power, factorial = jnp.ones_like(q), 1.0
    for k in range(_SERIES_TERMS):
        even = factorial  # (2k)!
        odd = factorial * (2 * k + 1)  # (2k+1)!
        m0 = m0 + he[2 * k] * power / (even * (2 * k + 1))
        m1 = m1 - he[2 * k + 1] * power / (odd * (2 * k + 3))
        m2 = m2 + he[2 * k] * power / (even * (2 * k + 3))
        power, factorial = power * q, odd * (2 * k + 2)
    return m0, m1 / m0, m2 / m0


def _mills_fraction(x: Array) -> tuple[Array, Array]:
    """The first two tails ``c₁, c₂`` of the Mills-ratio continued fraction at ``x > 0``:
    ``c_k = k/(x + c_{k+1})``, so that ``(1 − Φ(x))/φ(x) = 1/(x + c₁)`` and
    ``1 − x·c₁ = c₁·c₂`` without cancellation."""
    c = jnp.zeros_like(x)
    c2 = c
    for k in range(_CF_TERMS, 0, -1):
        c = k / (x + c)
        if k == 2:
            c2 = c
    return c, c2


def _tail_moments(alpha: Array, beta: Array) -> tuple[Array, Array, Array]:
    """Mean, variance and entropy of a standard normal on ``[α, β]`` with ``α ≥`` :data:`_TAIL`,
    free of the cancellation the general forms suffer there.

    With ``w = z − α`` on ``[0, L]``, ``A = R(α)``, ``B = ρ·R(β)`` (``R`` the Mills ratio,
    ``ρ = φ(β)/φ(α)``) and ``V_x = c₁(c₂ − c₁)`` the one-sided variance at ``x``:
    ``E[w] = (c₁(α)A − B(c₁(β) + L))/(A − B)``,
    ``Var = ((A − B)(A·V_α − B·V_β) − A·B·(L + c₁(β) − c₁(α))²)/(A − B)²``, and, since
    ``Z = φ(α)(A − B)``, the entropy ``log(A − B) + α E[w] + E[w²]/2``.
    An infinite ``β`` is the one-sided case (``B = 0``).
    """
    c1a, c2a = _mills_fraction(alpha)
    finite = jnp.isfinite(beta)
    b = jnp.where(finite, beta, alpha + 1.0)
    c1b, c2b = _mills_fraction(b)
    width = b - alpha
    a_mills, b_mills = 1.0 / (alpha + c1a), 1.0 / (b + c1b)
    rho = jnp.where(finite, jnp.exp(-width * (alpha + b) / 2.0), 0.0)
    big, small = a_mills, rho * b_mills
    va, vb = c1a * (c2a - c1a), c1b * (c2b - c1b)
    norm = big - small
    offset = (c1a * big - small * (c1b + width)) / norm
    var = (norm * (big * va - small * vb) - big * small * (width + c1b - c1a) ** 2) / norm**2
    entropy = jnp.log(norm) + alpha * offset + 0.5 * (var + offset**2)
    return alpha + offset, var, entropy


def _ndtri_log(log_p: Array) -> Array:
    """``Φ⁻¹(exp(log_p))`` for ``log_p ≤ log ½``, accurate where ``exp(log_p)`` underflows:
    Newton's method on ``log Φ(z) = log_p`` from ``ndtri`` (or the tail asymptote
    ``−√(−2 log_p)`` when ``exp(log_p)`` is not representable)."""
    start = jnp.where(
        log_p > -60.0,
        ndtri(jnp.exp(jnp.maximum(log_p, -60.0))),
        -jnp.sqrt(-2.0 * jnp.minimum(log_p, -60.0)),
    )

    def newton(z, _):
        lz = log_ndtr(z)
        slope = jnp.exp(_log_phi(z) - lz)  # d log Φ / dz
        return z - (lz - log_p) / slope, None

    z, _ = jax.lax.scan(newton, start, None, length=_NEWTON_STEPS)
    return z


[docs] class TruncatedGaussian(Distribution): """A diagonal Gaussian ``N(loc, diag(scale²))`` restricted to ``[low, high]`` (see module). Every leaf is ``(..., dim)``; ``low < high`` elementwise. """ #: The location of the untruncated Gaussian, ``(..., dim)``. loc: Array #: The standard deviation of the untruncated Gaussian, ``(..., dim)``. scale: Array #: The lower bound, ``(..., dim)``. low: Array #: The upper bound, ``(..., dim)``. high: Array
[docs] def __init__(self, loc: Array, scale: Array, low: Array, high: Array): arrays = [jnp.asarray(v, jnp.float32) for v in (loc, scale, low, high)] shape = jnp.broadcast_shapes(*(a.shape for a in arrays)) if not shape: raise ValueError( "TruncatedGaussian takes (..., dim) arrays: give loc, scale, low and high a " "trailing dimension axis" ) self.loc, self.scale, self.low, self.high = (jnp.broadcast_to(a, shape) for a in arrays)
@property def batch_shape(self) -> tuple[int, ...]: return self.loc.shape[:-1] @property def dim(self) -> int: return self.loc.shape[-1] # -------------------------------------------------------------- per-dimension pieces def _alpha_beta(self) -> tuple[Array, Array]: """``(α, β)`` per dimension, an infinite bound read :data:`_INFINITE` standard deviations beyond the other (both substituted before dividing, so gradients stay finite).""" fl, fh = jnp.isfinite(self.low), jnp.isfinite(self.high) low = jnp.where(fl, self.low, self.loc) high = jnp.where(fh, self.high, self.loc) alpha = (low - self.loc) / self.scale beta = (high - self.loc) / self.scale alpha = jnp.where(fl, alpha, jnp.minimum(-_INFINITE, jnp.where(fh, beta, 0.0) - _INFINITE)) beta = jnp.where(fh, beta, jnp.maximum(_INFINITE, alpha + _INFINITE)) return alpha, beta def _narrow(self) -> tuple[Array, Array, Array, Array, Array]: """``(narrow, c, q, M₀, M₁/M₀, M₂/M₀)`` pieces of the midpoint expansion; the series is evaluated at stand-in values where the box is not narrow.""" alpha, beta = self._alpha_beta() c, h = 0.5 * (alpha + beta), 0.5 * (beta - alpha) narrow = h * (1.0 + jnp.abs(c)) < _NARROW c, h = jnp.where(narrow, c, 0.0), jnp.where(narrow, h, 0.01) m0, r1, r2 = _narrow_series(c, h**2) return narrow, c, h, m0, r1, r2 def _standard(self) -> tuple[Array, Array, Array]: """``(α, β, log Z)`` per dimension; a narrow box's ``log Z`` is ``log φ(c) + log 2h + log M₀`` (:func:`_narrow_series`).""" alpha, beta = self._alpha_beta() narrow, c, h, m0, _, _ = self._narrow() wide = log_normaliser(jnp.where(narrow, -1.0, alpha), jnp.where(narrow, 1.0, beta)) log_z = jnp.where(narrow, _log_phi(c) + jnp.log(2.0 * h) + jnp.log(m0), wide) return alpha, beta, log_z def _ratios(self) -> tuple[Array, Array, Array, Array]: """``(φ(α)/Z, φ(β)/Z, αφ(α)/Z, βφ(β)/Z)`` per dimension.""" alpha, beta, log_z = self._standard() ra = jnp.exp(_log_phi(alpha) - log_z) rb = jnp.exp(_log_phi(beta) - log_z) return ra, rb, alpha * ra, beta * rb
[docs] def log_normalisers(self) -> Array: """``log Z`` per dimension, ``(..., dim)``.""" return self._standard()[2]
def _standard_moments(self) -> tuple[Array, Array]: """Mean and variance of the standardised variable, per dimension: the general forms near ``loc``, the tail forms (:func:`_tail_moments`, a lower-tail box reflected) beyond :data:`_TAIL` standard deviations. Narrow boxes are handled by :meth:`mean` and :meth:`variance` in the box's own units.""" alpha, beta = self._alpha_beta() ra, rb, a_ra, b_rb = self._ratios() mean, var = ra - rb, 1.0 + a_ra - b_rb - (ra - rb) ** 2 upper, lower = alpha >= _TAIL, beta <= -_TAIL up_mean, up_var, _ = _tail_moments( jnp.where(upper, alpha, _TAIL), jnp.where(upper, beta, _TAIL + 1.0) ) lo_mean, lo_var, _ = _tail_moments( jnp.where(lower, -beta, _TAIL), jnp.where(lower, -alpha, _TAIL + 1.0) ) mean = jnp.where(upper, up_mean, jnp.where(lower, -lo_mean, mean)) var = jnp.where(upper, up_var, jnp.where(lower, lo_var, var)) return mean, var def _half_width(self) -> Array: """Half the box's width in its own units (a stand-in where it is unbounded).""" finite = jnp.isfinite(self.low) & jnp.isfinite(self.high) return jnp.where(finite, 0.5 * (jnp.where(finite, self.high - self.low, 0.0)), 1.0)
[docs] def mean(self) -> Array: """The mean; a narrow box's is ``mid + half·h·M₁/M₀`` about its midpoint. Clamped into the box as a guard.""" mean_std, _ = self._standard_moments() narrow, _, h, _, r1, _ = self._narrow() half = self._half_width() mid = jnp.where(narrow, self.low + half, 0.0) mean = jnp.where(narrow, mid + half * h * r1, self.loc + self.scale * mean_std) return jnp.clip(mean, self.low, self.high)
[docs] def variance(self) -> Array: """Per-dimension variance, ``(..., dim)`` (the covariance is diagonal); a narrow box's is ``half²·(M₂/M₀ − h²(M₁/M₀)²)``, ``w²/12`` in the uniform limit. Clamped into ``(0, w²/4]`` as a guard.""" _, var_std = self._standard_moments() narrow, _, h, _, r1, r2 = self._narrow() half = self._half_width() var = jnp.where(narrow, half**2 * (r2 - h**2 * r1**2), self.scale**2 * var_std) bounded = jnp.isfinite(self.low) & jnp.isfinite(self.high) cap = jnp.where(bounded, half**2, jnp.inf) return jnp.clip(var, jnp.finfo(jnp.float32).tiny, cap)
@property def cov(self) -> Array: """The diagonal covariance as a matrix, ``(..., dim, dim)``.""" var = self.variance() return var[..., :, None] * jnp.eye(self.dim, dtype=var.dtype) # ------------------------------------------------------------------ the contract
[docs] def log_prob(self, x: Array) -> Array: """Log density at ``x``, summed over dimensions; ``−inf`` outside the box.""" x = jnp.asarray(x, jnp.float32) z = (x - self.loc) / self.scale per_dim = _log_phi(z) - jnp.log(self.scale) - self.log_normalisers() inside = jnp.all((x >= self.low) & (x <= self.high), axis=-1) return jnp.where(inside, jnp.sum(per_dim, axis=-1), -jnp.inf)
[docs] def sample(self, key: Array, sample_shape: tuple[int, ...] = ()) -> Array: """Reparameterised draws through the inverse CDF: ``u ~ U(0, 1)`` per dimension and ``x = loc + scale·Φ⁻¹(Φ(α) + u·Z)``, with ``Φ(α) + u·Z`` formed in log space and a box in the upper tail drawn from its reflection. Draws are clipped into the box against rounding. A box narrower than :data:`_TILT_HALF_WIDTH` standard deviations, where the inverse CDF runs out of float32 resolution and ``t²/2`` is below it, is drawn from its exponential tilt about the lower bound.""" u = jax.random.uniform(key, sample_shape + self.loc.shape, jnp.float32) u = jnp.clip(u, 1e-7, 1.0 - 1e-7) narrow, c, h, _, _, _ = self._narrow() narrow = narrow & (h < _TILT_HALF_WIDTH) tilted = jnp.abs(c) > 1e-6 a = jnp.where(tilted, c, 1.0) s_std = jnp.where(tilted, -jnp.log1p(u * jnp.expm1(-2.0 * a * h)) / a, 2.0 * h * u) thin = self.low + self.scale * s_std alpha, beta, log_z = self._standard() flip = alpha > 0.0 lo = jnp.where(flip, -beta, alpha) log_p = jnp.logaddexp(log_ndtr(lo), jnp.log(u) + log_z) # below Φ = ½ invert directly; above, invert the complement (less rounding) upper = log_p > -_LOG_2 log_q = _log1mexp(jnp.where(upper, log_p, -1.0)) z = jnp.where(upper, -_ndtri_log(log_q), _ndtri_log(jnp.where(upper, -1.0, log_p))) z = jnp.where(flip, -z, z) draw = jnp.where(narrow, jnp.where(narrow, thin, 0.0), self.loc + self.scale * z) return jnp.clip(draw, self.low, self.high)
[docs] def entropy(self) -> Interval: """The differential entropy, summed over dimensions; a box beyond :data:`_TAIL` standard deviations takes the tail form (:func:`_tail_moments`).""" alpha, beta = self._alpha_beta() _, _, a_ra, b_rb = self._ratios() standard = 0.5 * (1.0 + _LOG_2PI) + self.log_normalisers() + 0.5 * (a_ra - b_rb) narrow, c, h, m0, r1, r2 = self._narrow() # narrow: log 2h + log M₀ + c·E[t] + E[t²]/2, with t = z − c thin = jnp.log(2.0 * h) + jnp.log(m0) + c * h**2 * r1 + 0.5 * h**2 * r2 upper, lower = alpha >= _TAIL, beta <= -_TAIL _, _, up = _tail_moments( jnp.where(upper, alpha, _TAIL), jnp.where(upper, beta, _TAIL + 1.0) ) _, _, lo = _tail_moments( jnp.where(lower, -beta, _TAIL), jnp.where(lower, -alpha, _TAIL + 1.0) ) standard = jnp.where(upper, up, jnp.where(lower, lo, standard)) standard = jnp.where(narrow, thin, standard) return Interval.exact(jnp.sum(standard + jnp.log(self.scale), axis=-1))
[docs] def expected_square(self, centre: Array) -> Array: """``E[(x − centre)²]`` per dimension, from the closed mean and variance.""" return self.variance() + (self.mean() - centre) ** 2
[docs] def kl_to(self, other: "TruncatedGaussian") -> Interval: """``KL(self ‖ other)`` for two members on the same box, in closed form: ``−H[self] − E_self[log other]``, the cross term read off the closed first two moments. Members on different boxes raise (eagerly, or under a trace by ``equinox.error_if``): outside a shared box the KL is infinite or undefined. """ if not isinstance(other, TruncatedGaussian): raise TypeError( f"TruncatedGaussian.kl_to takes another TruncatedGaussian on the same box, got " f"{type(other).__name__}" ) self = _check_same_box(self, other) # noqa: PLW0642 per_dim = ( -0.5 * _LOG_2PI - jnp.log(other.scale) - other.log_normalisers() - 0.5 * self.expected_square(other.loc) / other.scale**2 ) return Interval.exact(-self.entropy().lower - jnp.sum(per_dim, axis=-1))
def _check_same_box(p: TruncatedGaussian, q: TruncatedGaussian) -> TruncatedGaussian: """``p``, checked to share ``q``'s box: concrete bounds raise here, traced ones when the computation runs (``equinox.error_if``).""" if p.dim != q.dim: raise ValueError( f"KL between truncated Gaussians of dimension {p.dim} and {q.dim}: pass two members " "on the same box" ) sentence = ( "KL between truncated Gaussians on different boxes is not finite: pass two members on " "the same box" ) same = jnp.all(p.low == q.low) & jnp.all(p.high == q.high) if not isinstance(same, jax.core.Tracer): if not bool(same): raise ValueError(sentence) return p return TruncatedGaussian(p.loc, p.scale, eqx.error_if(p.low, ~same, sentence), p.high)