Source code for hybrid_aif.distributions.mixture

"""Gaussian mixture with static component count K.

K is shape-derived and fixed; dead components are ``-inf`` log-weights, never a shape change,
so the object stays jit/vmap/scan-legal. ``entropy()`` returns the Kolchinsky & Tracey (2017)
pairwise-distance bound pair (Bhattacharyya lower, KL upper; gap ≤ H(weights) ≤ log K).
``moment_match()`` is the exact single-Gaussian collapse (GPB1); ``reduce(k)`` is Runnalls
(2007) greedy pairwise reduction to a static budget.

Design: docs/de-risk-report.md §2
"""

import math
from functools import partial

import equinox as eqx
import jax
import jax.numpy as jnp
from jax import Array, lax
from jax.scipy.linalg import solve_triangular
from jax.scipy.special import logsumexp

from hybrid_aif.distributions.base import Distribution
from hybrid_aif.distributions.gaussian import Gaussian
from hybrid_aif.interval import Interval


[docs] class MixtureGaussian(Distribution): """Gaussian mixture belief. ``log_weights`` are defined up to an additive gauge: every method is gauge-invariant in them and may return any gauge (``reduce`` returns normalized weights). """ #: The components' log weights, ``(..., K)``, up to an additive gauge. log_weights: Array #: The components' means, ``(..., K, dim)``. means: Array #: The components' lower-triangular Cholesky factors, ``(..., K, dim, dim)``. chols: Array @property def batch_shape(self) -> tuple[int, ...]: return self.log_weights.shape[:-1] @property def num_components(self) -> int: return self.log_weights.shape[-1] @property def dim(self) -> int: return self.means.shape[-1] @property def components(self) -> Gaussian: """The K components as a Gaussian with a trailing batch axis of size K.""" return Gaussian(loc=self.means, chol=self.chols)
[docs] @classmethod def from_gaussian(cls, g: Gaussian) -> "MixtureGaussian": """Promote a Gaussian to the equivalent single-component (K=1) mixture.""" return cls( log_weights=jnp.zeros(g.batch_shape + (1,), g.loc.dtype), means=g.loc[..., None, :], chols=g.chol[..., None, :, :], )
[docs] def log_prob(self, x: Array) -> Array: lp = self.components.log_prob(x[..., None, :]) # (..., K) return logsumexp(jax.nn.log_softmax(self.log_weights, axis=-1) + lp, axis=-1)
[docs] def sample(self, key: Array, sample_shape: tuple[int, ...] = ()) -> Array: key_z, key_x = jax.random.split(key) z = jax.random.categorical( key_z, self.log_weights, shape=sample_shape + self.batch_shape ).astype(jnp.int32) eps = jax.random.normal(key_x, sample_shape + self.batch_shape + (self.dim,)) # Gather the chosen component per draw straight from the K axis (the # unit sample axes broadcast inside the gather; nothing is materialised). lead = (None,) * len(sample_shape) means = jnp.take_along_axis(self.means[lead], z[..., None, None], axis=-2)[..., 0, :] chols = jnp.take_along_axis(self.chols[lead], z[..., None, None, None], axis=-3) return means + (chols[..., 0, :, :] @ eps[..., None])[..., 0]
[docs] def mean(self) -> Array: w = jax.nn.softmax(self.log_weights, axis=-1) return jnp.sum(w[..., None] * self.means, axis=-2)
[docs] def cov(self) -> Array: """Exact mixture covariance: E[Σ_z] + Cov[μ_z].""" w = jax.nn.softmax(self.log_weights, axis=-1) covs = self.chols @ jnp.swapaxes(self.chols, -1, -2) dev = self.means - self.mean()[..., None, :] spread = dev[..., :, None] * dev[..., None, :] return jnp.sum(w[..., None, None] * (covs + spread), axis=-3)
[docs] def moment_match(self) -> Gaussian: """Exact single-Gaussian collapse (preserves mean and covariance).""" return Gaussian.from_cov(loc=self.mean(), cov=self.cov())
[docs] def entropy(self) -> Interval: """Kolchinsky–Tracey pairwise bounds: Ĥ_D = Σᵢwᵢ H(pᵢ) − Σᵢwᵢ log Σⱼwⱼ e^{−D(pᵢ‖pⱼ)}. D = Bhattacharyya → lower bound; D = KL → upper bound. """ w = jax.nn.log_softmax(self.log_weights, axis=-1) comp_entropy = jnp.sum(jnp.exp(w) * self.components.entropy().lower, axis=-1) def bound(pairwise_d: Array) -> Array: inner = logsumexp(w[..., None, :] - pairwise_d, axis=-1) # (..., K) return comp_entropy - jnp.sum(jnp.exp(w) * inner, axis=-1) return Interval( lower=bound(self._pairwise_bhattacharyya()), upper=bound(self._pairwise_kl()), )
[docs] def cross_entropy_to(self, other: "MixtureGaussian | Gaussian") -> Interval: """CE(f, g) = −E_f[log g] bounds (Kolchinsky–Tracey §4), f = ``self``. - upper (variational): CE ≤ Σ_i w_i H(f_i) − Σ_i w_i log Σ_j v_j e^{−KL(f_i‖g_j)}; - lower (Jensen): E_{f_i}[−log g] ≥ −log Σ_j v_j Z_ij, with Z_ij = ∫ f_i g_j = N(μ_i; ν_j, Σ_i + Λ_j). A plain :class:`Gaussian` ``other`` is promoted via :meth:`from_gaussian`. When g has one component the upper bound equals the exact cross-entropy and the interval collapses to a point, which makes a Gaussian preference's utility over a mixture observation exact. """ if isinstance(other, Gaussian): other = MixtureGaussian.from_gaussian(other) lw_f = jax.nn.log_softmax(self.log_weights, axis=-1) lw_g = jax.nn.log_softmax(other.log_weights, axis=-1) w_f = jnp.exp(lw_f) gi = Gaussian(loc=self.means[..., :, None, :], chol=self.chols[..., :, None, :, :]) gj = Gaussian(loc=other.means[..., None, :, :], chol=other.chols[..., None, :, :, :]) pair_kl = gi.kl_to(gj).lower # (..., K_f, K_g) # log Z_ij: Gaussian density of mu_i - nu_j under N(0, Sigma_i + Lambda_j). cov_sum = gi.cov + gj.cov chol_sum = jnp.linalg.cholesky(cov_sum) dev = self.means[..., :, None, :] - other.means[..., None, :, :] z = solve_triangular(chol_sum, dev[..., None], lower=True)[..., 0] half_log_det = jnp.sum(jnp.log(jnp.diagonal(chol_sum, axis1=-2, axis2=-1)), axis=-1) log_z = -0.5 * (self.dim * jnp.log(2.0 * jnp.pi) + jnp.sum(z**2, axis=-1)) - half_log_det comp_h = jnp.sum(w_f * self.components.entropy().lower, axis=-1) ce_lower = -jnp.sum(w_f * logsumexp(lw_g[..., None, :] + log_z, axis=-1), axis=-1) ce_upper = comp_h - jnp.sum(w_f * logsumexp(lw_g[..., None, :] - pair_kl, axis=-1), axis=-1) if other.num_components == 1: # static: upper bound is the exact CE (docstring) ce_lower = ce_upper return Interval(lower=ce_lower, upper=ce_upper)
[docs] def kl_to(self, other: "MixtureGaussian | Gaussian") -> Interval: """Mixture-mixture KL bounds via KL(f‖g) = CE(f,g) − H(f) (Kolchinsky–Tracey §4). Lower = CE_lo − H_hi, clamped at 0; upper = CE_hi − H_lo. With a single-component ``other`` the width is f's entropy-bound width, zero when f is a single Gaussian. """ ce = self.cross_entropy_to(other) h_f = self.entropy() return Interval(lower=ce.lower - h_f.upper, upper=ce.upper - h_f.lower).clip_lower(0.0)
[docs] def reduce(self, k: int, aux=None): """Runnalls (2007) greedy reduction to exactly ``k`` components. Exactly K−k merge iterations (a fixed trip count). Each merges the valid pair of least cost B(i,j) = ½[(w_i+w_j)·logdet Σ_ij − w_i·logdet Σ_i − w_j·logdet Σ_j] (Σ_ij the moment-matched merge) into slot min(i,j) and retires slot max(i,j). Dead (``-inf``) components are merged first; a surplus of dead components survives as dead output slots. The output is compacted to the top-k slots by log-weight (descending, ties by slot index) and renormalized. Gradients are defined a.e. but discontinuous at merge-decision boundaries. ``aux`` is a per-component side statistic (a pytree of arrays, each with the component axis where ``log_weights`` has it, ``(..., K, ...)``): it rides through the reduction by the assignment that merges the moments, a merged slot holding its members' weight-average, and the call returns ``(mixture, aux)``. It never steers a merge, so the moments are those of the call without it (design: parameters-within-the-plan.md D2). """ n = self.num_components if not 1 <= k <= n: raise ValueError(f"reduce: need 1 <= k <= {n}, got {k}") if k == n: return self if aux is None else (self, aux) nb = len(self.batch_shape) if self.batch_shape: b = math.prod(self.batch_shape) lw = self.log_weights.reshape((b, n)) means = self.means.reshape((b, n, self.dim)) chols = self.chols.reshape((b, n, self.dim, self.dim)) if aux is None: out_lw, out_means, out_chols = jax.vmap(partial(_runnalls_reduce, k=k))( lw, means, chols ) else: flat = jax.tree.map(lambda a: a.reshape((b, n) + a.shape[nb + 1 :]), aux) out_lw, out_means, out_chols, out_aux = jax.vmap( lambda lw, mu, ch, ax: _runnalls_reduce(lw, mu, ch, k=k, aux=ax) )(lw, means, chols, flat) reduced = MixtureGaussian( log_weights=out_lw.reshape(self.batch_shape + (k,)), means=out_means.reshape(self.batch_shape + (k, self.dim)), chols=out_chols.reshape(self.batch_shape + (k, self.dim, self.dim)), ) if aux is None: return reduced return reduced, jax.tree.map( lambda a: a.reshape(self.batch_shape + (k,) + a.shape[2:]), out_aux ) if aux is None: out_lw, out_means, out_chols = _runnalls_reduce( self.log_weights, self.means, self.chols, k=k ) return MixtureGaussian(log_weights=out_lw, means=out_means, chols=out_chols) out_lw, out_means, out_chols, out_aux = _runnalls_reduce( self.log_weights, self.means, self.chols, k=k, aux=aux ) return MixtureGaussian(log_weights=out_lw, means=out_means, chols=out_chols), out_aux
[docs] def reduce_mode_aligned(self, k: int, num_modes: int, aux=None): """Mode-aligned grouped Runnalls reduction to ``k`` components (IMM/GPB2). On a mixture in the component-major grow layout of :func:`~hybrid_aif.inference.mixture_rollout.propagate` (slot ``s`` produced by mode ``s % num_modes``), Runnalls runs within each producing-mode group, reducing it to ``k // num_modes`` components. The output is contiguous blocks of ``k // num_modes`` slots per mode, so the mass of block ``g`` is the posterior mass of mode ``g``. ``k`` and the component count must both be multiples of ``num_modes``. ``aux`` is a per-component side statistic merged by the same assignment, as :meth:`reduce` takes it; the call then returns ``(mixture, aux)``. Design: docs/design/mixture-filtering.md D2; parameters-within-the-plan.md D2 """ n = self.num_components if num_modes < 1: raise ValueError(f"reduce_mode_aligned: num_modes must be >= 1, got {num_modes}") if k % num_modes != 0: raise ValueError( f"reduce_mode_aligned: k={k} must be a multiple of num_modes={num_modes} so " f"each mode owns k//num_modes slots; set mixture_k to a multiple of {num_modes}" ) if n % num_modes != 0: raise ValueError( f"reduce_mode_aligned: component count {n} must be a multiple of " f"num_modes={num_modes} (the component-major grow layout)" ) if not num_modes <= k <= n: raise ValueError(f"reduce_mode_aligned: need num_modes <= k <= {n}, got k={k}") nb = len(self.batch_shape) if self.batch_shape: b = math.prod(self.batch_shape) lw = self.log_weights.reshape((b, n)) means = self.means.reshape((b, n, self.dim)) chols = self.chols.reshape((b, n, self.dim, self.dim)) if aux is None: out_lw, out_means, out_chols = jax.vmap( partial(_mode_aligned_reduce, k=k, num_modes=num_modes) )(lw, means, chols) else: flat = jax.tree.map(lambda a: a.reshape((b, n) + a.shape[nb + 1 :]), aux) out_lw, out_means, out_chols, out_aux = jax.vmap( lambda lw, mu, ch, ax: _mode_aligned_reduce( lw, mu, ch, k=k, num_modes=num_modes, aux=ax ) )(lw, means, chols, flat) reduced = MixtureGaussian( log_weights=out_lw.reshape(self.batch_shape + (k,)), means=out_means.reshape(self.batch_shape + (k, self.dim)), chols=out_chols.reshape(self.batch_shape + (k, self.dim, self.dim)), ) if aux is None: return reduced return reduced, jax.tree.map( lambda a: a.reshape(self.batch_shape + (k,) + a.shape[2:]), out_aux ) if aux is None: out_lw, out_means, out_chols = _mode_aligned_reduce( self.log_weights, self.means, self.chols, k=k, num_modes=num_modes ) return MixtureGaussian(log_weights=out_lw, means=out_means, chols=out_chols) out_lw, out_means, out_chols, out_aux = _mode_aligned_reduce( self.log_weights, self.means, self.chols, k=k, num_modes=num_modes, aux=aux ) return MixtureGaussian(log_weights=out_lw, means=out_means, chols=out_chols), out_aux
def _pairwise_kl(self) -> Array: """KL(component_i ‖ component_j), shape (..., K, K).""" gi = Gaussian(loc=self.means[..., :, None, :], chol=self.chols[..., :, None, :, :]) gj = Gaussian(loc=self.means[..., None, :, :], chol=self.chols[..., None, :, :, :]) return gi.kl_to(gj).lower def _pairwise_bhattacharyya(self) -> Array: """Bhattacharyya distance between components, shape (..., K, K).""" covs = self.chols @ jnp.swapaxes(self.chols, -1, -2) cov_bar = 0.5 * (covs[..., :, None, :, :] + covs[..., None, :, :, :]) chol_bar = jnp.linalg.cholesky(cov_bar) dev = self.means[..., :, None, :] - self.means[..., None, :, :] z = solve_triangular(chol_bar, dev[..., None], lower=True)[..., 0] maha = jnp.sum(z**2, axis=-1) half_log_det = jnp.sum(jnp.log(jnp.diagonal(chol_bar, axis1=-2, axis2=-1)), axis=-1) comp_half_log_det = jnp.sum( jnp.log(jnp.diagonal(self.chols, axis1=-2, axis2=-1)), axis=-1 ) # (..., K) cross = 0.5 * (comp_half_log_det[..., :, None] + comp_half_log_det[..., None, :]) return 0.125 * maha + half_log_det - cross
def _log_det(covs: Array) -> Array: chol = jnp.linalg.cholesky(covs) return 2.0 * jnp.sum(jnp.log(jnp.diagonal(chol, axis1=-2, axis2=-1)), axis=-1) def _merge_moments( lw_i: Array, lw_j: Array, mu_i: Array, mu_j: Array, cov_i: Array, cov_j: Array ) -> tuple[Array, Array, Array]: """Moment-matched pair merge; ratios guarded so dead-dead pairs stay NaN-free.""" lw_sum = jnp.logaddexp(lw_i, lw_j) finite = jnp.isfinite(lw_sum) r_i = jnp.where(finite, jnp.exp(lw_i - jnp.where(finite, lw_sum, 0.0)), 0.5) r_j = 1.0 - r_i mu = r_i[..., None] * mu_i + r_j[..., None] * mu_j dev = mu_i - mu_j spread = dev[..., :, None] * dev[..., None, :] cov = r_i[..., None, None] * cov_i + r_j[..., None, None] * cov_j cov = cov + (r_i * r_j)[..., None, None] * spread return lw_sum, mu, cov def _pair_cost(lw_i, lw_j, mu_i, mu_j, cov_i, cov_j, ld_i, ld_j) -> Array: """Runnalls' merge cost for one pair, given each side's cached log-determinant.""" lw_sum, _, cov_m = _merge_moments(lw_i, lw_j, mu_i, mu_j, cov_i, cov_j) return 0.5 * (jnp.exp(lw_sum) * _log_det(cov_m) - jnp.exp(lw_i) * ld_i - jnp.exp(lw_j) * ld_j) def _merge_ratio(lw_i: Array, lw_j: Array) -> Array: """The weight share ``w_i / (w_i + w_j)`` :func:`_merge_moments` merges by (one half for a dead-dead pair).""" lw_sum = jnp.logaddexp(lw_i, lw_j) finite = jnp.isfinite(lw_sum) return jnp.where(finite, jnp.exp(lw_i - jnp.where(finite, lw_sum, 0.0)), 0.5) def _runnalls_merge(log_w: Array, means: Array, chols: Array, k: int, aux=None): """Runnalls merge core: n components -> the k live slots, dead slots dropped. Returns the k live slots' log-weights in the input gauge (unnormalized) and their covariances (not Cholesky factors); merge decisions are gauge- and scale-invariant. Robust to a fully dead input. The cost matrix and per-slot log-determinants are updated incrementally (only the merged slot's row and column change), so decisions match a full rebuild except on exact ties. ``aux`` (a pytree of ``(n, ...)`` arrays, or ``None``) is a per-component side statistic merged by the same assignment, a merged slot holding its members' weight-average; it is returned as a fourth value, the k slots' entries, and never steers a merge. """ n = log_w.shape[0] covs = chols @ jnp.swapaxes(chols, -1, -2) upper = jnp.triu(jnp.ones((n, n), dtype=bool), k=1) peak = jnp.max(log_w) # -inf only if every slot is dead offset = jnp.where(jnp.isfinite(peak), peak, 0.0) log_w = log_w - offset # bound exp() without changing merge decisions or gauge ld = _log_det(covs) cost = _pair_cost( log_w[:, None], log_w[None, :], means[:, None, :], means[None, :, :], covs[:, None, :, :], covs[None, :, :, :], ld[:, None], ld[None, :], ) def body(_, state): log_w, means, covs, ld, alive, cost, aux = state dead_pair = jnp.isneginf(log_w)[:, None] | jnp.isneginf(log_w)[None, :] valid = upper & alive[:, None] & alive[None, :] # Runnalls costs are >= 0, so -1 forces dead slots to be consumed first. masked = jnp.where(valid, jnp.where(dead_pair, -1.0, cost), jnp.inf) idx = jnp.argmin(masked.reshape(-1)) i, j = idx // n, idx % n # upper triangle only, so i < j if aux is not None: r_i = _merge_ratio(log_w[i], log_w[j]) aux = jax.tree.map(lambda a: a.at[i].set(r_i * a[i] + (1.0 - r_i) * a[j]), aux) lw_ij, mu_ij, cov_ij = _merge_moments( log_w[i], log_w[j], means[i], means[j], covs[i], covs[j] ) log_w = log_w.at[i].set(lw_ij).at[j].set(-jnp.inf) means = means.at[i].set(mu_ij) covs = covs.at[i].set(cov_ij) ld = ld.at[i].set(_log_det(cov_ij)) alive = alive.at[j].set(False) # only pairs involving the merged slot changed: recompute its row and column row = _pair_cost(log_w[i], log_w, means[i], means, covs[i], covs, ld[i], ld) cost = cost.at[i, :].set(row).at[:, i].set(row) return log_w, means, covs, ld, alive, cost, aux alive = jnp.ones(n, dtype=bool) log_w, means, covs, ld, alive, cost, aux = lax.fori_loop( 0, n - k, body, (log_w, means, covs, ld, alive, cost, aux) ) top_lw, top_idx = lax.top_k(log_w, k) if aux is None: return top_lw + offset, means[top_idx], covs[top_idx] return top_lw + offset, means[top_idx], covs[top_idx], jax.tree.map(lambda a: a[top_idx], aux) def _runnalls_reduce(log_w: Array, means: Array, chols: Array, k: int, aux=None): merged = _runnalls_merge(log_w, means, chols, k, aux) top_lw, means, covs = merged[:3] out = (jax.nn.log_softmax(top_lw), means, jnp.linalg.cholesky(covs)) return out if aux is None else (*out, merged[3]) def _mode_aligned_reduce( log_w: Array, means: Array, chols: Array, k: int, num_modes: int, aux=None ): """Grouped Runnalls: reduce each producing-mode group independently. The input is :func:`propagate`'s component-major grow output (slot ``s`` from mode ``s % num_modes``); each group is reduced to ``k // num_modes`` components with its total mass preserved, then the output is renormalized globally. Output layout: contiguous blocks of ``k // num_modes`` slots per mode. ``aux`` (``(n, ...)`` arrays) rides through each group's merge (:func:`_runnalls_merge`) and is returned fourth, in the output layout. """ n = log_w.shape[0] per_out = k // num_modes per_in = n // num_modes # component-major layout: slot s belongs to mode s % num_modes -> (num_modes, per_in, ...) g_lw = log_w.reshape(per_in, num_modes).T g_means = jnp.swapaxes(means.reshape(per_in, num_modes, -1), 0, 1) g_chols = jnp.swapaxes(chols.reshape(per_in, num_modes, *chols.shape[-2:]), 0, 1) mass = logsumexp(g_lw, axis=-1) # group total log-mass (may be -inf if fully dead) offset = jnp.where(jnp.isfinite(mass), mass, 0.0) if aux is None: red_lw, red_means, red_covs = jax.vmap(partial(_runnalls_merge, k=per_out))( g_lw - offset[:, None], g_means, g_chols ) else: g_aux = jax.tree.map( lambda a: jnp.swapaxes(a.reshape((per_in, num_modes) + a.shape[1:]), 0, 1), aux ) red_lw, red_means, red_covs, red_aux = jax.vmap( lambda lw, mu, ch, ax: _runnalls_merge(lw, mu, ch, per_out, ax) )(g_lw - offset[:, None], g_means, g_chols, g_aux) cat_lw = (red_lw + offset[:, None]).reshape(-1) # restore absolute group mass out_means = red_means.reshape(k, -1) out_covs = red_covs.reshape(k, *chols.shape[-2:]) out = (jax.nn.log_softmax(cat_lw), out_means, jnp.linalg.cholesky(out_covs)) if aux is None: return out return (*out, jax.tree.map(lambda a: a.reshape((k,) + a.shape[2:]), red_aux)) _LOG_2PI = math.log(2.0 * math.pi) def _spd_factors(m: Array) -> tuple[Array, Array]: """``(M^{-1}, log det M)`` of a batch of small symmetric positive-definite matrices.""" chol = jnp.linalg.cholesky(m) eye = jnp.broadcast_to(jnp.eye(m.shape[-1], dtype=m.dtype), m.shape) inv = jax.scipy.linalg.cho_solve((chol, True), eye) return inv, 2.0 * jnp.sum(jnp.log(jnp.diagonal(chol, axis1=-2, axis2=-1)), axis=-1)
[docs] class LowRankMixture(eqx.Module): """Gaussian mixture whose component covariances are diagonal plus low rank, ``S_i = diag(D_i) + U_i U_iᵀ``, with ``U_i`` of ``r`` columns in a ``p``-dimensional space. ``entropy()`` is the Kolchinsky-Tracey pair of :meth:`MixtureGaussian.entropy`, computed without forming a ``p`` by ``p`` array: the matrix determinant lemma gives every log determinant and the Woodbury identity every trace and quadratic form, from ``r`` by ``r`` and ``2r`` by ``2r`` factorisations. The cost is ``O(N² p r²)`` for ``N`` components against ``O(N² p³)`` dense. """ #: The components' log weights, ``(..., N)``. log_weights: Array #: The components' means, ``(..., N, p)``. means: Array #: The positive diagonal ``D_i`` of each covariance, ``(..., N, p)``. diag: Array #: The low-rank factors ``U_i``, ``(..., N, p, r)``. factors: Array @property def batch_shape(self) -> tuple[int, ...]: return self.log_weights.shape[:-1] @property def num_components(self) -> int: return self.log_weights.shape[-1] @property def dim(self) -> int: return self.means.shape[-1] def _prepare(self) -> tuple[Array, Array, Array]: """``(D^{-1}, M^{-1}, log det S)`` per component, ``M = I + Uᵀ D^{-1} U``.""" dinv = 1.0 / self.diag m = _capacitance(self.factors, dinv) m_inv, log_det_m = _spd_factors(m) return dinv, m_inv, jnp.sum(jnp.log(self.diag), axis=-1) + log_det_m
[docs] def component_entropies(self) -> Array: """``H`` of each component, ``½ log|2πe S_i|``, shape ``(..., N)``.""" _, _, log_det = self._prepare() return 0.5 * self.dim * (1.0 + _LOG_2PI) + 0.5 * log_det
[docs] def pairwise_kl(self) -> Array: """``KL(component_i ‖ component_j)``, shape ``(..., N, N)``, an exact zero on the diagonal. ``tr(S_j⁻¹ S_i) - p`` is formed from the differences ``D_i - D_j`` and ``Δ = U_i - U_j`` as ``tr(S_j⁻¹ (D_i - D_j)) + 2 tr(X_jᵀ Δ) + tr(Δᵀ S_j⁻¹ Δ)``, ``X_j = S_j⁻¹ U_j``, which is small for near-coincident components, where subtracting the two traces ``tr(S_j⁻¹ S_i)`` and ``p`` loses all precision in single precision.""" dinv, m_inv, log_det = self._prepare() u, d = self.factors, self.diag x = jnp.einsum("...jp,...jpa,...jab->...jpb", dinv, u, m_inv) # S_j^{-1} U_j s_diag = dinv - jnp.einsum("...jpa,...jpa->...jp", x, dinv[..., None] * u) d_diff = d[..., :, None, :] - d[..., None, :, :] # (i, j): D_i - D_j delta = u[..., :, None, :, :] - u[..., None, :, :, :] # (i, j): U_i - U_j dmu = self.means[..., None, :, :] - self.means[..., :, None, :] # (i, j): mu_j - mu_i def quad(v: Array) -> Array: # tr(vᵀ S_j^{-1} v), v (..., i, j, p, k) proj = jnp.einsum("...jpa,...jp,...ijpk->...ijak", u, dinv, v) return jnp.einsum("...jp,...ijpk->...ij", dinv, v**2) - jnp.einsum( "...ijak,...jab,...ijbk->...ij", proj, m_inv, proj ) trace_gap = ( jnp.einsum("...jp,...ijp->...ij", s_diag, d_diff) + 2.0 * jnp.einsum("...jpa,...ijpa->...ij", x, delta) + quad(delta) ) kl = 0.5 * ( trace_gap + quad(dmu[..., None]) + log_det[..., None, :] - log_det[..., :, None] ) return jnp.where(jnp.eye(self.num_components, dtype=bool), 0.0, kl)
[docs] def pairwise_bhattacharyya(self) -> Array: """Bhattacharyya distance between components, shape ``(..., N, N)``, an exact zero on the diagonal.""" _, _, log_det = self._prepare() d_bar = 0.5 * (self.diag[..., :, None, :] + self.diag[..., None, :, :]) # (..., N, N, p) e = 1.0 / d_bar u_i = jnp.broadcast_to( self.factors[..., :, None, :, :], d_bar.shape + (self.factors.shape[-1],) ) u_j = jnp.broadcast_to(self.factors[..., None, :, :, :], u_i.shape) w = jnp.concatenate([u_i, u_j], axis=-1) / jnp.sqrt(2.0) # (..., N, N, p, 2r) m_inv, log_det_m = _spd_factors(_capacitance(w, e)) log_det_bar = jnp.sum(jnp.log(d_bar), axis=-1) + log_det_m dmu = self.means[..., :, None, :] - self.means[..., None, :, :] proj = jnp.einsum("...ijpa,...ijp->...ija", w, e * dmu) maha = jnp.sum(e * dmu**2, axis=-1) - jnp.einsum( "...ija,...ijab,...ijb->...ij", proj, m_inv, proj ) dist = ( 0.125 * maha + 0.5 * log_det_bar - 0.25 * (log_det[..., :, None] + log_det[..., None, :]) ) return jnp.where(jnp.eye(self.num_components, dtype=bool), 0.0, dist)
[docs] def entropy(self) -> Interval: """Kolchinsky-Tracey pairwise bounds, as :meth:`MixtureGaussian.entropy`.""" w = jax.nn.log_softmax(self.log_weights, axis=-1) comp_entropy = jnp.sum(jnp.exp(w) * self.component_entropies(), axis=-1) def bound(pairwise_d: Array) -> Array: inner = logsumexp(w[..., None, :] - pairwise_d, axis=-1) return comp_entropy - jnp.sum(jnp.exp(w) * inner, axis=-1) return Interval(lower=bound(self.pairwise_bhattacharyya()), upper=bound(self.pairwise_kl()))
def _capacitance(u: Array, dinv: Array) -> Array: """``I + Uᵀ D^{-1} U`` over the leading axes of ``u`` ``(..., p, r)`` and ``dinv`` ``(..., p)``.""" r = u.shape[-1] return jnp.eye(r, dtype=u.dtype) + jnp.einsum("...pa,...p,...pb->...ab", u, dinv, u)