Source code for hybrid_aif.reference.envs.mountain_car

"""R-AIF's state-based Mountain Car: gymnasium's ``MountainCarContinuous`` physics with their
sparse reward and termination.

State and observation are ``(position, velocity)``; the action is ``Box(-1, 1, (1,))``.
Reward is −1 per raw step and 0 on the terminating one; the episode terminates at
``position >= 0.45`` (velocity ignored), which is also the success predicate. One agent
step is ``action_repeat = 2`` raw steps, breaking on termination, so a non-terminating
agent step scores −2 and the floor over ``TimeLimit(100)`` agent steps is −200. The time
limit is the caller's to enforce (:attr:`MountainCar.time_limit`). Reset draws
``position ~ U(-0.6, -0.4)``, ``velocity = 0``. Reward and termination ride on
:meth:`MountainCar.agent_step`, beside the :class:`~hybrid_aif.envs.base.Env` faces.

Design: ``docs/design/amortization-p0-addendum.md`` §1.
"""

import equinox as eqx
import jax
import jax.numpy as jnp
from jax import Array

from hybrid_aif.envs.base import Env, Obs, State


[docs] class MountainCar(Env): """Sparse-reward continuous Mountain Car. The dynamics constants are static fields.""" #: The force per unit of action, added to the velocity each raw step. power: float = eqx.field(static=True, default=0.0015) #: The bound on the absolute velocity. max_speed: float = eqx.field(static=True, default=0.07) #: The left wall (inelastic: hitting it stops inward velocity). min_position: float = eqx.field(static=True, default=-1.2) #: The right bound on the position. max_position: float = eqx.field(static=True, default=0.6) #: The position at or beyond which the goal is reached (termination and success). goal_position: float = eqx.field(static=True, default=0.45) #: The gravity term, ``gravity * cos(3 * position)`` subtracted from the velocity. gravity: float = eqx.field(static=True, default=0.0025) #: The raw steps per agent step (at least 1). action_repeat: int = eqx.field(static=True, default=2) #: The agent steps per episode before truncation (at least 1), read by the collectors. time_limit: int = eqx.field(static=True, default=100) #: The lower end of the uniform reset position. reset_low: float = eqx.field(static=True, default=-0.6) #: The upper end of the uniform reset position. reset_high: float = eqx.field(static=True, default=-0.4) def __check_init__(self) -> None: if self.action_repeat < 1: raise ValueError(f"action_repeat must be at least 1, got {self.action_repeat}") if self.time_limit < 1: raise ValueError(f"time_limit must be at least 1, got {self.time_limit}") @property def num_states(self) -> int: """One state factor: the ``(position, velocity)`` vector.""" return 1 @property def action_dim(self) -> int: return 1 # ------------------------------------------------------------------- dynamics
[docs] def physics_step(self, x: Array, action: Array) -> Array: """One **raw** env step of gymnasium's MountainCarContinuous dynamics.""" position, velocity = x[0], x[1] force = jnp.clip(jnp.asarray(action).reshape(-1)[0], -1.0, 1.0) velocity = velocity + force * self.power - self.gravity * jnp.cos(3.0 * position) velocity = jnp.clip(velocity, -self.max_speed, self.max_speed) position = jnp.clip(position + velocity, self.min_position, self.max_position) # the left wall is inelastic: hitting it kills inward velocity velocity = jnp.where((position <= self.min_position) & (velocity < 0.0), 0.0, velocity) return jnp.stack([position, velocity])
[docs] def reached_goal(self, state: State) -> Array: """``position >= goal_position``: the termination predicate and the success flag.""" return state[0][0] >= self.goal_position
# ------------------------------------------------------------------ env faces
[docs] def reset(self, key: Array, params=None) -> tuple[Obs, State]: """``position ~ U(-0.6, -0.4)``, ``velocity = 0`` (gymnasium's reset).""" position = jax.random.uniform( key, (), minval=self.reset_low, maxval=self.reset_high, dtype=jnp.float32 ) state = (jnp.stack([position, jnp.zeros((), jnp.float32)]),) return self.observe(state), state
[docs] def step(self, key: Array, state: State, action: Array, params=None) -> tuple[Obs, State]: """One agent step as the ``Env`` face: :meth:`agent_step` without reward and done.""" obs, state, _, _ = self.agent_step(key, state, action, params) return obs, state
[docs] def agent_step( self, key: Array, state: State, action: Array, params=None ) -> tuple[Obs, State, Array, Array]: """One agent step with the sparse reward: ``(obs, state, reward, done)``. Runs ``action_repeat`` raw steps, freezing the state once the goal is reached. ``reward`` is −1 per raw step taken before the goal; ``done`` is termination at or before this step. """ del key, params # the dynamics are deterministic; only reset draws x = state[0] done = self.reached_goal((x,)) reward = jnp.zeros((), jnp.float32) for _ in range(self.action_repeat): x_next = self.physics_step(x, action) hit = x_next[0] >= self.goal_position x = jnp.where(done, x, x_next) reward = reward + jnp.where(done, 0.0, jnp.where(hit, 0.0, -1.0)) done = done | hit state = (x,) return self.observe(state), state, reward, done
[docs] def observe(self, state: State) -> Obs: """The observation: the state itself, without noise.""" return (state[0],)