"""Format registry: a declarative read/write abstraction over dataset backends.
A ``Format`` ties together three concerns for one on-disk layout:
- ``name`` — the registry key, also usable as an explicit ``format=`` argument.
- ``detect(path)`` — does this path look like this format?
- ``open_reader(path)`` / ``open_writer(path, mode)`` — entry points.
To add a backend, subclass :class:`Format` in a module under ``data/formats/``
and decorate it with :func:`register_format`. Read-only formats omit
``open_writer``; write-only formats omit ``open_reader``.
Column convention
-----------------
An *episode* is a ``dict[str, np.ndarray]`` of named columns sharing a leading
time axis (``column.shape[0] == episode_length``). Keys starting with ``_``
are transient (e.g. debug info) and are skipped by all writers. Temporal
alignment of ``action`` relative to observations (whether ``action[t]`` was
taken *at* or *produced* ``obs[t]``) is the caller's concern: the pipeline
stores and windows columns positionally and never reinterprets them.
Writers are streaming: ``write_episodes`` consumes its iterable one episode at
a time with bounded memory. All writers share the modes in :data:`WRITE_MODES`.
"""
from __future__ import annotations
from collections.abc import Iterable, Iterator
from typing import Protocol, runtime_checkable
import numpy as np
FORMATS: dict[str, type[Format]] = {}
WRITE_MODES = ("append", "overwrite", "error")
"""Writer modes shared by all formats.
- ``append``: extend an existing dataset (schema-validated) or create one.
- ``overwrite``: drop any existing dataset and start fresh.
- ``error``: raise :class:`FileExistsError` if a dataset already exists.
"""
def validate_write_mode(mode: str) -> str:
if mode not in WRITE_MODES:
raise ValueError(f"write mode must be one of {WRITE_MODES}, got {mode!r}")
return mode
def register_format(cls: type[Format]) -> type[Format]:
"""Register a :class:`Format` subclass under its ``name`` class attribute, and return it
(usable as a class decorator). Raises :class:`ValueError` when ``name`` is empty or already
registered."""
name = getattr(cls, "name", None)
if not name:
raise ValueError(f"{cls.__name__} must set a non-empty `name` class attribute")
if name in FORMATS:
raise ValueError(f"format {name!r} is already registered")
FORMATS[name] = cls
return cls
def list_formats() -> list[str]:
"""The names of the registered formats, in registration order."""
return list(FORMATS)
def get_format(name: str) -> type[Format]:
"""The registered :class:`Format` subclass named ``name``. Raises :class:`ValueError`,
listing the registered names, when there is none."""
try:
return FORMATS[name]
except KeyError:
raise ValueError(f"unknown format {name!r}; available: {list_formats()}") from None
[docs]
class Reader(Protocol):
"""Random-access episode reader over flat per-column storage (a protocol).
``lengths`` and ``offsets`` give each stored episode's step count and its first step in the
flat storage; ``column_names`` the stored columns; ``read_slice(ep_idx, start, end)`` returns
steps ``start`` to ``end`` of one episode as a column dict. A :class:`Format`'s
``open_reader`` returns one.
"""
@property
def lengths(self) -> np.ndarray: ...
@property
def offsets(self) -> np.ndarray: ...
@property
def column_names(self) -> list[str]: ...
def read_slice(self, ep_idx: int, start: int, end: int) -> dict[str, np.ndarray]: ...
[docs]
@runtime_checkable
class Writer(Protocol):
"""Streaming episode writer — append episodes, finalize on exit.
``write_episodes`` must consume its iterable lazily (one episode resident
at a time); ``write_episode`` pushes a single episode. Keys starting with
``_`` are skipped.
"""
def __enter__(self) -> Writer: ...
def __exit__(self, *exc) -> None: ...
def write_episode(self, ep_data: dict[str, np.ndarray]) -> None: ...
def write_episodes(self, episodes: Iterable[dict[str, np.ndarray]]) -> None: ...
def public_items(ep_data: dict) -> Iterator[tuple[str, np.ndarray]]:
"""Yield (column, array) pairs, skipping ``_``-prefixed keys and coercing to numpy."""
for col, vals in ep_data.items():
if not col.startswith("_"):
yield col, np.asarray(vals)
def episode_schema(ep_data: dict) -> dict[str, tuple[tuple[int, ...], np.dtype]]:
"""Map each public column to its (per-step shape, dtype); validates the time axis."""
schema: dict[str, tuple[tuple[int, ...], np.dtype]] = {}
ep_len: int | None = None
for col, arr in public_items(ep_data):
if arr.ndim == 0:
raise ValueError(f"column {col!r} is scalar; expected a leading time axis")
if ep_len is None:
ep_len = arr.shape[0]
elif arr.shape[0] != ep_len:
raise ValueError(
f"column {col!r} has length {arr.shape[0]}, but other columns have {ep_len}"
)
schema[col] = (arr.shape[1:], arr.dtype)
if not schema:
raise ValueError("episode has no public (non-underscore) columns")
return schema
def check_schema(
expected: dict[str, tuple[tuple[int, ...], np.dtype]],
ep_data: dict,
context: str,
) -> None:
"""Raise if ``ep_data``'s public columns don't match ``expected`` exactly."""
incoming = episode_schema(ep_data)
if set(incoming) != set(expected):
raise ValueError(
f"{context}: schema mismatch — missing columns "
f"{sorted(set(expected) - set(incoming))}, "
f"unexpected columns {sorted(set(incoming) - set(expected))}"
)
for col, (shape, dtype) in incoming.items():
exp_shape, exp_dtype = expected[col]
if shape != exp_shape or dtype != exp_dtype:
raise ValueError(
f"{context}: column {col!r} mismatch — expected per-step shape "
f"{exp_shape} dtype {exp_dtype}, got shape {shape} dtype {dtype}"
)
__all__ = [
"FORMATS",
"WRITE_MODES",
"Format",
"Reader",
"Writer",
"check_schema",
"detect_format",
"episode_schema",
"get_format",
"list_formats",
"public_items",
"register_format",
"validate_write_mode",
]