Source code for hybrid_aif.data.dataset

"""Episode datasets with fixed-size clip windowing.

:class:`Dataset` turns per-episode ``lengths``/``offsets`` into a flat index
of all valid clips — ``clip_indices`` lists every ``(episode, start)`` whose
window of ``num_steps * frameskip`` raw steps fits inside the episode (shorter
episodes contribute nothing). Subclasses implement ``_load_slice`` to return
the dense raw slice; ``__getitem__`` applies frameskip and returns a dict of
numpy arrays. Converting to device arrays (``jnp.asarray``) is the caller's
job at the device boundary.
"""

from __future__ import annotations

import numpy as np

from hybrid_aif.data.format import Reader, detect_format, get_format


[docs] class Dataset: """Base class for episode-based datasets. Args: lengths: per-episode step counts. offsets: per-episode start offsets in the flat underlying storage. num_steps: observation frames per clip. frameskip: stride between observation frames within a clip. A clip spans ``num_steps * frameskip`` raw steps. ``__getitem__`` strides every column by ``frameskip`` except ``action``, which stays dense and is reshaped to ``(num_steps, frameskip * action_dim)`` when ``frameskip > 1`` so each observation frame carries the actions taken until the next one. """
[docs] def __init__(self, lengths, offsets, num_steps: int = 1, frameskip: int = 1) -> None: if num_steps <= 0 or frameskip <= 0: raise ValueError( f"num_steps and frameskip must be positive, got {num_steps=} {frameskip=}" ) self.lengths = np.asarray(lengths, dtype=np.int64) self.offsets = np.asarray(offsets, dtype=np.int64) self.num_steps = num_steps self.frameskip = frameskip self._clip_cum: np.ndarray | None = None self._clips: list[tuple[int, int]] | None = None
@property def span(self) -> int: return self.num_steps * self.frameskip @property def _windows(self) -> np.ndarray: """Cumulative valid-window counts over episodes, ``(num_episodes,)``, built once per change of ``lengths`` (a subclass that appends resets the cache).""" if self._clip_cum is None: self._clip_cum = np.cumsum(clip_window_counts(self.lengths, self.span)) return self._clip_cum @property def clip_indices(self) -> list[tuple[int, int]]: """Every valid ``(episode, start)`` in flat index order, as a list.""" if self._clips is None: self._clips = compute_clip_indices(self.lengths, self.span) return self._clips
[docs] def locate(self, idx) -> tuple[np.ndarray, np.ndarray]: """Flat clip indices → ``(episode, start)`` arrays, vectorised; negative indices count from the end, and an index outside ``[-len, len)`` raises.""" return locate_clips(self._windows, np.asarray(idx))
@property def column_names(self) -> list[str]: raise NotImplementedError def _load_slice(self, ep_idx: int, start: int, end: int) -> dict[str, np.ndarray]: """Return the dense raw slice ``[start:end)`` of one episode.""" raise NotImplementedError def __len__(self) -> int: cum = self._windows return int(cum[-1]) if cum.size else 0 def __getitem__(self, idx: int) -> dict[str, np.ndarray]: ep_idx, start = self.locate(idx) return self._clip(int(ep_idx), int(start)) def _clip(self, ep_idx: int, start: int) -> dict[str, np.ndarray]: steps = self._load_slice(ep_idx, start, start + self.span) if self.frameskip == 1: return steps out = {} for col, arr in steps.items(): if col == "action": out[col] = arr.reshape(self.num_steps, -1) else: out[col] = arr[:: self.frameskip] return out def load_episode(self, ep_idx: int) -> dict[str, np.ndarray]: return self._load_slice(ep_idx, 0, int(self.lengths[ep_idx]))
def clip_window_counts(lengths: np.ndarray, span: int) -> np.ndarray: """How many windows of ``span`` raw steps each episode holds, ``(num_episodes,)`` int64; an episode shorter than ``span`` holds none.""" lengths = np.asarray(lengths, dtype=np.int64).reshape(-1) return np.maximum(lengths - int(span) + 1, 0) def locate_clips(cum: np.ndarray, idx: np.ndarray) -> tuple[np.ndarray, np.ndarray]: """Flat clip indices → ``(episode, start)`` under cumulative window counts ``cum`` (:func:`clip_window_counts` summed), in the order :func:`compute_clip_indices` lists them. Negative indices count from the end; out-of-range ones raise.""" total = int(cum[-1]) if cum.size else 0 idx = np.asarray(idx, dtype=np.int64) flat = np.where(idx < 0, idx + total, idx) if flat.size and (flat.min() < 0 or flat.max() >= total): raise IndexError(f"clip index out of range for a dataset of {total} clips") eps = np.searchsorted(cum, flat, side="right") before = np.where(eps > 0, cum[np.maximum(eps - 1, 0)], 0) return eps, flat - before def compute_clip_indices(lengths: np.ndarray, span: int) -> list[tuple[int, int]]: """All valid ``(episode, start)`` windows of ``span`` raw steps, episodes in order and starts ascending within each.""" cum = np.cumsum(clip_window_counts(lengths, span)) total = int(cum[-1]) if cum.size else 0 eps, starts = locate_clips(cum, np.arange(total)) return list(zip(eps.tolist(), starts.tolist(), strict=True)) class ReaderDataset(Dataset): """Dataset over any registry :class:`Reader`.""" def __init__(self, reader: Reader, num_steps: int = 1, frameskip: int = 1) -> None: self.reader = reader super().__init__(reader.lengths, reader.offsets, num_steps, frameskip) @property def column_names(self) -> list[str]: return self.reader.column_names def _load_slice(self, ep_idx: int, start: int, end: int) -> dict[str, np.ndarray]: return self.reader.read_slice(ep_idx, start, end) class HDF5Dataset(ReaderDataset): def __init__(self, path, num_steps: int = 1, frameskip: int = 1) -> None: from hybrid_aif.data.formats.hdf5 import HDF5Reader super().__init__(HDF5Reader(path), num_steps, frameskip) class FolderDataset(ReaderDataset): def __init__(self, path, num_steps: int = 1, frameskip: int = 1) -> None: from hybrid_aif.data.formats.folder import FolderReader super().__init__(FolderReader(path), num_steps, frameskip)
[docs] def load_dataset( path, *, format: str | None = None, num_steps: int = 1, frameskip: int = 1 ) -> ReaderDataset: """Open ``path`` as a :class:`ReaderDataset`, through the registered format named ``format`` or, when it is ``None``, the first registered format that detects the path. ``num_steps`` is the number of observation frames per clip and ``frameskip`` the stride between them. Raises :class:`ValueError` when no format detects the path. """ import hybrid_aif.data.formats # noqa: F401 (registers built-in formats) fmt = get_format(format) if format is not None else detect_format(path) if fmt is None: raise ValueError(f"could not detect a registered dataset format at '{path}'") return ReaderDataset(fmt.open_reader(path), num_steps=num_steps, frameskip=frameskip)
__all__ = [ "Dataset", "FolderDataset", "HDF5Dataset", "ReaderDataset", "clip_window_counts", "compute_clip_indices", "load_dataset", "locate_clips", ]