From 2cb09076fd47f9ed8c517f078d0d7853ca160b7a Mon Sep 17 00:00:00 2001
From: sneakers-the-rat <sneakers-the-rat@protonmail.com>
Date: Fri, 20 Sep 2024 18:28:38 -0700
Subject: [PATCH 01/22] add ability to dump proxy classes to arrays, tests for
 doing so and json dumping

---
 src/numpydantic/interface/hdf5.py       |  9 +++++++-
 src/numpydantic/interface/video.py      |  4 ++++
 tests/test_interface/conftest.py        |  4 ++--
 tests/test_interface/test_interfaces.py | 30 +++++++++++++++++++++++++
 4 files changed, 44 insertions(+), 3 deletions(-)

diff --git a/src/numpydantic/interface/hdf5.py b/src/numpydantic/interface/hdf5.py
index 20cec0d..3696a6f 100644
--- a/src/numpydantic/interface/hdf5.py
+++ b/src/numpydantic/interface/hdf5.py
@@ -134,10 +134,17 @@ class H5Proxy:
             else:
                 return obj.dtype[self.field]
 
+    def __array__(self) -> np.ndarray:
+        """To a numpy array"""
+        with h5py.File(self.file, "r") as h5f:
+            obj = h5f.get(self.path)
+            return obj[:]
+
     def __getattr__(self, item: str):
         with h5py.File(self.file, "r") as h5f:
             obj = h5f.get(self.path)
-            return getattr(obj, item)
+            val = getattr(obj, item)
+            return val
 
     def __getitem__(
         self, item: Union[int, slice, Tuple[Union[int, slice], ...]]
diff --git a/src/numpydantic/interface/video.py b/src/numpydantic/interface/video.py
index f64457b..1660545 100644
--- a/src/numpydantic/interface/video.py
+++ b/src/numpydantic/interface/video.py
@@ -137,6 +137,10 @@ class VideoProxy:
             slice_ = slice(0, slice_.stop, slice_.step)
         return slice_
 
+    def __array__(self) -> np.ndarray:
+        """Whole video as a numpy array"""
+        return self[:]
+
     def __getitem__(self, item: Union[int, slice, tuple]) -> np.ndarray:
         if isinstance(item, int):
             # want a single frame
diff --git a/tests/test_interface/conftest.py b/tests/test_interface/conftest.py
index 63bdc4a..7e6f767 100644
--- a/tests/test_interface/conftest.py
+++ b/tests/test_interface/conftest.py
@@ -1,6 +1,6 @@
 import pytest
 
-from typing import Tuple, Callable
+from typing import Callable, Tuple, Type
 import numpy as np
 import dask.array as da
 import zarr
@@ -32,7 +32,7 @@ from numpydantic import interface, NDArray
         "video",
     ],
 )
-def interface_type(request) -> Tuple[NDArray, interface.Interface]:
+def interface_type(request) -> Tuple[NDArray, Type[interface.Interface]]:
     """
     Test cases for each interface's ``check`` method - each input should match the
     provided interface and that interface only
diff --git a/tests/test_interface/test_interfaces.py b/tests/test_interface/test_interfaces.py
index 36308c9..3b1370a 100644
--- a/tests/test_interface/test_interfaces.py
+++ b/tests/test_interface/test_interfaces.py
@@ -2,6 +2,10 @@
 Tests that should be applied to all interfaces
 """
 
+from typing import Callable
+import numpy as np
+from numpydantic.interface import Interface
+
 
 def test_interface_revalidate(all_interfaces):
     """
@@ -10,3 +14,29 @@ def test_interface_revalidate(all_interfaces):
     See: https://github.com/p2p-ld/numpydantic/pull/14
     """
     _ = type(all_interfaces)(array=all_interfaces.array)
+
+
+def test_interface_rematch(interface_type):
+    """
+    All interfaces should match the results of the object they return after validation
+    """
+    array, interface = interface_type
+    if isinstance(array, Callable):
+        array = array()
+
+    assert Interface.match(interface().validate(array)) is interface
+
+
+def test_interface_to_numpy_array(all_interfaces):
+    """
+    All interfaces should be able to have the output of their validation stage
+    coerced to a numpy array with np.array()
+    """
+    _ = np.array(all_interfaces.array)
+
+
+def test_interface_dump_json(all_interfaces):
+    """
+    All interfaces should be able to dump to json
+    """
+    all_interfaces.model_dump_json()

From b436d8d592ddac5aa18e038907ddf56d72f6c25c Mon Sep 17 00:00:00 2001
From: sneakers-the-rat <sneakers-the-rat@protonmail.com>
Date: Fri, 20 Sep 2024 23:44:59 -0700
Subject: [PATCH 02/22] Roundtrip json serialization using dataclasses for all
 interfaces. Separate JSON schema generation from core_schema generation. More
 consistent json dumping behavior - always return just the array unless
 `round_trip=True`.

---
 docs/api/monkeypatch.md                 |   6 --
 docs/index.md                           |   2 +-
 docs/serialization.md                   |   2 +
 pyproject.toml                          |   6 ++
 src/numpydantic/interface/__init__.py   |   3 +-
 src/numpydantic/interface/dask.py       |  72 +++++++++++--
 src/numpydantic/interface/hdf5.py       |  67 +++++++++---
 src/numpydantic/interface/interface.py  | 129 ++++++++++++++++++++++--
 src/numpydantic/interface/numpy.py      |  49 ++++++++-
 src/numpydantic/interface/video.py      |  43 +++++++-
 src/numpydantic/interface/zarr.py       |  86 ++++++++++++----
 src/numpydantic/ndarray.py              |  25 +++--
 src/numpydantic/schema.py               |  49 +++++++--
 tests/test_interface/test_hdf5.py       |  17 ++--
 tests/test_interface/test_interfaces.py |  26 +++++
 tests/test_interface/test_zarr.py       |  36 ++++---
 tests/test_ndarray.py                   |  13 +++
 17 files changed, 532 insertions(+), 99 deletions(-)
 delete mode 100644 docs/api/monkeypatch.md
 create mode 100644 docs/serialization.md

diff --git a/docs/api/monkeypatch.md b/docs/api/monkeypatch.md
deleted file mode 100644
index d397869..0000000
--- a/docs/api/monkeypatch.md
+++ /dev/null
@@ -1,6 +0,0 @@
-# monkeypatch
-
-```{eval-rst}
-.. automodule:: numpydantic.monkeypatch
-    :members:
-```
\ No newline at end of file
diff --git a/docs/index.md b/docs/index.md
index af2c908..0cea5b4 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -473,6 +473,7 @@ dumped = instance.model_dump_json(context={'zarr_dump_array': True})
 
 design
 syntax
+serialization
 interfaces
 todo
 changelog
@@ -489,7 +490,6 @@ api/dtype
 api/ndarray
 api/maps
 api/meta
-api/monkeypatch
 api/schema
 api/shape
 api/types
diff --git a/docs/serialization.md b/docs/serialization.md
new file mode 100644
index 0000000..d5162bf
--- /dev/null
+++ b/docs/serialization.md
@@ -0,0 +1,2 @@
+# Serialization
+
diff --git a/pyproject.toml b/pyproject.toml
index 4dddb69..f1b7173 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -109,6 +109,12 @@ filterwarnings = [
     # nptyping's alias warnings
     'ignore:.*deprecated alias.*Deprecated NumPy 1\.24.*'
 ]
+markers = [
+    "dtype: mark test related to dtype validation",
+    "shape: mark test related to shape validation",
+    "json_schema: mark test related to json schema generation",
+    "serialization: mark test related to serialization"
+]
 
 [tool.ruff]
 target-version = "py311"
diff --git a/src/numpydantic/interface/__init__.py b/src/numpydantic/interface/__init__.py
index 0a0c490..c5bd3f2 100644
--- a/src/numpydantic/interface/__init__.py
+++ b/src/numpydantic/interface/__init__.py
@@ -4,12 +4,13 @@ Interfaces between nptyping types and array backends
 
 from numpydantic.interface.dask import DaskInterface
 from numpydantic.interface.hdf5 import H5Interface
-from numpydantic.interface.interface import Interface
+from numpydantic.interface.interface import Interface, JsonDict
 from numpydantic.interface.numpy import NumpyInterface
 from numpydantic.interface.video import VideoInterface
 from numpydantic.interface.zarr import ZarrInterface
 
 __all__ = [
+    "JsonDict",
     "Interface",
     "DaskInterface",
     "H5Interface",
diff --git a/src/numpydantic/interface/dask.py b/src/numpydantic/interface/dask.py
index 7719e98..6f02025 100644
--- a/src/numpydantic/interface/dask.py
+++ b/src/numpydantic/interface/dask.py
@@ -2,26 +2,59 @@
 Interface for Dask arrays
 """
 
-from typing import Any, Optional
+from dataclasses import dataclass
+from typing import Any, Iterable, Literal, Optional
 
 import numpy as np
 from pydantic import SerializationInfo
 
-from numpydantic.interface.interface import Interface
+from numpydantic.interface.interface import Interface, JsonDict
 from numpydantic.types import DtypeType, NDArrayType
 
 try:
+    from dask.array import from_array
     from dask.array.core import Array as DaskArray
 except ImportError:  # pragma: no cover
     DaskArray = None
 
 
+def _as_tuple(a_list: list | Any) -> tuple:
+    """Make a list of list into a tuple of tuples"""
+    return tuple(
+        [_as_tuple(item) if isinstance(item, list) else item for item in a_list]
+    )
+
+
+@dataclass(kw_only=True)
+class DaskJsonDict(JsonDict):
+    """
+    Round-trip json serialized form of a dask array
+    """
+
+    type: Literal["dask"]
+    name: str
+    chunks: Iterable[tuple[int, ...]]
+    dtype: str
+    array: list
+
+    def to_array_input(self) -> DaskArray:
+        """Construct a dask array"""
+        np_array = np.array(self.array, dtype=self.dtype)
+        array = from_array(
+            np_array,
+            name=self.name,
+            chunks=_as_tuple(self.chunks),
+        )
+        return array
+
+
 class DaskInterface(Interface):
     """
     Interface for Dask :class:`~dask.array.core.Array`
     """
 
-    input_types = (DaskArray,)
+    name = "dask"
+    input_types = (DaskArray, dict)
     return_type = DaskArray
 
     @classmethod
@@ -29,7 +62,24 @@ class DaskInterface(Interface):
         """
         check if array is a dask array
         """
-        return DaskArray is not None and isinstance(array, DaskArray)
+        if DaskArray is None:
+            return False
+        elif isinstance(array, DaskArray):
+            return True
+        elif isinstance(array, dict):
+            return DaskJsonDict.is_valid(array)
+        else:
+            return False
+
+    def before_validation(self, array: Any) -> DaskArray:
+        """
+        If given a dict (like that from ``model_dump_json(round_trip=True)`` ),
+        re-cast to dask array
+        """
+        if isinstance(array, dict):
+            array = DaskJsonDict(**array).to_array_input()
+
+        return array
 
     def get_object_dtype(self, array: NDArrayType) -> DtypeType:
         """Dask arrays require a compute() call to retrieve a single value"""
@@ -43,7 +93,7 @@ class DaskInterface(Interface):
     @classmethod
     def to_json(
         cls, array: DaskArray, info: Optional[SerializationInfo] = None
-    ) -> list:
+    ) -> list | DaskJsonDict:
         """
         Convert an array to a JSON serializable array by first converting to a numpy
         array and then to a list.
@@ -56,4 +106,14 @@ class DaskInterface(Interface):
             method of serialization here using the python object itself rather than
             its JSON representation.
         """
-        return np.array(array).tolist()
+        np_array = np.array(array)
+        as_json = np_array.tolist()
+        if info.round_trip:
+            as_json = DaskJsonDict(
+                type=cls.name,
+                array=as_json,
+                name=array.name,
+                chunks=array.chunks,
+                dtype=str(np_array.dtype),
+            )
+        return as_json
diff --git a/src/numpydantic/interface/hdf5.py b/src/numpydantic/interface/hdf5.py
index 3696a6f..9a5bf93 100644
--- a/src/numpydantic/interface/hdf5.py
+++ b/src/numpydantic/interface/hdf5.py
@@ -40,6 +40,7 @@ as ``S32`` isoformatted byte strings (timezones optional) like:
 """
 
 import sys
+from dataclasses import dataclass
 from datetime import datetime
 from pathlib import Path
 from typing import Any, Iterable, List, NamedTuple, Optional, Tuple, TypeVar, Union
@@ -47,7 +48,7 @@ from typing import Any, Iterable, List, NamedTuple, Optional, Tuple, TypeVar, Un
 import numpy as np
 from pydantic import SerializationInfo
 
-from numpydantic.interface.interface import Interface
+from numpydantic.interface.interface import Interface, JsonDict
 from numpydantic.types import DtypeType, NDArrayType
 
 try:
@@ -76,6 +77,21 @@ class H5ArrayPath(NamedTuple):
     """Refer to a specific field within a compound dtype"""
 
 
+@dataclass
+class H5JsonDict(JsonDict):
+    """Round-trip Json-able version of an HDF5 dataset"""
+
+    file: str
+    path: str
+    field: Optional[str] = None
+
+    def to_array_input(self) -> H5ArrayPath:
+        """Construct an :class:`.H5ArrayPath`"""
+        return H5ArrayPath(
+            **{k: v for k, v in self.to_dict().items() if k in H5ArrayPath._fields}
+        )
+
+
 class H5Proxy:
     """
     Proxy class to mimic numpy-like array behavior with an HDF5 array
@@ -110,6 +126,7 @@ class H5Proxy:
         self.path = path
         self.field = field
         self._annotation_dtype = annotation_dtype
+        self._h5arraypath = H5ArrayPath(self.file, self.path, self.field)
 
     def array_exists(self) -> bool:
         """Check that there is in fact an array at :attr:`.path` within :attr:`.file`"""
@@ -212,6 +229,15 @@ class H5Proxy:
         """self.shape[0]"""
         return self.shape[0]
 
+    def __eq__(self, other: "H5Proxy") -> bool:
+        """
+        Check that we are referring to the same hdf5 array
+        """
+        if isinstance(other, H5Proxy):
+            return self._h5arraypath == other._h5arraypath
+        else:
+            raise ValueError("Can only compare equality of two H5Proxies")
+
     def open(self, mode: str = "r") -> "h5py.Dataset":
         """
         Return the opened :class:`h5py.Dataset` object
@@ -251,6 +277,7 @@ class H5Interface(Interface):
     passthrough numpy-like interface to the dataset.
     """
 
+    name = "hdf5"
     input_types = (H5ArrayPath, H5Arraylike, H5Proxy)
     return_type = H5Proxy
 
@@ -268,6 +295,13 @@ class H5Interface(Interface):
         if isinstance(array, (H5ArrayPath, H5Proxy)):
             return True
 
+        if isinstance(array, dict):
+            if array.get("type", False) == cls.name:
+                return True
+            # continue checking if dict contains an hdf5 file
+            file = array.get("file", "")
+            array = (file, "")
+
         if isinstance(array, (tuple, list)) and len(array) in (2, 3):
             # check that the first arg is an hdf5 file
             try:
@@ -294,6 +328,9 @@ class H5Interface(Interface):
 
     def before_validation(self, array: Any) -> NDArrayType:
         """Create an :class:`.H5Proxy` to use throughout validation"""
+        if isinstance(array, dict):
+            array = H5JsonDict(**array).to_array_input()
+
         if isinstance(array, H5ArrayPath):
             array = H5Proxy.from_h5array(h5array=array)
         elif isinstance(array, H5Proxy):
@@ -349,21 +386,27 @@ class H5Interface(Interface):
     @classmethod
     def to_json(cls, array: H5Proxy, info: Optional[SerializationInfo] = None) -> dict:
         """
-        Dump to a dictionary containing
+        Render HDF5 array as JSON
+
+        If ``round_trip == True``, we dump just the proxy info, a dictionary like:
 
         * ``file``: :attr:`.file`
         * ``path``: :attr:`.path`
         * ``attrs``: Any HDF5 attributes on the dataset
         * ``array``: The array as a list of lists
+
+        Otherwise, we dump the array as a list of lists
         """
-        try:
-            dset = array.open()
-            meta = {
-                "file": array.file,
-                "path": array.path,
-                "attrs": dict(dset.attrs),
-                "array": dset[:].tolist(),
+        if info.round_trip:
+            as_json = {
+                "type": cls.name,
             }
-            return meta
-        finally:
-            array.close()
+            as_json.update(array._h5arraypath._asdict())
+        else:
+            try:
+                dset = array.open()
+                as_json = dset[:].tolist()
+            finally:
+                array.close()
+
+        return as_json
diff --git a/src/numpydantic/interface/interface.py b/src/numpydantic/interface/interface.py
index 1ef307f..fa32b8e 100644
--- a/src/numpydantic/interface/interface.py
+++ b/src/numpydantic/interface/interface.py
@@ -2,12 +2,15 @@
 Base Interface metaclass
 """
 
+import inspect
 from abc import ABC, abstractmethod
+from dataclasses import asdict, dataclass
+from importlib.metadata import PackageNotFoundError, version
 from operator import attrgetter
-from typing import Any, Generic, Optional, Tuple, Type, TypeVar, Union
+from typing import Any, Generic, Tuple, Type, TypedDict, TypeVar, Union
 
 import numpy as np
-from pydantic import SerializationInfo
+from pydantic import SerializationInfo, TypeAdapter, ValidationError
 
 from numpydantic.exceptions import (
     DtypeError,
@@ -21,6 +24,60 @@ from numpydantic.types import DtypeType, NDArrayType, ShapeType
 T = TypeVar("T", bound=NDArrayType)
 
 
+class InterfaceMark(TypedDict):
+    """JSON-able mark to be able to round-trip json dumps"""
+
+    module: str
+    cls: str
+    version: str
+
+
+@dataclass(kw_only=True)
+class JsonDict:
+    """
+    Representation of array when dumped with round_trip == True.
+
+    Using a dataclass rather than a pydantic model to not tempt
+    us to use more sophisticated types than can be serialized to json.
+    """
+
+    type: str
+
+    @abstractmethod
+    def to_array_input(self) -> Any:
+        """
+        Convert this roundtrip specifier to the relevant input class
+        (one of the ``input_types`` of an interface).
+        """
+
+    def to_dict(self) -> dict:
+        """
+        Convenience method for casting dataclass to dict,
+        removing None-valued items
+        """
+        return {k: v for k, v in asdict(self).items() if v is not None}
+
+    @classmethod
+    def get_adapter(cls) -> TypeAdapter:
+        """Convenience method to get a typeadapter for this class"""
+        return TypeAdapter(cls)
+
+    @classmethod
+    def is_valid(cls, val: dict) -> bool:
+        """
+        Check whether a given dictionary matches this JsonDict specification
+
+        Returns:
+            bool - true if valid, false if not
+        """
+        adapter = cls.get_adapter()
+        try:
+            _ = adapter.validate_python(val)
+            return True
+        except ValidationError:
+            return False
+
+
 class Interface(ABC, Generic[T]):
     """
     Abstract parent class for interfaces to different array formats
@@ -30,7 +87,7 @@ class Interface(ABC, Generic[T]):
     return_type: Type[T]
     priority: int = 0
 
-    def __init__(self, shape: ShapeType, dtype: DtypeType) -> None:
+    def __init__(self, shape: ShapeType = Any, dtype: DtypeType = Any) -> None:
         self.shape = shape
         self.dtype = dtype
 
@@ -86,6 +143,7 @@ class Interface(ABC, Generic[T]):
         self.raise_for_shape(shape_valid, shape)
 
         array = self.after_validation(array)
+
         return array
 
     def before_validation(self, array: Any) -> NDArrayType:
@@ -117,8 +175,6 @@ class Interface(ABC, Generic[T]):
         """
         Validate the dtype of the given array, returning
         ``True`` if valid, ``False`` if not.
-
-
         """
         if self.dtype is Any:
             return True
@@ -196,6 +252,13 @@ class Interface(ABC, Generic[T]):
         """
         return array
 
+    def mark_input(self, array: Any) -> Any:
+        """
+        Preserve metadata about the interface and passed input when dumping with
+        ``round_trip``
+        """
+        return array
+
     @classmethod
     @abstractmethod
     def check(cls, array: Any) -> bool:
@@ -211,17 +274,40 @@ class Interface(ABC, Generic[T]):
         installed, etc.)
         """
 
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """
+        Short name for this interface
+        """
+
     @classmethod
-    def to_json(
-        cls, array: Type[T], info: Optional[SerializationInfo] = None
-    ) -> Union[list, dict]:
+    @abstractmethod
+    def to_json(cls, array: Type[T], info: SerializationInfo) -> Union[list, JsonDict]:
         """
         Convert an array of :attr:`.return_type` to a JSON-compatible format using
         base python types
         """
-        if not isinstance(array, np.ndarray):  # pragma: no cover
-            array = np.array(array)
-        return array.tolist()
+
+    @classmethod
+    def mark_json(cls, array: Union[list, dict]) -> dict:
+        """
+        When using ``model_dump_json`` with ``mark_interface: True`` in the ``context``,
+        add additional annotations that would allow the serialized array to be
+        roundtripped.
+
+        Default is just to add an :class:`.InterfaceMark`
+
+        Examples:
+
+            >>> from pprint import pprint
+            >>> pprint(Interface.mark_json([1.0, 2.0]))
+            {'interface': {'cls': 'Interface',
+                           'module': 'numpydantic.interface.interface',
+                           'version': '1.2.2'},
+             'value': [1.0, 2.0]}
+        """
+        return {"interface": cls.mark_interface(), "value": array}
 
     @classmethod
     def interfaces(
@@ -335,3 +421,24 @@ class Interface(ABC, Generic[T]):
             raise NoMatchError(f"No matching interfaces found for output {array}")
         else:
             return matches[0]
+
+    @classmethod
+    def mark_interface(cls) -> InterfaceMark:
+        """
+        Create an interface mark indicating this interface for validation after
+        JSON serialization with ``round_trip==True``
+        """
+        interface_module = inspect.getmodule(cls)
+        interface_module = (
+            None if interface_module is None else interface_module.__name__
+        )
+        try:
+            v = (
+                None
+                if interface_module is None
+                else version(interface_module.split(".")[0])
+            )
+        except PackageNotFoundError:
+            v = None
+        interface_name = cls.__name__
+        return InterfaceMark(module=interface_module, cls=interface_name, version=v)
diff --git a/src/numpydantic/interface/numpy.py b/src/numpydantic/interface/numpy.py
index 5ee988a..8c68f1e 100644
--- a/src/numpydantic/interface/numpy.py
+++ b/src/numpydantic/interface/numpy.py
@@ -2,9 +2,12 @@
 Interface to numpy arrays
 """
 
-from typing import Any
+from dataclasses import dataclass
+from typing import Any, Literal, Union
 
-from numpydantic.interface.interface import Interface
+from pydantic import SerializationInfo
+
+from numpydantic.interface.interface import Interface, JsonDict
 
 try:
     import numpy as np
@@ -18,11 +21,29 @@ except ImportError:  # pragma: no cover
     np = None
 
 
+@dataclass
+class NumpyJsonDict(JsonDict):
+    """
+    JSON-able roundtrip representation of numpy array
+    """
+
+    type: Literal["numpy"]
+    dtype: str
+    array: list
+
+    def to_array_input(self) -> ndarray:
+        """
+        Construct a numpy array
+        """
+        return np.array(self.array, dtype=self.dtype)
+
+
 class NumpyInterface(Interface):
     """
     Numpy :class:`~numpy.ndarray` s!
     """
 
+    name = "numpy"
     input_types = (ndarray, list)
     return_type = ndarray
     priority = -999
@@ -41,6 +62,8 @@ class NumpyInterface(Interface):
         """
         if isinstance(array, ndarray):
             return True
+        elif isinstance(array, dict):
+            return NumpyJsonDict.is_valid(array)
         else:
             try:
                 _ = np.array(array)
@@ -53,6 +76,9 @@ class NumpyInterface(Interface):
         Coerce to an ndarray. We have already checked if coercion is possible
         in :meth:`.check`
         """
+        if isinstance(array, dict):
+            array = NumpyJsonDict(**array).to_array_input()
+
         if not isinstance(array, ndarray):
             array = np.array(array)
         return array
@@ -61,3 +87,22 @@ class NumpyInterface(Interface):
     def enabled(cls) -> bool:
         """Check that numpy is present in the environment"""
         return ENABLED
+
+    @classmethod
+    def to_json(
+        cls, array: ndarray, info: SerializationInfo = None
+    ) -> Union[list, JsonDict]:
+        """
+        Convert an array of :attr:`.return_type` to a JSON-compatible format using
+        base python types
+        """
+        if not isinstance(array, np.ndarray):  # pragma: no cover
+            array = np.array(array)
+
+        json_array = array.tolist()
+
+        if info.round_trip:
+            json_array = NumpyJsonDict(
+                type=cls.name, dtype=str(array.dtype), array=json_array
+            )
+        return json_array
diff --git a/src/numpydantic/interface/video.py b/src/numpydantic/interface/video.py
index 1660545..3b3b499 100644
--- a/src/numpydantic/interface/video.py
+++ b/src/numpydantic/interface/video.py
@@ -2,11 +2,14 @@
 Interface to support treating videos like arrays using OpenCV
 """
 
+from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Optional, Tuple, Union
+from typing import Any, Literal, Optional, Tuple, Union
 
 import numpy as np
+from pydantic_core.core_schema import SerializationInfo
 
+from numpydantic.interface import JsonDict
 from numpydantic.interface.interface import Interface
 
 try:
@@ -19,6 +22,20 @@ except ImportError:  # pragma: no cover
 VIDEO_EXTENSIONS = (".mp4", ".avi", ".mov", ".mkv")
 
 
+@dataclass(kw_only=True)
+class VideoJsonDict(JsonDict):
+    """Json-able roundtrip representation of a video file"""
+
+    type: Literal["video"]
+    file: str
+
+    def to_array_input(self) -> "VideoProxy":
+        """
+        Construct a :class:`.VideoProxy`
+        """
+        return VideoProxy(path=Path(self.file))
+
+
 class VideoProxy:
     """
     Passthrough proxy class to interact with videos as arrays
@@ -184,6 +201,12 @@ class VideoProxy:
     def __getattr__(self, item: str):
         return getattr(self.video, item)
 
+    def __eq__(self, other: "VideoProxy") -> bool:
+        """Check if this is a proxy to the same video file"""
+        if not isinstance(other, VideoProxy):
+            raise TypeError("Can only compare equality of two VideoProxies")
+        return self.path == other.path
+
     def __len__(self) -> int:
         """Number of frames in the video"""
         return self.shape[0]
@@ -194,6 +217,7 @@ class VideoInterface(Interface):
     OpenCV interface to treat videos as arrays.
     """
 
+    name = "video"
     input_types = (str, Path, VideoCapture, VideoProxy)
     return_type = VideoProxy
 
@@ -213,6 +237,9 @@ class VideoInterface(Interface):
         ):
             return True
 
+        if isinstance(array, dict):
+            array = array.get("file", "")
+
         if isinstance(array, str):
             try:
                 array = Path(array)
@@ -224,10 +251,22 @@ class VideoInterface(Interface):
 
     def before_validation(self, array: Any) -> VideoProxy:
         """Get a :class:`.VideoProxy` object for this video"""
-        if isinstance(array, VideoCapture):
+        if isinstance(array, dict):
+            proxy = VideoJsonDict(**array).to_array_input()
+        elif isinstance(array, VideoCapture):
             proxy = VideoProxy(video=array)
         elif isinstance(array, VideoProxy):
             proxy = array
         else:
             proxy = VideoProxy(path=array)
         return proxy
+
+    @classmethod
+    def to_json(
+        cls, array: VideoProxy, info: SerializationInfo
+    ) -> Union[list, VideoJsonDict]:
+        """Return a json-representation of a video"""
+        if info.round_trip:
+            return VideoJsonDict(type=cls.name, file=str(array.path))
+        else:
+            return np.array(array).tolist()
diff --git a/src/numpydantic/interface/zarr.py b/src/numpydantic/interface/zarr.py
index 87f538a..1e5b612 100644
--- a/src/numpydantic/interface/zarr.py
+++ b/src/numpydantic/interface/zarr.py
@@ -5,12 +5,12 @@ Interface to zarr arrays
 import contextlib
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Optional, Sequence, Union
+from typing import Any, Literal, Optional, Sequence, Union
 
 import numpy as np
 from pydantic import SerializationInfo
 
-from numpydantic.interface.interface import Interface
+from numpydantic.interface.interface import Interface, JsonDict
 from numpydantic.types import DtypeType
 
 try:
@@ -56,11 +56,34 @@ class ZarrArrayPath:
             raise ValueError("Only len 1-2 iterables can be used for a ZarrArrayPath")
 
 
+@dataclass(kw_only=True)
+class ZarrJsonDict(JsonDict):
+    """Round-trip Json-able version of a Zarr Array"""
+
+    info: dict[str, str]
+    type: Literal["zarr"]
+    file: Optional[str] = None
+    path: Optional[str] = None
+    array: Optional[list] = None
+
+    def to_array_input(self) -> ZarrArray | ZarrArrayPath:
+        """
+        Construct a ZarrArrayPath if file and path are present,
+        otherwise a ZarrArray
+        """
+        if self.file:
+            array = ZarrArrayPath(file=self.file, path=self.path)
+        else:
+            array = zarr.array(self.array)
+        return array
+
+
 class ZarrInterface(Interface):
     """
     Interface to in-memory or on-disk zarr arrays
     """
 
+    name = "zarr"
     input_types = (Path, ZarrArray, ZarrArrayPath)
     return_type = ZarrArray
 
@@ -73,6 +96,9 @@ class ZarrInterface(Interface):
     def _get_array(
         array: Union[ZarrArray, str, Path, ZarrArrayPath, Sequence]
     ) -> ZarrArray:
+        if isinstance(array, dict):
+            array = ZarrJsonDict(**array).to_array_input()
+
         if isinstance(array, ZarrArray):
             return array
 
@@ -92,6 +118,12 @@ class ZarrInterface(Interface):
         if isinstance(array, ZarrArray):
             return True
 
+        if isinstance(array, dict):
+            if array.get("type", False) == cls.name:
+                return True
+            # continue checking if dict contains a zarr file
+            array = array.get("file", "")
+
         # See if can be coerced to ZarrArrayPath
         if isinstance(array, (Path, str)):
             array = ZarrArrayPath(file=array)
@@ -135,26 +167,46 @@ class ZarrInterface(Interface):
         cls,
         array: Union[ZarrArray, str, Path, ZarrArrayPath, Sequence],
         info: Optional[SerializationInfo] = None,
-    ) -> dict:
+    ) -> list | ZarrJsonDict:
         """
-        Dump just the metadata for an array from :meth:`zarr.core.Array.info_items`
-        plus the :meth:`zarr.core.Array.hexdigest`.
+        Dump a Zarr Array to JSON
+        
+        If ``info.round_trip == False``, dump the array as a list of lists.
+        This may be a memory-intensive operation.
+        
+        Otherwise, dump the metadata for an array from :meth:`zarr.core.Array.info_items`
+        plus the :meth:`zarr.core.Array.hexdigest` as a :class:`.ZarrJsonDict`
+        
+        If either the ``zarr_dump_array`` value in the context dictionary is ``True``
+        or the zarr array is an in-memory array, dump the array as well
+        (since without a persistent array it would be impossible to roundtrip and 
+        dumping to JSON would be meaningless) 
 
-        The full array can be returned by passing ``'zarr_dump_array': True`` to the
-        serialization ``context`` ::
+        Passing ``'zarr_dump_array': True`` to the serialization ``context`` looks like this::
 
             model.model_dump_json(context={'zarr_dump_array': True})
         """
-        dump_array = False
-        if info is not None and info.context is not None:
-            dump_array = info.context.get("zarr_dump_array", False)
-
         array = cls._get_array(array)
-        info = array.info_items()
-        info_dict = {i[0]: i[1] for i in info}
-        info_dict["hexdigest"] = array.hexdigest()
 
-        if dump_array:
-            info_dict["array"] = array[:].tolist()
+        if info.round_trip:
+            dump_array = False
+            if info is not None and info.context is not None:
+                dump_array = info.context.get("zarr_dump_array", False)
+            is_file = False
 
-        return info_dict
+            as_json = {"type": cls.name}
+            if hasattr(array.store, "dir_path"):
+                is_file = True
+                as_json["file"] = array.store.dir_path()
+                as_json["path"] = array.name
+            as_json["info"] = {i[0]: i[1] for i in array.info_items()}
+            as_json["info"]["hexdigest"] = array.hexdigest()
+
+            if dump_array or not is_file:
+                as_json["array"] = array[:].tolist()
+
+            as_json = ZarrJsonDict(**as_json)
+        else:
+            as_json = array[:].tolist()
+
+        return as_json
diff --git a/src/numpydantic/ndarray.py b/src/numpydantic/ndarray.py
index d951d3a..d494154 100644
--- a/src/numpydantic/ndarray.py
+++ b/src/numpydantic/ndarray.py
@@ -24,7 +24,6 @@ from numpydantic.exceptions import InterfaceError
 from numpydantic.interface import Interface
 from numpydantic.maps import python_to_nptyping
 from numpydantic.schema import (
-    _handler_type,
     _jsonize_array,
     get_validate_interface,
     make_json_schema,
@@ -41,6 +40,9 @@ from numpydantic.vendor.nptyping.typing_ import (
 
 if TYPE_CHECKING:  # pragma: no cover
     from nptyping.base_meta_classes import SubscriptableMeta
+    from pydantic._internal._schema_generation_shared import (
+        CallbackGetCoreSchemaHandler,
+    )
 
     from numpydantic import Shape
 
@@ -164,33 +166,34 @@ class NDArray(NPTypingType, metaclass=NDArrayMeta):
     def __get_pydantic_core_schema__(
         cls,
         _source_type: "NDArray",
-        _handler: _handler_type,
+        _handler: "CallbackGetCoreSchemaHandler",
     ) -> core_schema.CoreSchema:
         shape, dtype = _source_type.__args__
         shape: ShapeType
         dtype: DtypeType
 
-        # get pydantic core schema as a list of lists for JSON schema
-        list_schema = make_json_schema(shape, dtype, _handler)
+        # make core schema for json schema, store it and any model definitions
+        # note that there is a big of fragility in this function,
+        # as we need to access a private method of _handler to
+        # flatten out the json schema. See help(make_json_schema)
+        json_schema = make_json_schema(shape, dtype, _handler)
 
-        return core_schema.json_or_python_schema(
-            json_schema=list_schema,
-            python_schema=core_schema.with_info_plain_validator_function(
-                get_validate_interface(shape, dtype)
-            ),
+        return core_schema.with_info_plain_validator_function(
+            get_validate_interface(shape, dtype),
             serialization=core_schema.plain_serializer_function_ser_schema(
                 _jsonize_array, when_used="json", info_arg=True
             ),
+            metadata=json_schema,
         )
 
     @classmethod
     def __get_pydantic_json_schema__(
         cls, schema: core_schema.CoreSchema, handler: GetJsonSchemaHandler
     ) -> core_schema.JsonSchema:
-        json_schema = handler(schema)
+        shape, dtype = cls.__args__
+        json_schema = handler(schema["metadata"])
         json_schema = handler.resolve_ref_schema(json_schema)
 
-        dtype = cls.__args__[1]
         if not isinstance(dtype, tuple) and dtype.__module__ not in (
             "builtins",
             "typing",
diff --git a/src/numpydantic/schema.py b/src/numpydantic/schema.py
index d98f880..36d6812 100644
--- a/src/numpydantic/schema.py
+++ b/src/numpydantic/schema.py
@@ -13,19 +13,24 @@ from pydantic_core import CoreSchema, core_schema
 from pydantic_core.core_schema import ListSchema, ValidationInfo
 
 from numpydantic import dtype as dt
-from numpydantic.interface import Interface
+from numpydantic.interface import Interface, JsonDict
 from numpydantic.maps import np_to_python
 from numpydantic.types import DtypeType, NDArrayType, ShapeType
 from numpydantic.vendor.nptyping.structure import StructureMeta
 
 if TYPE_CHECKING:  # pragma: no cover
+    from pydantic._internal._schema_generation_shared import (
+        CallbackGetCoreSchemaHandler,
+    )
+
     from numpydantic import Shape
 
-_handler_type = Callable[[Any], core_schema.CoreSchema]
 _UNSUPPORTED_TYPES = (complex,)
 
 
-def _numeric_dtype(dtype: DtypeType, _handler: _handler_type) -> CoreSchema:
+def _numeric_dtype(
+    dtype: DtypeType, _handler: "CallbackGetCoreSchemaHandler"
+) -> CoreSchema:
     """Make a numeric dtype that respects min/max values from extended numpy types"""
     if dtype in (np.number,):
         dtype = float
@@ -36,14 +41,19 @@ def _numeric_dtype(dtype: DtypeType, _handler: _handler_type) -> CoreSchema:
     elif issubclass(dtype, np.integer):
         info = np.iinfo(dtype)
         schema = core_schema.int_schema(le=int(info.max), ge=int(info.min))
-
+    elif dtype is float:
+        schema = core_schema.float_schema()
+    elif dtype is int:
+        schema = core_schema.int_schema()
     else:
         schema = _handler.generate_schema(dtype)
 
     return schema
 
 
-def _lol_dtype(dtype: DtypeType, _handler: _handler_type) -> CoreSchema:
+def _lol_dtype(
+    dtype: DtypeType, _handler: "CallbackGetCoreSchemaHandler"
+) -> CoreSchema:
     """Get the innermost dtype schema to use in the generated pydantic schema"""
     if isinstance(dtype, StructureMeta):  # pragma: no cover
         raise NotImplementedError("Structured dtypes are currently unsupported")
@@ -84,6 +94,10 @@ def _lol_dtype(dtype: DtypeType, _handler: _handler_type) -> CoreSchema:
             # TODO: warn and log here
         elif python_type in (float, int):
             array_type = _numeric_dtype(dtype, _handler)
+        elif python_type is bool:
+            array_type = core_schema.bool_schema()
+        elif python_type is Any:
+            array_type = core_schema.any_schema()
         else:
             array_type = _handler.generate_schema(python_type)
 
@@ -208,14 +222,24 @@ def _unbounded_shape(
 
 
 def make_json_schema(
-    shape: ShapeType, dtype: DtypeType, _handler: _handler_type
+    shape: ShapeType, dtype: DtypeType, _handler: "CallbackGetCoreSchemaHandler"
 ) -> ListSchema:
     """
-    Make a list of list JSON schema from a shape and a dtype.
+    Make a list of list pydantic core schema for an array from a shape and a dtype.
+    Used to generate JSON schema in the containing model, but not for validation,
+    which is handled by interfaces.
 
     First resolves the dtype into a pydantic ``CoreSchema`` ,
     and then uses that with :func:`.list_of_lists_schema` .
 
+    .. admonition:: Potentially Fragile
+
+        Uses a private method from the handler to flatten out nested definitions
+        (e.g. when dtype is a pydantic model)
+        so that they are present in the generated schema directly rather than
+        as references. Otherwise, at the time __get_pydantic_json_schema__ is called,
+        the definition references are lost.
+
     Args:
         shape ( ShapeType ): Specification of a shape, as a tuple or
             an nptyping ``Shape``
@@ -234,6 +258,8 @@ def make_json_schema(
     else:
         list_schema = list_of_lists_schema(shape, dtype_schema)
 
+    list_schema = _handler._generate_schema.clean_schema(list_schema)
+
     return list_schema
 
 
@@ -257,4 +283,11 @@ def get_validate_interface(shape: ShapeType, dtype: DtypeType) -> Callable:
 def _jsonize_array(value: Any, info: SerializationInfo) -> Union[list, dict]:
     """Use an interface class to render an array as JSON"""
     interface_cls = Interface.match_output(value)
-    return interface_cls.to_json(value, info)
+    array = interface_cls.to_json(value, info)
+    if isinstance(array, JsonDict):
+        array = array.to_dict()
+
+    if info.context and info.context.get("mark_interface", False):
+        array = interface_cls.mark_json(array)
+
+    return array
diff --git a/tests/test_interface/test_hdf5.py b/tests/test_interface/test_hdf5.py
index 9ca9e94..b64d3fe 100644
--- a/tests/test_interface/test_hdf5.py
+++ b/tests/test_interface/test_hdf5.py
@@ -101,7 +101,8 @@ def test_assignment(hdf5_array, model_blank):
     assert (model.array[1:3, 2:4] == 10).all()
 
 
-def test_to_json(hdf5_array, array_model):
+@pytest.mark.parametrize("round_trip", (True, False))
+def test_to_json(hdf5_array, array_model, round_trip):
     """
     Test serialization of HDF5 arrays to JSON
     Args:
@@ -115,13 +116,13 @@ def test_to_json(hdf5_array, array_model):
 
     instance = model(array=array)  # type: BaseModel
 
-    json_str = instance.model_dump_json()
-    json_dict = json.loads(json_str)["array"]
-
-    assert json_dict["file"] == str(array.file)
-    assert json_dict["path"] == str(array.path)
-    assert json_dict["attrs"] == {}
-    assert json_dict["array"] == instance.array[:].tolist()
+    json_str = instance.model_dump_json(round_trip=round_trip)
+    json_dumped = json.loads(json_str)["array"]
+    if round_trip:
+        assert json_dumped["file"] == str(array.file)
+        assert json_dumped["path"] == str(array.path)
+    else:
+        assert json_dumped == instance.array[:].tolist()
 
 
 def test_compound_dtype(tmp_path):
diff --git a/tests/test_interface/test_interfaces.py b/tests/test_interface/test_interfaces.py
index 3b1370a..3d51ac0 100644
--- a/tests/test_interface/test_interfaces.py
+++ b/tests/test_interface/test_interfaces.py
@@ -2,8 +2,11 @@
 Tests that should be applied to all interfaces
 """
 
+import pytest
 from typing import Callable
 import numpy as np
+import dask.array as da
+from zarr.core import Array as ZarrArray
 from numpydantic.interface import Interface
 
 
@@ -35,8 +38,31 @@ def test_interface_to_numpy_array(all_interfaces):
     _ = np.array(all_interfaces.array)
 
 
+@pytest.mark.serialization
 def test_interface_dump_json(all_interfaces):
     """
     All interfaces should be able to dump to json
     """
     all_interfaces.model_dump_json()
+
+
+@pytest.mark.serialization
+@pytest.mark.parametrize("round_trip", [True, False])
+def test_interface_roundtrip_json(all_interfaces, round_trip):
+    """
+    All interfaces should be able to roundtrip to and from json
+    """
+    json = all_interfaces.model_dump_json(round_trip=round_trip)
+    model = all_interfaces.model_validate_json(json)
+    if round_trip:
+        assert type(model.array) is type(all_interfaces.array)
+        if isinstance(all_interfaces.array, (np.ndarray, ZarrArray)):
+            assert np.array_equal(model.array, np.array(all_interfaces.array))
+        elif isinstance(all_interfaces.array, da.Array):
+            assert np.all(da.equal(model.array, all_interfaces.array))
+        else:
+            assert model.array == all_interfaces.array
+
+        assert model.array.dtype == all_interfaces.array.dtype
+    else:
+        assert np.array_equal(model.array, np.array(all_interfaces.array))
diff --git a/tests/test_interface/test_zarr.py b/tests/test_interface/test_zarr.py
index 2e465f2..05eb8d5 100644
--- a/tests/test_interface/test_zarr.py
+++ b/tests/test_interface/test_zarr.py
@@ -123,7 +123,10 @@ def test_zarr_array_path_from_iterable(zarr_array):
     assert apath.path == inner_path
 
 
-def test_zarr_to_json(store, model_blank):
+@pytest.mark.serialization
+@pytest.mark.parametrize("dump_array", [True, False])
+@pytest.mark.parametrize("roundtrip", [True, False])
+def test_zarr_to_json(store, model_blank, roundtrip, dump_array):
     expected_fields = (
         "Type",
         "Data type",
@@ -137,17 +140,22 @@ def test_zarr_to_json(store, model_blank):
 
     array = zarr.array(lol_array, store=store)
     instance = model_blank(array=array)
-    as_json = json.loads(instance.model_dump_json())["array"]
-    assert "array" not in as_json
-    for field in expected_fields:
-        assert field in as_json
-    assert len(as_json["hexdigest"]) == 40
 
-    # dump the array itself too
-    as_json = json.loads(instance.model_dump_json(context={"zarr_dump_array": True}))[
-        "array"
-    ]
-    for field in expected_fields:
-        assert field in as_json
-    assert len(as_json["hexdigest"]) == 40
-    assert as_json["array"] == lol_array
+    context = {"zarr_dump_array": dump_array}
+    as_json = json.loads(
+        instance.model_dump_json(round_trip=roundtrip, context=context)
+    )["array"]
+
+    if roundtrip:
+        if dump_array:
+            assert as_json["array"] == lol_array
+        else:
+            if as_json.get("file", False):
+                assert "array" not in as_json
+
+        for field in expected_fields:
+            assert field in as_json["info"]
+        assert len(as_json["info"]["hexdigest"]) == 40
+
+    else:
+        assert as_json == lol_array
diff --git a/tests/test_ndarray.py b/tests/test_ndarray.py
index f92a66d..cef7dc3 100644
--- a/tests/test_ndarray.py
+++ b/tests/test_ndarray.py
@@ -15,6 +15,7 @@ from numpydantic import dtype
 from numpydantic.dtype import Number
 
 
+@pytest.mark.json_schema
 def test_ndarray_type():
     class Model(BaseModel):
         array: NDArray[Shape["2 x, * y"], Number]
@@ -40,6 +41,7 @@ def test_ndarray_type():
     instance = Model(array=np.zeros((2, 3)), array_any=np.ones((3, 4, 5)))
 
 
+@pytest.mark.json_schema
 def test_schema_unsupported_type():
     """
     Complex numbers should just be made with an `any` schema
@@ -55,6 +57,7 @@ def test_schema_unsupported_type():
     }
 
 
+@pytest.mark.json_schema
 def test_schema_tuple():
     """
     Types specified as tupled should have their schemas as a union
@@ -72,6 +75,7 @@ def test_schema_tuple():
     assert all([i["minimum"] == 0 for i in conditions])
 
 
+@pytest.mark.json_schema
 def test_schema_number():
     """
     np.numeric should just be the float schema
@@ -164,6 +168,7 @@ def test_ndarray_coercion():
         amod = Model(array=["a", "b", "c"])
 
 
+@pytest.mark.serialization
 def test_ndarray_serialize():
     """
     Arrays should be dumped to a list when using json, but kept as ndarray otherwise
@@ -188,6 +193,7 @@ _json_schema_types = [
 ]
 
 
+@pytest.mark.json_schema
 def test_json_schema_basic(array_model):
     """
     NDArray types should correctly generate a list of lists JSON schema
@@ -210,6 +216,8 @@ def test_json_schema_basic(array_model):
     assert inner["items"]["type"] == "number"
 
 
+@pytest.mark.dtype
+@pytest.mark.json_schema
 @pytest.mark.parametrize("dtype", [*dtype.Integer, *dtype.Float])
 def test_json_schema_dtype_single(dtype, array_model):
     """
@@ -240,6 +248,7 @@ def test_json_schema_dtype_single(dtype, array_model):
     )
 
 
+@pytest.mark.dtype
 @pytest.mark.parametrize(
     "dtype,expected",
     [
@@ -266,6 +275,8 @@ def test_json_schema_dtype_builtin(dtype, expected, array_model):
         assert inner_type["type"] == expected
 
 
+@pytest.mark.dtype
+@pytest.mark.json_schema
 def test_json_schema_dtype_model():
     """
     Pydantic models can be used in arrays as dtypes
@@ -314,6 +325,8 @@ def _recursive_array(schema):
     assert any_of[1]["minimum"] == 0
 
 
+@pytest.mark.shape
+@pytest.mark.json_schema
 def test_json_schema_ellipsis():
     """
     NDArray types should create a recursive JSON schema for any-shaped arrays

From 02855852b78277ccce1e0c9011eb5bb7529251e7 Mon Sep 17 00:00:00 2001
From: sneakers-the-rat <sneakers-the-rat@protonmail.com>
Date: Fri, 20 Sep 2024 23:58:40 -0700
Subject: [PATCH 03/22] lint

---
 src/numpydantic/interface/zarr.py | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/src/numpydantic/interface/zarr.py b/src/numpydantic/interface/zarr.py
index 1e5b612..c7a28e1 100644
--- a/src/numpydantic/interface/zarr.py
+++ b/src/numpydantic/interface/zarr.py
@@ -170,19 +170,21 @@ class ZarrInterface(Interface):
     ) -> list | ZarrJsonDict:
         """
         Dump a Zarr Array to JSON
-        
+
         If ``info.round_trip == False``, dump the array as a list of lists.
         This may be a memory-intensive operation.
-        
-        Otherwise, dump the metadata for an array from :meth:`zarr.core.Array.info_items`
+
+        Otherwise, dump the metadata for an array from
+        :meth:`zarr.core.Array.info_items`
         plus the :meth:`zarr.core.Array.hexdigest` as a :class:`.ZarrJsonDict`
-        
+
         If either the ``zarr_dump_array`` value in the context dictionary is ``True``
         or the zarr array is an in-memory array, dump the array as well
-        (since without a persistent array it would be impossible to roundtrip and 
-        dumping to JSON would be meaningless) 
+        (since without a persistent array it would be impossible to roundtrip and
+        dumping to JSON would be meaningless)
 
-        Passing ``'zarr_dump_array': True`` to the serialization ``context`` looks like this::
+        Passing ``'zarr_dump_array': True`` to the serialization ``context``
+        looks like this::
 
             model.model_dump_json(context={'zarr_dump_array': True})
         """

From d3ad8dac5c967977162009ce3786e81dc9db04ff Mon Sep 17 00:00:00 2001
From: sneakers-the-rat <sneakers-the-rat@protonmail.com>
Date: Sat, 21 Sep 2024 04:18:22 -0700
Subject: [PATCH 04/22] docs and docs and docs and docs

---
 docs/conf.py                           |   7 +-
 docs/data/test.avi                     | Bin 0 -> 5742 bytes
 docs/data/test.h5                      | Bin 0 -> 2080 bytes
 docs/data/test.zarr/.zarray            |  22 +
 docs/data/test.zarr/0.0                | Bin 0 -> 48 bytes
 docs/development.md                    |  84 +++
 docs/hooks.md                          |  11 -
 docs/index.md                          |  12 +-
 docs/serialization.md                  | 311 ++++++++
 pdm.lock                               | 993 ++++++++++++++++++++++++-
 pyproject.toml                         |   2 +
 src/numpydantic/interface/interface.py |  11 +-
 src/numpydantic/interface/video.py     |   3 +
 src/numpydantic/interface/zarr.py      |   6 +-
 src/numpydantic/ndarray.py             |   4 +-
 src/numpydantic/schema.py              |  19 +-
 src/numpydantic/serialization.py       |  94 +++
 17 files changed, 1525 insertions(+), 54 deletions(-)
 create mode 100644 docs/data/test.avi
 create mode 100644 docs/data/test.h5
 create mode 100644 docs/data/test.zarr/.zarray
 create mode 100644 docs/data/test.zarr/0.0
 create mode 100644 docs/development.md
 delete mode 100644 docs/hooks.md
 create mode 100644 src/numpydantic/serialization.py

diff --git a/docs/conf.py b/docs/conf.py
index bae1f75..963b0b3 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -25,7 +25,7 @@ extensions = [
     "sphinx.ext.doctest",
     "sphinx_design",
     "sphinxcontrib.mermaid",
-    "myst_parser",
+    "myst_nb",
     "sphinx.ext.todo",
 ]
 
@@ -77,3 +77,8 @@ napoleon_attr_annotations = True
 # todo
 todo_include_todos = True
 todo_link_only = True
+
+# myst
+# myst-nb
+nb_render_markdown_format = "myst"
+nb_execution_show_tb = True
diff --git a/docs/data/test.avi b/docs/data/test.avi
new file mode 100644
index 0000000000000000000000000000000000000000..880cc68be856a74765ee5244c428087ada7ddcd9
GIT binary patch
literal 5742
zcmeI0!D_-l6h&WR+Ui2In<AwPm##|Oh)e4tFjW+)Ex42pA%TdXCHNtJrXS$WzMFK!
zg<nwbB*V<yH}hWREV8&CCW_)Gk+(0?9-G<x@F<elO}5O_q%R_aVM78P(V8_8znqF!
zD#hAqf1zSPu6<Kv`?JZk7rFA(ZWsM9yxfO8Wv((eYMr#XJ-$}Z^KnN7bksukq$<ee
zzD=cbd;gq_T3Xx1!q9%7gPf2636KB@kN^pg011!)36KB@kN^qX1OfO1ftn6$yUO%U
zjyG}Z>z9O2^Q<8=iKb>2&FeZJFY~z9ebonE4QL%J*H2R=0TTG%1n`G=(JQXjc?u^L
euBzu(0yn?Af2gK**KMl~O^w&WdBr}rcG@p4jx$aG

literal 0
HcmV?d00001

diff --git a/docs/data/test.h5 b/docs/data/test.h5
new file mode 100644
index 0000000000000000000000000000000000000000..39fa06a71594170bace819fd8a5a2934fbe3c65e
GIT binary patch
literal 2080
zcmeD5aB<`1lHy_j0S*oZ76t(@6Gr@p0tF6;2#gPtPk=HQp>zk7Ucm%mFfxE31A_!q
zTo7tLy1I}cS62q0N|^aD8mf)KfCa*WIs+y=N{^5b@Njhu0C_b6>R(tYJpoN;uwY0@
zEJ*~hVd>EWCP606$iNCQ3u+)Eg9g|nMka^=%z9ijGcdh_R0;qSE+p+bfc3Kic_48n
zCWt{C&>X_d2vx?Q09J<}8W79@WCi>AyMS^u#4ijC3d{rOm{F@oLtr!nC<*~cDF!Tu
Npr*jGGqk#8004FkEnxrv

literal 0
HcmV?d00001

diff --git a/docs/data/test.zarr/.zarray b/docs/data/test.zarr/.zarray
new file mode 100644
index 0000000..8ec7941
--- /dev/null
+++ b/docs/data/test.zarr/.zarray
@@ -0,0 +1,22 @@
+{
+    "chunks": [
+        2,
+        2
+    ],
+    "compressor": {
+        "blocksize": 0,
+        "clevel": 5,
+        "cname": "lz4",
+        "id": "blosc",
+        "shuffle": 1
+    },
+    "dtype": "<i8",
+    "fill_value": 0,
+    "filters": null,
+    "order": "C",
+    "shape": [
+        2,
+        2
+    ],
+    "zarr_format": 2
+}
\ No newline at end of file
diff --git a/docs/data/test.zarr/0.0 b/docs/data/test.zarr/0.0
new file mode 100644
index 0000000000000000000000000000000000000000..cab0cbca723f5ff9fececdaba39bcca42c69cb3f
GIT binary patch
literal 48
gcmZQ#H0DrXU|;}Y10ZGuVh~`0Pz=mangvP&039&^xc~qF

literal 0
HcmV?d00001

diff --git a/docs/development.md b/docs/development.md
new file mode 100644
index 0000000..3b37086
--- /dev/null
+++ b/docs/development.md
@@ -0,0 +1,84 @@
+# Development
+
+## Versioning
+
+This package uses a colloquial form of [semantic versioning 2](https://semver.org/).
+
+Specifically:
+
+- Major version `2.*.*` is reserved for the transition from nptyping to using
+  `TypeVarTuple`, `Generic`, and `Protocol`. Until `2.*.*`...
+  - breaking changes will be indicated with an advance in `MINOR`
+      version, taking the place of `MAJOR` in semver
+  - backwards-compatible bugfixes **and** additions in functionality
+    will be indicated by a `PATCH` release, taking the place of `MINOR` and
+    `PATCH` in semver.
+- After `2.*.*`, semver as usual will resume
+
+You are encouraged to set an upper bound on your version dependencies until
+we pass `2.*.*`, as the major function of numpydantic is stable,
+but there is still a decent amount of jostling things around to be expected.
+
+
+### API Stability
+
+- All breaking changes to the **public API** will be signaled by a major
+  version's worth of deprecation warnings
+- All breaking changes to the **development API** will be signaled by a
+  minor version's worth of deprecation warnings.
+- Changes to the remainder of the package, whether marked as private with a
+  leading underscore or not, including the import structure of the package,
+  are not considered part of the API and should not be relied on as stable 
+  until explicitly marked otherwise.
+
+#### Public API
+
+**Only the {class}`.NDArray` and {class}`.Shape` classes should be considered
+part of the stable public API.**
+
+All associated functionality for validation should also be considered
+a stable part of the `NDArray` and `Shape` classes - functionality
+will only be added here, and the departure for the string-form of the 
+shape specifications (and its removal) will take place in `v3.*.*`
+
+End-users of numpydantic should pin an upper bound for the `MAJOR` version
+until after `v2.*.*`, after which time it is up to your discretion - 
+no breaking changes are planned, but they would be signaled by a major version change.
+
+#### Development API
+
+**Only the {class}`.Interface` class and its subclasses, 
+along with the Public API,
+should be considered part of the stable development API.**
+
+The `Interface` class is the primary point of external development expected
+for numpydantic. It is still somewhat in flux, but it is prioritized for stability
+and deprecation warnings above the rest of the package. 
+
+Dependent packages that define their own `Interface`s should pin an upper
+bound for the `PATCH` version until `2.*.*`, and afterwards likely pin a `MINOR` version.
+Tests are designed such that it should be easy to test major features against
+each interface, and that work is also ongoing. Once the test suite reaches
+maturity, it should be possible for any downstream interfaces to simply use those to
+ensure they are compatible with the latest version.
+
+## Release Schedule
+
+There is no release schedule. Versions are released according to need and available labor.
+
+## Contributing
+
+### Dev environment
+
+```{todo}
+Document dev environment
+
+Really it's very simple, you just clone a fork and install
+the `dev` environment like `pip install '.[dev]'`
+```
+
+### Pull Requests
+
+```{todo}
+Document pull requests if we ever receive one
+```
diff --git a/docs/hooks.md b/docs/hooks.md
deleted file mode 100644
index 42a3b81..0000000
--- a/docs/hooks.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# Hooks
-
-What hooks do we want to expose to downstream users so they can use this without needing
-to override everything?
-
-```{todo} 
-**NWB Compatibility**
-
-**Precision:** NWB allows for a sort of hierarchy of type specification - 
-a less precise type also allows the data to be specified in a more precise type
-```
\ No newline at end of file
diff --git a/docs/index.md b/docs/index.md
index 0cea5b4..c75ce74 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -86,8 +86,8 @@ isinstance(np.zeros((1,2,3), dtype=float), array_type)
   and a simple extension system to make it work with whatever else you want! Provides
   a uniform and transparent interface so you can both use common indexing operations
   and also access any special features of a given array library.
-- **Serialization** - Dump an array as a JSON-compatible array-of-arrays with enough metadata to be able to 
-  recreate the model in the native format
+- [**Serialization**](./serialization.md) - Dump an array as a JSON-compatible array-of-arrays with enough metadata to be able to 
+  recreate the model in the native format. Full roundtripping is supported :)
 - **Schema Generation** - Correct JSON Schema for arrays, complete with shape and dtype constraints, to
   make your models interoperable 
 
@@ -496,6 +496,14 @@ api/types
 
 ```
 
+```{toctree}
+:maxdepth: 2
+:caption: Meta
+:hidden: true
+
+development
+```
+
 ## See Also
 
 - [`jaxtyping`](https://docs.kidger.site/jaxtyping/)
diff --git a/docs/serialization.md b/docs/serialization.md
index d5162bf..f18b8f6 100644
--- a/docs/serialization.md
+++ b/docs/serialization.md
@@ -1,2 +1,313 @@
+---
+file_format: mystnb
+mystnb:
+    output_stderr: remove
+    render_text_lexer: python
+    render_markdown_format: myst
+myst:
+    enable_extensions: ["colon_fence"]
+---
+
 # Serialization
 
+## Python
+
+In most cases, dumping to python should work as expected.
+
+When a given array framework doesn't provide a tidy means of interacting
+with it from python, we substitute a proxy class like {class}`.hdf5.H5Proxy`,
+but aside from that numpydantic {class}`.NDArray` annotations
+should be passthrough when using {func}`~pydantic.BaseModel.model_dump` .
+
+## JSON
+
+JSON is the ~ ♥ fun one ♥ ~
+
+There isn't necessarily a single optimal way to represent all possible
+arrays in JSON. The standard way that n-dimensional arrays are rendered
+in json is as a list-of-lists (or array of arrays, in JSON parlance),
+but that's almost never what is desirable, especially for large arrays.
+
+### Normal Style[^normalstyle]
+
+Lists-of-lists are the standard, however, so it is the default behavior
+for all interfaces, and all interfaces must support it.
+
+```{code-cell}
+---
+tags: [hide-cell]
+---
+
+from pathlib import Path
+from pydantic import BaseModel
+from numpydantic import NDArray, Shape
+from numpydantic.interface.dask import DaskJsonDict
+from numpydantic.interface.numpy import NumpyJsonDict
+import numpy as np
+import dask.array as da
+import zarr
+import json
+from rich import print
+from rich.console import Console
+
+def print_json(string:str):
+    data = json.loads(string)
+    console = Console(width=74)
+    console.print(data)
+```
+
+For our humble model:
+
+```{code-cell}
+class MyModel(BaseModel):
+    array: NDArray
+```
+
+We should get the same thing for each interface:
+
+```{code-cell}
+model = MyModel(array=[[1,2],[3,4]])
+print(model.model_dump_json())
+```
+
+```{code-cell}
+model = MyModel(array=da.array([[1,2],[3,4]], dtype=int))
+print(model.model_dump_json())
+```
+
+```{code-cell}
+model = MyModel(array=zarr.array([[1,2],[3,4]], dtype=int))
+print(model.model_dump_json())
+```
+
+```{code-cell}
+model = MyModel(array="docs/data/test.avi")
+print(model.model_dump_json())
+```
+
+(ok maybe not that last one, since the video reader still incorrectly
+reads grayscale videos as BGR values for now, but you get the idea)
+
+Since by default arrays are dumped into unadorned JSON arrays,
+when they are re-validated, they will always be handled by the
+{class}`.NumpyInterface`
+
+```{code-cell}
+dask_array = da.array([[1,2],[3,4]], dtype=int)
+model = MyModel(array=dask_array)
+type(model.array)
+```
+
+```{code-cell}
+model_json = model.model_dump_json()
+deserialized_model = MyModel.model_validate_json(model_json)
+type(deserialized_model.array)
+```
+
+All information about `dtype` will be lost, and numbers will be either parsed
+as `int` ({class}`numpy.int64`) or `float` ({class}`numpy.float64`)
+
+## Roundtripping
+
+To roundtrip make arrays round-trippable, use the `round_trip` argument
+to {func}`~pydantic.BaseModel.model_dump_json`
+
+
+```{code-cell}
+print_json(model.model_dump_json(round_trip=True))
+```
+
+Each interface should[^notenforced] implement a dataclass that describes a
+json-able roundtrip form (see {class}`.interface.JsonDict`).
+
+That dataclass then has a {meth}`JsonDict.is_valid` method that checks
+whether an incoming dict matches its schema
+
+```{code-cell}
+roundtrip_json = json.loads(model.model_dump_json(round_trip=True))['array']
+DaskJsonDict.is_valid(roundtrip_json)
+```
+
+```{code-cell}
+NumpyJsonDict.is_valid(roundtrip_json)
+```
+
+#### Controlling paths
+
+When possible, the full content of the array is omitted in favor
+of the path to the file that provided it.
+
+```{code-cell}
+model = MyModel(array="docs/data/test.avi")
+print_json(model.model_dump_json(round_trip=True))
+```
+
+```{code-cell}
+model = MyModel(array=("docs/data/test.h5", "/data"))
+print_json(model.model_dump_json(round_trip=True))
+```
+
+You may notice the relative, rather than absolute paths.
+
+
+We expect that when people are dumping data to json in this roundtripped
+form that they are either working locally 
+(e.g. transmitting an array specification across a socket in multiprocessing
+or in a computing cluster),
+or exporting to some directory structure of data, 
+where they are making an index file that refers to datasets in a directory
+as part of a data standard or vernacular format.
+
+By default, numpydantic uses the current working directory as the root to find
+paths relative to, but this can be controlled by the [`relative_to`](#relative_to)
+context parameter:
+
+For example if you're working on data in many subdirectories,
+you might want to serialize relative to each of them:
+
+```{code-cell}
+print_json(
+  model.model_dump_json(
+    round_trip=True, 
+    context={"relative_to": Path('./docs/data')}
+  ))
+```
+
+Or in the other direction:
+
+```{code-cell}
+print_json(
+  model.model_dump_json(
+    round_trip=True, 
+    context={"relative_to": Path('../')}
+  ))
+```
+
+Or you might be working in some completely different place,
+numpydantic will try and find the way from here to there as long as it exists,
+even if it means traversing to the root of the readthedocs filesystem
+
+```{code-cell}
+print_json(
+  model.model_dump_json(
+    round_trip=True, 
+    context={"relative_to": Path('/a/long/distance/directory')}
+    ))
+```
+
+You can force absolute paths with the `absolute_paths` context parameter
+
+```{code-cell}
+print_json(
+  model.model_dump_json(
+    round_trip=True, 
+    context={"absolute_paths": True}
+    ))
+```
+
+#### Durable Interface Metadata
+
+Numpydantic tries to be [stable](./development.md#api-stability),
+but we're not perfect. To preserve the full information about the
+interface that's needed to load the data referred to by the value,
+use the `mark_interface` contest parameter:
+
+```{code-cell}
+print_json(
+  model.model_dump_json(
+    round_trip=True, 
+    context={"mark_interface": True}
+    ))
+```
+
+```{todo}
+We will also add a separate `mark_version` parameter for marking
+the specific version of the relevant interface package, like `zarr`, or `numpy`,
+patience.
+```
+
+## Context parameters
+
+A reference listing of all the things that can be passed to
+{func}`~pydantic.BaseModel.model_dump_json`
+
+
+### `mark_interface`
+
+Nest an additional layer of metadata for unambigous serialization that
+can be absolutely resolved across numpydantic versions 
+(for now for downstream metadata purposes only, 
+automatically resolving to a numpydantic version is not yet possible.)
+
+Supported interfaces:
+
+- (all)
+
+```{code-cell}
+model = MyModel(array=[[1,2],[3,4]])
+data = model.model_dump_json(
+  round_trip=True,
+  context={"mark_interface": True}
+)
+print_json(data)
+```
+
+### `absolute_paths`
+
+Make all paths (that exist) absolute.
+
+Supported interfaces:
+
+- (all)
+
+```{code-cell}
+model = MyModel(array=("docs/data/test.h5", "/data"))
+data = model.model_dump_json(
+    round_trip=True, 
+    context={"absolute_paths": True}
+    )
+print_json(data)
+```
+
+### `relative_to`
+
+Make all paths (that exist) relative to the given path
+
+Supported interfaces:
+
+- (all)
+
+```{code-cell}
+model = MyModel(array=("docs/data/test.h5", "/data"))
+data = model.model_dump_json(
+    round_trip=True, 
+    context={"relative_to": Path('../')}
+    )
+print_json(data)
+```
+
+### `dump_array`
+
+Dump the raw array contents when serializing to json inside an `array` field
+
+Supported interfaces:
+- {class}`.ZarrInterface`
+
+```{code-cell}
+model = MyModel(array=("docs/data/test.zarr",))
+data = model.model_dump_json(
+    round_trip=True, 
+    context={"dump_array": True}
+    )
+print_json(data)
+```
+
+
+
+[^normalstyle]: o ya we're posting JSON [normal style](https://normal.style)
+[^notenforced]: This is only *functionally* enforced at the moment, where 
+  a roundtrip test confirms that dtype and type are preserved,
+  but there is no formal test for each interface having its own serialization class
+
+
+
diff --git a/pdm.lock b/pdm.lock
index 41818a0..635d5b3 100644
--- a/pdm.lock
+++ b/pdm.lock
@@ -2,10 +2,13 @@
 # It is not intended for manual editing.
 
 [metadata]
-groups = ["default", "arrays", "dask", "dev", "docs", "hdf5", "tests", "video"]
+groups = ["default", "arrays", "dask", "dev", "docs", "hdf5", "tests", "video", "zarr"]
 strategy = ["cross_platform", "inherit_metadata"]
-lock_version = "4.4.2"
-content_hash = "sha256:62c153ae2eb738b173ac882538810a727eee2ac364ea674c0d8a1ef310bbefd6"
+lock_version = "4.5.0"
+content_hash = "sha256:607f6694faec9103ba99e8e51010e905ad9ce03612dfb492e0d3fbeb1cb4d562"
+
+[[metadata.targets]]
+requires_python = "~=3.9"
 
 [[package]]
 name = "alabaster"
@@ -23,7 +26,10 @@ name = "annotated-types"
 version = "0.7.0"
 requires_python = ">=3.8"
 summary = "Reusable constraint types to use with typing.Annotated"
-groups = ["arrays", "default", "dev", "docs", "tests"]
+groups = ["default", "dev", "docs"]
+dependencies = [
+    "typing-extensions>=4.0.0; python_version < \"3.9\"",
+]
 files = [
     {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"},
     {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"},
@@ -46,15 +52,55 @@ files = [
     {file = "anyio-4.4.0.tar.gz", hash = "sha256:5aadc6a1bbb7cdb0bede386cac5e2940f5e2ff3aa20277e991cf028e0585ce94"},
 ]
 
+[[package]]
+name = "appnope"
+version = "0.1.4"
+requires_python = ">=3.6"
+summary = "Disable App Nap on macOS >= 10.9"
+groups = ["dev", "docs"]
+marker = "platform_system == \"Darwin\""
+files = [
+    {file = "appnope-0.1.4-py2.py3-none-any.whl", hash = "sha256:502575ee11cd7a28c0205f379b525beefebab9d161b7c964670864014ed7213c"},
+    {file = "appnope-0.1.4.tar.gz", hash = "sha256:1de3860566df9caf38f01f86f65e0e13e379af54f9e4bee1e66b48f2efffd1ee"},
+]
+
 [[package]]
 name = "asciitree"
 version = "0.3.3"
 summary = "Draws ASCII trees."
-groups = ["arrays", "dev", "tests"]
+groups = ["arrays", "dev", "tests", "zarr"]
 files = [
     {file = "asciitree-0.3.3.tar.gz", hash = "sha256:4aa4b9b649f85e3fcb343363d97564aa1fb62e249677f2e18a96765145cc0f6e"},
 ]
 
+[[package]]
+name = "asttokens"
+version = "2.4.1"
+summary = "Annotate AST trees with source code positions"
+groups = ["dev", "docs"]
+dependencies = [
+    "six>=1.12.0",
+    "typing; python_version < \"3.5\"",
+]
+files = [
+    {file = "asttokens-2.4.1-py2.py3-none-any.whl", hash = "sha256:051ed49c3dcae8913ea7cd08e46a606dba30b79993209636c4875bc1d637bc24"},
+    {file = "asttokens-2.4.1.tar.gz", hash = "sha256:b03869718ba9a6eb027e134bfdf69f38a236d681c83c160d510768af11254ba0"},
+]
+
+[[package]]
+name = "attrs"
+version = "24.2.0"
+requires_python = ">=3.7"
+summary = "Classes Without Boilerplate"
+groups = ["dev", "docs"]
+dependencies = [
+    "importlib-metadata; python_version < \"3.8\"",
+]
+files = [
+    {file = "attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"},
+    {file = "attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346"},
+]
+
 [[package]]
 name = "autodoc-pydantic"
 version = "2.2.0"
@@ -63,6 +109,7 @@ summary = "Seamlessly integrate pydantic models in your Sphinx documentation."
 groups = ["dev", "docs"]
 dependencies = [
     "Sphinx>=4.0",
+    "importlib-metadata>1; python_version <= \"3.8\"",
     "pydantic-settings<3.0.0,>=2.0",
     "pydantic<3.0.0,>=2.0",
 ]
@@ -76,6 +123,9 @@ version = "2.15.0"
 requires_python = ">=3.8"
 summary = "Internationalization utilities"
 groups = ["dev", "docs"]
+dependencies = [
+    "pytz>=2015.7; python_version < \"3.9\"",
+]
 files = [
     {file = "Babel-2.15.0-py3-none-any.whl", hash = "sha256:08706bdad8d0a3413266ab61bd6c34d0c28d6e1e7badf40a2cebe67644e2e1fb"},
     {file = "babel-2.15.0.tar.gz", hash = "sha256:8daf0e265d05768bc6c7a314cf1321e9a123afc328cc635c18622a2f30a04413"},
@@ -142,6 +192,78 @@ files = [
     {file = "certifi-2024.7.4.tar.gz", hash = "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b"},
 ]
 
+[[package]]
+name = "cffi"
+version = "1.17.1"
+requires_python = ">=3.8"
+summary = "Foreign Function Interface for Python calling C code."
+groups = ["dev", "docs"]
+marker = "implementation_name == \"pypy\""
+dependencies = [
+    "pycparser",
+]
+files = [
+    {file = "cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14"},
+    {file = "cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67"},
+    {file = "cffi-1.17.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:edae79245293e15384b51f88b00613ba9f7198016a5948b5dddf4917d4d26382"},
+    {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45398b671ac6d70e67da8e4224a065cec6a93541bb7aebe1b198a61b58c7b702"},
+    {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad9413ccdeda48c5afdae7e4fa2192157e991ff761e7ab8fdd8926f40b160cc3"},
+    {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5da5719280082ac6bd9aa7becb3938dc9f9cbd57fac7d2871717b1feb0902ab6"},
+    {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bb1a08b8008b281856e5971307cc386a8e9c5b625ac297e853d36da6efe9c17"},
+    {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8"},
+    {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6883e737d7d9e4899a8a695e00ec36bd4e5e4f18fabe0aca0efe0a4b44cdb13e"},
+    {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6b8b4a92e1c65048ff98cfe1f735ef8f1ceb72e3d5f0c25fdb12087a23da22be"},
+    {file = "cffi-1.17.1-cp310-cp310-win32.whl", hash = "sha256:c9c3d058ebabb74db66e431095118094d06abf53284d9c81f27300d0e0d8bc7c"},
+    {file = "cffi-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:0f048dcf80db46f0098ccac01132761580d28e28bc0f78ae0d58048063317e15"},
+    {file = "cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401"},
+    {file = "cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf"},
+    {file = "cffi-1.17.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4"},
+    {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41"},
+    {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1"},
+    {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6"},
+    {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d"},
+    {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6"},
+    {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f"},
+    {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b"},
+    {file = "cffi-1.17.1-cp311-cp311-win32.whl", hash = "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655"},
+    {file = "cffi-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0"},
+    {file = "cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4"},
+    {file = "cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c"},
+    {file = "cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36"},
+    {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5"},
+    {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff"},
+    {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99"},
+    {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93"},
+    {file = "cffi-1.17.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3"},
+    {file = "cffi-1.17.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8"},
+    {file = "cffi-1.17.1-cp312-cp312-win32.whl", hash = "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65"},
+    {file = "cffi-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903"},
+    {file = "cffi-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e"},
+    {file = "cffi-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2"},
+    {file = "cffi-1.17.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3"},
+    {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683"},
+    {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5"},
+    {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4"},
+    {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd"},
+    {file = "cffi-1.17.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed"},
+    {file = "cffi-1.17.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9"},
+    {file = "cffi-1.17.1-cp313-cp313-win32.whl", hash = "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d"},
+    {file = "cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a"},
+    {file = "cffi-1.17.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b2ab587605f4ba0bf81dc0cb08a41bd1c0a5906bd59243d56bad7668a6fc6c16"},
+    {file = "cffi-1.17.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:28b16024becceed8c6dfbc75629e27788d8a3f9030691a1dbf9821a128b22c36"},
+    {file = "cffi-1.17.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1d599671f396c4723d016dbddb72fe8e0397082b0a77a4fab8028923bec050e8"},
+    {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca74b8dbe6e8e8263c0ffd60277de77dcee6c837a3d0881d8c1ead7268c9e576"},
+    {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7f5baafcc48261359e14bcd6d9bff6d4b28d9103847c9e136694cb0501aef87"},
+    {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:98e3969bcff97cae1b2def8ba499ea3d6f31ddfdb7635374834cf89a1a08ecf0"},
+    {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cdf5ce3acdfd1661132f2a9c19cac174758dc2352bfe37d98aa7512c6b7178b3"},
+    {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9755e4345d1ec879e3849e62222a18c7174d65a6a92d5b346b1863912168b595"},
+    {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f1e22e8c4419538cb197e4dd60acc919d7696e5ef98ee4da4e01d3f8cfa4cc5a"},
+    {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c03e868a0b3bc35839ba98e74211ed2b05d2119be4e8a0f224fba9384f1fe02e"},
+    {file = "cffi-1.17.1-cp39-cp39-win32.whl", hash = "sha256:e31ae45bc2e29f6b2abd0de1cc3b9d5205aa847cafaecb8af1476a609a2f6eb7"},
+    {file = "cffi-1.17.1-cp39-cp39-win_amd64.whl", hash = "sha256:d016c76bdd850f3c626af19b0542c9677ba156e4ee4fccfdd7848803533ef662"},
+    {file = "cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824"},
+]
+
 [[package]]
 name = "charset-normalizer"
 version = "3.3.2"
@@ -218,9 +340,10 @@ name = "click"
 version = "8.1.7"
 requires_python = ">=3.7"
 summary = "Composable command line interface toolkit"
-groups = ["arrays", "dask", "dev", "tests"]
+groups = ["arrays", "dask", "dev", "docs", "tests"]
 dependencies = [
     "colorama; platform_system == \"Windows\"",
+    "importlib-metadata; python_version < \"3.8\"",
 ]
 files = [
     {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"},
@@ -249,6 +372,20 @@ files = [
     {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"},
 ]
 
+[[package]]
+name = "comm"
+version = "0.2.2"
+requires_python = ">=3.8"
+summary = "Jupyter Python Comm implementation, for usage in ipykernel, xeus-python etc."
+groups = ["dev", "docs"]
+dependencies = [
+    "traitlets>=4",
+]
+files = [
+    {file = "comm-0.2.2-py3-none-any.whl", hash = "sha256:e6fb86cb70ff661ee8c9c14e7d36d6de3b4066f1441be4063df9c5009f0a64d3"},
+    {file = "comm-0.2.2.tar.gz", hash = "sha256:3fd7a84065306e07bea1773df6eb8282de51ba82f77c72f9c85716ab11fe980e"},
+]
+
 [[package]]
 name = "coverage"
 version = "6.5.0"
@@ -371,6 +508,44 @@ files = [
     {file = "dask-2024.7.1.tar.gz", hash = "sha256:dbaef2d50efee841a9d981a218cfeb50392fc9a95e0403b6d680450e4f50d531"},
 ]
 
+[[package]]
+name = "debugpy"
+version = "1.8.5"
+requires_python = ">=3.8"
+summary = "An implementation of the Debug Adapter Protocol for Python"
+groups = ["dev", "docs"]
+files = [
+    {file = "debugpy-1.8.5-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:7e4d594367d6407a120b76bdaa03886e9eb652c05ba7f87e37418426ad2079f7"},
+    {file = "debugpy-1.8.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4413b7a3ede757dc33a273a17d685ea2b0c09dbd312cc03f5534a0fd4d40750a"},
+    {file = "debugpy-1.8.5-cp310-cp310-win32.whl", hash = "sha256:dd3811bd63632bb25eda6bd73bea8e0521794cda02be41fa3160eb26fc29e7ed"},
+    {file = "debugpy-1.8.5-cp310-cp310-win_amd64.whl", hash = "sha256:b78c1250441ce893cb5035dd6f5fc12db968cc07f91cc06996b2087f7cefdd8e"},
+    {file = "debugpy-1.8.5-cp311-cp311-macosx_12_0_universal2.whl", hash = "sha256:606bccba19f7188b6ea9579c8a4f5a5364ecd0bf5a0659c8a5d0e10dcee3032a"},
+    {file = "debugpy-1.8.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db9fb642938a7a609a6c865c32ecd0d795d56c1aaa7a7a5722d77855d5e77f2b"},
+    {file = "debugpy-1.8.5-cp311-cp311-win32.whl", hash = "sha256:4fbb3b39ae1aa3e5ad578f37a48a7a303dad9a3d018d369bc9ec629c1cfa7408"},
+    {file = "debugpy-1.8.5-cp311-cp311-win_amd64.whl", hash = "sha256:345d6a0206e81eb68b1493ce2fbffd57c3088e2ce4b46592077a943d2b968ca3"},
+    {file = "debugpy-1.8.5-cp312-cp312-macosx_12_0_universal2.whl", hash = "sha256:5b5c770977c8ec6c40c60d6f58cacc7f7fe5a45960363d6974ddb9b62dbee156"},
+    {file = "debugpy-1.8.5-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0a65b00b7cdd2ee0c2cf4c7335fef31e15f1b7056c7fdbce9e90193e1a8c8cb"},
+    {file = "debugpy-1.8.5-cp312-cp312-win32.whl", hash = "sha256:c9f7c15ea1da18d2fcc2709e9f3d6de98b69a5b0fff1807fb80bc55f906691f7"},
+    {file = "debugpy-1.8.5-cp312-cp312-win_amd64.whl", hash = "sha256:28ced650c974aaf179231668a293ecd5c63c0a671ae6d56b8795ecc5d2f48d3c"},
+    {file = "debugpy-1.8.5-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:0a1029a2869d01cb777216af8c53cda0476875ef02a2b6ff8b2f2c9a4b04176c"},
+    {file = "debugpy-1.8.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e84c276489e141ed0b93b0af648eef891546143d6a48f610945416453a8ad406"},
+    {file = "debugpy-1.8.5-cp39-cp39-win32.whl", hash = "sha256:ad84b7cde7fd96cf6eea34ff6c4a1b7887e0fe2ea46e099e53234856f9d99a34"},
+    {file = "debugpy-1.8.5-cp39-cp39-win_amd64.whl", hash = "sha256:7b0fe36ed9d26cb6836b0a51453653f8f2e347ba7348f2bbfe76bfeb670bfb1c"},
+    {file = "debugpy-1.8.5-py2.py3-none-any.whl", hash = "sha256:55919dce65b471eff25901acf82d328bbd5b833526b6c1364bd5133754777a44"},
+    {file = "debugpy-1.8.5.zip", hash = "sha256:b2112cfeb34b4507399d298fe7023a16656fc553ed5246536060ca7bd0e668d0"},
+]
+
+[[package]]
+name = "decorator"
+version = "5.1.1"
+requires_python = ">=3.5"
+summary = "Decorators for Humans"
+groups = ["dev", "docs"]
+files = [
+    {file = "decorator-5.1.1-py3-none-any.whl", hash = "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186"},
+    {file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"},
+]
+
 [[package]]
 name = "docopt"
 version = "0.6.2"
@@ -396,25 +571,46 @@ name = "exceptiongroup"
 version = "1.2.2"
 requires_python = ">=3.7"
 summary = "Backport of PEP 654 (exception groups)"
-groups = ["dev", "tests"]
+groups = ["dev", "docs", "tests"]
 marker = "python_version < \"3.11\""
 files = [
     {file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"},
     {file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"},
 ]
 
+[[package]]
+name = "executing"
+version = "2.1.0"
+requires_python = ">=3.8"
+summary = "Get the currently executing AST node of a frame, and other information"
+groups = ["dev", "docs"]
+files = [
+    {file = "executing-2.1.0-py2.py3-none-any.whl", hash = "sha256:8d63781349375b5ebccc3142f4b30350c0cd9c79f921cde38be2be4637e98eaf"},
+    {file = "executing-2.1.0.tar.gz", hash = "sha256:8ea27ddd260da8150fa5a708269c4a10e76161e2496ec3e587da9e3c0fe4b9ab"},
+]
+
 [[package]]
 name = "fasteners"
 version = "0.19"
 requires_python = ">=3.6"
 summary = "A python package that provides useful locks"
-groups = ["arrays", "dev", "tests"]
+groups = ["arrays", "dev", "tests", "zarr"]
 marker = "sys_platform != \"emscripten\""
 files = [
     {file = "fasteners-0.19-py3-none-any.whl", hash = "sha256:758819cb5d94cdedf4e836988b74de396ceacb8e2794d21f82d131fd9ee77237"},
     {file = "fasteners-0.19.tar.gz", hash = "sha256:b4f37c3ac52d8a445af3a66bce57b33b5e90b97c696b7b984f530cf8f0ded09c"},
 ]
 
+[[package]]
+name = "fastjsonschema"
+version = "2.20.0"
+summary = "Fastest Python implementation of JSON schema"
+groups = ["dev", "docs"]
+files = [
+    {file = "fastjsonschema-2.20.0-py3-none-any.whl", hash = "sha256:5875f0b0fa7a0043a91e93a9b8f793bcbbba9691e7fd83dca95c28ba26d21f0a"},
+    {file = "fastjsonschema-2.20.0.tar.gz", hash = "sha256:3d48fc5300ee96f5d116f10fe6f28d938e6008f59a6a025c2649475b87f76a23"},
+]
+
 [[package]]
 name = "fsspec"
 version = "2024.6.1"
@@ -449,17 +645,87 @@ version = "1.2.0"
 requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 summary = "A backport of fstrings to python<3.6"
 groups = ["dev", "tests"]
+dependencies = [
+    "tokenize-rt>=3; python_version < \"3.6\"",
+]
 files = [
     {file = "future_fstrings-1.2.0-py2.py3-none-any.whl", hash = "sha256:90e49598b553d8746c4dc7d9442e0359d038c3039d802c91c0a55505da318c63"},
     {file = "future_fstrings-1.2.0.tar.gz", hash = "sha256:6cf41cbe97c398ab5a81168ce0dbb8ad95862d3caf23c21e4430627b90844089"},
 ]
 
+[[package]]
+name = "greenlet"
+version = "3.1.1"
+requires_python = ">=3.7"
+summary = "Lightweight in-process concurrent programming"
+groups = ["dev", "docs"]
+marker = "(platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\") and python_version < \"3.13\""
+files = [
+    {file = "greenlet-3.1.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:0bbae94a29c9e5c7e4a2b7f0aae5c17e8e90acbfd3bf6270eeba60c39fce3563"},
+    {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fde093fb93f35ca72a556cf72c92ea3ebfda3d79fc35bb19fbe685853869a83"},
+    {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:36b89d13c49216cadb828db8dfa6ce86bbbc476a82d3a6c397f0efae0525bdd0"},
+    {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:94b6150a85e1b33b40b1464a3f9988dcc5251d6ed06842abff82e42632fac120"},
+    {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93147c513fac16385d1036b7e5b102c7fbbdb163d556b791f0f11eada7ba65dc"},
+    {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:da7a9bff22ce038e19bf62c4dd1ec8391062878710ded0a845bcf47cc0200617"},
+    {file = "greenlet-3.1.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b2795058c23988728eec1f36a4e5e4ebad22f8320c85f3587b539b9ac84128d7"},
+    {file = "greenlet-3.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ed10eac5830befbdd0c32f83e8aa6288361597550ba669b04c48f0f9a2c843c6"},
+    {file = "greenlet-3.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:77c386de38a60d1dfb8e55b8c1101d68c79dfdd25c7095d51fec2dd800892b80"},
+    {file = "greenlet-3.1.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:e4d333e558953648ca09d64f13e6d8f0523fa705f51cae3f03b5983489958c70"},
+    {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09fc016b73c94e98e29af67ab7b9a879c307c6731a2c9da0db5a7d9b7edd1159"},
+    {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d5e975ca70269d66d17dd995dafc06f1b06e8cb1ec1e9ed54c1d1e4a7c4cf26e"},
+    {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3b2813dc3de8c1ee3f924e4d4227999285fd335d1bcc0d2be6dc3f1f6a318ec1"},
+    {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e347b3bfcf985a05e8c0b7d462ba6f15b1ee1c909e2dcad795e49e91b152c383"},
+    {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e8f8c9cb53cdac7ba9793c276acd90168f416b9ce36799b9b885790f8ad6c0a"},
+    {file = "greenlet-3.1.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:62ee94988d6b4722ce0028644418d93a52429e977d742ca2ccbe1c4f4a792511"},
+    {file = "greenlet-3.1.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1776fd7f989fc6b8d8c8cb8da1f6b82c5814957264d1f6cf818d475ec2bf6395"},
+    {file = "greenlet-3.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:48ca08c771c268a768087b408658e216133aecd835c0ded47ce955381105ba39"},
+    {file = "greenlet-3.1.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:4afe7ea89de619adc868e087b4d2359282058479d7cfb94970adf4b55284574d"},
+    {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f406b22b7c9a9b4f8aa9d2ab13d6ae0ac3e85c9a809bd590ad53fed2bf70dc79"},
+    {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c3a701fe5a9695b238503ce5bbe8218e03c3bcccf7e204e455e7462d770268aa"},
+    {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2846930c65b47d70b9d178e89c7e1a69c95c1f68ea5aa0a58646b7a96df12441"},
+    {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99cfaa2110534e2cf3ba31a7abcac9d328d1d9f1b95beede58294a60348fba36"},
+    {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1443279c19fca463fc33e65ef2a935a5b09bb90f978beab37729e1c3c6c25fe9"},
+    {file = "greenlet-3.1.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b7cede291382a78f7bb5f04a529cb18e068dd29e0fb27376074b6d0317bf4dd0"},
+    {file = "greenlet-3.1.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:23f20bb60ae298d7d8656c6ec6db134bca379ecefadb0b19ce6f19d1f232a942"},
+    {file = "greenlet-3.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:7124e16b4c55d417577c2077be379514321916d5790fa287c9ed6f23bd2ffd01"},
+    {file = "greenlet-3.1.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:05175c27cb459dcfc05d026c4232f9de8913ed006d42713cb8a5137bd49375f1"},
+    {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:935e943ec47c4afab8965954bf49bfa639c05d4ccf9ef6e924188f762145c0ff"},
+    {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:667a9706c970cb552ede35aee17339a18e8f2a87a51fba2ed39ceeeb1004798a"},
+    {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b8a678974d1f3aa55f6cc34dc480169d58f2e6d8958895d68845fa4ab566509e"},
+    {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efc0f674aa41b92da8c49e0346318c6075d734994c3c4e4430b1c3f853e498e4"},
+    {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0153404a4bb921f0ff1abeb5ce8a5131da56b953eda6e14b88dc6bbc04d2049e"},
+    {file = "greenlet-3.1.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:275f72decf9932639c1c6dd1013a1bc266438eb32710016a1c742df5da6e60a1"},
+    {file = "greenlet-3.1.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:c4aab7f6381f38a4b42f269057aee279ab0fc7bf2e929e3d4abfae97b682a12c"},
+    {file = "greenlet-3.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:b42703b1cf69f2aa1df7d1030b9d77d3e584a70755674d60e710f0af570f3761"},
+    {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1695e76146579f8c06c1509c7ce4dfe0706f49c6831a817ac04eebb2fd02011"},
+    {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7876452af029456b3f3549b696bb36a06db7c90747740c5302f74a9e9fa14b13"},
+    {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4ead44c85f8ab905852d3de8d86f6f8baf77109f9da589cb4fa142bd3b57b475"},
+    {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8320f64b777d00dd7ccdade271eaf0cad6636343293a25074cc5566160e4de7b"},
+    {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6510bf84a6b643dabba74d3049ead221257603a253d0a9873f55f6a59a65f822"},
+    {file = "greenlet-3.1.1-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:04b013dc07c96f83134b1e99888e7a79979f1a247e2a9f59697fa14b5862ed01"},
+    {file = "greenlet-3.1.1-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:411f015496fec93c1c8cd4e5238da364e1da7a124bcb293f085bf2860c32c6f6"},
+    {file = "greenlet-3.1.1-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:396979749bd95f018296af156201d6211240e7a23090f50a8d5d18c370084dc3"},
+    {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca9d0ff5ad43e785350894d97e13633a66e2b50000e8a183a50a88d834752d42"},
+    {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f6ff3b14f2df4c41660a7dec01045a045653998784bf8cfcb5a525bdffffbc8f"},
+    {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:94ebba31df2aa506d7b14866fed00ac141a867e63143fe5bca82a8e503b36437"},
+    {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73aaad12ac0ff500f62cebed98d8789198ea0e6f233421059fa68a5aa7220145"},
+    {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:63e4844797b975b9af3a3fb8f7866ff08775f5426925e1e0bbcfe7932059a12c"},
+    {file = "greenlet-3.1.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7939aa3ca7d2a1593596e7ac6d59391ff30281ef280d8632fa03d81f7c5f955e"},
+    {file = "greenlet-3.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d0028e725ee18175c6e422797c407874da24381ce0690d6b9396c204c7f7276e"},
+    {file = "greenlet-3.1.1-cp39-cp39-win32.whl", hash = "sha256:5e06afd14cbaf9e00899fae69b24a32f2196c19de08fcb9f4779dd4f004e5e7c"},
+    {file = "greenlet-3.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:3319aa75e0e0639bc15ff54ca327e8dc7a6fe404003496e3c6925cd3142e0e22"},
+    {file = "greenlet-3.1.1.tar.gz", hash = "sha256:4ce3ac6cdb6adf7946475d7ef31777c26d94bccc377e070a7986bd2d5c515467"},
+]
+
 [[package]]
 name = "h11"
 version = "0.14.0"
 requires_python = ">=3.7"
 summary = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1"
 groups = ["dev"]
+dependencies = [
+    "typing-extensions; python_version < \"3.8\"",
+]
 files = [
     {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"},
     {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"},
@@ -522,8 +788,8 @@ version = "8.2.0"
 requires_python = ">=3.8"
 summary = "Read metadata from Python packages"
 groups = ["arrays", "dask", "dev", "docs", "tests"]
-marker = "python_version < \"3.12\""
 dependencies = [
+    "typing-extensions>=3.6.4; python_version < \"3.8\"",
     "zipp>=0.5",
 ]
 files = [
@@ -542,6 +808,70 @@ files = [
     {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"},
 ]
 
+[[package]]
+name = "ipykernel"
+version = "6.29.5"
+requires_python = ">=3.8"
+summary = "IPython Kernel for Jupyter"
+groups = ["dev", "docs"]
+dependencies = [
+    "appnope; platform_system == \"Darwin\"",
+    "comm>=0.1.1",
+    "debugpy>=1.6.5",
+    "ipython>=7.23.1",
+    "jupyter-client>=6.1.12",
+    "jupyter-core!=5.0.*,>=4.12",
+    "matplotlib-inline>=0.1",
+    "nest-asyncio",
+    "packaging",
+    "psutil",
+    "pyzmq>=24",
+    "tornado>=6.1",
+    "traitlets>=5.4.0",
+]
+files = [
+    {file = "ipykernel-6.29.5-py3-none-any.whl", hash = "sha256:afdb66ba5aa354b09b91379bac28ae4afebbb30e8b39510c9690afb7a10421b5"},
+    {file = "ipykernel-6.29.5.tar.gz", hash = "sha256:f093a22c4a40f8828f8e330a9c297cb93dcab13bd9678ded6de8e5cf81c56215"},
+]
+
+[[package]]
+name = "ipython"
+version = "8.18.1"
+requires_python = ">=3.9"
+summary = "IPython: Productive Interactive Computing"
+groups = ["dev", "docs"]
+dependencies = [
+    "colorama; sys_platform == \"win32\"",
+    "decorator",
+    "exceptiongroup; python_version < \"3.11\"",
+    "jedi>=0.16",
+    "matplotlib-inline",
+    "pexpect>4.3; sys_platform != \"win32\"",
+    "prompt-toolkit<3.1.0,>=3.0.41",
+    "pygments>=2.4.0",
+    "stack-data",
+    "traitlets>=5",
+    "typing-extensions; python_version < \"3.10\"",
+]
+files = [
+    {file = "ipython-8.18.1-py3-none-any.whl", hash = "sha256:e8267419d72d81955ec1177f8a29aaa90ac80ad647499201119e2f05e99aa397"},
+    {file = "ipython-8.18.1.tar.gz", hash = "sha256:ca6f079bb33457c66e233e4580ebfc4128855b4cf6370dddd73842a9563e8a27"},
+]
+
+[[package]]
+name = "jedi"
+version = "0.19.1"
+requires_python = ">=3.6"
+summary = "An autocompletion tool for Python that can be used for text editors."
+groups = ["dev", "docs"]
+dependencies = [
+    "parso<0.9.0,>=0.8.3",
+]
+files = [
+    {file = "jedi-0.19.1-py2.py3-none-any.whl", hash = "sha256:e983c654fe5c02867aef4cdfce5a2fbb4a50adc0af145f70504238f18ef5e7e0"},
+    {file = "jedi-0.19.1.tar.gz", hash = "sha256:cf0496f3651bc65d7174ac1b7d043eff454892c708a87d1b683e57b569927ffd"},
+]
+
 [[package]]
 name = "jinja2"
 version = "3.1.4"
@@ -556,6 +886,96 @@ files = [
     {file = "jinja2-3.1.4.tar.gz", hash = "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369"},
 ]
 
+[[package]]
+name = "jsonschema"
+version = "4.23.0"
+requires_python = ">=3.8"
+summary = "An implementation of JSON Schema validation for Python"
+groups = ["dev", "docs"]
+dependencies = [
+    "attrs>=22.2.0",
+    "importlib-resources>=1.4.0; python_version < \"3.9\"",
+    "jsonschema-specifications>=2023.03.6",
+    "pkgutil-resolve-name>=1.3.10; python_version < \"3.9\"",
+    "referencing>=0.28.4",
+    "rpds-py>=0.7.1",
+]
+files = [
+    {file = "jsonschema-4.23.0-py3-none-any.whl", hash = "sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566"},
+    {file = "jsonschema-4.23.0.tar.gz", hash = "sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4"},
+]
+
+[[package]]
+name = "jsonschema-specifications"
+version = "2023.12.1"
+requires_python = ">=3.8"
+summary = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry"
+groups = ["dev", "docs"]
+dependencies = [
+    "importlib-resources>=1.4.0; python_version < \"3.9\"",
+    "referencing>=0.31.0",
+]
+files = [
+    {file = "jsonschema_specifications-2023.12.1-py3-none-any.whl", hash = "sha256:87e4fdf3a94858b8a2ba2778d9ba57d8a9cafca7c7489c46ba0d30a8bc6a9c3c"},
+    {file = "jsonschema_specifications-2023.12.1.tar.gz", hash = "sha256:48a76787b3e70f5ed53f1160d2b81f586e4ca6d1548c5de7085d1682674764cc"},
+]
+
+[[package]]
+name = "jupyter-cache"
+version = "1.0.0"
+requires_python = ">=3.9"
+summary = "A defined interface for working with a cache of jupyter notebooks."
+groups = ["dev", "docs"]
+dependencies = [
+    "attrs",
+    "click",
+    "importlib-metadata",
+    "nbclient>=0.2",
+    "nbformat",
+    "pyyaml",
+    "sqlalchemy<3,>=1.3.12",
+    "tabulate",
+]
+files = [
+    {file = "jupyter_cache-1.0.0-py3-none-any.whl", hash = "sha256:594b1c4e29b488b36547e12477645f489dbdc62cc939b2408df5679f79245078"},
+    {file = "jupyter_cache-1.0.0.tar.gz", hash = "sha256:d0fa7d7533cd5798198d8889318269a8c1382ed3b22f622c09a9356521f48687"},
+]
+
+[[package]]
+name = "jupyter-client"
+version = "8.6.3"
+requires_python = ">=3.8"
+summary = "Jupyter protocol implementation and client libraries"
+groups = ["dev", "docs"]
+dependencies = [
+    "importlib-metadata>=4.8.3; python_version < \"3.10\"",
+    "jupyter-core!=5.0.*,>=4.12",
+    "python-dateutil>=2.8.2",
+    "pyzmq>=23.0",
+    "tornado>=6.2",
+    "traitlets>=5.3",
+]
+files = [
+    {file = "jupyter_client-8.6.3-py3-none-any.whl", hash = "sha256:e8a19cc986cc45905ac3362915f410f3af85424b4c0905e94fa5f2cb08e8f23f"},
+    {file = "jupyter_client-8.6.3.tar.gz", hash = "sha256:35b3a0947c4a6e9d589eb97d7d4cd5e90f910ee73101611f01283732bd6d9419"},
+]
+
+[[package]]
+name = "jupyter-core"
+version = "5.7.2"
+requires_python = ">=3.8"
+summary = "Jupyter core package. A base package on which Jupyter projects rely."
+groups = ["dev", "docs"]
+dependencies = [
+    "platformdirs>=2.5",
+    "pywin32>=300; sys_platform == \"win32\" and platform_python_implementation != \"PyPy\"",
+    "traitlets>=5.3",
+]
+files = [
+    {file = "jupyter_core-5.7.2-py3-none-any.whl", hash = "sha256:4f7315d2f6b4bcf2e3e7cb6e46772eba760ae459cd1f59d29eb57b0a01bd7409"},
+    {file = "jupyter_core-5.7.2.tar.gz", hash = "sha256:aa5f8d32bbf6b431ac830496da7392035d6f61b4f54872f15c4bd2a9c3f536d9"},
+]
+
 [[package]]
 name = "locket"
 version = "1.0.0"
@@ -631,6 +1051,20 @@ files = [
     {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"},
 ]
 
+[[package]]
+name = "matplotlib-inline"
+version = "0.1.7"
+requires_python = ">=3.8"
+summary = "Inline Matplotlib backend for Jupyter"
+groups = ["dev", "docs"]
+dependencies = [
+    "traitlets",
+]
+files = [
+    {file = "matplotlib_inline-0.1.7-py3-none-any.whl", hash = "sha256:df192d39a4ff8f21b1895d72e6a13f5fcc5099f00fa84384e0ea28c2cc0653ca"},
+    {file = "matplotlib_inline-0.1.7.tar.gz", hash = "sha256:8423b23ec666be3d16e16b60bdd8ac4e86e840ebd1dd11a30b9f117f2fa0ab90"},
+]
+
 [[package]]
 name = "mdit-py-plugins"
 version = "0.4.1"
@@ -667,6 +1101,29 @@ files = [
     {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"},
 ]
 
+[[package]]
+name = "myst-nb"
+version = "1.1.1"
+requires_python = ">=3.9"
+summary = "A Jupyter Notebook Sphinx reader built on top of the MyST markdown parser."
+groups = ["dev", "docs"]
+dependencies = [
+    "importlib-metadata",
+    "ipykernel",
+    "ipython",
+    "jupyter-cache>=0.5",
+    "myst-parser>=1.0.0",
+    "nbclient",
+    "nbformat>=5.0",
+    "pyyaml",
+    "sphinx>=5",
+    "typing-extensions",
+]
+files = [
+    {file = "myst_nb-1.1.1-py3-none-any.whl", hash = "sha256:8b8f9085287d948eef46cb3764aafc21915e0e981882b8c742719f5b1a84c36f"},
+    {file = "myst_nb-1.1.1.tar.gz", hash = "sha256:74227c11f76d03494f43b7788659b161b94f4dedef230a2912412bc8c3c9e553"},
+]
+
 [[package]]
 name = "myst-parser"
 version = "2.0.0"
@@ -686,6 +1143,51 @@ files = [
     {file = "myst_parser-2.0.0.tar.gz", hash = "sha256:ea929a67a6a0b1683cdbe19b8d2e724cd7643f8aa3e7bb18dd65beac3483bead"},
 ]
 
+[[package]]
+name = "nbclient"
+version = "0.10.0"
+requires_python = ">=3.8.0"
+summary = "A client library for executing notebooks. Formerly nbconvert's ExecutePreprocessor."
+groups = ["dev", "docs"]
+dependencies = [
+    "jupyter-client>=6.1.12",
+    "jupyter-core!=5.0.*,>=4.12",
+    "nbformat>=5.1",
+    "traitlets>=5.4",
+]
+files = [
+    {file = "nbclient-0.10.0-py3-none-any.whl", hash = "sha256:f13e3529332a1f1f81d82a53210322476a168bb7090a0289c795fe9cc11c9d3f"},
+    {file = "nbclient-0.10.0.tar.gz", hash = "sha256:4b3f1b7dba531e498449c4db4f53da339c91d449dc11e9af3a43b4eb5c5abb09"},
+]
+
+[[package]]
+name = "nbformat"
+version = "5.10.4"
+requires_python = ">=3.8"
+summary = "The Jupyter Notebook format"
+groups = ["dev", "docs"]
+dependencies = [
+    "fastjsonschema>=2.15",
+    "jsonschema>=2.6",
+    "jupyter-core!=5.0.*,>=4.12",
+    "traitlets>=5.1",
+]
+files = [
+    {file = "nbformat-5.10.4-py3-none-any.whl", hash = "sha256:3b48d6c8fbca4b299bf3982ea7db1af21580e4fec269ad087b9e81588891200b"},
+    {file = "nbformat-5.10.4.tar.gz", hash = "sha256:322168b14f937a5d11362988ecac2a4952d3d8e3a2cbeb2319584631226d5b3a"},
+]
+
+[[package]]
+name = "nest-asyncio"
+version = "1.6.0"
+requires_python = ">=3.5"
+summary = "Patch asyncio to allow nested event loops"
+groups = ["dev", "docs"]
+files = [
+    {file = "nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c"},
+    {file = "nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe"},
+]
+
 [[package]]
 name = "networkx"
 version = "3.2.1"
@@ -702,7 +1204,7 @@ name = "numcodecs"
 version = "0.12.1"
 requires_python = ">=3.8"
 summary = "A Python package providing buffer compression and transformation codecs for use in data storage and communication applications."
-groups = ["arrays", "dev", "tests"]
+groups = ["arrays", "dev", "tests", "zarr"]
 dependencies = [
     "numpy>=1.7",
 ]
@@ -731,7 +1233,7 @@ name = "numpy"
 version = "2.0.1"
 requires_python = ">=3.9"
 summary = "Fundamental package for array computing in Python"
-groups = ["arrays", "default", "dev", "hdf5", "tests", "video"]
+groups = ["default", "arrays", "dev", "hdf5", "tests", "video", "zarr"]
 files = [
     {file = "numpy-2.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0fbb536eac80e27a2793ffd787895242b7f18ef792563d742c2d673bfcb75134"},
     {file = "numpy-2.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:69ff563d43c69b1baba77af455dd0a839df8d25e8590e79c90fcbe1499ebde42"},
@@ -787,6 +1289,7 @@ requires_python = ">=3.6"
 summary = "Wrapper package for OpenCV python bindings."
 groups = ["arrays", "dev", "tests", "video"]
 dependencies = [
+    "numpy>=1.13.3; python_version < \"3.7\"",
     "numpy>=1.17.0; python_version >= \"3.7\"",
     "numpy>=1.17.3; python_version >= \"3.8\"",
     "numpy>=1.19.3; python_version >= \"3.6\" and platform_system == \"Linux\" and platform_machine == \"aarch64\"",
@@ -818,6 +1321,17 @@ files = [
     {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"},
 ]
 
+[[package]]
+name = "parso"
+version = "0.8.4"
+requires_python = ">=3.6"
+summary = "A Python Parser"
+groups = ["dev", "docs"]
+files = [
+    {file = "parso-0.8.4-py2.py3-none-any.whl", hash = "sha256:a418670a20291dacd2dddc80c377c5c3791378ee1e8d12bffc35420643d43f18"},
+    {file = "parso-0.8.4.tar.gz", hash = "sha256:eb3a7b58240fb99099a345571deecc0f9540ea5f4dd2fe14c2a99d6b281ab92d"},
+]
+
 [[package]]
 name = "partd"
 version = "1.4.2"
@@ -844,12 +1358,26 @@ files = [
     {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"},
 ]
 
+[[package]]
+name = "pexpect"
+version = "4.9.0"
+summary = "Pexpect allows easy control of interactive console applications."
+groups = ["dev", "docs"]
+marker = "sys_platform != \"win32\""
+dependencies = [
+    "ptyprocess>=0.5",
+]
+files = [
+    {file = "pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523"},
+    {file = "pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f"},
+]
+
 [[package]]
 name = "platformdirs"
 version = "4.2.2"
 requires_python = ">=3.8"
 summary = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`."
-groups = ["dev"]
+groups = ["dev", "docs"]
 files = [
     {file = "platformdirs-4.2.2-py3-none-any.whl", hash = "sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee"},
     {file = "platformdirs-4.2.2.tar.gz", hash = "sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3"},
@@ -866,12 +1394,76 @@ files = [
     {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"},
 ]
 
+[[package]]
+name = "prompt-toolkit"
+version = "3.0.47"
+requires_python = ">=3.7.0"
+summary = "Library for building powerful interactive command lines in Python"
+groups = ["dev", "docs"]
+dependencies = [
+    "wcwidth",
+]
+files = [
+    {file = "prompt_toolkit-3.0.47-py3-none-any.whl", hash = "sha256:0d7bfa67001d5e39d02c224b663abc33687405033a8c422d0d675a5a13361d10"},
+    {file = "prompt_toolkit-3.0.47.tar.gz", hash = "sha256:1e1b29cb58080b1e69f207c893a1a7bf16d127a5c30c9d17a25a5d77792e5360"},
+]
+
+[[package]]
+name = "psutil"
+version = "6.0.0"
+requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7"
+summary = "Cross-platform lib for process and system monitoring in Python."
+groups = ["dev", "docs"]
+files = [
+    {file = "psutil-6.0.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:c588a7e9b1173b6e866756dde596fd4cad94f9399daf99ad8c3258b3cb2b47a0"},
+    {file = "psutil-6.0.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ed2440ada7ef7d0d608f20ad89a04ec47d2d3ab7190896cd62ca5fc4fe08bf0"},
+    {file = "psutil-6.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fd9a97c8e94059b0ef54a7d4baf13b405011176c3b6ff257c247cae0d560ecd"},
+    {file = "psutil-6.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2e8d0054fc88153ca0544f5c4d554d42e33df2e009c4ff42284ac9ebdef4132"},
+    {file = "psutil-6.0.0-cp37-abi3-win32.whl", hash = "sha256:a495580d6bae27291324fe60cea0b5a7c23fa36a7cd35035a16d93bdcf076b9d"},
+    {file = "psutil-6.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:33ea5e1c975250a720b3a6609c490db40dae5d83a4eb315170c4fe0d8b1f34b3"},
+    {file = "psutil-6.0.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:ffe7fc9b6b36beadc8c322f84e1caff51e8703b88eee1da46d1e3a6ae11b4fd0"},
+    {file = "psutil-6.0.0.tar.gz", hash = "sha256:8faae4f310b6d969fa26ca0545338b21f73c6b15db7c4a8d934a5482faa818f2"},
+]
+
+[[package]]
+name = "ptyprocess"
+version = "0.7.0"
+summary = "Run a subprocess in a pseudo terminal"
+groups = ["dev", "docs"]
+marker = "sys_platform != \"win32\""
+files = [
+    {file = "ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35"},
+    {file = "ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220"},
+]
+
+[[package]]
+name = "pure-eval"
+version = "0.2.3"
+summary = "Safely evaluate AST nodes without side effects"
+groups = ["dev", "docs"]
+files = [
+    {file = "pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0"},
+    {file = "pure_eval-0.2.3.tar.gz", hash = "sha256:5f4e983f40564c576c7c8635ae88db5956bb2229d7e9237d03b3c0b0190eaf42"},
+]
+
+[[package]]
+name = "pycparser"
+version = "2.22"
+requires_python = ">=3.8"
+summary = "C parser in Python"
+groups = ["dev", "docs"]
+marker = "implementation_name == \"pypy\""
+files = [
+    {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"},
+    {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"},
+]
+
 [[package]]
 name = "pydantic"
 version = "2.8.2"
 requires_python = ">=3.8"
 summary = "Data validation using Python type hints"
-groups = ["arrays", "default", "dev", "docs", "tests"]
+groups = ["default", "dev", "docs"]
 dependencies = [
     "annotated-types>=0.4.0",
     "pydantic-core==2.20.1",
@@ -888,7 +1480,7 @@ name = "pydantic-core"
 version = "2.20.1"
 requires_python = ">=3.8"
 summary = "Core functionality for Pydantic validation and serialization"
-groups = ["arrays", "default", "dev", "docs", "tests"]
+groups = ["default", "dev", "docs"]
 dependencies = [
     "typing-extensions!=4.7.0,>=4.6.0",
 ]
@@ -1048,6 +1640,20 @@ files = [
     {file = "pytest_depends-1.0.1-py3-none-any.whl", hash = "sha256:a1df072bcc93d77aca3f0946903f5fed8af2d9b0056db1dfc9ed5ac164ab0642"},
 ]
 
+[[package]]
+name = "python-dateutil"
+version = "2.9.0.post0"
+requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
+summary = "Extensions to the standard Python datetime module"
+groups = ["dev", "docs"]
+dependencies = [
+    "six>=1.5",
+]
+files = [
+    {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"},
+    {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"},
+]
+
 [[package]]
 name = "python-dotenv"
 version = "1.0.1"
@@ -1059,6 +1665,25 @@ files = [
     {file = "python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"},
 ]
 
+[[package]]
+name = "pywin32"
+version = "306"
+summary = "Python for Window Extensions"
+groups = ["dev", "docs"]
+marker = "sys_platform == \"win32\" and platform_python_implementation != \"PyPy\""
+files = [
+    {file = "pywin32-306-cp310-cp310-win32.whl", hash = "sha256:06d3420a5155ba65f0b72f2699b5bacf3109f36acbe8923765c22938a69dfc8d"},
+    {file = "pywin32-306-cp310-cp310-win_amd64.whl", hash = "sha256:84f4471dbca1887ea3803d8848a1616429ac94a4a8d05f4bc9c5dcfd42ca99c8"},
+    {file = "pywin32-306-cp311-cp311-win32.whl", hash = "sha256:e65028133d15b64d2ed8f06dd9fbc268352478d4f9289e69c190ecd6818b6407"},
+    {file = "pywin32-306-cp311-cp311-win_amd64.whl", hash = "sha256:a7639f51c184c0272e93f244eb24dafca9b1855707d94c192d4a0b4c01e1100e"},
+    {file = "pywin32-306-cp311-cp311-win_arm64.whl", hash = "sha256:70dba0c913d19f942a2db25217d9a1b726c278f483a919f1abfed79c9cf64d3a"},
+    {file = "pywin32-306-cp312-cp312-win32.whl", hash = "sha256:383229d515657f4e3ed1343da8be101000562bf514591ff383ae940cad65458b"},
+    {file = "pywin32-306-cp312-cp312-win_amd64.whl", hash = "sha256:37257794c1ad39ee9be652da0462dc2e394c8159dfd913a8a4e8eb6fd346da0e"},
+    {file = "pywin32-306-cp312-cp312-win_arm64.whl", hash = "sha256:5821ec52f6d321aa59e2db7e0a35b997de60c201943557d108af9d4ae1ec7040"},
+    {file = "pywin32-306-cp39-cp39-win32.whl", hash = "sha256:e25fd5b485b55ac9c057f67d94bc203f3f6595078d1fb3b458c9c28b7153a802"},
+    {file = "pywin32-306-cp39-cp39-win_amd64.whl", hash = "sha256:39b61c15272833b5c329a2989999dcae836b1eed650252ab1b7bfbe1d59f30f4"},
+]
+
 [[package]]
 name = "pyyaml"
 version = "6.0.1"
@@ -1100,6 +1725,113 @@ files = [
     {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"},
 ]
 
+[[package]]
+name = "pyzmq"
+version = "26.2.0"
+requires_python = ">=3.7"
+summary = "Python bindings for 0MQ"
+groups = ["dev", "docs"]
+dependencies = [
+    "cffi; implementation_name == \"pypy\"",
+]
+files = [
+    {file = "pyzmq-26.2.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:ddf33d97d2f52d89f6e6e7ae66ee35a4d9ca6f36eda89c24591b0c40205a3629"},
+    {file = "pyzmq-26.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dacd995031a01d16eec825bf30802fceb2c3791ef24bcce48fa98ce40918c27b"},
+    {file = "pyzmq-26.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89289a5ee32ef6c439086184529ae060c741334b8970a6855ec0b6ad3ff28764"},
+    {file = "pyzmq-26.2.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5506f06d7dc6ecf1efacb4a013b1f05071bb24b76350832c96449f4a2d95091c"},
+    {file = "pyzmq-26.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8ea039387c10202ce304af74def5021e9adc6297067f3441d348d2b633e8166a"},
+    {file = "pyzmq-26.2.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:a2224fa4a4c2ee872886ed00a571f5e967c85e078e8e8c2530a2fb01b3309b88"},
+    {file = "pyzmq-26.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:28ad5233e9c3b52d76196c696e362508959741e1a005fb8fa03b51aea156088f"},
+    {file = "pyzmq-26.2.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:1c17211bc037c7d88e85ed8b7d8f7e52db6dc8eca5590d162717c654550f7282"},
+    {file = "pyzmq-26.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b8f86dd868d41bea9a5f873ee13bf5551c94cf6bc51baebc6f85075971fe6eea"},
+    {file = "pyzmq-26.2.0-cp310-cp310-win32.whl", hash = "sha256:46a446c212e58456b23af260f3d9fb785054f3e3653dbf7279d8f2b5546b21c2"},
+    {file = "pyzmq-26.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:49d34ab71db5a9c292a7644ce74190b1dd5a3475612eefb1f8be1d6961441971"},
+    {file = "pyzmq-26.2.0-cp310-cp310-win_arm64.whl", hash = "sha256:bfa832bfa540e5b5c27dcf5de5d82ebc431b82c453a43d141afb1e5d2de025fa"},
+    {file = "pyzmq-26.2.0-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:8f7e66c7113c684c2b3f1c83cdd3376103ee0ce4c49ff80a648643e57fb22218"},
+    {file = "pyzmq-26.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3a495b30fc91db2db25120df5847d9833af237546fd59170701acd816ccc01c4"},
+    {file = "pyzmq-26.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77eb0968da535cba0470a5165468b2cac7772cfb569977cff92e240f57e31bef"},
+    {file = "pyzmq-26.2.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ace4f71f1900a548f48407fc9be59c6ba9d9aaf658c2eea6cf2779e72f9f317"},
+    {file = "pyzmq-26.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:92a78853d7280bffb93df0a4a6a2498cba10ee793cc8076ef797ef2f74d107cf"},
+    {file = "pyzmq-26.2.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:689c5d781014956a4a6de61d74ba97b23547e431e9e7d64f27d4922ba96e9d6e"},
+    {file = "pyzmq-26.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0aca98bc423eb7d153214b2df397c6421ba6373d3397b26c057af3c904452e37"},
+    {file = "pyzmq-26.2.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:1f3496d76b89d9429a656293744ceca4d2ac2a10ae59b84c1da9b5165f429ad3"},
+    {file = "pyzmq-26.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5c2b3bfd4b9689919db068ac6c9911f3fcb231c39f7dd30e3138be94896d18e6"},
+    {file = "pyzmq-26.2.0-cp311-cp311-win32.whl", hash = "sha256:eac5174677da084abf378739dbf4ad245661635f1600edd1221f150b165343f4"},
+    {file = "pyzmq-26.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:5a509df7d0a83a4b178d0f937ef14286659225ef4e8812e05580776c70e155d5"},
+    {file = "pyzmq-26.2.0-cp311-cp311-win_arm64.whl", hash = "sha256:c0e6091b157d48cbe37bd67233318dbb53e1e6327d6fc3bb284afd585d141003"},
+    {file = "pyzmq-26.2.0-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:ded0fc7d90fe93ae0b18059930086c51e640cdd3baebdc783a695c77f123dcd9"},
+    {file = "pyzmq-26.2.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:17bf5a931c7f6618023cdacc7081f3f266aecb68ca692adac015c383a134ca52"},
+    {file = "pyzmq-26.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55cf66647e49d4621a7e20c8d13511ef1fe1efbbccf670811864452487007e08"},
+    {file = "pyzmq-26.2.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4661c88db4a9e0f958c8abc2b97472e23061f0bc737f6f6179d7a27024e1faa5"},
+    {file = "pyzmq-26.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ea7f69de383cb47522c9c208aec6dd17697db7875a4674c4af3f8cfdac0bdeae"},
+    {file = "pyzmq-26.2.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:7f98f6dfa8b8ccaf39163ce872bddacca38f6a67289116c8937a02e30bbe9711"},
+    {file = "pyzmq-26.2.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:e3e0210287329272539eea617830a6a28161fbbd8a3271bf4150ae3e58c5d0e6"},
+    {file = "pyzmq-26.2.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:6b274e0762c33c7471f1a7471d1a2085b1a35eba5cdc48d2ae319f28b6fc4de3"},
+    {file = "pyzmq-26.2.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:29c6a4635eef69d68a00321e12a7d2559fe2dfccfa8efae3ffb8e91cd0b36a8b"},
+    {file = "pyzmq-26.2.0-cp312-cp312-win32.whl", hash = "sha256:989d842dc06dc59feea09e58c74ca3e1678c812a4a8a2a419046d711031f69c7"},
+    {file = "pyzmq-26.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:2a50625acdc7801bc6f74698c5c583a491c61d73c6b7ea4dee3901bb99adb27a"},
+    {file = "pyzmq-26.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:4d29ab8592b6ad12ebbf92ac2ed2bedcfd1cec192d8e559e2e099f648570e19b"},
+    {file = "pyzmq-26.2.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9dd8cd1aeb00775f527ec60022004d030ddc51d783d056e3e23e74e623e33726"},
+    {file = "pyzmq-26.2.0-cp313-cp313-macosx_10_15_universal2.whl", hash = "sha256:28c812d9757fe8acecc910c9ac9dafd2ce968c00f9e619db09e9f8f54c3a68a3"},
+    {file = "pyzmq-26.2.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4d80b1dd99c1942f74ed608ddb38b181b87476c6a966a88a950c7dee118fdf50"},
+    {file = "pyzmq-26.2.0-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8c997098cc65e3208eca09303630e84d42718620e83b733d0fd69543a9cab9cb"},
+    {file = "pyzmq-26.2.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ad1bc8d1b7a18497dda9600b12dc193c577beb391beae5cd2349184db40f187"},
+    {file = "pyzmq-26.2.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:bea2acdd8ea4275e1278350ced63da0b166421928276c7c8e3f9729d7402a57b"},
+    {file = "pyzmq-26.2.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:23f4aad749d13698f3f7b64aad34f5fc02d6f20f05999eebc96b89b01262fb18"},
+    {file = "pyzmq-26.2.0-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:a4f96f0d88accc3dbe4a9025f785ba830f968e21e3e2c6321ccdfc9aef755115"},
+    {file = "pyzmq-26.2.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ced65e5a985398827cc9276b93ef6dfabe0273c23de8c7931339d7e141c2818e"},
+    {file = "pyzmq-26.2.0-cp313-cp313-win32.whl", hash = "sha256:31507f7b47cc1ead1f6e86927f8ebb196a0bab043f6345ce070f412a59bf87b5"},
+    {file = "pyzmq-26.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:70fc7fcf0410d16ebdda9b26cbd8bf8d803d220a7f3522e060a69a9c87bf7bad"},
+    {file = "pyzmq-26.2.0-cp313-cp313-win_arm64.whl", hash = "sha256:c3789bd5768ab5618ebf09cef6ec2b35fed88709b104351748a63045f0ff9797"},
+    {file = "pyzmq-26.2.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:034da5fc55d9f8da09015d368f519478a52675e558c989bfcb5cf6d4e16a7d2a"},
+    {file = "pyzmq-26.2.0-cp313-cp313t-macosx_10_15_universal2.whl", hash = "sha256:c92d73464b886931308ccc45b2744e5968cbaade0b1d6aeb40d8ab537765f5bc"},
+    {file = "pyzmq-26.2.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:794a4562dcb374f7dbbfb3f51d28fb40123b5a2abadee7b4091f93054909add5"},
+    {file = "pyzmq-26.2.0-cp313-cp313t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aee22939bb6075e7afededabad1a56a905da0b3c4e3e0c45e75810ebe3a52672"},
+    {file = "pyzmq-26.2.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ae90ff9dad33a1cfe947d2c40cb9cb5e600d759ac4f0fd22616ce6540f72797"},
+    {file = "pyzmq-26.2.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:43a47408ac52647dfabbc66a25b05b6a61700b5165807e3fbd40063fcaf46386"},
+    {file = "pyzmq-26.2.0-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:25bf2374a2a8433633c65ccb9553350d5e17e60c8eb4de4d92cc6bd60f01d306"},
+    {file = "pyzmq-26.2.0-cp313-cp313t-musllinux_1_1_i686.whl", hash = "sha256:007137c9ac9ad5ea21e6ad97d3489af654381324d5d3ba614c323f60dab8fae6"},
+    {file = "pyzmq-26.2.0-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:470d4a4f6d48fb34e92d768b4e8a5cc3780db0d69107abf1cd7ff734b9766eb0"},
+    {file = "pyzmq-26.2.0-cp39-cp39-macosx_10_15_universal2.whl", hash = "sha256:b1d464cb8d72bfc1a3adc53305a63a8e0cac6bc8c5a07e8ca190ab8d3faa43c2"},
+    {file = "pyzmq-26.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4da04c48873a6abdd71811c5e163bd656ee1b957971db7f35140a2d573f6949c"},
+    {file = "pyzmq-26.2.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:d049df610ac811dcffdc147153b414147428567fbbc8be43bb8885f04db39d98"},
+    {file = "pyzmq-26.2.0-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:05590cdbc6b902101d0e65d6a4780af14dc22914cc6ab995d99b85af45362cc9"},
+    {file = "pyzmq-26.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c811cfcd6a9bf680236c40c6f617187515269ab2912f3d7e8c0174898e2519db"},
+    {file = "pyzmq-26.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:6835dd60355593de10350394242b5757fbbd88b25287314316f266e24c61d073"},
+    {file = "pyzmq-26.2.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc6bee759a6bddea5db78d7dcd609397449cb2d2d6587f48f3ca613b19410cfc"},
+    {file = "pyzmq-26.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c530e1eecd036ecc83c3407f77bb86feb79916d4a33d11394b8234f3bd35b940"},
+    {file = "pyzmq-26.2.0-cp39-cp39-win32.whl", hash = "sha256:367b4f689786fca726ef7a6c5ba606958b145b9340a5e4808132cc65759abd44"},
+    {file = "pyzmq-26.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:e6fa2e3e683f34aea77de8112f6483803c96a44fd726d7358b9888ae5bb394ec"},
+    {file = "pyzmq-26.2.0-cp39-cp39-win_arm64.whl", hash = "sha256:7445be39143a8aa4faec43b076e06944b8f9d0701b669df4af200531b21e40bb"},
+    {file = "pyzmq-26.2.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:706e794564bec25819d21a41c31d4df2d48e1cc4b061e8d345d7fb4dd3e94072"},
+    {file = "pyzmq-26.2.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b435f2753621cd36e7c1762156815e21c985c72b19135dac43a7f4f31d28dd1"},
+    {file = "pyzmq-26.2.0-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:160c7e0a5eb178011e72892f99f918c04a131f36056d10d9c1afb223fc952c2d"},
+    {file = "pyzmq-26.2.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c4a71d5d6e7b28a47a394c0471b7e77a0661e2d651e7ae91e0cab0a587859ca"},
+    {file = "pyzmq-26.2.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:90412f2db8c02a3864cbfc67db0e3dcdbda336acf1c469526d3e869394fe001c"},
+    {file = "pyzmq-26.2.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:cdeabcff45d1c219636ee2e54d852262e5c2e085d6cb476d938aee8d921356b3"},
+    {file = "pyzmq-26.2.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35cffef589bcdc587d06f9149f8d5e9e8859920a071df5a2671de2213bef592a"},
+    {file = "pyzmq-26.2.0-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18c8dc3b7468d8b4bdf60ce9d7141897da103c7a4690157b32b60acb45e333e6"},
+    {file = "pyzmq-26.2.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7133d0a1677aec369d67dd78520d3fa96dd7f3dcec99d66c1762870e5ea1a50a"},
+    {file = "pyzmq-26.2.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:6a96179a24b14fa6428cbfc08641c779a53f8fcec43644030328f44034c7f1f4"},
+    {file = "pyzmq-26.2.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:4f78c88905461a9203eac9faac157a2a0dbba84a0fd09fd29315db27be40af9f"},
+    {file = "pyzmq-26.2.0.tar.gz", hash = "sha256:070672c258581c8e4f640b5159297580a9974b026043bd4ab0470be9ed324f1f"},
+]
+
+[[package]]
+name = "referencing"
+version = "0.35.1"
+requires_python = ">=3.8"
+summary = "JSON Referencing + Python"
+groups = ["dev", "docs"]
+dependencies = [
+    "attrs>=22.2.0",
+    "rpds-py>=0.7.0",
+]
+files = [
+    {file = "referencing-0.35.1-py3-none-any.whl", hash = "sha256:eda6d3234d62814d1c64e305c1331c9a3a6132da475ab6382eaa997b21ee75de"},
+    {file = "referencing-0.35.1.tar.gz", hash = "sha256:25b42124a6c8b632a425174f24087783efb348a6f1e0008e63cd4466fedf703c"},
+]
+
 [[package]]
 name = "requests"
 version = "2.32.3"
@@ -1117,6 +1849,105 @@ files = [
     {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"},
 ]
 
+[[package]]
+name = "rpds-py"
+version = "0.20.0"
+requires_python = ">=3.8"
+summary = "Python bindings to Rust's persistent data structures (rpds)"
+groups = ["dev", "docs"]
+files = [
+    {file = "rpds_py-0.20.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:3ad0fda1635f8439cde85c700f964b23ed5fc2d28016b32b9ee5fe30da5c84e2"},
+    {file = "rpds_py-0.20.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9bb4a0d90fdb03437c109a17eade42dfbf6190408f29b2744114d11586611d6f"},
+    {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c6377e647bbfd0a0b159fe557f2c6c602c159fc752fa316572f012fc0bf67150"},
+    {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eb851b7df9dda52dc1415ebee12362047ce771fc36914586b2e9fcbd7d293b3e"},
+    {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1e0f80b739e5a8f54837be5d5c924483996b603d5502bfff79bf33da06164ee2"},
+    {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5a8c94dad2e45324fc74dce25e1645d4d14df9a4e54a30fa0ae8bad9a63928e3"},
+    {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8e604fe73ba048c06085beaf51147eaec7df856824bfe7b98657cf436623daf"},
+    {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:df3de6b7726b52966edf29663e57306b23ef775faf0ac01a3e9f4012a24a4140"},
+    {file = "rpds_py-0.20.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cf258ede5bc22a45c8e726b29835b9303c285ab46fc7c3a4cc770736b5304c9f"},
+    {file = "rpds_py-0.20.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:55fea87029cded5df854ca7e192ec7bdb7ecd1d9a3f63d5c4eb09148acf4a7ce"},
+    {file = "rpds_py-0.20.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ae94bd0b2f02c28e199e9bc51485d0c5601f58780636185660f86bf80c89af94"},
+    {file = "rpds_py-0.20.0-cp310-none-win32.whl", hash = "sha256:28527c685f237c05445efec62426d285e47a58fb05ba0090a4340b73ecda6dee"},
+    {file = "rpds_py-0.20.0-cp310-none-win_amd64.whl", hash = "sha256:238a2d5b1cad28cdc6ed15faf93a998336eb041c4e440dd7f902528b8891b399"},
+    {file = "rpds_py-0.20.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:ac2f4f7a98934c2ed6505aead07b979e6f999389f16b714448fb39bbaa86a489"},
+    {file = "rpds_py-0.20.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:220002c1b846db9afd83371d08d239fdc865e8f8c5795bbaec20916a76db3318"},
+    {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d7919548df3f25374a1f5d01fbcd38dacab338ef5f33e044744b5c36729c8db"},
+    {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:758406267907b3781beee0f0edfe4a179fbd97c0be2e9b1154d7f0a1279cf8e5"},
+    {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3d61339e9f84a3f0767b1995adfb171a0d00a1185192718a17af6e124728e0f5"},
+    {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1259c7b3705ac0a0bd38197565a5d603218591d3f6cee6e614e380b6ba61c6f6"},
+    {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c1dc0f53856b9cc9a0ccca0a7cc61d3d20a7088201c0937f3f4048c1718a209"},
+    {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7e60cb630f674a31f0368ed32b2a6b4331b8350d67de53c0359992444b116dd3"},
+    {file = "rpds_py-0.20.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:dbe982f38565bb50cb7fb061ebf762c2f254ca3d8c20d4006878766e84266272"},
+    {file = "rpds_py-0.20.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:514b3293b64187172bc77c8fb0cdae26981618021053b30d8371c3a902d4d5ad"},
+    {file = "rpds_py-0.20.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d0a26ffe9d4dd35e4dfdd1e71f46401cff0181c75ac174711ccff0459135fa58"},
+    {file = "rpds_py-0.20.0-cp311-none-win32.whl", hash = "sha256:89c19a494bf3ad08c1da49445cc5d13d8fefc265f48ee7e7556839acdacf69d0"},
+    {file = "rpds_py-0.20.0-cp311-none-win_amd64.whl", hash = "sha256:c638144ce971df84650d3ed0096e2ae7af8e62ecbbb7b201c8935c370df00a2c"},
+    {file = "rpds_py-0.20.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a84ab91cbe7aab97f7446652d0ed37d35b68a465aeef8fc41932a9d7eee2c1a6"},
+    {file = "rpds_py-0.20.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:56e27147a5a4c2c21633ff8475d185734c0e4befd1c989b5b95a5d0db699b21b"},
+    {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2580b0c34583b85efec8c5c5ec9edf2dfe817330cc882ee972ae650e7b5ef739"},
+    {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b80d4a7900cf6b66bb9cee5c352b2d708e29e5a37fe9bf784fa97fc11504bf6c"},
+    {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:50eccbf054e62a7b2209b28dc7a22d6254860209d6753e6b78cfaeb0075d7bee"},
+    {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:49a8063ea4296b3a7e81a5dfb8f7b2d73f0b1c20c2af401fb0cdf22e14711a96"},
+    {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ea438162a9fcbee3ecf36c23e6c68237479f89f962f82dae83dc15feeceb37e4"},
+    {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:18d7585c463087bddcfa74c2ba267339f14f2515158ac4db30b1f9cbdb62c8ef"},
+    {file = "rpds_py-0.20.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d4c7d1a051eeb39f5c9547e82ea27cbcc28338482242e3e0b7768033cb083821"},
+    {file = "rpds_py-0.20.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e4df1e3b3bec320790f699890d41c59d250f6beda159ea3c44c3f5bac1976940"},
+    {file = "rpds_py-0.20.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2cf126d33a91ee6eedc7f3197b53e87a2acdac63602c0f03a02dd69e4b138174"},
+    {file = "rpds_py-0.20.0-cp312-none-win32.whl", hash = "sha256:8bc7690f7caee50b04a79bf017a8d020c1f48c2a1077ffe172abec59870f1139"},
+    {file = "rpds_py-0.20.0-cp312-none-win_amd64.whl", hash = "sha256:0e13e6952ef264c40587d510ad676a988df19adea20444c2b295e536457bc585"},
+    {file = "rpds_py-0.20.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:aa9a0521aeca7d4941499a73ad7d4f8ffa3d1affc50b9ea11d992cd7eff18a29"},
+    {file = "rpds_py-0.20.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4a1f1d51eccb7e6c32ae89243cb352389228ea62f89cd80823ea7dd1b98e0b91"},
+    {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8a86a9b96070674fc88b6f9f71a97d2c1d3e5165574615d1f9168ecba4cecb24"},
+    {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6c8ef2ebf76df43f5750b46851ed1cdf8f109d7787ca40035fe19fbdc1acc5a7"},
+    {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b74b25f024b421d5859d156750ea9a65651793d51b76a2e9238c05c9d5f203a9"},
+    {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:57eb94a8c16ab08fef6404301c38318e2c5a32216bf5de453e2714c964c125c8"},
+    {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1940dae14e715e2e02dfd5b0f64a52e8374a517a1e531ad9412319dc3ac7879"},
+    {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d20277fd62e1b992a50c43f13fbe13277a31f8c9f70d59759c88f644d66c619f"},
+    {file = "rpds_py-0.20.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:06db23d43f26478303e954c34c75182356ca9aa7797d22c5345b16871ab9c45c"},
+    {file = "rpds_py-0.20.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b2a5db5397d82fa847e4c624b0c98fe59d2d9b7cf0ce6de09e4d2e80f8f5b3f2"},
+    {file = "rpds_py-0.20.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5a35df9f5548fd79cb2f52d27182108c3e6641a4feb0f39067911bf2adaa3e57"},
+    {file = "rpds_py-0.20.0-cp313-none-win32.whl", hash = "sha256:fd2d84f40633bc475ef2d5490b9c19543fbf18596dcb1b291e3a12ea5d722f7a"},
+    {file = "rpds_py-0.20.0-cp313-none-win_amd64.whl", hash = "sha256:9bc2d153989e3216b0559251b0c260cfd168ec78b1fac33dd485750a228db5a2"},
+    {file = "rpds_py-0.20.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:3fde368e9140312b6e8b6c09fb9f8c8c2f00999d1823403ae90cc00480221b22"},
+    {file = "rpds_py-0.20.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9824fb430c9cf9af743cf7aaf6707bf14323fb51ee74425c380f4c846ea70789"},
+    {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:11ef6ce74616342888b69878d45e9f779b95d4bd48b382a229fe624a409b72c5"},
+    {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c52d3f2f82b763a24ef52f5d24358553e8403ce05f893b5347098014f2d9eff2"},
+    {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d35cef91e59ebbeaa45214861874bc6f19eb35de96db73e467a8358d701a96c"},
+    {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d72278a30111e5b5525c1dd96120d9e958464316f55adb030433ea905866f4de"},
+    {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4c29cbbba378759ac5786730d1c3cb4ec6f8ababf5c42a9ce303dc4b3d08cda"},
+    {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6632f2d04f15d1bd6fe0eedd3b86d9061b836ddca4c03d5cf5c7e9e6b7c14580"},
+    {file = "rpds_py-0.20.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:d0b67d87bb45ed1cd020e8fbf2307d449b68abc45402fe1a4ac9e46c3c8b192b"},
+    {file = "rpds_py-0.20.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:ec31a99ca63bf3cd7f1a5ac9fe95c5e2d060d3c768a09bc1d16e235840861420"},
+    {file = "rpds_py-0.20.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:22e6c9976e38f4d8c4a63bd8a8edac5307dffd3ee7e6026d97f3cc3a2dc02a0b"},
+    {file = "rpds_py-0.20.0-cp39-none-win32.whl", hash = "sha256:569b3ea770c2717b730b61998b6c54996adee3cef69fc28d444f3e7920313cf7"},
+    {file = "rpds_py-0.20.0-cp39-none-win_amd64.whl", hash = "sha256:e6900ecdd50ce0facf703f7a00df12374b74bbc8ad9fe0f6559947fb20f82364"},
+    {file = "rpds_py-0.20.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:617c7357272c67696fd052811e352ac54ed1d9b49ab370261a80d3b6ce385045"},
+    {file = "rpds_py-0.20.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:9426133526f69fcaba6e42146b4e12d6bc6c839b8b555097020e2b78ce908dcc"},
+    {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:deb62214c42a261cb3eb04d474f7155279c1a8a8c30ac89b7dcb1721d92c3c02"},
+    {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fcaeb7b57f1a1e071ebd748984359fef83ecb026325b9d4ca847c95bc7311c92"},
+    {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d454b8749b4bd70dd0a79f428731ee263fa6995f83ccb8bada706e8d1d3ff89d"},
+    {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d807dc2051abe041b6649681dce568f8e10668e3c1c6543ebae58f2d7e617855"},
+    {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3c20f0ddeb6e29126d45f89206b8291352b8c5b44384e78a6499d68b52ae511"},
+    {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b7f19250ceef892adf27f0399b9e5afad019288e9be756d6919cb58892129f51"},
+    {file = "rpds_py-0.20.0-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:4f1ed4749a08379555cebf4650453f14452eaa9c43d0a95c49db50c18b7da075"},
+    {file = "rpds_py-0.20.0-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:dcedf0b42bcb4cfff4101d7771a10532415a6106062f005ab97d1d0ab5681c60"},
+    {file = "rpds_py-0.20.0-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:39ed0d010457a78f54090fafb5d108501b5aa5604cc22408fc1c0c77eac14344"},
+    {file = "rpds_py-0.20.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:bb273176be34a746bdac0b0d7e4e2c467323d13640b736c4c477881a3220a989"},
+    {file = "rpds_py-0.20.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f918a1a130a6dfe1d7fe0f105064141342e7dd1611f2e6a21cd2f5c8cb1cfb3e"},
+    {file = "rpds_py-0.20.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:f60012a73aa396be721558caa3a6fd49b3dd0033d1675c6d59c4502e870fcf0c"},
+    {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d2b1ad682a3dfda2a4e8ad8572f3100f95fad98cb99faf37ff0ddfe9cbf9d03"},
+    {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:614fdafe9f5f19c63ea02817fa4861c606a59a604a77c8cdef5aa01d28b97921"},
+    {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fa518bcd7600c584bf42e6617ee8132869e877db2f76bcdc281ec6a4113a53ab"},
+    {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f0475242f447cc6cb8a9dd486d68b2ef7fbee84427124c232bff5f63b1fe11e5"},
+    {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f90a4cd061914a60bd51c68bcb4357086991bd0bb93d8aa66a6da7701370708f"},
+    {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:def7400461c3a3f26e49078302e1c1b38f6752342c77e3cf72ce91ca69fb1bc1"},
+    {file = "rpds_py-0.20.0-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:65794e4048ee837494aea3c21a28ad5fc080994dfba5b036cf84de37f7ad5074"},
+    {file = "rpds_py-0.20.0-pp39-pypy39_pp73-musllinux_1_2_i686.whl", hash = "sha256:faefcc78f53a88f3076b7f8be0a8f8d35133a3ecf7f3770895c25f8813460f08"},
+    {file = "rpds_py-0.20.0-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:5b4f105deeffa28bbcdff6c49b34e74903139afa690e35d2d9e3c2c2fba18cec"},
+    {file = "rpds_py-0.20.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:fdfc3a892927458d98f3d55428ae46b921d1f7543b89382fdb483f5640daaec8"},
+    {file = "rpds_py-0.20.0.tar.gz", hash = "sha256:d72a210824facfdaf8768cf2d7ca25a042c30320b3020de2fa04640920d4e121"},
+]
+
 [[package]]
 name = "ruff"
 version = "0.5.5"
@@ -1144,6 +1975,17 @@ files = [
     {file = "ruff-0.5.5.tar.gz", hash = "sha256:cc5516bdb4858d972fbc31d246bdb390eab8df1a26e2353be2dbc0c2d7f5421a"},
 ]
 
+[[package]]
+name = "six"
+version = "1.16.0"
+requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
+summary = "Python 2 and 3 compatibility utilities"
+groups = ["dev", "docs"]
+files = [
+    {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"},
+    {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
+]
+
 [[package]]
 name = "sniffio"
 version = "1.3.1"
@@ -1331,6 +2173,69 @@ files = [
     {file = "sphinxcontrib_serializinghtml-2.0.0.tar.gz", hash = "sha256:e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d"},
 ]
 
+[[package]]
+name = "sqlalchemy"
+version = "2.0.35"
+requires_python = ">=3.7"
+summary = "Database Abstraction Library"
+groups = ["dev", "docs"]
+dependencies = [
+    "greenlet!=0.4.17; (platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\") and python_version < \"3.13\"",
+    "importlib-metadata; python_version < \"3.8\"",
+    "typing-extensions>=4.6.0",
+]
+files = [
+    {file = "SQLAlchemy-2.0.35-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:67219632be22f14750f0d1c70e62f204ba69d28f62fd6432ba05ab295853de9b"},
+    {file = "SQLAlchemy-2.0.35-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4668bd8faf7e5b71c0319407b608f278f279668f358857dbfd10ef1954ac9f90"},
+    {file = "SQLAlchemy-2.0.35-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb8bea573863762bbf45d1e13f87c2d2fd32cee2dbd50d050f83f87429c9e1ea"},
+    {file = "SQLAlchemy-2.0.35-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f552023710d4b93d8fb29a91fadf97de89c5926c6bd758897875435f2a939f33"},
+    {file = "SQLAlchemy-2.0.35-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:016b2e665f778f13d3c438651dd4de244214b527a275e0acf1d44c05bc6026a9"},
+    {file = "SQLAlchemy-2.0.35-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7befc148de64b6060937231cbff8d01ccf0bfd75aa26383ffdf8d82b12ec04ff"},
+    {file = "SQLAlchemy-2.0.35-cp310-cp310-win32.whl", hash = "sha256:22b83aed390e3099584b839b93f80a0f4a95ee7f48270c97c90acd40ee646f0b"},
+    {file = "SQLAlchemy-2.0.35-cp310-cp310-win_amd64.whl", hash = "sha256:a29762cd3d116585278ffb2e5b8cc311fb095ea278b96feef28d0b423154858e"},
+    {file = "SQLAlchemy-2.0.35-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e21f66748ab725ade40fa7af8ec8b5019c68ab00b929f6643e1b1af461eddb60"},
+    {file = "SQLAlchemy-2.0.35-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8a6219108a15fc6d24de499d0d515c7235c617b2540d97116b663dade1a54d62"},
+    {file = "SQLAlchemy-2.0.35-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:042622a5306c23b972192283f4e22372da3b8ddf5f7aac1cc5d9c9b222ab3ff6"},
+    {file = "SQLAlchemy-2.0.35-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:627dee0c280eea91aed87b20a1f849e9ae2fe719d52cbf847c0e0ea34464b3f7"},
+    {file = "SQLAlchemy-2.0.35-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4fdcd72a789c1c31ed242fd8c1bcd9ea186a98ee8e5408a50e610edfef980d71"},
+    {file = "SQLAlchemy-2.0.35-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:89b64cd8898a3a6f642db4eb7b26d1b28a497d4022eccd7717ca066823e9fb01"},
+    {file = "SQLAlchemy-2.0.35-cp311-cp311-win32.whl", hash = "sha256:6a93c5a0dfe8d34951e8a6f499a9479ffb9258123551fa007fc708ae2ac2bc5e"},
+    {file = "SQLAlchemy-2.0.35-cp311-cp311-win_amd64.whl", hash = "sha256:c68fe3fcde03920c46697585620135b4ecfdfc1ed23e75cc2c2ae9f8502c10b8"},
+    {file = "SQLAlchemy-2.0.35-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:eb60b026d8ad0c97917cb81d3662d0b39b8ff1335e3fabb24984c6acd0c900a2"},
+    {file = "SQLAlchemy-2.0.35-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6921ee01caf375363be5e9ae70d08ce7ca9d7e0e8983183080211a062d299468"},
+    {file = "SQLAlchemy-2.0.35-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8cdf1a0dbe5ced887a9b127da4ffd7354e9c1a3b9bb330dce84df6b70ccb3a8d"},
+    {file = "SQLAlchemy-2.0.35-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93a71c8601e823236ac0e5d087e4f397874a421017b3318fd92c0b14acf2b6db"},
+    {file = "SQLAlchemy-2.0.35-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e04b622bb8a88f10e439084486f2f6349bf4d50605ac3e445869c7ea5cf0fa8c"},
+    {file = "SQLAlchemy-2.0.35-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1b56961e2d31389aaadf4906d453859f35302b4eb818d34a26fab72596076bb8"},
+    {file = "SQLAlchemy-2.0.35-cp312-cp312-win32.whl", hash = "sha256:0f9f3f9a3763b9c4deb8c5d09c4cc52ffe49f9876af41cc1b2ad0138878453cf"},
+    {file = "SQLAlchemy-2.0.35-cp312-cp312-win_amd64.whl", hash = "sha256:25b0f63e7fcc2a6290cb5f7f5b4fc4047843504983a28856ce9b35d8f7de03cc"},
+    {file = "SQLAlchemy-2.0.35-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ccae5de2a0140d8be6838c331604f91d6fafd0735dbdcee1ac78fc8fbaba76b4"},
+    {file = "SQLAlchemy-2.0.35-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2a275a806f73e849e1c309ac11108ea1a14cd7058577aba962cd7190e27c9e3c"},
+    {file = "SQLAlchemy-2.0.35-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:732e026240cdd1c1b2e3ac515c7a23820430ed94292ce33806a95869c46bd139"},
+    {file = "SQLAlchemy-2.0.35-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:890da8cd1941fa3dab28c5bac3b9da8502e7e366f895b3b8e500896f12f94d11"},
+    {file = "SQLAlchemy-2.0.35-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:c0d8326269dbf944b9201911b0d9f3dc524d64779a07518199a58384c3d37a44"},
+    {file = "SQLAlchemy-2.0.35-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b76d63495b0508ab9fc23f8152bac63205d2a704cd009a2b0722f4c8e0cba8e0"},
+    {file = "SQLAlchemy-2.0.35-cp39-cp39-win32.whl", hash = "sha256:69683e02e8a9de37f17985905a5eca18ad651bf592314b4d3d799029797d0eb3"},
+    {file = "SQLAlchemy-2.0.35-cp39-cp39-win_amd64.whl", hash = "sha256:aee110e4ef3c528f3abbc3c2018c121e708938adeeff9006428dd7c8555e9b3f"},
+    {file = "SQLAlchemy-2.0.35-py3-none-any.whl", hash = "sha256:2ab3f0336c0387662ce6221ad30ab3a5e6499aab01b9790879b6578fd9b8faa1"},
+    {file = "sqlalchemy-2.0.35.tar.gz", hash = "sha256:e11d7ea4d24f0a262bccf9a7cd6284c976c5369dac21db237cff59586045ab9f"},
+]
+
+[[package]]
+name = "stack-data"
+version = "0.6.3"
+summary = "Extract data from python stack frames and tracebacks for informative displays"
+groups = ["dev", "docs"]
+dependencies = [
+    "asttokens>=2.1.0",
+    "executing>=1.2.0",
+    "pure-eval",
+]
+files = [
+    {file = "stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695"},
+    {file = "stack_data-0.6.3.tar.gz", hash = "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9"},
+]
+
 [[package]]
 name = "starlette"
 version = "0.38.2"
@@ -1346,6 +2251,17 @@ files = [
     {file = "starlette-0.38.2.tar.gz", hash = "sha256:c7c0441065252160993a1a37cf2a73bb64d271b17303e0b0c1eb7191cfb12d75"},
 ]
 
+[[package]]
+name = "tabulate"
+version = "0.9.0"
+requires_python = ">=3.7"
+summary = "Pretty-print tabular data"
+groups = ["dev", "docs"]
+files = [
+    {file = "tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f"},
+    {file = "tabulate-0.9.0.tar.gz", hash = "sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c"},
+]
+
 [[package]]
 name = "tomli"
 version = "2.0.1"
@@ -1369,12 +2285,43 @@ files = [
     {file = "toolz-0.12.1.tar.gz", hash = "sha256:ecca342664893f177a13dac0e6b41cbd8ac25a358e5f215316d43e2100224f4d"},
 ]
 
+[[package]]
+name = "tornado"
+version = "6.4.1"
+requires_python = ">=3.8"
+summary = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed."
+groups = ["dev", "docs"]
+files = [
+    {file = "tornado-6.4.1-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:163b0aafc8e23d8cdc3c9dfb24c5368af84a81e3364745ccb4427669bf84aec8"},
+    {file = "tornado-6.4.1-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:6d5ce3437e18a2b66fbadb183c1d3364fb03f2be71299e7d10dbeeb69f4b2a14"},
+    {file = "tornado-6.4.1-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2e20b9113cd7293f164dc46fffb13535266e713cdb87bd2d15ddb336e96cfc4"},
+    {file = "tornado-6.4.1-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ae50a504a740365267b2a8d1a90c9fbc86b780a39170feca9bcc1787ff80842"},
+    {file = "tornado-6.4.1-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:613bf4ddf5c7a95509218b149b555621497a6cc0d46ac341b30bd9ec19eac7f3"},
+    {file = "tornado-6.4.1-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:25486eb223babe3eed4b8aecbac33b37e3dd6d776bc730ca14e1bf93888b979f"},
+    {file = "tornado-6.4.1-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:454db8a7ecfcf2ff6042dde58404164d969b6f5d58b926da15e6b23817950fc4"},
+    {file = "tornado-6.4.1-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a02a08cc7a9314b006f653ce40483b9b3c12cda222d6a46d4ac63bb6c9057698"},
+    {file = "tornado-6.4.1-cp38-abi3-win32.whl", hash = "sha256:d9a566c40b89757c9aa8e6f032bcdb8ca8795d7c1a9762910c722b1635c9de4d"},
+    {file = "tornado-6.4.1-cp38-abi3-win_amd64.whl", hash = "sha256:b24b8982ed444378d7f21d563f4180a2de31ced9d8d84443907a0a64da2072e7"},
+    {file = "tornado-6.4.1.tar.gz", hash = "sha256:92d3ab53183d8c50f8204a51e6f91d18a15d5ef261e84d452800d4ff6fc504e9"},
+]
+
+[[package]]
+name = "traitlets"
+version = "5.14.3"
+requires_python = ">=3.8"
+summary = "Traitlets Python configuration system"
+groups = ["dev", "docs"]
+files = [
+    {file = "traitlets-5.14.3-py3-none-any.whl", hash = "sha256:b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f"},
+    {file = "traitlets-5.14.3.tar.gz", hash = "sha256:9ed0579d3502c94b4b3732ac120375cda96f923114522847de4b3bb98b96b6b7"},
+]
+
 [[package]]
 name = "typing-extensions"
 version = "4.12.2"
 requires_python = ">=3.8"
 summary = "Backported and Experimental Type Hints for Python 3.8+"
-groups = ["arrays", "default", "dev", "docs", "tests"]
+groups = ["default", "dev", "docs"]
 files = [
     {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"},
     {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"},
@@ -1482,6 +2429,19 @@ files = [
     {file = "watchfiles-0.22.0.tar.gz", hash = "sha256:988e981aaab4f3955209e7e28c7794acdb690be1efa7f16f8ea5aba7ffdadacb"},
 ]
 
+[[package]]
+name = "wcwidth"
+version = "0.2.13"
+summary = "Measures the displayed width of unicode strings in a terminal"
+groups = ["dev", "docs"]
+dependencies = [
+    "backports-functools-lru-cache>=1.2.1; python_version < \"3.2\"",
+]
+files = [
+    {file = "wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859"},
+    {file = "wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"},
+]
+
 [[package]]
 name = "websockets"
 version = "12.0"
@@ -1557,7 +2517,7 @@ name = "zarr"
 version = "2.18.2"
 requires_python = ">=3.9"
 summary = "An implementation of chunked, compressed, N-dimensional arrays for Python"
-groups = ["arrays", "dev", "tests"]
+groups = ["arrays", "dev", "tests", "zarr"]
 dependencies = [
     "asciitree",
     "fasteners; sys_platform != \"emscripten\"",
@@ -1575,7 +2535,6 @@ version = "3.19.2"
 requires_python = ">=3.8"
 summary = "Backport of pathlib-compatible object wrapper for zip files"
 groups = ["arrays", "dask", "dev", "docs", "tests"]
-marker = "python_version < \"3.12\""
 files = [
     {file = "zipp-3.19.2-py3-none-any.whl", hash = "sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c"},
     {file = "zipp-3.19.2.tar.gz", hash = "sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19"},
diff --git a/pyproject.toml b/pyproject.toml
index f1b7173..0190678 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -73,12 +73,14 @@ tests = [
     "coveralls<4.0.0,>=3.3.1",
 ]
 docs = [
+    "numpydantic[arrays]",
     "sphinx<8.0.0,>=7.2.6",
     "furo>=2024.1.29",
     "myst-parser<3.0.0,>=2.0.0",
     "autodoc-pydantic<3.0.0,>=2.0.1",
     "sphinx-design<1.0.0,>=0.5.0",
     "sphinxcontrib-mermaid>=0.9.2",
+    "myst-nb>=1.1.1",
 ]
 dev = [
     "numpydantic[tests,docs]",
diff --git a/src/numpydantic/interface/interface.py b/src/numpydantic/interface/interface.py
index fa32b8e..94223eb 100644
--- a/src/numpydantic/interface/interface.py
+++ b/src/numpydantic/interface/interface.py
@@ -63,10 +63,15 @@ class JsonDict:
         return TypeAdapter(cls)
 
     @classmethod
-    def is_valid(cls, val: dict) -> bool:
+    def is_valid(cls, val: dict, raise_on_error: bool = False) -> bool:
         """
         Check whether a given dictionary matches this JsonDict specification
 
+        Args:
+            val (dict): The dictionary to check for validity
+            raise_on_error (bool): If ``True``, raise the validation error
+                rather than returning a bool. (default: ``False``)
+
         Returns:
             bool - true if valid, false if not
         """
@@ -74,7 +79,9 @@ class JsonDict:
         try:
             _ = adapter.validate_python(val)
             return True
-        except ValidationError:
+        except ValidationError as e:
+            if raise_on_error:
+                raise e
             return False
 
 
diff --git a/src/numpydantic/interface/video.py b/src/numpydantic/interface/video.py
index 3b3b499..7685628 100644
--- a/src/numpydantic/interface/video.py
+++ b/src/numpydantic/interface/video.py
@@ -159,6 +159,9 @@ class VideoProxy:
         return self[:]
 
     def __getitem__(self, item: Union[int, slice, tuple]) -> np.ndarray:
+        if not self.path.exists():
+            raise FileNotFoundError(f"Video file {self.path} does not exist!")
+
         if isinstance(item, int):
             # want a single frame
             return self._get_frame(item)
diff --git a/src/numpydantic/interface/zarr.py b/src/numpydantic/interface/zarr.py
index c7a28e1..9158f24 100644
--- a/src/numpydantic/interface/zarr.py
+++ b/src/numpydantic/interface/zarr.py
@@ -178,12 +178,12 @@ class ZarrInterface(Interface):
         :meth:`zarr.core.Array.info_items`
         plus the :meth:`zarr.core.Array.hexdigest` as a :class:`.ZarrJsonDict`
 
-        If either the ``zarr_dump_array`` value in the context dictionary is ``True``
+        If either the ``dump_array`` value in the context dictionary is ``True``
         or the zarr array is an in-memory array, dump the array as well
         (since without a persistent array it would be impossible to roundtrip and
         dumping to JSON would be meaningless)
 
-        Passing ``'zarr_dump_array': True`` to the serialization ``context``
+        Passing ```dump_array': True`` to the serialization ``context``
         looks like this::
 
             model.model_dump_json(context={'zarr_dump_array': True})
@@ -193,7 +193,7 @@ class ZarrInterface(Interface):
         if info.round_trip:
             dump_array = False
             if info is not None and info.context is not None:
-                dump_array = info.context.get("zarr_dump_array", False)
+                dump_array = info.context.get("dump_array", False)
             is_file = False
 
             as_json = {"type": cls.name}
diff --git a/src/numpydantic/ndarray.py b/src/numpydantic/ndarray.py
index d494154..fb81f69 100644
--- a/src/numpydantic/ndarray.py
+++ b/src/numpydantic/ndarray.py
@@ -24,10 +24,10 @@ from numpydantic.exceptions import InterfaceError
 from numpydantic.interface import Interface
 from numpydantic.maps import python_to_nptyping
 from numpydantic.schema import (
-    _jsonize_array,
     get_validate_interface,
     make_json_schema,
 )
+from numpydantic.serialization import jsonize_array
 from numpydantic.types import DtypeType, NDArrayType, ShapeType
 from numpydantic.vendor.nptyping.error import InvalidArgumentsError
 from numpydantic.vendor.nptyping.ndarray import NDArrayMeta as _NDArrayMeta
@@ -181,7 +181,7 @@ class NDArray(NPTypingType, metaclass=NDArrayMeta):
         return core_schema.with_info_plain_validator_function(
             get_validate_interface(shape, dtype),
             serialization=core_schema.plain_serializer_function_ser_schema(
-                _jsonize_array, when_used="json", info_arg=True
+                jsonize_array, when_used="json", info_arg=True
             ),
             metadata=json_schema,
         )
diff --git a/src/numpydantic/schema.py b/src/numpydantic/schema.py
index 36d6812..fff9d1d 100644
--- a/src/numpydantic/schema.py
+++ b/src/numpydantic/schema.py
@@ -5,15 +5,15 @@ Helper functions for use with :class:`~numpydantic.NDArray` - see the note in
 
 import hashlib
 import json
-from typing import TYPE_CHECKING, Any, Callable, Optional, Union
+from typing import TYPE_CHECKING, Any, Callable, Optional
 
 import numpy as np
-from pydantic import BaseModel, SerializationInfo
+from pydantic import BaseModel
 from pydantic_core import CoreSchema, core_schema
 from pydantic_core.core_schema import ListSchema, ValidationInfo
 
 from numpydantic import dtype as dt
-from numpydantic.interface import Interface, JsonDict
+from numpydantic.interface import Interface
 from numpydantic.maps import np_to_python
 from numpydantic.types import DtypeType, NDArrayType, ShapeType
 from numpydantic.vendor.nptyping.structure import StructureMeta
@@ -278,16 +278,3 @@ def get_validate_interface(shape: ShapeType, dtype: DtypeType) -> Callable:
         return value
 
     return validate_interface
-
-
-def _jsonize_array(value: Any, info: SerializationInfo) -> Union[list, dict]:
-    """Use an interface class to render an array as JSON"""
-    interface_cls = Interface.match_output(value)
-    array = interface_cls.to_json(value, info)
-    if isinstance(array, JsonDict):
-        array = array.to_dict()
-
-    if info.context and info.context.get("mark_interface", False):
-        array = interface_cls.mark_json(array)
-
-    return array
diff --git a/src/numpydantic/serialization.py b/src/numpydantic/serialization.py
new file mode 100644
index 0000000..a2b2507
--- /dev/null
+++ b/src/numpydantic/serialization.py
@@ -0,0 +1,94 @@
+from pathlib import Path
+from typing import Any, Callable, TypeVar, Union
+
+from pydantic_core.core_schema import SerializationInfo
+
+from numpydantic.interface import Interface, JsonDict
+
+T = TypeVar("T")
+U = TypeVar("U")
+
+
+def jsonize_array(value: Any, info: SerializationInfo) -> Union[list, dict]:
+    """Use an interface class to render an array as JSON"""
+    interface_cls = Interface.match_output(value)
+    array = interface_cls.to_json(value, info)
+    if isinstance(array, JsonDict):
+        array = array.to_dict()
+
+    if info.context:
+        if info.context.get("mark_interface", False):
+            array = interface_cls.mark_json(array)
+        if info.context.get("absolute_paths", False):
+            array = _absolutize_paths(array)
+        else:
+            relative_to = info.context.get("relative_to", ".")
+            array = _relativize_paths(array, relative_to)
+
+    return array
+
+
+def _relativize_paths(value: dict, relative_to: str = ".") -> dict:
+    """
+    Make paths relative to either the current directory or the provided
+    ``relative_to`` directory, if provided in the context
+    """
+    relative_to = Path(relative_to).resolve()
+
+    def _r_path(v: Any) -> Any:
+        try:
+            path = Path(v)
+            if not path.exists():
+                return v
+            return str(relative_path(path, relative_to))
+        except:
+            return v
+
+    return _walk_and_apply(value, _r_path)
+
+
+def _absolutize_paths(value: dict) -> dict:
+    def _a_path(v: Any) -> Any:
+        try:
+            path = Path(v)
+            if not path.exists():
+                return v
+            return str(path.resolve())
+        except:
+            return v
+
+    return _walk_and_apply(value, _a_path)
+
+
+def _walk_and_apply(value: T, f: Callable[[U], U]) -> T:
+    """
+    Walk an object, applying a function
+    """
+    if isinstance(value, dict):
+        for k, v in value.items():
+            if isinstance(v, dict):
+                _walk_and_apply(v, f)
+            elif isinstance(v, list):
+                value[k] = [_walk_and_apply(sub_v, f) for sub_v in v]
+            else:
+                value[k] = f(v)
+    elif isinstance(value, list):
+        value = [_walk_and_apply(v, f) for v in value]
+    else:
+        value = f(value)
+    return value
+
+
+def relative_path(target: Path, origin: Path) -> Path:
+    """
+    return path of target relative to origin, even if they're
+    not in the same subpath
+
+    References:
+        - https://stackoverflow.com/a/71874881
+    """
+    try:
+        return Path(target).resolve().relative_to(Path(origin).resolve())
+    except ValueError:  # target does not start with origin
+        # recursion with origin (eventually origin is root so try will succeed)
+        return Path("..").joinpath(relative_path(target, Path(origin).parent))

From 7b0c64a7f6c8e9e5dd1292cb07720f63979b3edc Mon Sep 17 00:00:00 2001
From: sneakers-the-rat <sneakers-the-rat@protonmail.com>
Date: Sat, 21 Sep 2024 04:22:20 -0700
Subject: [PATCH 05/22] how could i ever not depend on rich

---
 pdm.lock       | 44 ++++++++++++++++++++++++++++++--------------
 pyproject.toml |  1 +
 2 files changed, 31 insertions(+), 14 deletions(-)

diff --git a/pdm.lock b/pdm.lock
index 635d5b3..6673204 100644
--- a/pdm.lock
+++ b/pdm.lock
@@ -5,7 +5,7 @@
 groups = ["default", "arrays", "dask", "dev", "docs", "hdf5", "tests", "video", "zarr"]
 strategy = ["cross_platform", "inherit_metadata"]
 lock_version = "4.5.0"
-content_hash = "sha256:607f6694faec9103ba99e8e51010e905ad9ce03612dfb492e0d3fbeb1cb4d562"
+content_hash = "sha256:cc2b0fb32896c6df0ad747ddb5dee89af22f5c4c4643ee7a52db47fef30da936"
 
 [[metadata.targets]]
 requires_python = "~=3.9"
@@ -68,7 +68,7 @@ files = [
 name = "asciitree"
 version = "0.3.3"
 summary = "Draws ASCII trees."
-groups = ["arrays", "dev", "tests", "zarr"]
+groups = ["arrays", "dev", "docs", "tests", "zarr"]
 files = [
     {file = "asciitree-0.3.3.tar.gz", hash = "sha256:4aa4b9b649f85e3fcb343363d97564aa1fb62e249677f2e18a96765145cc0f6e"},
 ]
@@ -355,7 +355,7 @@ name = "cloudpickle"
 version = "3.0.0"
 requires_python = ">=3.8"
 summary = "Pickler class to extend the standard pickle.Pickler functionality"
-groups = ["arrays", "dask", "dev", "tests"]
+groups = ["arrays", "dask", "dev", "docs", "tests"]
 files = [
     {file = "cloudpickle-3.0.0-py3-none-any.whl", hash = "sha256:246ee7d0c295602a036e86369c77fecda4ab17b506496730f2f576d9016fd9c7"},
     {file = "cloudpickle-3.0.0.tar.gz", hash = "sha256:996d9a482c6fb4f33c1a35335cf8afd065d2a56e973270364840712d9131a882"},
@@ -492,7 +492,7 @@ name = "dask"
 version = "2024.7.1"
 requires_python = ">=3.9"
 summary = "Parallel PyData with Task Scheduling"
-groups = ["arrays", "dask", "dev", "tests"]
+groups = ["arrays", "dask", "dev", "docs", "tests"]
 dependencies = [
     "click>=8.1",
     "cloudpickle>=1.5.0",
@@ -594,7 +594,7 @@ name = "fasteners"
 version = "0.19"
 requires_python = ">=3.6"
 summary = "A python package that provides useful locks"
-groups = ["arrays", "dev", "tests", "zarr"]
+groups = ["arrays", "dev", "docs", "tests", "zarr"]
 marker = "sys_platform != \"emscripten\""
 files = [
     {file = "fasteners-0.19-py3-none-any.whl", hash = "sha256:758819cb5d94cdedf4e836988b74de396ceacb8e2794d21f82d131fd9ee77237"},
@@ -616,7 +616,7 @@ name = "fsspec"
 version = "2024.6.1"
 requires_python = ">=3.8"
 summary = "File-system specification"
-groups = ["arrays", "dask", "dev", "tests"]
+groups = ["arrays", "dask", "dev", "docs", "tests"]
 files = [
     {file = "fsspec-2024.6.1-py3-none-any.whl", hash = "sha256:3cb443f8bcd2efb31295a5b9fdb02aee81d8452c80d28f97a6d0959e6cee101e"},
     {file = "fsspec-2024.6.1.tar.gz", hash = "sha256:fad7d7e209dd4c1208e3bbfda706620e0da5142bebbd9c384afb95b07e798e49"},
@@ -736,7 +736,7 @@ name = "h5py"
 version = "3.11.0"
 requires_python = ">=3.8"
 summary = "Read and write HDF5 files from Python"
-groups = ["arrays", "dev", "hdf5", "tests"]
+groups = ["arrays", "dev", "docs", "hdf5", "tests"]
 dependencies = [
     "numpy>=1.17.3",
 ]
@@ -981,7 +981,7 @@ name = "locket"
 version = "1.0.0"
 requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 summary = "File-based locks for Python on Linux and Windows"
-groups = ["arrays", "dask", "dev", "tests"]
+groups = ["arrays", "dask", "dev", "docs", "tests"]
 files = [
     {file = "locket-1.0.0-py2.py3-none-any.whl", hash = "sha256:b6c819a722f7b6bd955b80781788e4a66a55628b858d347536b7e81325a3a5e3"},
     {file = "locket-1.0.0.tar.gz", hash = "sha256:5c0d4c052a8bbbf750e056a8e65ccd309086f4f0f18a2eac306a8dfa4112a632"},
@@ -1204,7 +1204,7 @@ name = "numcodecs"
 version = "0.12.1"
 requires_python = ">=3.8"
 summary = "A Python package providing buffer compression and transformation codecs for use in data storage and communication applications."
-groups = ["arrays", "dev", "tests", "zarr"]
+groups = ["arrays", "dev", "docs", "tests", "zarr"]
 dependencies = [
     "numpy>=1.7",
 ]
@@ -1233,7 +1233,7 @@ name = "numpy"
 version = "2.0.1"
 requires_python = ">=3.9"
 summary = "Fundamental package for array computing in Python"
-groups = ["default", "arrays", "dev", "hdf5", "tests", "video", "zarr"]
+groups = ["default", "arrays", "dev", "docs", "hdf5", "tests", "video", "zarr"]
 files = [
     {file = "numpy-2.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0fbb536eac80e27a2793ffd787895242b7f18ef792563d742c2d673bfcb75134"},
     {file = "numpy-2.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:69ff563d43c69b1baba77af455dd0a839df8d25e8590e79c90fcbe1499ebde42"},
@@ -1287,7 +1287,7 @@ name = "opencv-python"
 version = "4.10.0.84"
 requires_python = ">=3.6"
 summary = "Wrapper package for OpenCV python bindings."
-groups = ["arrays", "dev", "tests", "video"]
+groups = ["arrays", "dev", "docs", "tests", "video"]
 dependencies = [
     "numpy>=1.13.3; python_version < \"3.7\"",
     "numpy>=1.17.0; python_version >= \"3.7\"",
@@ -1337,7 +1337,7 @@ name = "partd"
 version = "1.4.2"
 requires_python = ">=3.9"
 summary = "Appendable key-value storage"
-groups = ["arrays", "dask", "dev", "tests"]
+groups = ["arrays", "dask", "dev", "docs", "tests"]
 dependencies = [
     "locket",
     "toolz",
@@ -1849,6 +1849,22 @@ files = [
     {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"},
 ]
 
+[[package]]
+name = "rich"
+version = "13.8.1"
+requires_python = ">=3.7.0"
+summary = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal"
+groups = ["dev", "docs"]
+dependencies = [
+    "markdown-it-py>=2.2.0",
+    "pygments<3.0.0,>=2.13.0",
+    "typing-extensions<5.0,>=4.0.0; python_version < \"3.9\"",
+]
+files = [
+    {file = "rich-13.8.1-py3-none-any.whl", hash = "sha256:1760a3c0848469b97b558fc61c85233e3dafb69c7a071b4d60c38099d3cd4c06"},
+    {file = "rich-13.8.1.tar.gz", hash = "sha256:8260cda28e3db6bf04d2d1ef4dbc03ba80a824c88b0e7668a0f23126a424844a"},
+]
+
 [[package]]
 name = "rpds-py"
 version = "0.20.0"
@@ -2279,7 +2295,7 @@ name = "toolz"
 version = "0.12.1"
 requires_python = ">=3.7"
 summary = "List processing tools and functional utilities"
-groups = ["arrays", "dask", "dev", "tests"]
+groups = ["arrays", "dask", "dev", "docs", "tests"]
 files = [
     {file = "toolz-0.12.1-py3-none-any.whl", hash = "sha256:d22731364c07d72eea0a0ad45bafb2c2937ab6fd38a3507bf55eae8744aa7d85"},
     {file = "toolz-0.12.1.tar.gz", hash = "sha256:ecca342664893f177a13dac0e6b41cbd8ac25a358e5f215316d43e2100224f4d"},
@@ -2517,7 +2533,7 @@ name = "zarr"
 version = "2.18.2"
 requires_python = ">=3.9"
 summary = "An implementation of chunked, compressed, N-dimensional arrays for Python"
-groups = ["arrays", "dev", "tests", "zarr"]
+groups = ["arrays", "dev", "docs", "tests", "zarr"]
 dependencies = [
     "asciitree",
     "fasteners; sys_platform != \"emscripten\"",
diff --git a/pyproject.toml b/pyproject.toml
index 0190678..79d93ba 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -81,6 +81,7 @@ docs = [
     "sphinx-design<1.0.0,>=0.5.0",
     "sphinxcontrib-mermaid>=0.9.2",
     "myst-nb>=1.1.1",
+    "rich>=13.8.1",
 ]
 dev = [
     "numpydantic[tests,docs]",

From 2f453f1d0372fee50554d53bc65ed0dd86df3da7 Mon Sep 17 00:00:00 2001
From: sneakers-the-rat <sneakers-the-rat@protonmail.com>
Date: Sat, 21 Sep 2024 04:27:53 -0700
Subject: [PATCH 06/22] better video file not found position

---
 src/numpydantic/interface/video.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/numpydantic/interface/video.py b/src/numpydantic/interface/video.py
index 7685628..60f0846 100644
--- a/src/numpydantic/interface/video.py
+++ b/src/numpydantic/interface/video.py
@@ -69,6 +69,9 @@ class VideoProxy:
                     "and it cant be reopened since source path cant be gotten "
                     "from VideoCapture objects"
                 )
+            if not self.path.exists():
+                raise FileNotFoundError(f"Video file {self.path} does not exist!")
+
             self._video = VideoCapture(str(self.path))
         return self._video
 
@@ -159,9 +162,6 @@ class VideoProxy:
         return self[:]
 
     def __getitem__(self, item: Union[int, slice, tuple]) -> np.ndarray:
-        if not self.path.exists():
-            raise FileNotFoundError(f"Video file {self.path} does not exist!")
-
         if isinstance(item, int):
             # want a single frame
             return self._get_frame(item)

From 135b74aa2e4d6e5a755c27d3bb7fe170650058d3 Mon Sep 17 00:00:00 2001
From: sneakers-the-rat <sneakers-the-rat@protonmail.com>
Date: Sat, 21 Sep 2024 04:37:13 -0700
Subject: [PATCH 07/22] omg does myst_nb execute in the cwd of the file, that's
 even better

---
 docs/serialization.md | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/docs/serialization.md b/docs/serialization.md
index f18b8f6..5812f30 100644
--- a/docs/serialization.md
+++ b/docs/serialization.md
@@ -81,7 +81,7 @@ print(model.model_dump_json())
 ```
 
 ```{code-cell}
-model = MyModel(array="docs/data/test.avi")
+model = MyModel(array="data/test.avi")
 print(model.model_dump_json())
 ```
 
@@ -138,12 +138,12 @@ When possible, the full content of the array is omitted in favor
 of the path to the file that provided it.
 
 ```{code-cell}
-model = MyModel(array="docs/data/test.avi")
+model = MyModel(array="data/test.avi")
 print_json(model.model_dump_json(round_trip=True))
 ```
 
 ```{code-cell}
-model = MyModel(array=("docs/data/test.h5", "/data"))
+model = MyModel(array=("data/test.h5", "/data"))
 print_json(model.model_dump_json(round_trip=True))
 ```
 
@@ -169,7 +169,7 @@ you might want to serialize relative to each of them:
 print_json(
   model.model_dump_json(
     round_trip=True, 
-    context={"relative_to": Path('./docs/data')}
+    context={"relative_to": Path('./data')}
   ))
 ```
 
@@ -261,7 +261,7 @@ Supported interfaces:
 - (all)
 
 ```{code-cell}
-model = MyModel(array=("docs/data/test.h5", "/data"))
+model = MyModel(array=("data/test.h5", "/data"))
 data = model.model_dump_json(
     round_trip=True, 
     context={"absolute_paths": True}
@@ -278,7 +278,7 @@ Supported interfaces:
 - (all)
 
 ```{code-cell}
-model = MyModel(array=("docs/data/test.h5", "/data"))
+model = MyModel(array=("data/test.h5", "/data"))
 data = model.model_dump_json(
     round_trip=True, 
     context={"relative_to": Path('../')}
@@ -294,7 +294,7 @@ Supported interfaces:
 - {class}`.ZarrInterface`
 
 ```{code-cell}
-model = MyModel(array=("docs/data/test.zarr",))
+model = MyModel(array=("data/test.zarr",))
 data = model.model_dump_json(
     round_trip=True, 
     context={"dump_array": True}

From 70bf254dddc8de454bde2214e023c62aca6541e8 Mon Sep 17 00:00:00 2001
From: sneakers-the-rat <sneakers-the-rat@protonmail.com>
Date: Sat, 21 Sep 2024 18:26:25 -0700
Subject: [PATCH 08/22] lint, fix zarr dump test

---
 docs/api/serialization.md         | 7 +++++++
 docs/index.md                     | 1 +
 src/numpydantic/serialization.py  | 9 +++++++--
 tests/test_interface/test_zarr.py | 2 +-
 4 files changed, 16 insertions(+), 3 deletions(-)
 create mode 100644 docs/api/serialization.md

diff --git a/docs/api/serialization.md b/docs/api/serialization.md
new file mode 100644
index 0000000..f070ea0
--- /dev/null
+++ b/docs/api/serialization.md
@@ -0,0 +1,7 @@
+# serialization
+
+```{eval-rst}
+.. automodule:: numpydantic.serialization
+    :members:
+    :undoc-members:
+```
\ No newline at end of file
diff --git a/docs/index.md b/docs/index.md
index c75ce74..003f5c3 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -491,6 +491,7 @@ api/ndarray
 api/maps
 api/meta
 api/schema
+api/serialization
 api/shape
 api/types
 
diff --git a/src/numpydantic/serialization.py b/src/numpydantic/serialization.py
index a2b2507..2f5b578 100644
--- a/src/numpydantic/serialization.py
+++ b/src/numpydantic/serialization.py
@@ -1,3 +1,8 @@
+"""
+Serialization helpers for :func:`pydantic.BaseModel.model_dump`
+and :func:`pydantic.BaseModel.model_dump_json` .
+"""
+
 from pathlib import Path
 from typing import Any, Callable, TypeVar, Union
 
@@ -41,7 +46,7 @@ def _relativize_paths(value: dict, relative_to: str = ".") -> dict:
             if not path.exists():
                 return v
             return str(relative_path(path, relative_to))
-        except:
+        except (TypeError, ValueError):
             return v
 
     return _walk_and_apply(value, _r_path)
@@ -54,7 +59,7 @@ def _absolutize_paths(value: dict) -> dict:
             if not path.exists():
                 return v
             return str(path.resolve())
-        except:
+        except (TypeError, ValueError):
             return v
 
     return _walk_and_apply(value, _a_path)
diff --git a/tests/test_interface/test_zarr.py b/tests/test_interface/test_zarr.py
index 05eb8d5..fca15ae 100644
--- a/tests/test_interface/test_zarr.py
+++ b/tests/test_interface/test_zarr.py
@@ -141,7 +141,7 @@ def test_zarr_to_json(store, model_blank, roundtrip, dump_array):
     array = zarr.array(lol_array, store=store)
     instance = model_blank(array=array)
 
-    context = {"zarr_dump_array": dump_array}
+    context = {"dump_array": dump_array}
     as_json = json.loads(
         instance.model_dump_json(round_trip=roundtrip, context=context)
     )["array"]

From 74f03b10bf0d1cc8a4fb4a8829876cc891e305af Mon Sep 17 00:00:00 2001
From: sneakers-the-rat <sneakers-the-rat@protonmail.com>
Date: Sat, 21 Sep 2024 21:24:32 -0700
Subject: [PATCH 09/22] use pydantic model not dataclass

---
 src/numpydantic/interface/dask.py      |  4 ++--
 src/numpydantic/interface/hdf5.py      |  6 +++---
 src/numpydantic/interface/interface.py | 24 +++---------------------
 src/numpydantic/interface/numpy.py     |  4 ++--
 src/numpydantic/interface/video.py     |  4 ++--
 src/numpydantic/interface/zarr.py      |  5 +++--
 src/numpydantic/serialization.py       |  2 +-
 7 files changed, 16 insertions(+), 33 deletions(-)

diff --git a/src/numpydantic/interface/dask.py b/src/numpydantic/interface/dask.py
index 6f02025..13c6c1a 100644
--- a/src/numpydantic/interface/dask.py
+++ b/src/numpydantic/interface/dask.py
@@ -2,7 +2,6 @@
 Interface for Dask arrays
 """
 
-from dataclasses import dataclass
 from typing import Any, Iterable, Literal, Optional
 
 import numpy as np
@@ -25,7 +24,6 @@ def _as_tuple(a_list: list | Any) -> tuple:
     )
 
 
-@dataclass(kw_only=True)
 class DaskJsonDict(JsonDict):
     """
     Round-trip json serialized form of a dask array
@@ -78,6 +76,8 @@ class DaskInterface(Interface):
         """
         if isinstance(array, dict):
             array = DaskJsonDict(**array).to_array_input()
+        elif isinstance(array, DaskJsonDict):
+            array = array.to_array_input()
 
         return array
 
diff --git a/src/numpydantic/interface/hdf5.py b/src/numpydantic/interface/hdf5.py
index 9a5bf93..4ce16ce 100644
--- a/src/numpydantic/interface/hdf5.py
+++ b/src/numpydantic/interface/hdf5.py
@@ -40,7 +40,6 @@ as ``S32`` isoformatted byte strings (timezones optional) like:
 """
 
 import sys
-from dataclasses import dataclass
 from datetime import datetime
 from pathlib import Path
 from typing import Any, Iterable, List, NamedTuple, Optional, Tuple, TypeVar, Union
@@ -77,7 +76,6 @@ class H5ArrayPath(NamedTuple):
     """Refer to a specific field within a compound dtype"""
 
 
-@dataclass
 class H5JsonDict(JsonDict):
     """Round-trip Json-able version of an HDF5 dataset"""
 
@@ -88,7 +86,7 @@ class H5JsonDict(JsonDict):
     def to_array_input(self) -> H5ArrayPath:
         """Construct an :class:`.H5ArrayPath`"""
         return H5ArrayPath(
-            **{k: v for k, v in self.to_dict().items() if k in H5ArrayPath._fields}
+            **{k: v for k, v in self.model_dump().items() if k in H5ArrayPath._fields}
         )
 
 
@@ -330,6 +328,8 @@ class H5Interface(Interface):
         """Create an :class:`.H5Proxy` to use throughout validation"""
         if isinstance(array, dict):
             array = H5JsonDict(**array).to_array_input()
+        elif isinstance(array, H5JsonDict):
+            array = array.to_array_input()
 
         if isinstance(array, H5ArrayPath):
             array = H5Proxy.from_h5array(h5array=array)
diff --git a/src/numpydantic/interface/interface.py b/src/numpydantic/interface/interface.py
index 94223eb..ebcb950 100644
--- a/src/numpydantic/interface/interface.py
+++ b/src/numpydantic/interface/interface.py
@@ -4,13 +4,12 @@ Base Interface metaclass
 
 import inspect
 from abc import ABC, abstractmethod
-from dataclasses import asdict, dataclass
 from importlib.metadata import PackageNotFoundError, version
 from operator import attrgetter
 from typing import Any, Generic, Tuple, Type, TypedDict, TypeVar, Union
 
 import numpy as np
-from pydantic import SerializationInfo, TypeAdapter, ValidationError
+from pydantic import BaseModel, SerializationInfo, ValidationError
 
 from numpydantic.exceptions import (
     DtypeError,
@@ -32,13 +31,9 @@ class InterfaceMark(TypedDict):
     version: str
 
 
-@dataclass(kw_only=True)
-class JsonDict:
+class JsonDict(BaseModel):
     """
     Representation of array when dumped with round_trip == True.
-
-    Using a dataclass rather than a pydantic model to not tempt
-    us to use more sophisticated types than can be serialized to json.
     """
 
     type: str
@@ -50,18 +45,6 @@ class JsonDict:
         (one of the ``input_types`` of an interface).
         """
 
-    def to_dict(self) -> dict:
-        """
-        Convenience method for casting dataclass to dict,
-        removing None-valued items
-        """
-        return {k: v for k, v in asdict(self).items() if v is not None}
-
-    @classmethod
-    def get_adapter(cls) -> TypeAdapter:
-        """Convenience method to get a typeadapter for this class"""
-        return TypeAdapter(cls)
-
     @classmethod
     def is_valid(cls, val: dict, raise_on_error: bool = False) -> bool:
         """
@@ -75,9 +58,8 @@ class JsonDict:
         Returns:
             bool - true if valid, false if not
         """
-        adapter = cls.get_adapter()
         try:
-            _ = adapter.validate_python(val)
+            _ = cls.model_validate(val)
             return True
         except ValidationError as e:
             if raise_on_error:
diff --git a/src/numpydantic/interface/numpy.py b/src/numpydantic/interface/numpy.py
index 8c68f1e..a1e0b94 100644
--- a/src/numpydantic/interface/numpy.py
+++ b/src/numpydantic/interface/numpy.py
@@ -2,7 +2,6 @@
 Interface to numpy arrays
 """
 
-from dataclasses import dataclass
 from typing import Any, Literal, Union
 
 from pydantic import SerializationInfo
@@ -21,7 +20,6 @@ except ImportError:  # pragma: no cover
     np = None
 
 
-@dataclass
 class NumpyJsonDict(JsonDict):
     """
     JSON-able roundtrip representation of numpy array
@@ -78,6 +76,8 @@ class NumpyInterface(Interface):
         """
         if isinstance(array, dict):
             array = NumpyJsonDict(**array).to_array_input()
+        elif isinstance(array, NumpyJsonDict):
+            array = array.to_array_input()
 
         if not isinstance(array, ndarray):
             array = np.array(array)
diff --git a/src/numpydantic/interface/video.py b/src/numpydantic/interface/video.py
index 60f0846..23f940d 100644
--- a/src/numpydantic/interface/video.py
+++ b/src/numpydantic/interface/video.py
@@ -2,7 +2,6 @@
 Interface to support treating videos like arrays using OpenCV
 """
 
-from dataclasses import dataclass
 from pathlib import Path
 from typing import Any, Literal, Optional, Tuple, Union
 
@@ -22,7 +21,6 @@ except ImportError:  # pragma: no cover
 VIDEO_EXTENSIONS = (".mp4", ".avi", ".mov", ".mkv")
 
 
-@dataclass(kw_only=True)
 class VideoJsonDict(JsonDict):
     """Json-able roundtrip representation of a video file"""
 
@@ -256,6 +254,8 @@ class VideoInterface(Interface):
         """Get a :class:`.VideoProxy` object for this video"""
         if isinstance(array, dict):
             proxy = VideoJsonDict(**array).to_array_input()
+        elif isinstance(array, VideoJsonDict):
+            proxy = array.to_array_input()
         elif isinstance(array, VideoCapture):
             proxy = VideoProxy(video=array)
         elif isinstance(array, VideoProxy):
diff --git a/src/numpydantic/interface/zarr.py b/src/numpydantic/interface/zarr.py
index 9158f24..922d806 100644
--- a/src/numpydantic/interface/zarr.py
+++ b/src/numpydantic/interface/zarr.py
@@ -56,7 +56,6 @@ class ZarrArrayPath:
             raise ValueError("Only len 1-2 iterables can be used for a ZarrArrayPath")
 
 
-@dataclass(kw_only=True)
 class ZarrJsonDict(JsonDict):
     """Round-trip Json-able version of a Zarr Array"""
 
@@ -94,10 +93,12 @@ class ZarrInterface(Interface):
 
     @staticmethod
     def _get_array(
-        array: Union[ZarrArray, str, Path, ZarrArrayPath, Sequence]
+        array: Union[ZarrArray, str, dict, ZarrJsonDict, Path, ZarrArrayPath, Sequence]
     ) -> ZarrArray:
         if isinstance(array, dict):
             array = ZarrJsonDict(**array).to_array_input()
+        elif isinstance(array, ZarrJsonDict):
+            array = array.to_array_input()
 
         if isinstance(array, ZarrArray):
             return array
diff --git a/src/numpydantic/serialization.py b/src/numpydantic/serialization.py
index 2f5b578..f645239 100644
--- a/src/numpydantic/serialization.py
+++ b/src/numpydantic/serialization.py
@@ -19,7 +19,7 @@ def jsonize_array(value: Any, info: SerializationInfo) -> Union[list, dict]:
     interface_cls = Interface.match_output(value)
     array = interface_cls.to_json(value, info)
     if isinstance(array, JsonDict):
-        array = array.to_dict()
+        array = array.model_dump(exclude_none=True)
 
     if info.context:
         if info.context.get("mark_interface", False):

From 705de53838dbe9a7e8fc4aa0c41bbbb65ba4dd6b Mon Sep 17 00:00:00 2001
From: sneakers-the-rat <sneakers-the-rat@protonmail.com>
Date: Sat, 21 Sep 2024 21:30:05 -0700
Subject: [PATCH 10/22] python 3.9 compat

---
 src/numpydantic/interface/dask.py | 6 +++---
 src/numpydantic/interface/zarr.py | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/numpydantic/interface/dask.py b/src/numpydantic/interface/dask.py
index 13c6c1a..960c94b 100644
--- a/src/numpydantic/interface/dask.py
+++ b/src/numpydantic/interface/dask.py
@@ -2,7 +2,7 @@
 Interface for Dask arrays
 """
 
-from typing import Any, Iterable, Literal, Optional
+from typing import Any, Iterable, List, Literal, Optional, Union
 
 import numpy as np
 from pydantic import SerializationInfo
@@ -17,7 +17,7 @@ except ImportError:  # pragma: no cover
     DaskArray = None
 
 
-def _as_tuple(a_list: list | Any) -> tuple:
+def _as_tuple(a_list: Any) -> tuple:
     """Make a list of list into a tuple of tuples"""
     return tuple(
         [_as_tuple(item) if isinstance(item, list) else item for item in a_list]
@@ -93,7 +93,7 @@ class DaskInterface(Interface):
     @classmethod
     def to_json(
         cls, array: DaskArray, info: Optional[SerializationInfo] = None
-    ) -> list | DaskJsonDict:
+    ) -> Union[List, DaskJsonDict]:
         """
         Convert an array to a JSON serializable array by first converting to a numpy
         array and then to a list.
diff --git a/src/numpydantic/interface/zarr.py b/src/numpydantic/interface/zarr.py
index 922d806..d79491c 100644
--- a/src/numpydantic/interface/zarr.py
+++ b/src/numpydantic/interface/zarr.py
@@ -65,7 +65,7 @@ class ZarrJsonDict(JsonDict):
     path: Optional[str] = None
     array: Optional[list] = None
 
-    def to_array_input(self) -> ZarrArray | ZarrArrayPath:
+    def to_array_input(self) -> Union[ZarrArray, ZarrArrayPath]:
         """
         Construct a ZarrArrayPath if file and path are present,
         otherwise a ZarrArray
@@ -168,7 +168,7 @@ class ZarrInterface(Interface):
         cls,
         array: Union[ZarrArray, str, Path, ZarrArrayPath, Sequence],
         info: Optional[SerializationInfo] = None,
-    ) -> list | ZarrJsonDict:
+    ) -> Union[list, ZarrJsonDict]:
         """
         Dump a Zarr Array to JSON
 

From 9026eb700f1f33a0d6659aec818ece688a83ba7a Mon Sep 17 00:00:00 2001
From: sneakers-the-rat <sneakers-the-rat@protonmail.com>
Date: Sat, 21 Sep 2024 21:58:11 -0700
Subject: [PATCH 11/22] add json_model abstract class attr, make deserialize
 validation method

---
 docs/interfaces.md                     |  5 +++
 src/numpydantic/interface/dask.py      | 13 +-------
 src/numpydantic/interface/hdf5.py      |  6 +---
 src/numpydantic/interface/interface.py | 43 +++++++++++++++++++++++++-
 src/numpydantic/interface/numpy.py     |  6 +---
 src/numpydantic/interface/video.py     |  7 ++---
 src/numpydantic/interface/zarr.py      |  6 +---
 7 files changed, 53 insertions(+), 33 deletions(-)

diff --git a/docs/interfaces.md b/docs/interfaces.md
index c4d873d..b26ebba 100644
--- a/docs/interfaces.md
+++ b/docs/interfaces.md
@@ -46,6 +46,11 @@ for interfaces to implement custom behavior that matches the array format.
 
 {meth}`.Interface.validate` calls the following methods, in order:
 
+A method to deserialize the array dumped with a {func}`~pydantic.BaseModel.model_dump_json`
+with `round_trip = True` (see [serialization](./serialization.md))
+ 
+- {meth}`.Interface.deserialize`
+
 An initial hook for modifying the input data before validation, eg.
 if it needs to be coerced or wrapped in some proxy class. This method
 should accept all and only the types specified in that interface's
diff --git a/src/numpydantic/interface/dask.py b/src/numpydantic/interface/dask.py
index 960c94b..bc12a13 100644
--- a/src/numpydantic/interface/dask.py
+++ b/src/numpydantic/interface/dask.py
@@ -54,6 +54,7 @@ class DaskInterface(Interface):
     name = "dask"
     input_types = (DaskArray, dict)
     return_type = DaskArray
+    json_model = DaskJsonDict
 
     @classmethod
     def check(cls, array: Any) -> bool:
@@ -69,18 +70,6 @@ class DaskInterface(Interface):
         else:
             return False
 
-    def before_validation(self, array: Any) -> DaskArray:
-        """
-        If given a dict (like that from ``model_dump_json(round_trip=True)`` ),
-        re-cast to dask array
-        """
-        if isinstance(array, dict):
-            array = DaskJsonDict(**array).to_array_input()
-        elif isinstance(array, DaskJsonDict):
-            array = array.to_array_input()
-
-        return array
-
     def get_object_dtype(self, array: NDArrayType) -> DtypeType:
         """Dask arrays require a compute() call to retrieve a single value"""
         return type(array.ravel()[0].compute())
diff --git a/src/numpydantic/interface/hdf5.py b/src/numpydantic/interface/hdf5.py
index 4ce16ce..67b9899 100644
--- a/src/numpydantic/interface/hdf5.py
+++ b/src/numpydantic/interface/hdf5.py
@@ -278,6 +278,7 @@ class H5Interface(Interface):
     name = "hdf5"
     input_types = (H5ArrayPath, H5Arraylike, H5Proxy)
     return_type = H5Proxy
+    json_model = H5JsonDict
 
     @classmethod
     def enabled(cls) -> bool:
@@ -326,11 +327,6 @@ class H5Interface(Interface):
 
     def before_validation(self, array: Any) -> NDArrayType:
         """Create an :class:`.H5Proxy` to use throughout validation"""
-        if isinstance(array, dict):
-            array = H5JsonDict(**array).to_array_input()
-        elif isinstance(array, H5JsonDict):
-            array = array.to_array_input()
-
         if isinstance(array, H5ArrayPath):
             array = H5Proxy.from_h5array(h5array=array)
         elif isinstance(array, H5Proxy):
diff --git a/src/numpydantic/interface/interface.py b/src/numpydantic/interface/interface.py
index ebcb950..2cb61f4 100644
--- a/src/numpydantic/interface/interface.py
+++ b/src/numpydantic/interface/interface.py
@@ -21,6 +21,9 @@ from numpydantic.shape import check_shape
 from numpydantic.types import DtypeType, NDArrayType, ShapeType
 
 T = TypeVar("T", bound=NDArrayType)
+U = TypeVar("U", bound="JsonDict")
+V = TypeVar("V")  # input type
+W = TypeVar("W")  # Any type in handle_input
 
 
 class InterfaceMark(TypedDict):
@@ -39,7 +42,7 @@ class JsonDict(BaseModel):
     type: str
 
     @abstractmethod
-    def to_array_input(self) -> Any:
+    def to_array_input(self) -> V:
         """
         Convert this roundtrip specifier to the relevant input class
         (one of the ``input_types`` of an interface).
@@ -66,6 +69,20 @@ class JsonDict(BaseModel):
                 raise e
             return False
 
+    @classmethod
+    def handle_input(cls: Type[U], value: Union[dict, U, W]) -> Union[V, W]:
+        """
+        Handle input that is the json serialized roundtrip version
+        (from :func:`~pydantic.BaseModel.model_dump` with ``round_trip=True``)
+        converting it to the input format with :meth:`.JsonDict.to_array_input`
+        or passing it through if not applicable
+        """
+        if isinstance(value, dict):
+            value = cls(**value).to_array_input()
+        elif isinstance(value, cls):
+            value = value.to_array_input()
+        return value
+
 
 class Interface(ABC, Generic[T]):
     """
@@ -86,6 +103,7 @@ class Interface(ABC, Generic[T]):
 
         Calls the methods, in order:
 
+        * array = :meth:`.deserialize` (array)
         * array = :meth:`.before_validation` (array)
         * dtype = :meth:`.get_dtype` (array) - get the dtype from the array,
             override if eg. the dtype is not contained in ``array.dtype``
@@ -120,6 +138,8 @@ class Interface(ABC, Generic[T]):
             :class:`.DtypeError` and :class:`.ShapeError` (both of which are children
             of :class:`.InterfaceError` )
         """
+        array = self.deserialize(array)
+
         array = self.before_validation(array)
 
         dtype = self.get_dtype(array)
@@ -135,6 +155,19 @@ class Interface(ABC, Generic[T]):
 
         return array
 
+    def deserialize(self, array: Any) -> Union[V, Any]:
+        """
+        If given a JSON serialized version of the array,
+        deserialize it first
+
+        Args:
+            array:
+
+        Returns:
+
+        """
+        return self.json_model.handle_input(array)
+
     def before_validation(self, array: Any) -> NDArrayType:
         """
         Optional step pre-validation that coerces the input into a type that can be
@@ -270,6 +303,14 @@ class Interface(ABC, Generic[T]):
         Short name for this interface
         """
 
+    @property
+    @abstractmethod
+    def json_model(self) -> JsonDict:
+        """
+        The :class:`.JsonDict` model used for roundtripping
+        JSON serialization
+        """
+
     @classmethod
     @abstractmethod
     def to_json(cls, array: Type[T], info: SerializationInfo) -> Union[list, JsonDict]:
diff --git a/src/numpydantic/interface/numpy.py b/src/numpydantic/interface/numpy.py
index a1e0b94..ad97474 100644
--- a/src/numpydantic/interface/numpy.py
+++ b/src/numpydantic/interface/numpy.py
@@ -44,6 +44,7 @@ class NumpyInterface(Interface):
     name = "numpy"
     input_types = (ndarray, list)
     return_type = ndarray
+    json_model = NumpyJsonDict
     priority = -999
     """
     The numpy interface is usually the interface of last resort.
@@ -74,11 +75,6 @@ class NumpyInterface(Interface):
         Coerce to an ndarray. We have already checked if coercion is possible
         in :meth:`.check`
         """
-        if isinstance(array, dict):
-            array = NumpyJsonDict(**array).to_array_input()
-        elif isinstance(array, NumpyJsonDict):
-            array = array.to_array_input()
-
         if not isinstance(array, ndarray):
             array = np.array(array)
         return array
diff --git a/src/numpydantic/interface/video.py b/src/numpydantic/interface/video.py
index 23f940d..b214a74 100644
--- a/src/numpydantic/interface/video.py
+++ b/src/numpydantic/interface/video.py
@@ -221,6 +221,7 @@ class VideoInterface(Interface):
     name = "video"
     input_types = (str, Path, VideoCapture, VideoProxy)
     return_type = VideoProxy
+    json_model = VideoJsonDict
 
     @classmethod
     def enabled(cls) -> bool:
@@ -252,11 +253,7 @@ class VideoInterface(Interface):
 
     def before_validation(self, array: Any) -> VideoProxy:
         """Get a :class:`.VideoProxy` object for this video"""
-        if isinstance(array, dict):
-            proxy = VideoJsonDict(**array).to_array_input()
-        elif isinstance(array, VideoJsonDict):
-            proxy = array.to_array_input()
-        elif isinstance(array, VideoCapture):
+        if isinstance(array, VideoCapture):
             proxy = VideoProxy(video=array)
         elif isinstance(array, VideoProxy):
             proxy = array
diff --git a/src/numpydantic/interface/zarr.py b/src/numpydantic/interface/zarr.py
index d79491c..41cad03 100644
--- a/src/numpydantic/interface/zarr.py
+++ b/src/numpydantic/interface/zarr.py
@@ -85,6 +85,7 @@ class ZarrInterface(Interface):
     name = "zarr"
     input_types = (Path, ZarrArray, ZarrArrayPath)
     return_type = ZarrArray
+    json_model = ZarrJsonDict
 
     @classmethod
     def enabled(cls) -> bool:
@@ -95,11 +96,6 @@ class ZarrInterface(Interface):
     def _get_array(
         array: Union[ZarrArray, str, dict, ZarrJsonDict, Path, ZarrArrayPath, Sequence]
     ) -> ZarrArray:
-        if isinstance(array, dict):
-            array = ZarrJsonDict(**array).to_array_input()
-        elif isinstance(array, ZarrJsonDict):
-            array = array.to_array_input()
-
         if isinstance(array, ZarrArray):
             return array
 

From 8cc2574399fdc25e3789c43b1c201c7eaa87adea Mon Sep 17 00:00:00 2001
From: sneakers-the-rat <sneakers-the-rat@protonmail.com>
Date: Mon, 23 Sep 2024 13:28:38 -0700
Subject: [PATCH 12/22] add marks to all tests

---
 pyproject.toml                          |  8 +++-
 src/numpydantic/interface/dask.py       |  2 +-
 tests/conftest.py                       |  2 +
 tests/fixtures.py                       |  5 +++
 tests/test_interface/conftest.py        | 56 +++++++++++++++++--------
 tests/test_interface/test_dask.py       |  7 +++-
 tests/test_interface/test_dunder.py     | 10 -----
 tests/test_interface/test_hdf5.py       | 12 ++++++
 tests/test_interface/test_interfaces.py |  7 ++++
 tests/test_interface/test_numpy.py      |  4 ++
 tests/test_interface/test_video.py      |  6 +++
 tests/test_interface/test_zarr.py       |  5 ++-
 tests/test_ndarray.py                   | 11 +++--
 tests/test_shape.py                     |  2 +
 14 files changed, 101 insertions(+), 36 deletions(-)
 delete mode 100644 tests/test_interface/test_dunder.py

diff --git a/pyproject.toml b/pyproject.toml
index 79d93ba..a53ef1a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -116,7 +116,13 @@ markers = [
     "dtype: mark test related to dtype validation",
     "shape: mark test related to shape validation",
     "json_schema: mark test related to json schema generation",
-    "serialization: mark test related to serialization"
+    "serialization: mark test related to serialization",
+    "proxy: test for proxy class in any interface",
+    "dask: dask interface",
+    "hdf5: hdf5 interface",
+    "numpy: numpy interface",
+    "video: video interface",
+    "zarr: zarr interface",
 ]
 
 [tool.ruff]
diff --git a/src/numpydantic/interface/dask.py b/src/numpydantic/interface/dask.py
index bc12a13..cd36a65 100644
--- a/src/numpydantic/interface/dask.py
+++ b/src/numpydantic/interface/dask.py
@@ -61,7 +61,7 @@ class DaskInterface(Interface):
         """
         check if array is a dask array
         """
-        if DaskArray is None:
+        if DaskArray is None:  # pragma: no cover - no tests for interface deps atm
             return False
         elif isinstance(array, DaskArray):
             return True
diff --git a/tests/conftest.py b/tests/conftest.py
index 0467f25..3870be2 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -83,6 +83,7 @@ STRING: TypeAlias = NDArray[Shape["*, *, *"], str]
 MODEL: TypeAlias = NDArray[Shape["*, *, *"], BasicModel]
 
 
+@pytest.mark.shape
 @pytest.fixture(
     scope="module",
     params=[
@@ -120,6 +121,7 @@ def shape_cases(request) -> ValidationCase:
     return request.param
 
 
+@pytest.mark.dtype
 @pytest.fixture(
     scope="module",
     params=[
diff --git a/tests/fixtures.py b/tests/fixtures.py
index 89359ae..fb393b5 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -104,6 +104,7 @@ def model_blank() -> Type[BaseModel]:
     return BlankModel
 
 
+@pytest.mark.hdf5
 @pytest.fixture(scope="function")
 def hdf5_file(tmp_output_dir_func) -> h5py.File:
     h5f_file = tmp_output_dir_func / "h5f.h5"
@@ -112,6 +113,7 @@ def hdf5_file(tmp_output_dir_func) -> h5py.File:
     h5f.close()
 
 
+@pytest.mark.hdf5
 @pytest.fixture(scope="function")
 def hdf5_array(
     hdf5_file, request
@@ -154,6 +156,7 @@ def hdf5_array(
     return _hdf5_array
 
 
+@pytest.mark.zarr
 @pytest.fixture(scope="function")
 def zarr_nested_array(tmp_output_dir_func) -> ZarrArrayPath:
     """Zarr array within a nested array"""
@@ -164,6 +167,7 @@ def zarr_nested_array(tmp_output_dir_func) -> ZarrArrayPath:
     return ZarrArrayPath(file=file, path=path)
 
 
+@pytest.mark.zarr
 @pytest.fixture(scope="function")
 def zarr_array(tmp_output_dir_func) -> Path:
     file = tmp_output_dir_func / "array.zarr"
@@ -172,6 +176,7 @@ def zarr_array(tmp_output_dir_func) -> Path:
     return file
 
 
+@pytest.mark.video
 @pytest.fixture(scope="function")
 def avi_video(tmp_path) -> Callable[[Tuple[int, int], int, bool], Path]:
     video_path = tmp_path / "test.avi"
diff --git a/tests/test_interface/conftest.py b/tests/test_interface/conftest.py
index 7e6f767..5d36fa5 100644
--- a/tests/test_interface/conftest.py
+++ b/tests/test_interface/conftest.py
@@ -12,24 +12,44 @@ from numpydantic import interface, NDArray
 @pytest.fixture(
     scope="function",
     params=[
-        ([[1, 2], [3, 4]], interface.NumpyInterface),
-        (np.zeros((3, 4)), interface.NumpyInterface),
-        ("hdf5_array", interface.H5Interface),
-        (da.random.random((10, 10)), interface.DaskInterface),
-        (zarr.ones((10, 10)), interface.ZarrInterface),
-        ("zarr_nested_array", interface.ZarrInterface),
-        ("zarr_array", interface.ZarrInterface),
-        ("avi_video", interface.VideoInterface),
-    ],
-    ids=[
-        "numpy_list",
-        "numpy",
-        "H5ArrayPath",
-        "dask",
-        "zarr_memory",
-        "zarr_nested",
-        "zarr_array",
-        "video",
+        pytest.param(
+            ([[1, 2], [3, 4]], interface.NumpyInterface),
+            marks=pytest.mark.numpy,
+            id="numpy-list",
+        ),
+        pytest.param(
+            (np.zeros((3, 4)), interface.NumpyInterface),
+            marks=pytest.mark.numpy,
+            id="numpy",
+        ),
+        pytest.param(
+            ("hdf5_array", interface.H5Interface),
+            marks=pytest.mark.hdf5,
+            id="h5-array-path",
+        ),
+        pytest.param(
+            (da.random.random((10, 10)), interface.DaskInterface),
+            marks=pytest.mark.dask,
+            id="dask",
+        ),
+        pytest.param(
+            (zarr.ones((10, 10)), interface.ZarrInterface),
+            marks=pytest.mark.zarr,
+            id="zarr-memory",
+        ),
+        pytest.param(
+            ("zarr_nested_array", interface.ZarrInterface),
+            marks=pytest.mark.zarr,
+            id="zarr-nested",
+        ),
+        pytest.param(
+            ("zarr_array", interface.ZarrInterface),
+            marks=pytest.mark.zarr,
+            id="zarr-array",
+        ),
+        pytest.param(
+            ("avi_video", interface.VideoInterface), marks=pytest.mark.video, id="video"
+        ),
     ],
 )
 def interface_type(request) -> Tuple[NDArray, Type[interface.Interface]]:
diff --git a/tests/test_interface/test_dask.py b/tests/test_interface/test_dask.py
index fb1e4cb..c3b70e0 100644
--- a/tests/test_interface/test_dask.py
+++ b/tests/test_interface/test_dask.py
@@ -1,5 +1,3 @@
-import pdb
-
 import pytest
 import json
 
@@ -11,6 +9,8 @@ from numpydantic.exceptions import DtypeError, ShapeError
 
 from tests.conftest import ValidationCase
 
+pytestmark = pytest.mark.dask
+
 
 def dask_array(case: ValidationCase) -> da.Array:
     if issubclass(case.dtype, BaseModel):
@@ -42,14 +42,17 @@ def test_dask_check(interface_type):
         assert not DaskInterface.check(interface_type[0])
 
 
+@pytest.mark.shape
 def test_dask_shape(shape_cases):
     _test_dask_case(shape_cases)
 
 
+@pytest.mark.dtype
 def test_dask_dtype(dtype_cases):
     _test_dask_case(dtype_cases)
 
 
+@pytest.mark.serialization
 def test_dask_to_json(array_model):
     array_list = [[1, 2, 3], [4, 5, 6], [7, 8, 9]]
     array = da.array(array_list)
diff --git a/tests/test_interface/test_dunder.py b/tests/test_interface/test_dunder.py
deleted file mode 100644
index 60f42d8..0000000
--- a/tests/test_interface/test_dunder.py
+++ /dev/null
@@ -1,10 +0,0 @@
-"""
-Tests for dunder methods on all interfaces
-"""
-
-
-def test_dunder_len(all_interfaces):
-    """
-    Each interface or proxy type should support __len__
-    """
-    assert len(all_interfaces.array) == all_interfaces.array.shape[0]
diff --git a/tests/test_interface/test_hdf5.py b/tests/test_interface/test_hdf5.py
index b64d3fe..f88b411 100644
--- a/tests/test_interface/test_hdf5.py
+++ b/tests/test_interface/test_hdf5.py
@@ -14,6 +14,8 @@ from numpydantic.exceptions import DtypeError, ShapeError
 
 from tests.conftest import ValidationCase
 
+pytestmark = pytest.mark.hdf5
+
 
 def hdf5_array_case(
     case: ValidationCase, array_func, compound: bool = False
@@ -72,11 +74,13 @@ def test_hdf5_check_not_hdf5(tmp_path):
     assert not H5Interface.check(spec)
 
 
+@pytest.mark.shape
 @pytest.mark.parametrize("compound", [True, False])
 def test_hdf5_shape(shape_cases, hdf5_array, compound):
     _test_hdf5_case(shape_cases, hdf5_array, compound)
 
 
+@pytest.mark.dtype
 @pytest.mark.parametrize("compound", [True, False])
 def test_hdf5_dtype(dtype_cases, hdf5_array, compound):
     _test_hdf5_case(dtype_cases, hdf5_array, compound)
@@ -90,6 +94,7 @@ def test_hdf5_dataset_not_exists(hdf5_array, model_blank):
         assert "no array found" in e
 
 
+@pytest.mark.proxy
 def test_assignment(hdf5_array, model_blank):
     array = hdf5_array()
 
@@ -101,6 +106,7 @@ def test_assignment(hdf5_array, model_blank):
     assert (model.array[1:3, 2:4] == 10).all()
 
 
+@pytest.mark.serialization
 @pytest.mark.parametrize("round_trip", (True, False))
 def test_to_json(hdf5_array, array_model, round_trip):
     """
@@ -125,6 +131,8 @@ def test_to_json(hdf5_array, array_model, round_trip):
         assert json_dumped == instance.array[:].tolist()
 
 
+@pytest.mark.dtype
+@pytest.mark.proxy
 def test_compound_dtype(tmp_path):
     """
     hdf5 proxy indexes compound dtypes as single fields when field is given
@@ -159,6 +167,8 @@ def test_compound_dtype(tmp_path):
     assert all(instance.array[1] == 2)
 
 
+@pytest.mark.dtype
+@pytest.mark.proxy
 @pytest.mark.parametrize("compound", [True, False])
 def test_strings(hdf5_array, compound):
     """
@@ -178,6 +188,8 @@ def test_strings(hdf5_array, compound):
     assert all(instance.array[1] == "sup")
 
 
+@pytest.mark.dtype
+@pytest.mark.proxy
 @pytest.mark.parametrize("compound", [True, False])
 def test_datetime(hdf5_array, compound):
     """
diff --git a/tests/test_interface/test_interfaces.py b/tests/test_interface/test_interfaces.py
index 3d51ac0..01709d7 100644
--- a/tests/test_interface/test_interfaces.py
+++ b/tests/test_interface/test_interfaces.py
@@ -66,3 +66,10 @@ def test_interface_roundtrip_json(all_interfaces, round_trip):
         assert model.array.dtype == all_interfaces.array.dtype
     else:
         assert np.array_equal(model.array, np.array(all_interfaces.array))
+
+
+def test_dunder_len(all_interfaces):
+    """
+    Each interface or proxy type should support __len__
+    """
+    assert len(all_interfaces.array) == all_interfaces.array.shape[0]
diff --git a/tests/test_interface/test_numpy.py b/tests/test_interface/test_numpy.py
index 6a34b98..bfb4c4d 100644
--- a/tests/test_interface/test_numpy.py
+++ b/tests/test_interface/test_numpy.py
@@ -5,6 +5,8 @@ from numpydantic.exceptions import DtypeError, ShapeError
 
 from tests.conftest import ValidationCase
 
+pytestmark = pytest.mark.numpy
+
 
 def numpy_array(case: ValidationCase) -> np.ndarray:
     if issubclass(case.dtype, BaseModel):
@@ -22,10 +24,12 @@ def _test_np_case(case: ValidationCase):
             case.model(array=array)
 
 
+@pytest.mark.shape
 def test_numpy_shape(shape_cases):
     _test_np_case(shape_cases)
 
 
+@pytest.mark.dtype
 def test_numpy_dtype(dtype_cases):
     _test_np_case(dtype_cases)
 
diff --git a/tests/test_interface/test_video.py b/tests/test_interface/test_video.py
index d5ef1b3..44c8b9a 100644
--- a/tests/test_interface/test_video.py
+++ b/tests/test_interface/test_video.py
@@ -14,6 +14,8 @@ from numpydantic import NDArray, Shape
 from numpydantic import dtype as dt
 from numpydantic.interface.video import VideoProxy
 
+pytestmark = pytest.mark.video
+
 
 @pytest.mark.parametrize("input_type", [str, Path])
 def test_video_validation(avi_video, input_type):
@@ -49,6 +51,7 @@ def test_video_from_videocapture(avi_video):
         opened_vid.release()
 
 
+@pytest.mark.shape
 def test_video_wrong_shape(avi_video):
     shape = (100, 50)
 
@@ -65,6 +68,7 @@ def test_video_wrong_shape(avi_video):
         instance = MyModel(array=vid)
 
 
+@pytest.mark.proxy
 def test_video_getitem(avi_video):
     """
     Should be able to get individual frames and slices as if it were a normal array
@@ -127,6 +131,7 @@ def test_video_getitem(avi_video):
         instance.array[5] = 10
 
 
+@pytest.mark.proxy
 def test_video_attrs(avi_video):
     """Should be able to access opencv properties"""
     shape = (100, 50)
@@ -142,6 +147,7 @@ def test_video_attrs(avi_video):
     assert int(instance.array.get(cv2.CAP_PROP_POS_FRAMES)) == 5
 
 
+@pytest.mark.proxy
 def test_video_close(avi_video):
     """Should close and reopen video file if needed"""
     shape = (100, 50)
diff --git a/tests/test_interface/test_zarr.py b/tests/test_interface/test_zarr.py
index fca15ae..ed5c252 100644
--- a/tests/test_interface/test_zarr.py
+++ b/tests/test_interface/test_zarr.py
@@ -6,13 +6,14 @@ import zarr
 from pydantic import BaseModel, ValidationError
 from numcodecs import Pickle
 
-
 from numpydantic.interface import ZarrInterface
 from numpydantic.interface.zarr import ZarrArrayPath
 from numpydantic.exceptions import DtypeError, ShapeError
 
 from tests.conftest import ValidationCase
 
+pytestmark = pytest.mark.zarr
+
 
 @pytest.fixture()
 def dir_array(tmp_output_dir_func) -> zarr.DirectoryStore:
@@ -87,10 +88,12 @@ def test_zarr_check(interface_type):
         assert not ZarrInterface.check(interface_type[0])
 
 
+@pytest.mark.shape
 def test_zarr_shape(store, shape_cases):
     _test_zarr_case(shape_cases, store)
 
 
+@pytest.mark.dtype
 def test_zarr_dtype(dtype_cases, store):
     _test_zarr_case(dtype_cases, store)
 
diff --git a/tests/test_ndarray.py b/tests/test_ndarray.py
index cef7dc3..73abedf 100644
--- a/tests/test_ndarray.py
+++ b/tests/test_ndarray.py
@@ -41,6 +41,7 @@ def test_ndarray_type():
     instance = Model(array=np.zeros((2, 3)), array_any=np.ones((3, 4, 5)))
 
 
+@pytest.mark.dtype
 @pytest.mark.json_schema
 def test_schema_unsupported_type():
     """
@@ -57,10 +58,11 @@ def test_schema_unsupported_type():
     }
 
 
+@pytest.mark.dtype
 @pytest.mark.json_schema
 def test_schema_tuple():
     """
-    Types specified as tupled should have their schemas as a union
+    Types specified as tuples should have their schemas as a union
     """
 
     class Model(BaseModel):
@@ -75,6 +77,7 @@ def test_schema_tuple():
     assert all([i["minimum"] == 0 for i in conditions])
 
 
+@pytest.mark.dtype
 @pytest.mark.json_schema
 def test_schema_number():
     """
@@ -119,12 +122,12 @@ def test_ndarray_union():
         instance = Model(array=np.random.random((5, 10, 4, 6)))
 
 
+@pytest.mark.shape
+@pytest.mark.dtype
 @pytest.mark.parametrize("dtype", dtype.Number)
 def test_ndarray_unparameterized(dtype):
     """
     NDArray without any parameters is any shape, any type
-    Returns:
-
     """
 
     class Model(BaseModel):
@@ -138,6 +141,7 @@ def test_ndarray_unparameterized(dtype):
         _ = Model(array=np.zeros(dim_sizes, dtype=dtype))
 
 
+@pytest.mark.shape
 def test_ndarray_any():
     """
     using :class:`typing.Any` in for the shape means any shape
@@ -249,6 +253,7 @@ def test_json_schema_dtype_single(dtype, array_model):
 
 
 @pytest.mark.dtype
+@pytest.mark.json_schema
 @pytest.mark.parametrize(
     "dtype,expected",
     [
diff --git a/tests/test_shape.py b/tests/test_shape.py
index 3abff19..03de477 100644
--- a/tests/test_shape.py
+++ b/tests/test_shape.py
@@ -9,6 +9,8 @@ import numpy as np
 
 from numpydantic import NDArray, Shape
 
+pytestmark = pytest.mark.shape
+
 
 @pytest.mark.parametrize(
     "shape,valid",

From 708e6e81d8b9cb1b61a121003adc7eb6944f00a4 Mon Sep 17 00:00:00 2001
From: sneakers-the-rat <sneakers-the-rat@protonmail.com>
Date: Mon, 23 Sep 2024 15:54:55 -0700
Subject: [PATCH 13/22] roundtripping marked arrays, roundtrip or not

---
 src/numpydantic/exceptions.py               |   4 +
 src/numpydantic/interface/__init__.py       |  13 +-
 src/numpydantic/interface/hdf5.py           |   5 +-
 src/numpydantic/interface/interface.py      | 137 +++++++++++++++++---
 src/numpydantic/interface/video.py          |   4 +-
 src/numpydantic/serialization.py            |   3 +-
 tests/conftest.py                           |   2 -
 tests/fixtures.py                           |   5 -
 tests/test_interface/test_hdf5.py           |  26 ++++
 tests/test_interface/test_interface_base.py |  50 ++++++-
 tests/test_interface/test_interfaces.py     |  93 ++++++++++---
 tests/test_interface/test_video.py          |  39 ++++++
 12 files changed, 328 insertions(+), 53 deletions(-)

diff --git a/src/numpydantic/exceptions.py b/src/numpydantic/exceptions.py
index a61258f..c23b96f 100644
--- a/src/numpydantic/exceptions.py
+++ b/src/numpydantic/exceptions.py
@@ -25,3 +25,7 @@ class NoMatchError(MatchError):
 
 class TooManyMatchesError(MatchError):
     """Too many matches found by :class:`.Interface.match`"""
+
+
+class MarkMismatchError(MatchError):
+    """A serialized :class:`.InterfaceMark` doesn't match the receiving interface"""
diff --git a/src/numpydantic/interface/__init__.py b/src/numpydantic/interface/__init__.py
index c5bd3f2..36c7d97 100644
--- a/src/numpydantic/interface/__init__.py
+++ b/src/numpydantic/interface/__init__.py
@@ -4,16 +4,23 @@ Interfaces between nptyping types and array backends
 
 from numpydantic.interface.dask import DaskInterface
 from numpydantic.interface.hdf5 import H5Interface
-from numpydantic.interface.interface import Interface, JsonDict
+from numpydantic.interface.interface import (
+    Interface,
+    InterfaceMark,
+    JsonDict,
+    MarkedJson,
+)
 from numpydantic.interface.numpy import NumpyInterface
 from numpydantic.interface.video import VideoInterface
 from numpydantic.interface.zarr import ZarrInterface
 
 __all__ = [
-    "JsonDict",
-    "Interface",
     "DaskInterface",
     "H5Interface",
+    "Interface",
+    "InterfaceMark",
+    "JsonDict",
+    "MarkedJson",
     "NumpyInterface",
     "VideoInterface",
     "ZarrInterface",
diff --git a/src/numpydantic/interface/hdf5.py b/src/numpydantic/interface/hdf5.py
index 67b9899..9215ec2 100644
--- a/src/numpydantic/interface/hdf5.py
+++ b/src/numpydantic/interface/hdf5.py
@@ -120,7 +120,7 @@ class H5Proxy:
         annotation_dtype: Optional[DtypeType] = None,
     ):
         self._h5f = None
-        self.file = Path(file)
+        self.file = Path(file).resolve()
         self.path = path
         self.field = field
         self._annotation_dtype = annotation_dtype
@@ -156,6 +156,9 @@ class H5Proxy:
             return obj[:]
 
     def __getattr__(self, item: str):
+        if item == "__name__":
+            # special case for H5Proxies that don't refer to a real file during testing
+            return "H5Proxy"
         with h5py.File(self.file, "r") as h5f:
             obj = h5f.get(self.path)
             val = getattr(obj, item)
diff --git a/src/numpydantic/interface/interface.py b/src/numpydantic/interface/interface.py
index 2cb61f4..6b1e5a4 100644
--- a/src/numpydantic/interface/interface.py
+++ b/src/numpydantic/interface/interface.py
@@ -3,16 +3,19 @@ Base Interface metaclass
 """
 
 import inspect
+import warnings
 from abc import ABC, abstractmethod
+from functools import lru_cache
 from importlib.metadata import PackageNotFoundError, version
 from operator import attrgetter
-from typing import Any, Generic, Tuple, Type, TypedDict, TypeVar, Union
+from typing import Any, Generic, Optional, Tuple, Type, TypeVar, Union
 
 import numpy as np
 from pydantic import BaseModel, SerializationInfo, ValidationError
 
 from numpydantic.exceptions import (
     DtypeError,
+    MarkMismatchError,
     NoMatchError,
     ShapeError,
     TooManyMatchesError,
@@ -26,13 +29,49 @@ V = TypeVar("V")  # input type
 W = TypeVar("W")  # Any type in handle_input
 
 
-class InterfaceMark(TypedDict):
+class InterfaceMark(BaseModel):
     """JSON-able mark to be able to round-trip json dumps"""
 
     module: str
     cls: str
+    name: str
     version: str
 
+    def is_valid(self, cls: Type["Interface"], raise_on_error: bool = False) -> bool:
+        """
+        Check that a given interface matches the mark.
+
+        Args:
+            cls (Type): Interface type to check
+            raise_on_error (bool): Raise an ``MarkMismatchError`` when the match
+                is incorrect
+
+        Returns:
+            bool
+
+        Raises:
+            :class:`.MarkMismatchError` if requested by ``raise_on_error``
+            for an invalid match
+        """
+        mark = cls.mark_interface()
+        valid = self == mark
+        if not valid and raise_on_error:
+            raise MarkMismatchError(
+                "Mismatch between serialized mark and current interface, "
+                f"Serialized: {self}; current: {cls}"
+            )
+        return valid
+
+    def match_by_name(self) -> Optional[Type["Interface"]]:
+        """
+        Try to find a matching interface by its name, returning it if found,
+        or None if not found.
+        """
+        for i in Interface.interfaces(sort=False):
+            if i.name == self.name:
+                return i
+        return None
+
 
 class JsonDict(BaseModel):
     """
@@ -84,6 +123,29 @@ class JsonDict(BaseModel):
         return value
 
 
+class MarkedJson(BaseModel):
+    """
+    Model of JSON dumped with an additional interface mark
+    with ``model_dump_json({'mark_interface': True})``
+    """
+
+    interface: InterfaceMark
+    value: Union[list, dict]
+    """
+    Inner value of the array, we don't validate for JsonDict here, 
+    that should be downstream from us for performance reasons 
+    """
+
+    @classmethod
+    def try_cast(cls, value: Union[V, dict]) -> Union[V, "MarkedJson"]:
+        """
+        Try to cast to MarkedJson if applicable, otherwise return input
+        """
+        if isinstance(value, dict) and "interface" in value and "value" in value:
+            value = MarkedJson(**value)
+        return value
+
+
 class Interface(ABC, Generic[T]):
     """
     Abstract parent class for interfaces to different array formats
@@ -158,14 +220,24 @@ class Interface(ABC, Generic[T]):
     def deserialize(self, array: Any) -> Union[V, Any]:
         """
         If given a JSON serialized version of the array,
-        deserialize it first
+        deserialize it first.
 
-        Args:
-            array:
-
-        Returns:
+        If a roundtrip-serialized :class:`.JsonDict`,
+        pass to :meth:`.JsonDict.handle_input`.
 
+        If a roundtrip-serialized :class:`.MarkedJson`,
+        unpack mark, check for validity, warn if not,
+        and try to continue with validation
         """
+        if isinstance(marked_array := MarkedJson.try_cast(array), MarkedJson):
+            try:
+                marked_array.interface.is_valid(self.__class__, raise_on_error=True)
+            except MarkMismatchError as e:
+                warnings.warn(
+                    str(e) + "\nAttempting to continue validation...", stacklevel=2
+                )
+            array = marked_array.value
+
         return self.json_model.handle_input(array)
 
     def before_validation(self, array: Any) -> NDArrayType:
@@ -274,13 +346,6 @@ class Interface(ABC, Generic[T]):
         """
         return array
 
-    def mark_input(self, array: Any) -> Any:
-        """
-        Preserve metadata about the interface and passed input when dumping with
-        ``round_trip``
-        """
-        return array
-
     @classmethod
     @abstractmethod
     def check(cls, array: Any) -> bool:
@@ -320,7 +385,7 @@ class Interface(ABC, Generic[T]):
         """
 
     @classmethod
-    def mark_json(cls, array: Union[list, dict]) -> dict:
+    def mark_json(cls, array: Union[list, dict]) -> MarkedJson:
         """
         When using ``model_dump_json`` with ``mark_interface: True`` in the ``context``,
         add additional annotations that would allow the serialized array to be
@@ -337,7 +402,7 @@ class Interface(ABC, Generic[T]):
                            'version': '1.2.2'},
              'value': [1.0, 2.0]}
         """
-        return {"interface": cls.mark_interface(), "value": array}
+        return MarkedJson.model_construct(interface=cls.mark_interface(), value=array)
 
     @classmethod
     def interfaces(
@@ -390,6 +455,28 @@ class Interface(ABC, Generic[T]):
 
         return tuple(in_types)
 
+    @classmethod
+    def match_mark(cls, array: Any) -> Optional[Type["Interface"]]:
+        """
+        Match a marked JSON dump of this array to the interface that it indicates.
+
+        First find an interface that matches by name, and then run its
+        ``check`` method, because arrays can be dumped with a mark
+        but without ``round_trip == True`` (and thus can't necessarily
+        use the same interface that they were dumped with)
+
+        Returns:
+            Interface if match found, None otherwise
+        """
+        mark = MarkedJson.try_cast(array)
+        if not isinstance(mark, MarkedJson):
+            return None
+
+        interface = mark.interface.match_by_name()
+        if interface is not None and interface.check(mark.value):
+            return interface
+        return None
+
     @classmethod
     def match(cls, array: Any, fast: bool = False) -> Type["Interface"]:
         """
@@ -407,11 +494,18 @@ class Interface(ABC, Generic[T]):
               check each interface (as ordered by its ``priority`` , decreasing),
               and return on the first match.
         """
+        # Shortcircuit match if this is a marked json dump
+        array = MarkedJson.try_cast(array)
+        if (match := cls.match_mark(array)) is not None:
+            return match
+        elif isinstance(array, MarkedJson):
+            array = array.value
+
         # first try and find a non-numpy interface, since the numpy interface
         # will try and load the array into memory in its check method
         interfaces = cls.interfaces()
-        non_np_interfaces = [i for i in interfaces if i.__name__ != "NumpyInterface"]
-        np_interface = [i for i in interfaces if i.__name__ == "NumpyInterface"][0]
+        non_np_interfaces = [i for i in interfaces if i.name != "numpy"]
+        np_interface = [i for i in interfaces if i.name == "numpy"][0]
 
         if fast:
             matches = []
@@ -453,6 +547,7 @@ class Interface(ABC, Generic[T]):
             return matches[0]
 
     @classmethod
+    @lru_cache(maxsize=32)
     def mark_interface(cls) -> InterfaceMark:
         """
         Create an interface mark indicating this interface for validation after
@@ -470,5 +565,7 @@ class Interface(ABC, Generic[T]):
             )
         except PackageNotFoundError:
             v = None
-        interface_name = cls.__name__
-        return InterfaceMark(module=interface_module, cls=interface_name, version=v)
+
+        return InterfaceMark(
+            module=interface_module, cls=cls.__name__, name=cls.name, version=v
+        )
diff --git a/src/numpydantic/interface/video.py b/src/numpydantic/interface/video.py
index b214a74..53a3ba5 100644
--- a/src/numpydantic/interface/video.py
+++ b/src/numpydantic/interface/video.py
@@ -48,7 +48,7 @@ class VideoProxy:
             )
 
         if path is not None:
-            path = Path(path)
+            path = Path(path).resolve()
         self.path = path
 
         self._video = video  # type: Optional[VideoCapture]
@@ -200,6 +200,8 @@ class VideoProxy:
         raise NotImplementedError("Setting pixel values on videos is not supported!")
 
     def __getattr__(self, item: str):
+        if item == "__name__":
+            return "VideoProxy"
         return getattr(self.video, item)
 
     def __eq__(self, other: "VideoProxy") -> bool:
diff --git a/src/numpydantic/serialization.py b/src/numpydantic/serialization.py
index f645239..f5c7b35 100644
--- a/src/numpydantic/serialization.py
+++ b/src/numpydantic/serialization.py
@@ -23,7 +23,8 @@ def jsonize_array(value: Any, info: SerializationInfo) -> Union[list, dict]:
 
     if info.context:
         if info.context.get("mark_interface", False):
-            array = interface_cls.mark_json(array)
+            array = interface_cls.mark_json(array).model_dump()
+
         if info.context.get("absolute_paths", False):
             array = _absolutize_paths(array)
         else:
diff --git a/tests/conftest.py b/tests/conftest.py
index 3870be2..0467f25 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -83,7 +83,6 @@ STRING: TypeAlias = NDArray[Shape["*, *, *"], str]
 MODEL: TypeAlias = NDArray[Shape["*, *, *"], BasicModel]
 
 
-@pytest.mark.shape
 @pytest.fixture(
     scope="module",
     params=[
@@ -121,7 +120,6 @@ def shape_cases(request) -> ValidationCase:
     return request.param
 
 
-@pytest.mark.dtype
 @pytest.fixture(
     scope="module",
     params=[
diff --git a/tests/fixtures.py b/tests/fixtures.py
index fb393b5..89359ae 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -104,7 +104,6 @@ def model_blank() -> Type[BaseModel]:
     return BlankModel
 
 
-@pytest.mark.hdf5
 @pytest.fixture(scope="function")
 def hdf5_file(tmp_output_dir_func) -> h5py.File:
     h5f_file = tmp_output_dir_func / "h5f.h5"
@@ -113,7 +112,6 @@ def hdf5_file(tmp_output_dir_func) -> h5py.File:
     h5f.close()
 
 
-@pytest.mark.hdf5
 @pytest.fixture(scope="function")
 def hdf5_array(
     hdf5_file, request
@@ -156,7 +154,6 @@ def hdf5_array(
     return _hdf5_array
 
 
-@pytest.mark.zarr
 @pytest.fixture(scope="function")
 def zarr_nested_array(tmp_output_dir_func) -> ZarrArrayPath:
     """Zarr array within a nested array"""
@@ -167,7 +164,6 @@ def zarr_nested_array(tmp_output_dir_func) -> ZarrArrayPath:
     return ZarrArrayPath(file=file, path=path)
 
 
-@pytest.mark.zarr
 @pytest.fixture(scope="function")
 def zarr_array(tmp_output_dir_func) -> Path:
     file = tmp_output_dir_func / "array.zarr"
@@ -176,7 +172,6 @@ def zarr_array(tmp_output_dir_func) -> Path:
     return file
 
 
-@pytest.mark.video
 @pytest.fixture(scope="function")
 def avi_video(tmp_path) -> Callable[[Tuple[int, int], int, bool], Path]:
     video_path = tmp_path / "test.avi"
diff --git a/tests/test_interface/test_hdf5.py b/tests/test_interface/test_hdf5.py
index f88b411..c412e7a 100644
--- a/tests/test_interface/test_hdf5.py
+++ b/tests/test_interface/test_hdf5.py
@@ -231,3 +231,29 @@ def test_empty_dataset(dtype, tmp_path):
         array: NDArray[Any, dtype]
 
     _ = MyModel(array=(array_path, "/data"))
+
+
+@pytest.mark.proxy
+@pytest.mark.parametrize(
+    "comparison,valid",
+    [
+        (H5Proxy(file="test_file.h5", path="/subpath", field="sup"), True),
+        (H5Proxy(file="test_file.h5", path="/subpath"), False),
+        (H5Proxy(file="different_file.h5", path="/subpath"), False),
+        (("different_file.h5", "/subpath", "sup"), ValueError),
+        ("not even a proxy-like thing", ValueError),
+    ],
+)
+def test_proxy_eq(comparison, valid):
+    """
+    test the __eq__ method of H5ArrayProxy matches proxies to the same
+    dataset (and path), or raises a ValueError
+    """
+    proxy_a = H5Proxy(file="test_file.h5", path="/subpath", field="sup")
+    if valid is True:
+        assert proxy_a == comparison
+    elif valid is False:
+        assert proxy_a != comparison
+    else:
+        with pytest.raises(valid):
+            assert proxy_a == comparison
diff --git a/tests/test_interface/test_interface_base.py b/tests/test_interface/test_interface_base.py
index baacc60..1c18e73 100644
--- a/tests/test_interface/test_interface_base.py
+++ b/tests/test_interface/test_interface_base.py
@@ -4,11 +4,26 @@ for tests that should apply to all interfaces, use ``test_interfaces.py``
 """
 
 import gc
+from typing import Literal
 
 import pytest
 import numpy as np
 
-from numpydantic.interface import Interface
+from numpydantic.interface import Interface, JsonDict
+from pydantic import ValidationError
+
+from numpydantic.interface.interface import V
+
+
+class MyJsonDict(JsonDict):
+    type: Literal["my_json_dict"]
+    field: str
+    number: int
+
+    def to_array_input(self) -> V:
+        dumped = self.model_dump()
+        dumped["extra_input_param"] = True
+        return dumped
 
 
 @pytest.fixture(scope="module")
@@ -162,3 +177,36 @@ def test_interface_recursive(interfaces):
     assert issubclass(interfaces.interface3, interfaces.interface1)
     assert issubclass(interfaces.interface1, Interface)
     assert interfaces.interface4 in ifaces
+
+
+@pytest.mark.serialization
+def test_jsondict_is_valid():
+    """
+    A JsonDict should return a bool true/false if it is valid or not,
+    and raise an error when requested
+    """
+    invalid = {"doesnt": "have", "the": "props"}
+    valid = {"type": "my_json_dict", "field": "a_field", "number": 1}
+    assert MyJsonDict.is_valid(valid)
+    assert not MyJsonDict.is_valid(invalid)
+    with pytest.raises(ValidationError):
+        assert not MyJsonDict.is_valid(invalid, raise_on_error=True)
+
+
+@pytest.mark.serialization
+def test_jsondict_handle_input():
+    """
+    JsonDict should be able to parse a valid dict and return it to the input format
+    """
+    valid = {"type": "my_json_dict", "field": "a_field", "number": 1}
+    instantiated = MyJsonDict(**valid)
+    expected = {
+        "type": "my_json_dict",
+        "field": "a_field",
+        "number": 1,
+        "extra_input_param": True,
+    }
+
+    for item in (valid, instantiated):
+        result = MyJsonDict.handle_input(item)
+        assert result == expected
diff --git a/tests/test_interface/test_interfaces.py b/tests/test_interface/test_interfaces.py
index 01709d7..faec0d8 100644
--- a/tests/test_interface/test_interfaces.py
+++ b/tests/test_interface/test_interfaces.py
@@ -4,10 +4,38 @@ Tests that should be applied to all interfaces
 
 import pytest
 from typing import Callable
+from importlib.metadata import version
+import json
+
 import numpy as np
 import dask.array as da
 from zarr.core import Array as ZarrArray
-from numpydantic.interface import Interface
+from pydantic import BaseModel
+
+from numpydantic.interface import Interface, InterfaceMark, MarkedJson
+
+
+def _test_roundtrip(source: BaseModel, target: BaseModel, round_trip: bool):
+    """Test model equality for roundtrip tests"""
+    if round_trip:
+        assert type(target.array) is type(source.array)
+        if isinstance(source.array, (np.ndarray, ZarrArray)):
+            assert np.array_equal(target.array, np.array(source.array))
+        elif isinstance(source.array, da.Array):
+            assert np.all(da.equal(target.array, source.array))
+        else:
+            assert target.array == source.array
+
+        assert target.array.dtype == source.array.dtype
+    else:
+        assert np.array_equal(target.array, np.array(source.array))
+
+
+def test_dunder_len(all_interfaces):
+    """
+    Each interface or proxy type should support __len__
+    """
+    assert len(all_interfaces.array) == all_interfaces.array.shape[0]
 
 
 def test_interface_revalidate(all_interfaces):
@@ -52,24 +80,51 @@ def test_interface_roundtrip_json(all_interfaces, round_trip):
     """
     All interfaces should be able to roundtrip to and from json
     """
-    json = all_interfaces.model_dump_json(round_trip=round_trip)
-    model = all_interfaces.model_validate_json(json)
-    if round_trip:
-        assert type(model.array) is type(all_interfaces.array)
-        if isinstance(all_interfaces.array, (np.ndarray, ZarrArray)):
-            assert np.array_equal(model.array, np.array(all_interfaces.array))
-        elif isinstance(all_interfaces.array, da.Array):
-            assert np.all(da.equal(model.array, all_interfaces.array))
-        else:
-            assert model.array == all_interfaces.array
+    dumped_json = all_interfaces.model_dump_json(round_trip=round_trip)
+    model = all_interfaces.model_validate_json(dumped_json)
+    _test_roundtrip(all_interfaces, model, round_trip)
 
-        assert model.array.dtype == all_interfaces.array.dtype
+
+@pytest.mark.serialization
+@pytest.mark.parametrize("an_interface", Interface.interfaces())
+def test_interface_mark_interface(an_interface):
+    """
+    All interfaces should be able to mark the current version and interface info
+    """
+    mark = an_interface.mark_interface()
+    assert isinstance(mark, InterfaceMark)
+    assert mark.name == an_interface.name
+    assert mark.cls == an_interface.__name__
+    assert mark.module == an_interface.__module__
+    assert mark.version == version(mark.module.split(".")[0])
+
+
+@pytest.mark.serialization
+@pytest.mark.parametrize("valid", [True, False])
+@pytest.mark.parametrize("round_trip", [True, False])
+@pytest.mark.filterwarnings("ignore:Mismatch between serialized mark")
+def test_interface_mark_roundtrip(all_interfaces, valid, round_trip):
+    """
+    All interfaces should be able to roundtrip with the marked interface,
+    and a mismatch should raise a warning and attempt to proceed
+    """
+    dumped_json = all_interfaces.model_dump_json(
+        round_trip=round_trip, context={"mark_interface": True}
+    )
+
+    data = json.loads(dumped_json)
+
+    # ensure that we are a MarkedJson
+    _ = MarkedJson.model_validate_json(json.dumps(data["array"]))
+
+    if not valid:
+        # ruin the version
+        data["array"]["interface"]["version"] = "v99999999"
+        dumped_json = json.dumps(data)
+
+        with pytest.warns(match="Mismatch.*"):
+            model = all_interfaces.model_validate_json(dumped_json)
     else:
-        assert np.array_equal(model.array, np.array(all_interfaces.array))
+        model = all_interfaces.model_validate_json(dumped_json)
 
-
-def test_dunder_len(all_interfaces):
-    """
-    Each interface or proxy type should support __len__
-    """
-    assert len(all_interfaces.array) == all_interfaces.array.shape[0]
+    _test_roundtrip(all_interfaces, model, round_trip)
diff --git a/tests/test_interface/test_video.py b/tests/test_interface/test_video.py
index 44c8b9a..5f03a57 100644
--- a/tests/test_interface/test_video.py
+++ b/tests/test_interface/test_video.py
@@ -164,3 +164,42 @@ def test_video_close(avi_video):
     assert instance.array._video is None
     # reopen
     assert isinstance(instance.array.video, cv2.VideoCapture)
+
+
+@pytest.mark.proxy
+def test_video_not_exists(tmp_path):
+    """
+    A video file that doesn't exist should raise an error
+    """
+    video = VideoProxy(tmp_path / "not_real.avi")
+    with pytest.raises(FileNotFoundError):
+        _ = video.video
+
+
+@pytest.mark.proxy
+@pytest.mark.parametrize(
+    "comparison,valid",
+    [
+        (VideoProxy("test_video.avi"), True),
+        (VideoProxy("not_real_video.avi"), False),
+        ("not even a video proxy", TypeError),
+    ],
+)
+def test_video_proxy_eq(comparison, valid):
+    """
+    Comparing a video proxy's equality should be valid if the path matches
+    Args:
+        comparison:
+        valid:
+
+    Returns:
+
+    """
+    proxy_a = VideoProxy("test_video.avi")
+    if valid is True:
+        assert proxy_a == comparison
+    elif valid is False:
+        assert proxy_a != comparison
+    else:
+        with pytest.raises(valid):
+            assert proxy_a == comparison

From 6253c47e37fca4cc2d7f798dfa040c1551e28bb7 Mon Sep 17 00:00:00 2001
From: sneakers-the-rat <sneakers-the-rat@protonmail.com>
Date: Mon, 23 Sep 2024 16:08:56 -0700
Subject: [PATCH 14/22] check if video recursion error coming from windows
 pathing

---
 src/numpydantic/interface/interface.py | 10 +++++++---
 src/numpydantic/serialization.py       |  8 ++++----
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/src/numpydantic/interface/interface.py b/src/numpydantic/interface/interface.py
index 6b1e5a4..d3c9119 100644
--- a/src/numpydantic/interface/interface.py
+++ b/src/numpydantic/interface/interface.py
@@ -142,7 +142,11 @@ class MarkedJson(BaseModel):
         Try to cast to MarkedJson if applicable, otherwise return input
         """
         if isinstance(value, dict) and "interface" in value and "value" in value:
-            value = MarkedJson(**value)
+            try:
+                value = MarkedJson(**value)
+            except ValidationError:
+                # fine, just not a MarkedJson dict even if it looks like one
+                return value
         return value
 
 
@@ -385,7 +389,7 @@ class Interface(ABC, Generic[T]):
         """
 
     @classmethod
-    def mark_json(cls, array: Union[list, dict]) -> MarkedJson:
+    def mark_json(cls, array: Union[list, dict]) -> dict:
         """
         When using ``model_dump_json`` with ``mark_interface: True`` in the ``context``,
         add additional annotations that would allow the serialized array to be
@@ -402,7 +406,7 @@ class Interface(ABC, Generic[T]):
                            'version': '1.2.2'},
              'value': [1.0, 2.0]}
         """
-        return MarkedJson.model_construct(interface=cls.mark_interface(), value=array)
+        return {"interface": cls.mark_interface(), "value": array}
 
     @classmethod
     def interfaces(
diff --git a/src/numpydantic/serialization.py b/src/numpydantic/serialization.py
index f5c7b35..c60136c 100644
--- a/src/numpydantic/serialization.py
+++ b/src/numpydantic/serialization.py
@@ -23,13 +23,13 @@ def jsonize_array(value: Any, info: SerializationInfo) -> Union[list, dict]:
 
     if info.context:
         if info.context.get("mark_interface", False):
-            array = interface_cls.mark_json(array).model_dump()
+            array = interface_cls.mark_json(array)
 
         if info.context.get("absolute_paths", False):
             array = _absolutize_paths(array)
-        else:
-            relative_to = info.context.get("relative_to", ".")
-            array = _relativize_paths(array, relative_to)
+        # else:
+        #     relative_to = info.context.get("relative_to", ".")
+        #     array = _relativize_paths(array, relative_to)
 
     return array
 

From 3d326dfc8b76dcaf123e4819b8a198e8aaf42f29 Mon Sep 17 00:00:00 2001
From: sneakers-the-rat <sneakers-the-rat@protonmail.com>
Date: Mon, 23 Sep 2024 16:22:02 -0700
Subject: [PATCH 15/22] test whether the py3.12 relative_to works on windows

---
 src/numpydantic/serialization.py | 49 +++++++++++++++++++++++++++++---
 1 file changed, 45 insertions(+), 4 deletions(-)

diff --git a/src/numpydantic/serialization.py b/src/numpydantic/serialization.py
index c60136c..5ff04b8 100644
--- a/src/numpydantic/serialization.py
+++ b/src/numpydantic/serialization.py
@@ -27,9 +27,9 @@ def jsonize_array(value: Any, info: SerializationInfo) -> Union[list, dict]:
 
         if info.context.get("absolute_paths", False):
             array = _absolutize_paths(array)
-        # else:
-        #     relative_to = info.context.get("relative_to", ".")
-        #     array = _relativize_paths(array, relative_to)
+        else:
+            relative_to = info.context.get("relative_to", ".")
+            array = _relativize_paths(array, relative_to)
 
     return array
 
@@ -97,4 +97,45 @@ def relative_path(target: Path, origin: Path) -> Path:
         return Path(target).resolve().relative_to(Path(origin).resolve())
     except ValueError:  # target does not start with origin
         # recursion with origin (eventually origin is root so try will succeed)
-        return Path("..").joinpath(relative_path(target, Path(origin).parent))
+        try:
+            return Path("..").joinpath(relative_path(target, Path(origin).parent))
+        except ValueError:
+            # break recursion in windows when
+            pass
+
+
+def relative_to(self: Path, other: Path, walk_up=True) -> Path:
+    """
+    "Backport" of :meth:`pathlib.Path.relative_to` with ``walk_up=True``
+    that's not available pre 3.12.
+
+    Return the relative path to another path identified by the passed
+    arguments.  If the operation is not possible (because this is not
+    related to the other path), raise ValueError.
+
+    The *walk_up* parameter controls whether `..` may be used to resolve
+    the path.
+    """
+    if not isinstance(other, Path):
+        other = self.with_segments(other)
+    anchor0, parts0 = self._stack
+    anchor1, parts1 = other._stack
+    if anchor0 != anchor1:
+        raise ValueError(
+            f"{self._raw_path!r} and {other._raw_path!r} have different anchors"
+        )
+    while parts0 and parts1 and parts0[-1] == parts1[-1]:
+        parts0.pop()
+        parts1.pop()
+    for part in parts1:
+        if not part or part == ".":
+            pass
+        elif not walk_up:
+            raise ValueError(
+                f"{self._raw_path!r} is not in the subpath of {other._raw_path!r}"
+            )
+        elif part == "..":
+            raise ValueError(f"'..' segment in {other._raw_path!r} cannot be walked")
+        else:
+            parts0.append("..")
+    return self.with_segments("", *reversed(parts0))

From ea0e8127fa042ed6776919ca71b072c3b1c7740a Mon Sep 17 00:00:00 2001
From: sneakers-the-rat <sneakers-the-rat@protonmail.com>
Date: Mon, 23 Sep 2024 16:24:35 -0700
Subject: [PATCH 16/22] actually need to swap it out

---
 src/numpydantic/serialization.py | 39 +++++++++++++++++---------------
 1 file changed, 21 insertions(+), 18 deletions(-)

diff --git a/src/numpydantic/serialization.py b/src/numpydantic/serialization.py
index 5ff04b8..0929ab9 100644
--- a/src/numpydantic/serialization.py
+++ b/src/numpydantic/serialization.py
@@ -85,26 +85,26 @@ def _walk_and_apply(value: T, f: Callable[[U], U]) -> T:
     return value
 
 
-def relative_path(target: Path, origin: Path) -> Path:
-    """
-    return path of target relative to origin, even if they're
-    not in the same subpath
-
-    References:
-        - https://stackoverflow.com/a/71874881
-    """
-    try:
-        return Path(target).resolve().relative_to(Path(origin).resolve())
-    except ValueError:  # target does not start with origin
-        # recursion with origin (eventually origin is root so try will succeed)
-        try:
-            return Path("..").joinpath(relative_path(target, Path(origin).parent))
-        except ValueError:
-            # break recursion in windows when
-            pass
+# def relative_path(target: Path, origin: Path) -> Path:
+#     """
+#     return path of target relative to origin, even if they're
+#     not in the same subpath
+# 
+#     References:
+#         - https://stackoverflow.com/a/71874881
+#     """
+#     try:
+#         return Path(target).resolve().relative_to(Path(origin).resolve())
+#     except ValueError:  # target does not start with origin
+#         # recursion with origin (eventually origin is root so try will succeed)
+#         try:
+#             return Path("..").joinpath(relative_path(target, Path(origin).parent))
+#         except ValueError:
+#             # break recursion in windows when
+#             pass
 
 
-def relative_to(self: Path, other: Path, walk_up=True) -> Path:
+def relative_path(self: Path, other: Path, walk_up: bool = True) -> Path:
     """
     "Backport" of :meth:`pathlib.Path.relative_to` with ``walk_up=True``
     that's not available pre 3.12.
@@ -115,6 +115,9 @@ def relative_to(self: Path, other: Path, walk_up=True) -> Path:
 
     The *walk_up* parameter controls whether `..` may be used to resolve
     the path.
+
+    References:
+        https://github.com/python/cpython/blob/8a2baedc4bcb606da937e4e066b4b3a18961cace/Lib/pathlib/_abc.py#L244-L270
     """
     if not isinstance(other, Path):
         other = self.with_segments(other)

From 3afeb9bf3f2270fed495bd507df50da13a2ea7f3 Mon Sep 17 00:00:00 2001
From: sneakers-the-rat <sneakers-the-rat@protonmail.com>
Date: Mon, 23 Sep 2024 16:32:13 -0700
Subject: [PATCH 17/22] valueerrors for attrs not available outside of
 PurePathBase

---
 src/numpydantic/serialization.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/numpydantic/serialization.py b/src/numpydantic/serialization.py
index 0929ab9..85ee43f 100644
--- a/src/numpydantic/serialization.py
+++ b/src/numpydantic/serialization.py
@@ -89,7 +89,7 @@ def _walk_and_apply(value: T, f: Callable[[U], U]) -> T:
 #     """
 #     return path of target relative to origin, even if they're
 #     not in the same subpath
-# 
+#
 #     References:
 #         - https://stackoverflow.com/a/71874881
 #     """
@@ -121,11 +121,13 @@ def relative_path(self: Path, other: Path, walk_up: bool = True) -> Path:
     """
     if not isinstance(other, Path):
         other = self.with_segments(other)
-    anchor0, parts0 = self._stack
-    anchor1, parts1 = other._stack
+    self_parts = self.parts
+    other_parts = other.parts
+    anchor0, parts0 = self_parts[0], list(reversed(self_parts[1:]))
+    anchor1, parts1 = other_parts[0], list(reversed(other_parts[1:]))
     if anchor0 != anchor1:
         raise ValueError(
-            f"{self._raw_path!r} and {other._raw_path!r} have different anchors"
+            f"{self!r} and {other!r} have different anchors"
         )
     while parts0 and parts1 and parts0[-1] == parts1[-1]:
         parts0.pop()
@@ -135,10 +137,10 @@ def relative_path(self: Path, other: Path, walk_up: bool = True) -> Path:
             pass
         elif not walk_up:
             raise ValueError(
-                f"{self._raw_path!r} is not in the subpath of {other._raw_path!r}"
+                f"{self!r} is not in the subpath of {other!r}"
             )
         elif part == "..":
-            raise ValueError(f"'..' segment in {other._raw_path!r} cannot be walked")
+            raise ValueError(f"'..' segment in {other!r} cannot be walked")
         else:
             parts0.append("..")
     return self.with_segments("", *reversed(parts0))

From 66fffc49f87bfaaa2f4d05bf1730c343b10c9cc6 Mon Sep 17 00:00:00 2001
From: sneakers-the-rat <sneakers-the-rat@protonmail.com>
Date: Mon, 23 Sep 2024 16:36:40 -0700
Subject: [PATCH 18/22] more value errors

---
 src/numpydantic/serialization.py | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/src/numpydantic/serialization.py b/src/numpydantic/serialization.py
index 85ee43f..fab3816 100644
--- a/src/numpydantic/serialization.py
+++ b/src/numpydantic/serialization.py
@@ -120,15 +120,13 @@ def relative_path(self: Path, other: Path, walk_up: bool = True) -> Path:
         https://github.com/python/cpython/blob/8a2baedc4bcb606da937e4e066b4b3a18961cace/Lib/pathlib/_abc.py#L244-L270
     """
     if not isinstance(other, Path):
-        other = self.with_segments(other)
+        other = Path(other)
     self_parts = self.parts
     other_parts = other.parts
     anchor0, parts0 = self_parts[0], list(reversed(self_parts[1:]))
     anchor1, parts1 = other_parts[0], list(reversed(other_parts[1:]))
     if anchor0 != anchor1:
-        raise ValueError(
-            f"{self!r} and {other!r} have different anchors"
-        )
+        raise ValueError(f"{self!r} and {other!r} have different anchors")
     while parts0 and parts1 and parts0[-1] == parts1[-1]:
         parts0.pop()
         parts1.pop()
@@ -136,11 +134,9 @@ def relative_path(self: Path, other: Path, walk_up: bool = True) -> Path:
         if not part or part == ".":
             pass
         elif not walk_up:
-            raise ValueError(
-                f"{self!r} is not in the subpath of {other!r}"
-            )
+            raise ValueError(f"{self!r} is not in the subpath of {other!r}")
         elif part == "..":
             raise ValueError(f"'..' segment in {other!r} cannot be walked")
         else:
             parts0.append("..")
-    return self.with_segments("", *reversed(parts0))
+    return Path(*reversed(parts0))

From f9a992843e133fc9d7951a4640e157bd8f60943c Mon Sep 17 00:00:00 2001
From: sneakers-the-rat <sneakers-the-rat@protonmail.com>
Date: Mon, 23 Sep 2024 17:30:06 -0700
Subject: [PATCH 19/22] tests for paths

---
 src/numpydantic/interface/interface.py      |  4 +-
 src/numpydantic/schema.py                   | 11 +--
 src/numpydantic/serialization.py            | 28 ++----
 tests/test_interface/test_hdf5.py           |  4 +-
 tests/test_interface/test_interface_base.py | 38 ++++++++-
 tests/test_serialization.py                 | 95 +++++++++++++++++++++
 6 files changed, 146 insertions(+), 34 deletions(-)
 create mode 100644 tests/test_serialization.py

diff --git a/src/numpydantic/interface/interface.py b/src/numpydantic/interface/interface.py
index d3c9119..bee85b6 100644
--- a/src/numpydantic/interface/interface.py
+++ b/src/numpydantic/interface/interface.py
@@ -567,7 +567,9 @@ class Interface(ABC, Generic[T]):
                 if interface_module is None
                 else version(interface_module.split(".")[0])
             )
-        except PackageNotFoundError:
+        except (
+            PackageNotFoundError
+        ):  # pragma: no cover - no tests for missing interface deps
             v = None
 
         return InterfaceMark(
diff --git a/src/numpydantic/schema.py b/src/numpydantic/schema.py
index fff9d1d..cafa1f4 100644
--- a/src/numpydantic/schema.py
+++ b/src/numpydantic/schema.py
@@ -25,8 +25,6 @@ if TYPE_CHECKING:  # pragma: no cover
 
     from numpydantic import Shape
 
-_UNSUPPORTED_TYPES = (complex,)
-
 
 def _numeric_dtype(
     dtype: DtypeType, _handler: "CallbackGetCoreSchemaHandler"
@@ -41,10 +39,6 @@ def _numeric_dtype(
     elif issubclass(dtype, np.integer):
         info = np.iinfo(dtype)
         schema = core_schema.int_schema(le=int(info.max), ge=int(info.min))
-    elif dtype is float:
-        schema = core_schema.float_schema()
-    elif dtype is int:
-        schema = core_schema.int_schema()
     else:
         schema = _handler.generate_schema(dtype)
 
@@ -89,10 +83,7 @@ def _lol_dtype(
                 # does this need a warning?
                 python_type = Any
 
-        if python_type in _UNSUPPORTED_TYPES:
-            array_type = core_schema.any_schema()
-            # TODO: warn and log here
-        elif python_type in (float, int):
+        if python_type in (float, int):
             array_type = _numeric_dtype(dtype, _handler)
         elif python_type is bool:
             array_type = core_schema.bool_schema()
diff --git a/src/numpydantic/serialization.py b/src/numpydantic/serialization.py
index fab3816..1f1edd0 100644
--- a/src/numpydantic/serialization.py
+++ b/src/numpydantic/serialization.py
@@ -30,6 +30,9 @@ def jsonize_array(value: Any, info: SerializationInfo) -> Union[list, dict]:
         else:
             relative_to = info.context.get("relative_to", ".")
             array = _relativize_paths(array, relative_to)
+    else:
+        # relativize paths by default
+        array = _relativize_paths(array, ".")
 
     return array
 
@@ -40,6 +43,7 @@ def _relativize_paths(value: dict, relative_to: str = ".") -> dict:
     ``relative_to`` directory, if provided in the context
     """
     relative_to = Path(relative_to).resolve()
+    # pdb.set_trace()
 
     def _r_path(v: Any) -> Any:
         try:
@@ -85,25 +89,6 @@ def _walk_and_apply(value: T, f: Callable[[U], U]) -> T:
     return value
 
 
-# def relative_path(target: Path, origin: Path) -> Path:
-#     """
-#     return path of target relative to origin, even if they're
-#     not in the same subpath
-#
-#     References:
-#         - https://stackoverflow.com/a/71874881
-#     """
-#     try:
-#         return Path(target).resolve().relative_to(Path(origin).resolve())
-#     except ValueError:  # target does not start with origin
-#         # recursion with origin (eventually origin is root so try will succeed)
-#         try:
-#             return Path("..").joinpath(relative_path(target, Path(origin).parent))
-#         except ValueError:
-#             # break recursion in windows when
-#             pass
-
-
 def relative_path(self: Path, other: Path, walk_up: bool = True) -> Path:
     """
     "Backport" of :meth:`pathlib.Path.relative_to` with ``walk_up=True``
@@ -119,7 +104,8 @@ def relative_path(self: Path, other: Path, walk_up: bool = True) -> Path:
     References:
         https://github.com/python/cpython/blob/8a2baedc4bcb606da937e4e066b4b3a18961cace/Lib/pathlib/_abc.py#L244-L270
     """
-    if not isinstance(other, Path):
+    # pdb.set_trace()
+    if not isinstance(other, Path):  # pragma: no cover - ripped from cpython
         other = Path(other)
     self_parts = self.parts
     other_parts = other.parts
@@ -130,7 +116,7 @@ def relative_path(self: Path, other: Path, walk_up: bool = True) -> Path:
     while parts0 and parts1 and parts0[-1] == parts1[-1]:
         parts0.pop()
         parts1.pop()
-    for part in parts1:
+    for part in parts1:  # pragma: no cover - not testing, ripped off from cpython
         if not part or part == ".":
             pass
         elif not walk_up:
diff --git a/tests/test_interface/test_hdf5.py b/tests/test_interface/test_hdf5.py
index c412e7a..bd94810 100644
--- a/tests/test_interface/test_hdf5.py
+++ b/tests/test_interface/test_hdf5.py
@@ -122,7 +122,9 @@ def test_to_json(hdf5_array, array_model, round_trip):
 
     instance = model(array=array)  # type: BaseModel
 
-    json_str = instance.model_dump_json(round_trip=round_trip)
+    json_str = instance.model_dump_json(
+        round_trip=round_trip, context={"absolute_paths": True}
+    )
     json_dumped = json.loads(json_str)["array"]
     if round_trip:
         assert json_dumped["file"] == str(array.file)
diff --git a/tests/test_interface/test_interface_base.py b/tests/test_interface/test_interface_base.py
index 1c18e73..0b99ae6 100644
--- a/tests/test_interface/test_interface_base.py
+++ b/tests/test_interface/test_interface_base.py
@@ -9,7 +9,13 @@ from typing import Literal
 import pytest
 import numpy as np
 
-from numpydantic.interface import Interface, JsonDict
+from numpydantic.interface import (
+    Interface,
+    JsonDict,
+    InterfaceMark,
+    NumpyInterface,
+    MarkedJson,
+)
 from pydantic import ValidationError
 
 from numpydantic.interface.interface import V
@@ -210,3 +216,33 @@ def test_jsondict_handle_input():
     for item in (valid, instantiated):
         result = MyJsonDict.handle_input(item)
         assert result == expected
+
+
+@pytest.mark.serialization
+@pytest.mark.parametrize("interface", Interface.interfaces())
+def test_interface_mark_match_by_name(interface):
+    """
+    Interface mark should match an interface by its name
+    """
+    # other parts don't matter
+    mark = InterfaceMark(module="fake", cls="fake", version="fake", name=interface.name)
+    fake_mark = InterfaceMark(
+        module="fake", cls="fake", version="fake", name="also_fake"
+    )
+    assert mark.match_by_name() is interface
+    assert fake_mark.match_by_name() is None
+
+
+@pytest.mark.serialization
+def test_marked_json_try_cast():
+    """
+    MarkedJson.try_cast should try and cast to a markedjson!
+    returning the value unchanged if it's not a match
+    """
+    valid = {"interface": NumpyInterface.mark_interface(), "value": [[1, 2], [3, 4]]}
+    invalid = [1, 2, 3, 4, 5]
+    mimic = {"interface": "not really", "value": "still not really"}
+
+    assert isinstance(MarkedJson.try_cast(valid), MarkedJson)
+    assert MarkedJson.try_cast(invalid) is invalid
+    assert MarkedJson.try_cast(mimic) is mimic
diff --git a/tests/test_serialization.py b/tests/test_serialization.py
new file mode 100644
index 0000000..569168c
--- /dev/null
+++ b/tests/test_serialization.py
@@ -0,0 +1,95 @@
+"""
+Test serialization-specific functionality that doesn't need to be
+applied across every interface (use test_interface/test_interfaces for that
+"""
+
+import pdb
+
+import h5py
+import pytest
+from pathlib import Path
+from typing import Callable
+import numpy as np
+import json
+
+
+@pytest.fixture(scope="module")
+def hdf5_at_path() -> Callable[[Path], None]:
+    _path = ""
+
+    def _hdf5_at_path(path: Path) -> None:
+        nonlocal _path
+        _path = path
+        h5f = h5py.File(path, "w")
+        _ = h5f.create_dataset("/data", data=np.array([[1, 2], [3, 4]]))
+        _ = h5f.create_dataset("subpath/to/dataset", data=np.array([[1, 2], [4, 5]]))
+        h5f.close()
+
+    yield _hdf5_at_path
+
+    Path(_path).unlink(missing_ok=True)
+
+
+def test_relative_path(hdf5_at_path, tmp_output_dir, model_blank):
+    """
+    By default, we should make all paths relative to the cwd
+    """
+    out_path = tmp_output_dir / "relative.h5"
+    hdf5_at_path(out_path)
+    model = model_blank(array=(out_path, "/data"))
+    rt = model.model_dump_json(round_trip=True)
+    file = json.loads(rt)["array"]["file"]
+
+    # should not be absolute
+    assert not Path(file).is_absolute()
+    # should be relative to cwd
+    out_file = (Path.cwd() / file).resolve()
+    assert out_file == out_path.resolve()
+
+
+def test_relative_to_path(hdf5_at_path, tmp_output_dir, model_blank):
+    """
+    When explicitly passed a path to be ``relative_to`` ,
+    relative to that instead of cwd
+    """
+    out_path = tmp_output_dir / "relative.h5"
+    relative_to_path = Path(__file__) / "fake_dir" / "sub_fake_dir"
+    expected_path = "../../../__tmp__/relative.h5"
+
+    hdf5_at_path(out_path)
+    model = model_blank(array=(out_path, "/data"))
+    rt = model.model_dump_json(
+        round_trip=True, context={"relative_to": str(relative_to_path)}
+    )
+    data = json.loads(rt)["array"]
+    file = data["file"]
+
+    # should not be absolute
+    assert not Path(file).is_absolute()
+    # should be expected path and reach the file
+    assert file == expected_path
+    assert (relative_to_path / file).resolve() == out_path.resolve()
+
+    # we shouldn't have touched `/data` even though it is pathlike
+    assert data["path"] == "/data"
+
+
+def test_relative_to_path(hdf5_at_path, tmp_output_dir, model_blank):
+    """
+    When told, we make paths absolute
+    """
+    out_path = tmp_output_dir / "relative.h5"
+    expected_dataset = "subpath/to/dataset"
+
+    hdf5_at_path(out_path)
+    model = model_blank(array=(out_path, expected_dataset))
+    rt = model.model_dump_json(round_trip=True, context={"absolute_paths": True})
+    data = json.loads(rt)["array"]
+    file = data["file"]
+
+    # should be absolute and equal to out_path
+    assert Path(file).is_absolute()
+    assert Path(file) == out_path.resolve()
+
+    # shouldn't have absolutized subpath even if it's pathlike
+    assert data["path"] == expected_dataset

From 16b0eb05422cda1877720b42c075945b0068aba9 Mon Sep 17 00:00:00 2001
From: sneakers-the-rat <sneakers-the-rat@protonmail.com>
Date: Mon, 23 Sep 2024 17:35:24 -0700
Subject: [PATCH 20/22] test shape ellipsis at last

---
 tests/test_ndarray.py       | 12 ++++++++++++
 tests/test_serialization.py |  2 ++
 2 files changed, 14 insertions(+)

diff --git a/tests/test_ndarray.py b/tests/test_ndarray.py
index 73abedf..7be03bb 100644
--- a/tests/test_ndarray.py
+++ b/tests/test_ndarray.py
@@ -172,6 +172,18 @@ def test_ndarray_coercion():
         amod = Model(array=["a", "b", "c"])
 
 
+@pytest.mark.shape
+def test_shape_ellipsis():
+    """
+    Test that ellipsis is a wildcard, rather than "repeat the last index"
+    """
+
+    class MyModel(BaseModel):
+        array: NDArray[Shape["1, 2, ..."], Number]
+
+    _ = MyModel(array=np.zeros((1, 2, 3, 4, 5)))
+
+
 @pytest.mark.serialization
 def test_ndarray_serialize():
     """
diff --git a/tests/test_serialization.py b/tests/test_serialization.py
index 569168c..dc0ef06 100644
--- a/tests/test_serialization.py
+++ b/tests/test_serialization.py
@@ -12,6 +12,8 @@ from typing import Callable
 import numpy as np
 import json
 
+pytestmark = pytest.mark.serialization
+
 
 @pytest.fixture(scope="module")
 def hdf5_at_path() -> Callable[[Path], None]:

From 0a175d17c07ccc2406758d125d7d94ed66cb23d2 Mon Sep 17 00:00:00 2001
From: sneakers-the-rat <sneakers-the-rat@protonmail.com>
Date: Mon, 23 Sep 2024 18:15:10 -0700
Subject: [PATCH 21/22] changelog, bump version, remove pdb

---
 docs/changelog.md           | 67 +++++++++++++++++++++++++++++++++++++
 docs/serialization.md       | 35 ++++++++++++++-----
 pyproject.toml              |  2 +-
 tests/conftest.py           |  1 -
 tests/test_ndarray.py       |  2 --
 tests/test_serialization.py |  2 --
 tests/test_shape.py         |  2 --
 7 files changed, 95 insertions(+), 16 deletions(-)

diff --git a/docs/changelog.md b/docs/changelog.md
index 684602d..af6375e 100644
--- a/docs/changelog.md
+++ b/docs/changelog.md
@@ -2,6 +2,73 @@
 
 ## 1.*
 
+### 1.6.*
+
+#### 1.6.0 - 24-09-23 - Roundtrip JSON Serialization
+
+Roundtrip JSON serialization is here - with serialization to list of lists,
+as well as file references that don't require copying the whole array if 
+used in data modeling, control over path relativization, and stamping of
+interface version for the extra provenance conscious.
+
+Please see [serialization](./serialization.md) for narrative documentation :)
+
+**Potentially Breaking Changes**
+- See [development](./development.md) for a statement about API stability
+- An additional {meth}`.Interface.deserialize` method has been added to
+  {meth}`.Interface.validate` - downstream users are not intended to override the
+  `validate method`, but if they have, then JSON deserialization will not work for them.
+- `Interface` subclasses now require a `name` attribute, a short string identifier for that interface,
+  and a `json_model` that inherits from {class}`.interface.JsonDict`. Interfaces without
+  these attributes will not be able to be instantiated.
+- {meth}`.Interface.to_json` is now an abstract method that all interfaces must define.
+
+**Features**
+- Roundtrip JSON serialization - by default dump to a list of list arrays, but
+  support the `round_trip` keyword in `model_dump_json` for provenance-preserving dumps
+- JSON Schema generation has been separated from `core_schema` generation in {class}`.NDArray`.
+  Downstream interfaces can customize json schema generation without compromising ability to validate.
+- All proxy classes must have an `__eq__` dunder method to compare equality -
+  in proxy classes, these compare equality of arguments, since the arrays that
+  are referenced on disk should be equal by definition. Direct array comparison
+  should use {func}`numpy.array_equal`
+- Interfaces previously couldn't be instantiated without explicit shape and dtype arguments,
+  these have been given `Any` defaults.
+- New {mod}`numpydantic.serialization` module to contain serialization logic.
+
+**New Classes**
+See the docstrings for descriptions of each class
+- `MarkMismatchError` for when an array serialized with `mark_interface` doesn't match
+  the interface that's deserializing it
+- {class}`.interface.InterfaceMark`
+- {class}`.interface.MarkedJson`
+- {class}`.interface.JsonDict`
+  - {class}`.dask.DaskJsonDict`
+  - {class}`.hdf5.H5JsonDict`
+  - {class}`.numpy.NumpyJsonDict`
+  - {class}`.video.VideoJsonDict`
+  - {class}`.zarr.ZarrJsonDict`
+
+**Bugfix**
+- [`#17`](https://github.com/p2p-ld/numpydantic/issues/17) - Arrays are re-validated as lists, rather than arrays
+- Some proxy classes would fail to be serialized becauase they lacked an `__array__` method.
+  `__array__` methods have been added, and tests for coercing to an array to prevent regression.
+- Some proxy classes lacked a `__name__` attribute, which caused failures to serialize
+  when the `__getattr__` methods attempted to pass it through. These have been added where needed.
+
+**Docs**
+- Add statement about versioning and API stability to [development](./development.md)
+- Add docs for serialization!
+- Remove stranded docs from hooks and monkeypatch
+- Added `myst_nb` to docs dependencies for direct rendering of code and output
+
+**Tests**
+- Marks have been added for running subsets of the tests for a given interface,
+  package feature, etc.
+- Tests for all the above functionality
+
+
+
 ### 1.5.*
 
 #### 1.5.3 - 24-09-03 - Bugfix, type checking for empty HDF5 datasets
diff --git a/docs/serialization.md b/docs/serialization.md
index 5812f30..ccad606 100644
--- a/docs/serialization.md
+++ b/docs/serialization.md
@@ -110,14 +110,17 @@ as `int` ({class}`numpy.int64`) or `float` ({class}`numpy.float64`)
 ## Roundtripping
 
 To roundtrip make arrays round-trippable, use the `round_trip` argument
-to {func}`~pydantic.BaseModel.model_dump_json`
+to {func}`~pydantic.BaseModel.model_dump_json`.
 
+All the following should return an equivalent array from the same
+file/etc. as the source array when using 
+`{func}`~pydantic.BaseModel.model_validate_json`` .
 
 ```{code-cell}
 print_json(model.model_dump_json(round_trip=True))
 ```
 
-Each interface should[^notenforced] implement a dataclass that describes a
+Each interface must implement a dataclass that describes a
 json-able roundtrip form (see {class}`.interface.JsonDict`).
 
 That dataclass then has a {meth}`JsonDict.is_valid` method that checks
@@ -220,12 +223,34 @@ print_json(
     ))
 ```
 
+When an array marked with the interface is deserialized,
+it short-circuits the {meth}`.Interface.match` method,
+attempting to directly return the indicated interface as long as the
+array dumped in `value` still satisfies that interface's {meth}`.Interface.check`
+method. Arrays dumped *without* `round_trip=True` might *not* validate with
+the originating model, even when marked -- eg. an array dumped without `round_trip`
+will be revalidated as a numpy array for the same reasons it is everywhere else,
+since all connection to the source file is lost.
+
+```{todo}
+Currently, the version of the package the interface is from (usually `numpydantic`)
+will be stored, but there is no means of resolving it on the fly. 
+If there is a mismatch between the marked interface description and the interface
+that was matched on revalidation, a warning is emitted, but validation
+attempts to proceed as normal.
+
+This feature is for extra-verbose provenance, rather than airtight serialization
+and deserialization, but PRs welcome if you would like to make it be that way.
+```
+
 ```{todo}
 We will also add a separate `mark_version` parameter for marking
 the specific version of the relevant interface package, like `zarr`, or `numpy`,
 patience.
 ```
 
+
+
 ## Context parameters
 
 A reference listing of all the things that can be passed to
@@ -305,9 +330,3 @@ print_json(data)
 
 
 [^normalstyle]: o ya we're posting JSON [normal style](https://normal.style)
-[^notenforced]: This is only *functionally* enforced at the moment, where 
-  a roundtrip test confirms that dtype and type are preserved,
-  but there is no formal test for each interface having its own serialization class
-
-
-
diff --git a/pyproject.toml b/pyproject.toml
index a53ef1a..0e6926b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "numpydantic"
-version = "1.5.3"
+version = "1.6.0"
 description = "Type and shape validation and serialization for arbitrary array types in pydantic models"
 authors = [
     {name = "sneakers-the-rat", email = "sneakers-the-rat@protonmail.com"},
diff --git a/tests/conftest.py b/tests/conftest.py
index 0467f25..c9035f4 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,4 +1,3 @@
-import pdb
 import sys
 
 import pytest
diff --git a/tests/test_ndarray.py b/tests/test_ndarray.py
index 7be03bb..cda092c 100644
--- a/tests/test_ndarray.py
+++ b/tests/test_ndarray.py
@@ -1,5 +1,3 @@
-import pdb
-
 import pytest
 
 from typing import Union, Optional, Any
diff --git a/tests/test_serialization.py b/tests/test_serialization.py
index dc0ef06..702dc1a 100644
--- a/tests/test_serialization.py
+++ b/tests/test_serialization.py
@@ -3,8 +3,6 @@ Test serialization-specific functionality that doesn't need to be
 applied across every interface (use test_interface/test_interfaces for that
 """
 
-import pdb
-
 import h5py
 import pytest
 from pathlib import Path
diff --git a/tests/test_shape.py b/tests/test_shape.py
index 03de477..b521054 100644
--- a/tests/test_shape.py
+++ b/tests/test_shape.py
@@ -1,5 +1,3 @@
-import pdb
-
 import pytest
 
 from typing import Any

From 4d1557a2f90b1bb8115a146e0a49ad4110ad3707 Mon Sep 17 00:00:00 2001
From: sneakers-the-rat <sneakers-the-rat@protonmail.com>
Date: Mon, 23 Sep 2024 18:17:30 -0700
Subject: [PATCH 22/22] move meta things to meta in docs

---
 docs/index.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/index.md b/docs/index.md
index 003f5c3..0880a59 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -475,8 +475,6 @@ design
 syntax
 serialization
 interfaces
-todo
-changelog
 ```
 
 ```{toctree}
@@ -502,7 +500,9 @@ api/types
 :caption: Meta
 :hidden: true
 
+changelog
 development
+todo
 ```
 
 ## See Also