From a079fbcafcceadb22b16ddbc154ea5befafc4a86 Mon Sep 17 00:00:00 2001 From: sneakers-the-rat Date: Wed, 8 May 2024 22:06:41 -0700 Subject: [PATCH] working zarr impl --- src/numpydantic/dtype.py | 3 +- tests/test_interface/conftest.py | 12 +-- tests/test_interface/test_dask.py | 40 ++++----- tests/test_interface/test_zarr.py | 56 +++++++----- tests/test_linkml/__init__.py | 0 tests/test_linkml/test_pydanticgen.py | 120 -------------------------- 6 files changed, 63 insertions(+), 168 deletions(-) delete mode 100644 tests/test_linkml/__init__.py delete mode 100644 tests/test_linkml/test_pydanticgen.py diff --git a/src/numpydantic/dtype.py b/src/numpydantic/dtype.py index 76631a2..d25bcd1 100644 --- a/src/numpydantic/dtype.py +++ b/src/numpydantic/dtype.py @@ -52,7 +52,7 @@ LongLong = np.longlong Timedelta64 = np.timedelta64 SignedInteger = (np.int8, np.int16, np.int32, np.int64, np.short) UnsignedInteger = (np.uint8, np.uint16, np.uint32, np.uint64, np.ushort) -Integer = tuple([np.integer, *SignedInteger, *UnsignedInteger]) +Integer = tuple([*SignedInteger, *UnsignedInteger]) Int = Integer # Int should translate to the "generic" int type. Float16 = np.float16 @@ -68,7 +68,6 @@ Float = ( np.float16, np.float32, np.float64, - np.floating, np.single, np.double, ) diff --git a/tests/test_interface/conftest.py b/tests/test_interface/conftest.py index b2021c3..e3bf0f7 100644 --- a/tests/test_interface/conftest.py +++ b/tests/test_interface/conftest.py @@ -5,7 +5,6 @@ import dask.array as da import zarr from numpydantic import interface -from tests.fixtures import hdf5_array, zarr_nested_array, zarr_array @pytest.fixture( @@ -13,11 +12,11 @@ from tests.fixtures import hdf5_array, zarr_nested_array, zarr_array params=[ ([[1, 2], [3, 4]], interface.NumpyInterface), (np.zeros((3, 4)), interface.NumpyInterface), - (hdf5_array, interface.H5Interface), + ("hdf5_array", interface.H5Interface), (da.random.random((10, 10)), interface.DaskInterface), (zarr.ones((10, 10)), interface.ZarrInterface), - (zarr_nested_array, interface.ZarrInterface), - (zarr_array, interface.ZarrInterface), + ("zarr_nested_array", interface.ZarrInterface), + ("zarr_array", interface.ZarrInterface), ], ids=[ "numpy_list", @@ -34,4 +33,7 @@ def interface_type(request): Test cases for each interface's ``check`` method - each input should match the provided interface and that interface only """ - return request.param + if isinstance(request.param[0], str): + return (request.getfixturevalue(request.param[0]), request.param[1]) + else: + return request.param diff --git a/tests/test_interface/test_dask.py b/tests/test_interface/test_dask.py index 51122d2..8584ff2 100644 --- a/tests/test_interface/test_dask.py +++ b/tests/test_interface/test_dask.py @@ -4,6 +4,22 @@ import dask.array as da from pydantic import ValidationError from numpydantic.interface import DaskInterface +from numpydantic.exceptions import DtypeError, ShapeError + +from tests.conftest import ValidationCase + + +def dask_array(case: ValidationCase) -> da.Array: + return da.zeros(shape=case.shape, dtype=case.dtype, chunks=10) + + +def _test_dask_case(case: ValidationCase): + array = dask_array(case) + if case.passes: + case.model(array=array) + else: + with pytest.raises((ValidationError, DtypeError, ShapeError)): + case.model(array=array) def test_dask_enabled(): @@ -20,25 +36,9 @@ def test_dask_check(interface_type): assert not DaskInterface.check(interface_type[0]) -@pytest.mark.parametrize( - "array,passes", - [ - (da.random.random((5, 10)), True), - (da.random.random((5, 10, 3)), True), - (da.random.random((5, 10, 3, 4)), True), - (da.random.random((5, 10, 4)), False), - (da.random.random((5, 10, 3, 6)), False), - (da.random.random((5, 10, 4, 6)), False), - ], -) -def test_dask_shape(model_rgb, array, passes): - if passes: - model_rgb(array=array) - else: - with pytest.raises(ValidationError): - model_rgb(array=array) +def test_dask_shape(shape_cases): + _test_dask_case(shape_cases) -@pytest.mark.skip("TODO") -def test_dask_dtype(): - pass +def test_dask_dtype(dtype_cases): + _test_dask_case(dtype_cases) diff --git a/tests/test_interface/test_zarr.py b/tests/test_interface/test_zarr.py index f1d2458..6913b40 100644 --- a/tests/test_interface/test_zarr.py +++ b/tests/test_interface/test_zarr.py @@ -4,6 +4,9 @@ import zarr from pydantic import ValidationError from numpydantic.interface import ZarrInterface +from numpydantic.exceptions import DtypeError, ShapeError + +from tests.conftest import ValidationCase @pytest.fixture() @@ -24,11 +27,33 @@ def nested_dir_array(tmp_output_dir_func) -> zarr.NestedDirectoryStore: return store -STORES = ( - dir_array, - zip_array, +def zarr_array(case: ValidationCase, store) -> zarr.core.Array: + return zarr.zeros(shape=case.shape, dtype=case.dtype, store=store) + + +def _test_zarr_case(case: ValidationCase, store): + array = zarr_array(case, store) + if case.passes: + case.model(array=array) + else: + with pytest.raises((ValidationError, DtypeError, ShapeError)): + case.model(array=array) + + +@pytest.fixture( + params=[ + None, # use the default store + "dir_array", + "zip_array", + "nested_dir_array", + ], + ids=["MutableMapping", "DirectoryStore", "ZipStore", "NestedDirectoryStore"], ) -"""stores for single arrays""" +def store(request): + if isinstance(request.param, str): + return request.getfixturevalue(request.param) + else: + return request.param def test_zarr_enabled(): @@ -45,20 +70,9 @@ def test_zarr_check(interface_type): assert not ZarrInterface.check(interface_type[0]) -@pytest.mark.parametrize( - "array,passes", - [ - (zarr.zeros((5, 10)), True), - (zarr.zeros((5, 10, 3)), True), - (zarr.zeros((5, 10, 3, 4)), True), - (zarr.zeros((5, 10, 4)), False), - (zarr.zeros((5, 10, 3, 6)), False), - (zarr.zeros((5, 10, 4, 6)), False), - ], -) -def test_zarr_shape(model_rgb, array, passes): - if passes: - model_rgb(array=array) - else: - with pytest.raises(ValidationError): - model_rgb(array=array) +def test_zarr_shape(store, shape_cases): + _test_zarr_case(shape_cases, store) + + +def test_zarr_dtype(dtype_cases, store): + _test_zarr_case(dtype_cases, store) diff --git a/tests/test_linkml/__init__.py b/tests/test_linkml/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_linkml/test_pydanticgen.py b/tests/test_linkml/test_pydanticgen.py deleted file mode 100644 index d8de13f..0000000 --- a/tests/test_linkml/test_pydanticgen.py +++ /dev/null @@ -1,120 +0,0 @@ -""" -Test custom features of the pydantic generator - -Note that since this is largely a subclass, we don't test all of the functionality of the generator -because it's tested in the base linkml package. -""" -import re -import sys -import typing - -import numpy as np -import pytest -from pydantic import BaseModel - - -# def test_arraylike(imported_schema): -# """ -# Arraylike classes are converted to slots that specify nptyping arrays -# -# array: Optional[Union[ -# NDArray[Shape["* x, * y"], Number], -# NDArray[Shape["* x, * y, 3 z"], Number], -# NDArray[Shape["* x, * y, 3 z, 4 a"], Number] -# ]] = Field(None) -# """ -# # check that we have gotten an NDArray annotation and its shape is correct -# array = imported_schema["core"].MainTopLevel.model_fields["array"].annotation -# args = typing.get_args(array) -# for i, shape in enumerate(("* x, * y", "* x, * y, 3 z", "* x, * y, 3 z, 4 a")): -# assert isinstance(args[i], NDArrayMeta) -# assert args[i].__args__[0].__args__ -# assert args[i].__args__[1] == np.number -# -# # we shouldn't have an actual class for the array -# assert not hasattr(imported_schema["core"], "MainTopLevel__Array") -# assert not hasattr(imported_schema["core"], "MainTopLevelArray") -# -# -# def test_inject_fields(imported_schema): -# """ -# Our root model should have the special fields we injected -# """ -# base = imported_schema["core"].ConfiguredBaseModel -# assert "hdf5_path" in base.model_fields -# assert "object_id" in base.model_fields -# -# -# def test_linkml_meta(imported_schema): -# """ -# We should be able to store some linkml metadata with our classes -# """ -# meta = imported_schema["core"].LinkML_Meta -# assert "tree_root" in meta.model_fields -# assert imported_schema["core"].MainTopLevel.linkml_meta.default.tree_root == True -# assert imported_schema["core"].OtherClass.linkml_meta.default.tree_root == False -# -# -# def test_skip(linkml_schema): -# """ -# We can skip slots and classes -# """ -# modules = generate_and_import( -# linkml_schema, -# split=False, -# generator_kwargs={ -# "SKIP_SLOTS": ("SkippableSlot",), -# "SKIP_CLASSES": ("Skippable", "skippable"), -# }, -# ) -# assert not hasattr(modules["core"], "Skippable") -# assert "SkippableSlot" not in modules["core"].MainTopLevel.model_fields -# -# -# def test_inline_with_identifier(imported_schema): -# """ -# By default, if a class has an identifier attribute, it is inlined -# as a string rather than its class. We overrode that to be able to make dictionaries of collections -# """ -# main = imported_schema["core"].MainTopLevel -# inline = main.model_fields["inline_dict"].annotation -# assert typing.get_origin(typing.get_args(inline)[0]) == dict -# # god i hate pythons typing interface -# otherclass, stillanother = typing.get_args( -# typing.get_args(typing.get_args(inline)[0])[1] -# ) -# assert otherclass is imported_schema["core"].OtherClass -# assert stillanother is imported_schema["core"].StillAnotherClass -# -# -# def test_namespace(imported_schema): -# """ -# Namespace schema import all classes from the other schema -# Returns: -# -# """ -# ns = imported_schema["namespace"] -# -# for classname, modname in ( -# ("MainThing", "test_schema.imported"), -# ("Arraylike", "test_schema.imported"), -# ("MainTopLevel", "test_schema.core"), -# ("Skippable", "test_schema.core"), -# ("OtherClass", "test_schema.core"), -# ("StillAnotherClass", "test_schema.core"), -# ): -# assert hasattr(ns, classname) -# if imported_schema["split"]: -# assert getattr(ns, classname).__module__ == modname -# -# -# def test_get_set_item(imported_schema): -# """We can get and set without explicitly addressing array""" -# cls_ = imported_schema["core"].MainTopLevel(array=np.array([[1, 2, 3], [4, 5, 6]])) -# cls_[0] = 50 -# assert (cls_[0] == 50).all() -# assert (cls_.array[0] == 50).all() -# -# cls_[1, 1] = 100 -# assert cls_[1, 1] == 100 -# assert cls_.array[1, 1] == 100