p2p-ld/numpydantic
1
0
Fork 0
mirror of https://github.com/p2p-ld/numpydantic.git synced 2024-11-12 17:54:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

working zarr impl

Browse source
This commit is contained in:
sneakers-the-rat 2024-05-08 22:06:41 -07:00
parent b82a49df1b
commit a079fbcafc
Signed by untrusted user who does not match committer: jonny
GPG key ID: 6DCB96EF1E4D232D
6 changed files with 63 additions and 168 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
src/numpydantic/dtype.py
View file

@ -52,7 +52,7 @@ LongLong = np.longlong
Timedelta64 = np.timedelta64
SignedInteger = (np.int8, np.int16, np.int32, np.int64, np.short)
UnsignedInteger = (np.uint8, np.uint16, np.uint32, np.uint64, np.ushort)
Integer = tuple([np.integer, *SignedInteger, *UnsignedInteger])
Integer = tuple([*SignedInteger, *UnsignedInteger])
Int = Integer # Int should translate to the "generic" int type.
Float16 = np.float16
@ -68,7 +68,6 @@ Float = (
np.float16,
np.float32,
np.float64,
np.floating,
np.single,
np.double,
)

12
tests/test_interface/conftest.py
View file

@ -5,7 +5,6 @@ import dask.array as da
import zarr
from numpydantic import interface
from tests.fixtures import hdf5_array, zarr_nested_array, zarr_array
@pytest.fixture(
@ -13,11 +12,11 @@ from tests.fixtures import hdf5_array, zarr_nested_array, zarr_array
params=[
([[1, 2], [3, 4]], interface.NumpyInterface),
(np.zeros((3, 4)), interface.NumpyInterface),
(hdf5_array, interface.H5Interface),
("hdf5_array", interface.H5Interface),
(da.random.random((10, 10)), interface.DaskInterface),
(zarr.ones((10, 10)), interface.ZarrInterface),
(zarr_nested_array, interface.ZarrInterface),
(zarr_array, interface.ZarrInterface),
("zarr_nested_array", interface.ZarrInterface),
("zarr_array", interface.ZarrInterface),
],
ids=[
"numpy_list",
@ -34,4 +33,7 @@ def interface_type(request):
Test cases for each interface's ``check`` method - each input should match the
provided interface and that interface only
"""
return request.param
if isinstance(request.param[0], str):
return (request.getfixturevalue(request.param[0]), request.param[1])
else:
return request.param

40
tests/test_interface/test_dask.py
View file

@ -4,6 +4,22 @@ import dask.array as da
from pydantic import ValidationError
from numpydantic.interface import DaskInterface
from numpydantic.exceptions import DtypeError, ShapeError
from tests.conftest import ValidationCase
def dask_array(case: ValidationCase) -> da.Array:
return da.zeros(shape=case.shape, dtype=case.dtype, chunks=10)
def _test_dask_case(case: ValidationCase):
array = dask_array(case)
if case.passes:
case.model(array=array)
else:
with pytest.raises((ValidationError, DtypeError, ShapeError)):
case.model(array=array)
def test_dask_enabled():
@ -20,25 +36,9 @@ def test_dask_check(interface_type):
assert not DaskInterface.check(interface_type[0])
@pytest.mark.parametrize(
"array,passes",
[
(da.random.random((5, 10)), True),
(da.random.random((5, 10, 3)), True),
(da.random.random((5, 10, 3, 4)), True),
(da.random.random((5, 10, 4)), False),
(da.random.random((5, 10, 3, 6)), False),
(da.random.random((5, 10, 4, 6)), False),
],
)
def test_dask_shape(model_rgb, array, passes):
if passes:
model_rgb(array=array)
else:
with pytest.raises(ValidationError):
model_rgb(array=array)
def test_dask_shape(shape_cases):
_test_dask_case(shape_cases)
@pytest.mark.skip("TODO")
def test_dask_dtype():
pass
def test_dask_dtype(dtype_cases):
_test_dask_case(dtype_cases)

56
tests/test_interface/test_zarr.py
View file

@ -4,6 +4,9 @@ import zarr
from pydantic import ValidationError
from numpydantic.interface import ZarrInterface
from numpydantic.exceptions import DtypeError, ShapeError
from tests.conftest import ValidationCase
@pytest.fixture()
@ -24,11 +27,33 @@ def nested_dir_array(tmp_output_dir_func) -> zarr.NestedDirectoryStore:
return store
STORES = (
dir_array,
zip_array,
def zarr_array(case: ValidationCase, store) -> zarr.core.Array:
return zarr.zeros(shape=case.shape, dtype=case.dtype, store=store)
def _test_zarr_case(case: ValidationCase, store):
array = zarr_array(case, store)
if case.passes:
case.model(array=array)
else:
with pytest.raises((ValidationError, DtypeError, ShapeError)):
case.model(array=array)
@pytest.fixture(
params=[
None, # use the default store
"dir_array",
"zip_array",
"nested_dir_array",
],
ids=["MutableMapping", "DirectoryStore", "ZipStore", "NestedDirectoryStore"],
)
"""stores for single arrays"""
def store(request):
if isinstance(request.param, str):
return request.getfixturevalue(request.param)
else:
return request.param
def test_zarr_enabled():
@ -45,20 +70,9 @@ def test_zarr_check(interface_type):
assert not ZarrInterface.check(interface_type[0])
@pytest.mark.parametrize(
"array,passes",
[
(zarr.zeros((5, 10)), True),
(zarr.zeros((5, 10, 3)), True),
(zarr.zeros((5, 10, 3, 4)), True),
(zarr.zeros((5, 10, 4)), False),
(zarr.zeros((5, 10, 3, 6)), False),
(zarr.zeros((5, 10, 4, 6)), False),
],
)
def test_zarr_shape(model_rgb, array, passes):
if passes:
model_rgb(array=array)
else:
with pytest.raises(ValidationError):
model_rgb(array=array)
def test_zarr_shape(store, shape_cases):
_test_zarr_case(shape_cases, store)
def test_zarr_dtype(dtype_cases, store):
_test_zarr_case(dtype_cases, store)

0
tests/test_linkml/__init__.py
View file

120
tests/test_linkml/test_pydanticgen.py
View file

@ -1,120 +0,0 @@
"""
Test custom features of the pydantic generator
Note that since this is largely a subclass, we don't test all of the functionality of the generator
because it's tested in the base linkml package.
"""
import re
import sys
import typing
import numpy as np
import pytest
from pydantic import BaseModel
# def test_arraylike(imported_schema):
# """
# Arraylike classes are converted to slots that specify nptyping arrays
#
# array: Optional[Union[
# NDArray[Shape["* x, * y"], Number],
# NDArray[Shape["* x, * y, 3 z"], Number],
# NDArray[Shape["* x, * y, 3 z, 4 a"], Number]
# ]] = Field(None)
# """
# # check that we have gotten an NDArray annotation and its shape is correct
# array = imported_schema["core"].MainTopLevel.model_fields["array"].annotation
# args = typing.get_args(array)
# for i, shape in enumerate(("* x, * y", "* x, * y, 3 z", "* x, * y, 3 z, 4 a")):
# assert isinstance(args[i], NDArrayMeta)
# assert args[i].__args__[0].__args__
# assert args[i].__args__[1] == np.number
#
# # we shouldn't have an actual class for the array
# assert not hasattr(imported_schema["core"], "MainTopLevel__Array")
# assert not hasattr(imported_schema["core"], "MainTopLevelArray")
#
#
# def test_inject_fields(imported_schema):
# """
# Our root model should have the special fields we injected
# """
# base = imported_schema["core"].ConfiguredBaseModel
# assert "hdf5_path" in base.model_fields
# assert "object_id" in base.model_fields
#
#
# def test_linkml_meta(imported_schema):
# """
# We should be able to store some linkml metadata with our classes
# """
# meta = imported_schema["core"].LinkML_Meta
# assert "tree_root" in meta.model_fields
# assert imported_schema["core"].MainTopLevel.linkml_meta.default.tree_root == True
# assert imported_schema["core"].OtherClass.linkml_meta.default.tree_root == False
#
#
# def test_skip(linkml_schema):
# """
# We can skip slots and classes
# """
# modules = generate_and_import(
# linkml_schema,
# split=False,
# generator_kwargs={
# "SKIP_SLOTS": ("SkippableSlot",),
# "SKIP_CLASSES": ("Skippable", "skippable"),
# },
# )
# assert not hasattr(modules["core"], "Skippable")
# assert "SkippableSlot" not in modules["core"].MainTopLevel.model_fields
#
#
# def test_inline_with_identifier(imported_schema):
# """
# By default, if a class has an identifier attribute, it is inlined
# as a string rather than its class. We overrode that to be able to make dictionaries of collections
# """
# main = imported_schema["core"].MainTopLevel
# inline = main.model_fields["inline_dict"].annotation
# assert typing.get_origin(typing.get_args(inline)[0]) == dict
# # god i hate pythons typing interface
# otherclass, stillanother = typing.get_args(
# typing.get_args(typing.get_args(inline)[0])[1]
# )
# assert otherclass is imported_schema["core"].OtherClass
# assert stillanother is imported_schema["core"].StillAnotherClass
#
#
# def test_namespace(imported_schema):
# """
# Namespace schema import all classes from the other schema
# Returns:
#
# """
# ns = imported_schema["namespace"]
#
# for classname, modname in (
# ("MainThing", "test_schema.imported"),
# ("Arraylike", "test_schema.imported"),
# ("MainTopLevel", "test_schema.core"),
# ("Skippable", "test_schema.core"),
# ("OtherClass", "test_schema.core"),
# ("StillAnotherClass", "test_schema.core"),
# ):
# assert hasattr(ns, classname)
# if imported_schema["split"]:
# assert getattr(ns, classname).__module__ == modname
#
#
# def test_get_set_item(imported_schema):
# """We can get and set without explicitly addressing array"""
# cls_ = imported_schema["core"].MainTopLevel(array=np.array([[1, 2, 3], [4, 5, 6]]))
# cls_[0] = 50
# assert (cls_[0] == 50).all()
# assert (cls_.array[0] == 50).all()
#
# cls_[1, 1] = 100
# assert cls_[1, 1] == 100
# assert cls_.array[1, 1] == 100