Merge pull request #12 from p2p-ld/interface-len

Support `len()` across all interfaces
This commit is contained in:
Jonny Saunders 2024-09-02 18:24:16 -07:00 committed by GitHub
commit f699d2ab7b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 88 additions and 36 deletions

View file

@ -2,6 +2,22 @@
## 1.*
### 1.4.1 - 24-09-02 - `len()` support and dunder method testing
It's pretty natural to want to do `len(array)` as a shorthand for `array.shape[0]`,
but since some of the numpydantic classes are passthrough proxy objects,
they don't implement all the dunder methods of the classes they wrap
(though they should attempt to via `__getattr__`).
This PR adds `__len__` to the two interfaces that are missing it,
and adds fixtures and makes a testing module specifically for testing dunder methods
that should be true across all interfaces.
Previously we have had fixtures that test all of a set of dtype and shape cases for each interface,
but we haven't had a way of asserting that something should be true for all interfaces.
There is a certain combinatoric explosion when we start testing across all interfaces,
for all input types, for all dtype and all shape cases,
but for now numpydantic is fast enough that this doesn't matter <3.
### 1.4.0 - 24-09-02 - HDF5 Compound Dtype Support
HDF5 can have compound dtypes like:

View file

@ -1,6 +1,6 @@
[project]
name = "numpydantic"
version = "1.4.0"
version = "1.4.1"
description = "Type and shape validation and serialization for numpy arrays in pydantic models"
authors = [
{name = "sneakers-the-rat", email = "sneakers-the-rat@protonmail.com"},

View file

@ -121,6 +121,10 @@ class H5Proxy:
else:
obj[key, self.field] = value
def __len__(self) -> int:
"""self.shape[0]"""
return self.shape[0]
def open(self, mode: str = "r") -> "h5py.Dataset":
"""
Return the opened :class:`h5py.Dataset` object

View file

@ -180,6 +180,10 @@ class VideoProxy:
def __getattr__(self, item: str):
return getattr(self.video, item)
def __len__(self) -> int:
"""Number of frames in the video"""
return self.shape[0]
class VideoInterface(Interface):
"""

View file

@ -8,6 +8,7 @@ import numpy as np
import pytest
from pydantic import BaseModel, Field
import zarr
import cv2
from numpydantic.interface.hdf5 import H5ArrayPath
from numpydantic.interface.zarr import ZarrArrayPath
@ -150,3 +151,32 @@ def zarr_array(tmp_output_dir_func) -> Path:
array = zarr.open(str(file), mode="w", shape=(100, 100), chunks=(10, 10))
array[:] = 0
return file
@pytest.fixture(scope="function")
def avi_video(tmp_path) -> Callable[[Tuple[int, int], int, bool], Path]:
video_path = tmp_path / "test.avi"
def _make_video(shape=(100, 50), frames=10, is_color=True) -> Path:
writer = cv2.VideoWriter(
str(video_path),
cv2.VideoWriter_fourcc(*"RGBA"), # raw video for testing purposes
30,
(shape[1], shape[0]),
is_color,
)
if is_color:
shape = (*shape, 3)
for i in range(frames):
# make fresh array every time bc opencv eats them
array = np.zeros(shape, dtype=np.uint8)
if not is_color:
array[i, i] = i
else:
array[i, i, :] = i
writer.write(array)
writer.release()
return video_path
return _make_video

View file

@ -1,10 +1,12 @@
import pytest
from typing import Tuple, Callable
import numpy as np
import dask.array as da
import zarr
from pydantic import BaseModel
from numpydantic import interface
from numpydantic import interface, NDArray
@pytest.fixture(
@ -17,6 +19,7 @@ from numpydantic import interface
(zarr.ones((10, 10)), interface.ZarrInterface),
("zarr_nested_array", interface.ZarrInterface),
("zarr_array", interface.ZarrInterface),
("avi_video", interface.VideoInterface),
],
ids=[
"numpy_list",
@ -26,9 +29,10 @@ from numpydantic import interface
"zarr_memory",
"zarr_nested",
"zarr_array",
"video",
],
)
def interface_type(request):
def interface_type(request) -> Tuple[NDArray, interface.Interface]:
"""
Test cases for each interface's ``check`` method - each input should match the
provided interface and that interface only
@ -37,3 +41,20 @@ def interface_type(request):
return (request.getfixturevalue(request.param[0]), request.param[1])
else:
return request.param
@pytest.fixture()
def all_interfaces(interface_type) -> BaseModel:
"""
An instantiated version of each interface within a basemodel,
with the array in an `array` field
"""
array, interface = interface_type
if isinstance(array, Callable):
array = array()
class MyModel(BaseModel):
array: NDArray
instance = MyModel(array=array)
return instance

View file

@ -0,0 +1,10 @@
"""
Tests for dunder methods on all interfaces
"""
def test_dunder_len(all_interfaces):
"""
Each interface or proxy type should support __len__
"""
assert len(all_interfaces.array) == all_interfaces.array.shape[0]

View file

@ -2,8 +2,6 @@
Needs to be refactored to DRY, but works for now
"""
import pdb
import numpy as np
import pytest
@ -17,37 +15,6 @@ from numpydantic import dtype as dt
from numpydantic.interface.video import VideoProxy
@pytest.fixture(scope="function")
def avi_video(tmp_path):
video_path = tmp_path / "test.avi"
def _make_video(shape=(100, 50), frames=10, is_color=True) -> Path:
writer = cv2.VideoWriter(
str(video_path),
cv2.VideoWriter_fourcc(*"RGBA"), # raw video for testing purposes
30,
(shape[1], shape[0]),
is_color,
)
if is_color:
shape = (*shape, 3)
for i in range(frames):
# make fresh array every time bc opencv eats them
array = np.zeros(shape, dtype=np.uint8)
if not is_color:
array[i, i] = i
else:
array[i, i, :] = i
writer.write(array)
writer.release()
return video_path
yield _make_video
video_path.unlink(missing_ok=True)
@pytest.mark.parametrize("input_type", [str, Path])
def test_video_validation(avi_video, input_type):
"""Color videos should validate for normal uint8 shape specs"""