From b1c8d3e422b7afd7addded76798213e7514b6d16 Mon Sep 17 00:00:00 2001 From: sneakers-the-rat Date: Mon, 2 Sep 2024 18:13:28 -0700 Subject: [PATCH 1/3] Add len method to video and hdf5 interfaces, make dunder test module to test dunder methods across all interfaces --- src/numpydantic/interface/hdf5.py | 4 ++++ src/numpydantic/interface/video.py | 4 ++++ tests/fixtures.py | 32 ++++++++++++++++++++++++++++ tests/test_interface/conftest.py | 25 ++++++++++++++++++++-- tests/test_interface/test_dunder.py | 10 +++++++++ tests/test_interface/test_video.py | 33 ----------------------------- 6 files changed, 73 insertions(+), 35 deletions(-) create mode 100644 tests/test_interface/test_dunder.py diff --git a/src/numpydantic/interface/hdf5.py b/src/numpydantic/interface/hdf5.py index ec32b0e..656273d 100644 --- a/src/numpydantic/interface/hdf5.py +++ b/src/numpydantic/interface/hdf5.py @@ -121,6 +121,10 @@ class H5Proxy: else: obj[key, self.field] = value + def __len__(self) -> int: + """self.shape[0]""" + return self.shape[0] + def open(self, mode: str = "r") -> "h5py.Dataset": """ Return the opened :class:`h5py.Dataset` object diff --git a/src/numpydantic/interface/video.py b/src/numpydantic/interface/video.py index 4f2048d..0ac7c66 100644 --- a/src/numpydantic/interface/video.py +++ b/src/numpydantic/interface/video.py @@ -180,6 +180,10 @@ class VideoProxy: def __getattr__(self, item: str): return getattr(self.video, item) + def __len__(self) -> int: + """Number of frames in the video""" + return self.shape[0] + class VideoInterface(Interface): """ diff --git a/tests/fixtures.py b/tests/fixtures.py index 347152c..e74c9a6 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -8,6 +8,7 @@ import numpy as np import pytest from pydantic import BaseModel, Field import zarr +import cv2 from numpydantic.interface.hdf5 import H5ArrayPath from numpydantic.interface.zarr import ZarrArrayPath @@ -150,3 +151,34 @@ def zarr_array(tmp_output_dir_func) -> Path: array = zarr.open(str(file), mode="w", shape=(100, 100), chunks=(10, 10)) array[:] = 0 return file + + +@pytest.fixture(scope="function") +def avi_video(tmp_path) -> Callable[[Tuple[int, int], int, bool], Path]: + video_path = tmp_path / "test.avi" + + def _make_video(shape=(100, 50), frames=10, is_color=True) -> Path: + writer = cv2.VideoWriter( + str(video_path), + cv2.VideoWriter_fourcc(*"RGBA"), # raw video for testing purposes + 30, + (shape[1], shape[0]), + is_color, + ) + if is_color: + shape = (*shape, 3) + + for i in range(frames): + # make fresh array every time bc opencv eats them + array = np.zeros(shape, dtype=np.uint8) + if not is_color: + array[i, i] = i + else: + array[i, i, :] = i + writer.write(array) + writer.release() + return video_path + + yield _make_video + + video_path.unlink(missing_ok=True) diff --git a/tests/test_interface/conftest.py b/tests/test_interface/conftest.py index e3bf0f7..63bdc4a 100644 --- a/tests/test_interface/conftest.py +++ b/tests/test_interface/conftest.py @@ -1,10 +1,12 @@ import pytest +from typing import Tuple, Callable import numpy as np import dask.array as da import zarr +from pydantic import BaseModel -from numpydantic import interface +from numpydantic import interface, NDArray @pytest.fixture( @@ -17,6 +19,7 @@ from numpydantic import interface (zarr.ones((10, 10)), interface.ZarrInterface), ("zarr_nested_array", interface.ZarrInterface), ("zarr_array", interface.ZarrInterface), + ("avi_video", interface.VideoInterface), ], ids=[ "numpy_list", @@ -26,9 +29,10 @@ from numpydantic import interface "zarr_memory", "zarr_nested", "zarr_array", + "video", ], ) -def interface_type(request): +def interface_type(request) -> Tuple[NDArray, interface.Interface]: """ Test cases for each interface's ``check`` method - each input should match the provided interface and that interface only @@ -37,3 +41,20 @@ def interface_type(request): return (request.getfixturevalue(request.param[0]), request.param[1]) else: return request.param + + +@pytest.fixture() +def all_interfaces(interface_type) -> BaseModel: + """ + An instantiated version of each interface within a basemodel, + with the array in an `array` field + """ + array, interface = interface_type + if isinstance(array, Callable): + array = array() + + class MyModel(BaseModel): + array: NDArray + + instance = MyModel(array=array) + return instance diff --git a/tests/test_interface/test_dunder.py b/tests/test_interface/test_dunder.py new file mode 100644 index 0000000..60f42d8 --- /dev/null +++ b/tests/test_interface/test_dunder.py @@ -0,0 +1,10 @@ +""" +Tests for dunder methods on all interfaces +""" + + +def test_dunder_len(all_interfaces): + """ + Each interface or proxy type should support __len__ + """ + assert len(all_interfaces.array) == all_interfaces.array.shape[0] diff --git a/tests/test_interface/test_video.py b/tests/test_interface/test_video.py index bfd7f7d..d5ef1b3 100644 --- a/tests/test_interface/test_video.py +++ b/tests/test_interface/test_video.py @@ -2,8 +2,6 @@ Needs to be refactored to DRY, but works for now """ -import pdb - import numpy as np import pytest @@ -17,37 +15,6 @@ from numpydantic import dtype as dt from numpydantic.interface.video import VideoProxy -@pytest.fixture(scope="function") -def avi_video(tmp_path): - video_path = tmp_path / "test.avi" - - def _make_video(shape=(100, 50), frames=10, is_color=True) -> Path: - writer = cv2.VideoWriter( - str(video_path), - cv2.VideoWriter_fourcc(*"RGBA"), # raw video for testing purposes - 30, - (shape[1], shape[0]), - is_color, - ) - if is_color: - shape = (*shape, 3) - - for i in range(frames): - # make fresh array every time bc opencv eats them - array = np.zeros(shape, dtype=np.uint8) - if not is_color: - array[i, i] = i - else: - array[i, i, :] = i - writer.write(array) - writer.release() - return video_path - - yield _make_video - - video_path.unlink(missing_ok=True) - - @pytest.mark.parametrize("input_type", [str, Path]) def test_video_validation(avi_video, input_type): """Color videos should validate for normal uint8 shape specs""" From fa5e4defe2fd14470350446174467ac5aeccd4c2 Mon Sep 17 00:00:00 2001 From: sneakers-the-rat Date: Mon, 2 Sep 2024 18:18:57 -0700 Subject: [PATCH 2/3] changelog and bump version --- docs/changelog.md | 16 ++++++++++++++++ pyproject.toml | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/changelog.md b/docs/changelog.md index 07276c3..b93adb2 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,22 @@ ## 1.* +### 1.4.1 - 24-09-02 - `len()` support and dunder method testing + +It's pretty natural to want to do `len(array)` as a shorthand for `array.shape[0]`, +but since some of the numpydantic classes are passthrough proxy objects, +they don't implement all the dunder methods of the classes they wrap +(though they should attempt to via `__getattr__`). + +This PR adds `__len__` to the two interfaces that are missing it, +and adds fixtures and makes a testing module specifically for testing dunder methods +that should be true across all interfaces. +Previously we have had fixtures that test all of a set of dtype and shape cases for each interface, +but we haven't had a way of asserting that something should be true for all interfaces. +There is a certain combinatoric explosion when we start testing across all interfaces, +for all input types, for all dtype and all shape cases, +but for now numpydantic is fast enough that this doesn't matter <3. + ### 1.4.0 - 24-09-02 - HDF5 Compound Dtype Support HDF5 can have compound dtypes like: diff --git a/pyproject.toml b/pyproject.toml index f5c8542..57ae6be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "numpydantic" -version = "1.4.0" +version = "1.4.1" description = "Type and shape validation and serialization for numpy arrays in pydantic models" authors = [ {name = "sneakers-the-rat", email = "sneakers-the-rat@protonmail.com"}, From 8883076e5cd75d2c5b60219e16c6b62740df68b6 Mon Sep 17 00:00:00 2001 From: sneakers-the-rat Date: Mon, 2 Sep 2024 18:20:55 -0700 Subject: [PATCH 3/3] we actually don't need to manually delete that since the tmp_dir should delete it on its own --- tests/fixtures.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/fixtures.py b/tests/fixtures.py index e74c9a6..6fbc5ad 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -179,6 +179,4 @@ def avi_video(tmp_path) -> Callable[[Tuple[int, int], int, bool], Path]: writer.release() return video_path - yield _make_video - - video_path.unlink(missing_ok=True) + return _make_video