From 91b2abf07e32e951a08f62a9d2318c5f5a9935d7 Mon Sep 17 00:00:00 2001 From: sneakers-the-rat Date: Wed, 11 Sep 2024 21:04:41 -0700 Subject: [PATCH] working thru tests for nwb file --- docs/meta/todo.md | 4 + nwb_linkml/conftest.py | 2 +- nwb_linkml/src/nwb_linkml/includes/hdmf.py | 4 +- nwb_linkml/tests/test_io/test_io_nwb.py | 86 +++++++++++++++++++ .../hdmf_common/v1_1_0/hdmf_common_table.py | 4 +- .../hdmf_common/v1_1_2/hdmf_common_table.py | 4 +- .../hdmf_common/v1_1_3/hdmf_common_table.py | 4 +- .../hdmf_common/v1_2_0/hdmf_common_table.py | 4 +- .../hdmf_common/v1_2_1/hdmf_common_table.py | 4 +- .../hdmf_common/v1_3_0/hdmf_common_table.py | 4 +- .../hdmf_common/v1_4_0/hdmf_common_table.py | 4 +- .../hdmf_common/v1_5_0/hdmf_common_table.py | 4 +- .../hdmf_common/v1_5_1/hdmf_common_table.py | 4 +- .../hdmf_common/v1_6_0/hdmf_common_table.py | 4 +- .../hdmf_common/v1_7_0/hdmf_common_table.py | 4 +- .../hdmf_common/v1_8_0/hdmf_common_table.py | 4 +- 16 files changed, 117 insertions(+), 27 deletions(-) diff --git a/docs/meta/todo.md b/docs/meta/todo.md index d2bf9ac..dd9f750 100644 --- a/docs/meta/todo.md +++ b/docs/meta/todo.md @@ -49,6 +49,10 @@ Remove monkeypatches/overrides once PRs are closed Tests - [ ] Ensure schemas and pydantic modules in repos are up to date +Loading +- [ ] Top-level containers are still a little janky, eg. how `ProcessingModule` just accepts + extra args rather than properly abstracting `value` as a `__getitem__(self, key) -> T:` + ## Docs TODOs ```{todolist} diff --git a/nwb_linkml/conftest.py b/nwb_linkml/conftest.py index 450875f..88c09a6 100644 --- a/nwb_linkml/conftest.py +++ b/nwb_linkml/conftest.py @@ -71,7 +71,7 @@ adapter_parser = Sybil( doctest_parser = Sybil( parsers=[DocTestParser(optionflags=ELLIPSIS + NORMALIZE_WHITESPACE), PythonCodeBlockParser()], - patterns=["*.py"], + patterns=["providers/git.py"], ) pytest_collect_file = (adapter_parser + doctest_parser).pytest() diff --git a/nwb_linkml/src/nwb_linkml/includes/hdmf.py b/nwb_linkml/src/nwb_linkml/includes/hdmf.py index 19b8848..b64e0f1 100644 --- a/nwb_linkml/src/nwb_linkml/includes/hdmf.py +++ b/nwb_linkml/src/nwb_linkml/includes/hdmf.py @@ -141,7 +141,7 @@ class DynamicTableMixin(BaseModel): # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -627,7 +627,7 @@ class AlignedDynamicTableMixin(BaseModel): ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) + ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id")) tables = [ids] for category_name, category in self._categories.items(): table = category[item] diff --git a/nwb_linkml/tests/test_io/test_io_nwb.py b/nwb_linkml/tests/test_io/test_io_nwb.py index f5fff62..f758855 100644 --- a/nwb_linkml/tests/test_io/test_io_nwb.py +++ b/nwb_linkml/tests/test_io/test_io_nwb.py @@ -2,7 +2,16 @@ Placeholder test module to test reading from pynwb-generated NWB file """ +import pytest +from datetime import datetime +from numpydantic.interface.hdf5 import H5Proxy from nwb_linkml.io.hdf5 import HDF5IO +from nwb_models.models import NWBFile +from pydantic import BaseModel +import numpy as np +import pandas as pd + +from pynwb import NWBHDF5IO, NWBFile as PyNWBFile def test_read_from_nwbfile(nwb_file): @@ -15,6 +24,83 @@ def test_read_from_nwbfile(nwb_file): res = HDF5IO(nwb_file).read() +@pytest.fixture(scope="module") +def read_nwbfile(nwb_file) -> NWBFile: + res = HDF5IO(nwb_file).read() + return res + + +@pytest.fixture(scope="module") +def read_pynwb(nwb_file) -> PyNWBFile: + nwbf = NWBHDF5IO(nwb_file, "r") + res = nwbf.read() + yield res + nwbf.close() + + +def _compare_attrs(model: BaseModel, pymodel: object): + for field, value in model.model_dump().items(): + if isinstance(value, (dict, H5Proxy)): + continue + if hasattr(pymodel, field): + pynwb_val = getattr(pymodel, field) + if isinstance(pynwb_val, list): + if isinstance(pynwb_val[0], datetime): + # need to normalize UTC numpy.datetime64 with datetime with tz + continue + assert all([val == pval for val, pval in zip(value, pynwb_val)]) + else: + if not pynwb_val: + # pynwb instantiates some stuff as empty dicts where we use ``None`` + assert bool(pynwb_val) == bool(value) + else: + assert value == pynwb_val + + +def test_nwbfile_base(read_nwbfile, read_pynwb): + """ + Base attributes on top-level nwbfile are correct + """ + _compare_attrs(read_nwbfile, read_pynwb) + + +def test_timeseries(read_nwbfile, read_pynwb): + py_acq = read_pynwb.get_acquisition("test_timeseries") + acq = read_nwbfile.acquisition["test_timeseries"] + _compare_attrs(acq, py_acq) + # data and timeseries should be equal + assert np.array_equal(acq.data[:], py_acq.data[:]) + assert np.array_equal(acq.timestamps[:], py_acq.timestamps[:]) + + +def test_position(read_nwbfile, read_pynwb): + trials = read_nwbfile.intervals.trials[:] + py_trials = read_pynwb.trials.to_dataframe() + pd.testing.assert_frame_equal(py_trials, trials) + + spatial = read_nwbfile.processing["behavior"].Position.SpatialSeries + py_spatial = read_pynwb.processing["behavior"]["Position"]["SpatialSeries"] + _compare_attrs(spatial, py_spatial) + assert np.array_equal(spatial[:], py_spatial.data[:]) + assert np.array_equal(spatial.timestamps[:], py_spatial.timestamps[:]) + + +def test_ecephys(read_nwbfile, read_pynwb): + pass + + +def test_units(read_nwbfile, read_pynwb): + pass + + +def test_icephys(read_nwbfile, read_pynwb): + pass + + +def test_ca_imaging(read_nwbfile, read_pynwb): + pass + + def test_read_from_yaml(nwb_file): """ Read data from a yaml-fied NWB file diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_0/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_0/hdmf_common_table.py index c1dd417..00ced23 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_0/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_0/hdmf_common_table.py @@ -417,7 +417,7 @@ class DynamicTableMixin(BaseModel): # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -704,7 +704,7 @@ class AlignedDynamicTableMixin(BaseModel): ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) + ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id")) tables = [ids] for category_name, category in self._categories.items(): table = category[item] diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_2/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_2/hdmf_common_table.py index 7c9cede..2bc9dde 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_2/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_2/hdmf_common_table.py @@ -417,7 +417,7 @@ class DynamicTableMixin(BaseModel): # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -704,7 +704,7 @@ class AlignedDynamicTableMixin(BaseModel): ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) + ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id")) tables = [ids] for category_name, category in self._categories.items(): table = category[item] diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_3/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_3/hdmf_common_table.py index 3ce5c1c..cf7c150 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_3/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_3/hdmf_common_table.py @@ -417,7 +417,7 @@ class DynamicTableMixin(BaseModel): # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -704,7 +704,7 @@ class AlignedDynamicTableMixin(BaseModel): ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) + ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id")) tables = [ids] for category_name, category in self._categories.items(): table = category[item] diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/hdmf_common_table.py index 08fa840..663d58b 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/hdmf_common_table.py @@ -419,7 +419,7 @@ class DynamicTableMixin(BaseModel): # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -706,7 +706,7 @@ class AlignedDynamicTableMixin(BaseModel): ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) + ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id")) tables = [ids] for category_name, category in self._categories.items(): table = category[item] diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/hdmf_common_table.py index fc08eaf..b61981d 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/hdmf_common_table.py @@ -419,7 +419,7 @@ class DynamicTableMixin(BaseModel): # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -706,7 +706,7 @@ class AlignedDynamicTableMixin(BaseModel): ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) + ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id")) tables = [ids] for category_name, category in self._categories.items(): table = category[item] diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_table.py index 7bbf3fd..2d1973b 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_table.py @@ -419,7 +419,7 @@ class DynamicTableMixin(BaseModel): # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -706,7 +706,7 @@ class AlignedDynamicTableMixin(BaseModel): ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) + ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id")) tables = [ids] for category_name, category in self._categories.items(): table = category[item] diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/hdmf_common_table.py index a2a0a05..804d424 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/hdmf_common_table.py @@ -419,7 +419,7 @@ class DynamicTableMixin(BaseModel): # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -706,7 +706,7 @@ class AlignedDynamicTableMixin(BaseModel): ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) + ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id")) tables = [ids] for category_name, category in self._categories.items(): table = category[item] diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/hdmf_common_table.py index 4e94a01..9e8c2ad 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/hdmf_common_table.py @@ -419,7 +419,7 @@ class DynamicTableMixin(BaseModel): # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -706,7 +706,7 @@ class AlignedDynamicTableMixin(BaseModel): ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) + ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id")) tables = [ids] for category_name, category in self._categories.items(): table = category[item] diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/hdmf_common_table.py index 75eadd1..d3c2f22 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/hdmf_common_table.py @@ -419,7 +419,7 @@ class DynamicTableMixin(BaseModel): # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -706,7 +706,7 @@ class AlignedDynamicTableMixin(BaseModel): ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) + ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id")) tables = [ids] for category_name, category in self._categories.items(): table = category[item] diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/hdmf_common_table.py index 7536e5c..7dc1868 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/hdmf_common_table.py @@ -419,7 +419,7 @@ class DynamicTableMixin(BaseModel): # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -706,7 +706,7 @@ class AlignedDynamicTableMixin(BaseModel): ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) + ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id")) tables = [ids] for category_name, category in self._categories.items(): table = category[item] diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/hdmf_common_table.py index 92f64a1..0780f0e 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/hdmf_common_table.py @@ -419,7 +419,7 @@ class DynamicTableMixin(BaseModel): # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -706,7 +706,7 @@ class AlignedDynamicTableMixin(BaseModel): ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) + ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id")) tables = [ids] for category_name, category in self._categories.items(): table = category[item] diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/hdmf_common_table.py index 06a391f..024e442 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/hdmf_common_table.py @@ -419,7 +419,7 @@ class DynamicTableMixin(BaseModel): # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -706,7 +706,7 @@ class AlignedDynamicTableMixin(BaseModel): ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) + ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id")) tables = [ids] for category_name, category in self._categories.items(): table = category[item]