working thru tests for nwb file

docs/meta/todo.md

@@ -49,6 +49,10 @@ Remove monkeypatches/overrides once PRs are closed
 Tests
 - [ ] Ensure schemas and pydantic modules in repos are up to date
 
+Loading
+- [ ] Top-level containers are still a little janky, eg. how `ProcessingModule` just accepts 
+  extra args rather than properly abstracting `value` as a `__getitem__(self, key) -> T:` 
+
 ## Docs TODOs
 
 ```{todolist}

nwb_linkml/conftest.py

@@ -71,7 +71,7 @@ adapter_parser = Sybil(
 
 doctest_parser = Sybil(
     parsers=[DocTestParser(optionflags=ELLIPSIS + NORMALIZE_WHITESPACE), PythonCodeBlockParser()],
-    patterns=["*.py"],
+    patterns=["providers/git.py"],
 )
 
 pytest_collect_file = (adapter_parser + doctest_parser).pytest()

nwb_linkml/src/nwb_linkml/includes/hdmf.py

@@ -141,7 +141,7 @@ class DynamicTableMixin(BaseModel):
         # cast to DF
         if not isinstance(index, Iterable):
             index = [index]
-        index = pd.Index(data=index)
+        index = pd.Index(data=index, name="id")
         return pd.DataFrame(data, index=index)
 
     def _slice_range(
@@ -627,7 +627,7 @@ class AlignedDynamicTableMixin(BaseModel):
             ids = self.id[item]
             if not isinstance(ids, Iterable):
                 ids = pd.Series([ids])
-            ids = pd.DataFrame({"id": ids})
+            ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id"))
             tables = [ids]
             for category_name, category in self._categories.items():
                 table = category[item]

nwb_linkml/tests/test_io/test_io_nwb.py

@@ -2,7 +2,16 @@
 Placeholder test module to test reading from pynwb-generated NWB file
 """
 
+import pytest
+from datetime import datetime
+from numpydantic.interface.hdf5 import H5Proxy
 from nwb_linkml.io.hdf5 import HDF5IO
+from nwb_models.models import NWBFile
+from pydantic import BaseModel
+import numpy as np
+import pandas as pd
+
+from pynwb import NWBHDF5IO, NWBFile as PyNWBFile
 
 
 def test_read_from_nwbfile(nwb_file):
@@ -15,6 +24,83 @@ def test_read_from_nwbfile(nwb_file):
     res = HDF5IO(nwb_file).read()
 
 
+@pytest.fixture(scope="module")
+def read_nwbfile(nwb_file) -> NWBFile:
+    res = HDF5IO(nwb_file).read()
+    return res
+
+
+@pytest.fixture(scope="module")
+def read_pynwb(nwb_file) -> PyNWBFile:
+    nwbf = NWBHDF5IO(nwb_file, "r")
+    res = nwbf.read()
+    yield res
+    nwbf.close()
+
+
+def _compare_attrs(model: BaseModel, pymodel: object):
+    for field, value in model.model_dump().items():
+        if isinstance(value, (dict, H5Proxy)):
+            continue
+        if hasattr(pymodel, field):
+            pynwb_val = getattr(pymodel, field)
+            if isinstance(pynwb_val, list):
+                if isinstance(pynwb_val[0], datetime):
+                    # need to normalize UTC numpy.datetime64 with datetime with tz
+                    continue
+                assert all([val == pval for val, pval in zip(value, pynwb_val)])
+            else:
+                if not pynwb_val:
+                    # pynwb instantiates some stuff as empty dicts where we use ``None``
+                    assert bool(pynwb_val) == bool(value)
+                else:
+                    assert value == pynwb_val
+
+
+def test_nwbfile_base(read_nwbfile, read_pynwb):
+    """
+    Base attributes on top-level nwbfile are correct
+    """
+    _compare_attrs(read_nwbfile, read_pynwb)
+
+
+def test_timeseries(read_nwbfile, read_pynwb):
+    py_acq = read_pynwb.get_acquisition("test_timeseries")
+    acq = read_nwbfile.acquisition["test_timeseries"]
+    _compare_attrs(acq, py_acq)
+    # data and timeseries should be equal
+    assert np.array_equal(acq.data[:], py_acq.data[:])
+    assert np.array_equal(acq.timestamps[:], py_acq.timestamps[:])
+
+
+def test_position(read_nwbfile, read_pynwb):
+    trials = read_nwbfile.intervals.trials[:]
+    py_trials = read_pynwb.trials.to_dataframe()
+    pd.testing.assert_frame_equal(py_trials, trials)
+
+    spatial = read_nwbfile.processing["behavior"].Position.SpatialSeries
+    py_spatial = read_pynwb.processing["behavior"]["Position"]["SpatialSeries"]
+    _compare_attrs(spatial, py_spatial)
+    assert np.array_equal(spatial[:], py_spatial.data[:])
+    assert np.array_equal(spatial.timestamps[:], py_spatial.timestamps[:])
+
+
+def test_ecephys(read_nwbfile, read_pynwb):
+    pass
+
+
+def test_units(read_nwbfile, read_pynwb):
+    pass
+
+
+def test_icephys(read_nwbfile, read_pynwb):
+    pass
+
+
+def test_ca_imaging(read_nwbfile, read_pynwb):
+    pass
+
+
 def test_read_from_yaml(nwb_file):
     """
     Read data from a yaml-fied NWB file

nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_0/hdmf_common_table.py

@@ -417,7 +417,7 @@ class DynamicTableMixin(BaseModel):
         # cast to DF
         if not isinstance(index, Iterable):
             index = [index]
-        index = pd.Index(data=index)
+        index = pd.Index(data=index, name="id")
         return pd.DataFrame(data, index=index)
 
     def _slice_range(
@@ -704,7 +704,7 @@ class AlignedDynamicTableMixin(BaseModel):
             ids = self.id[item]
             if not isinstance(ids, Iterable):
                 ids = pd.Series([ids])
-            ids = pd.DataFrame({"id": ids})
+            ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id"))
             tables = [ids]
             for category_name, category in self._categories.items():
                 table = category[item]

nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_2/hdmf_common_table.py

@@ -417,7 +417,7 @@ class DynamicTableMixin(BaseModel):
         # cast to DF
         if not isinstance(index, Iterable):
             index = [index]
-        index = pd.Index(data=index)
+        index = pd.Index(data=index, name="id")
         return pd.DataFrame(data, index=index)
 
     def _slice_range(
@@ -704,7 +704,7 @@ class AlignedDynamicTableMixin(BaseModel):
             ids = self.id[item]
             if not isinstance(ids, Iterable):
                 ids = pd.Series([ids])
-            ids = pd.DataFrame({"id": ids})
+            ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id"))
             tables = [ids]
             for category_name, category in self._categories.items():
                 table = category[item]

nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_3/hdmf_common_table.py

@@ -417,7 +417,7 @@ class DynamicTableMixin(BaseModel):
         # cast to DF
         if not isinstance(index, Iterable):
             index = [index]
-        index = pd.Index(data=index)
+        index = pd.Index(data=index, name="id")
         return pd.DataFrame(data, index=index)
 
     def _slice_range(
@@ -704,7 +704,7 @@ class AlignedDynamicTableMixin(BaseModel):
             ids = self.id[item]
             if not isinstance(ids, Iterable):
                 ids = pd.Series([ids])
-            ids = pd.DataFrame({"id": ids})
+            ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id"))
             tables = [ids]
             for category_name, category in self._categories.items():
                 table = category[item]

nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/hdmf_common_table.py

@@ -419,7 +419,7 @@ class DynamicTableMixin(BaseModel):
         # cast to DF
         if not isinstance(index, Iterable):
             index = [index]
-        index = pd.Index(data=index)
+        index = pd.Index(data=index, name="id")
         return pd.DataFrame(data, index=index)
 
     def _slice_range(
@@ -706,7 +706,7 @@ class AlignedDynamicTableMixin(BaseModel):
             ids = self.id[item]
             if not isinstance(ids, Iterable):
                 ids = pd.Series([ids])
-            ids = pd.DataFrame({"id": ids})
+            ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id"))
             tables = [ids]
             for category_name, category in self._categories.items():
                 table = category[item]

nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/hdmf_common_table.py

@@ -419,7 +419,7 @@ class DynamicTableMixin(BaseModel):
         # cast to DF
         if not isinstance(index, Iterable):
             index = [index]
-        index = pd.Index(data=index)
+        index = pd.Index(data=index, name="id")
         return pd.DataFrame(data, index=index)
 
     def _slice_range(
@@ -706,7 +706,7 @@ class AlignedDynamicTableMixin(BaseModel):
             ids = self.id[item]
             if not isinstance(ids, Iterable):
                 ids = pd.Series([ids])
-            ids = pd.DataFrame({"id": ids})
+            ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id"))
             tables = [ids]
             for category_name, category in self._categories.items():
                 table = category[item]

nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_table.py

@@ -419,7 +419,7 @@ class DynamicTableMixin(BaseModel):
         # cast to DF
         if not isinstance(index, Iterable):
             index = [index]
-        index = pd.Index(data=index)
+        index = pd.Index(data=index, name="id")
         return pd.DataFrame(data, index=index)
 
     def _slice_range(
@@ -706,7 +706,7 @@ class AlignedDynamicTableMixin(BaseModel):
             ids = self.id[item]
             if not isinstance(ids, Iterable):
                 ids = pd.Series([ids])
-            ids = pd.DataFrame({"id": ids})
+            ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id"))
             tables = [ids]
             for category_name, category in self._categories.items():
                 table = category[item]

nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/hdmf_common_table.py

@@ -419,7 +419,7 @@ class DynamicTableMixin(BaseModel):
         # cast to DF
         if not isinstance(index, Iterable):
             index = [index]
-        index = pd.Index(data=index)
+        index = pd.Index(data=index, name="id")
         return pd.DataFrame(data, index=index)
 
     def _slice_range(
@@ -706,7 +706,7 @@ class AlignedDynamicTableMixin(BaseModel):
             ids = self.id[item]
             if not isinstance(ids, Iterable):
                 ids = pd.Series([ids])
-            ids = pd.DataFrame({"id": ids})
+            ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id"))
             tables = [ids]
             for category_name, category in self._categories.items():
                 table = category[item]

nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/hdmf_common_table.py

@@ -419,7 +419,7 @@ class DynamicTableMixin(BaseModel):
         # cast to DF
         if not isinstance(index, Iterable):
             index = [index]
-        index = pd.Index(data=index)
+        index = pd.Index(data=index, name="id")
         return pd.DataFrame(data, index=index)
 
     def _slice_range(
@@ -706,7 +706,7 @@ class AlignedDynamicTableMixin(BaseModel):
             ids = self.id[item]
             if not isinstance(ids, Iterable):
                 ids = pd.Series([ids])
-            ids = pd.DataFrame({"id": ids})
+            ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id"))
             tables = [ids]
             for category_name, category in self._categories.items():
                 table = category[item]

nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/hdmf_common_table.py

@@ -419,7 +419,7 @@ class DynamicTableMixin(BaseModel):
         # cast to DF
         if not isinstance(index, Iterable):
             index = [index]
-        index = pd.Index(data=index)
+        index = pd.Index(data=index, name="id")
         return pd.DataFrame(data, index=index)
 
     def _slice_range(
@@ -706,7 +706,7 @@ class AlignedDynamicTableMixin(BaseModel):
             ids = self.id[item]
             if not isinstance(ids, Iterable):
                 ids = pd.Series([ids])
-            ids = pd.DataFrame({"id": ids})
+            ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id"))
             tables = [ids]
             for category_name, category in self._categories.items():
                 table = category[item]

nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/hdmf_common_table.py

@@ -419,7 +419,7 @@ class DynamicTableMixin(BaseModel):
         # cast to DF
         if not isinstance(index, Iterable):
             index = [index]
-        index = pd.Index(data=index)
+        index = pd.Index(data=index, name="id")
         return pd.DataFrame(data, index=index)
 
     def _slice_range(
@@ -706,7 +706,7 @@ class AlignedDynamicTableMixin(BaseModel):
             ids = self.id[item]
             if not isinstance(ids, Iterable):
                 ids = pd.Series([ids])
-            ids = pd.DataFrame({"id": ids})
+            ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id"))
             tables = [ids]
             for category_name, category in self._categories.items():
                 table = category[item]

nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/hdmf_common_table.py

@@ -419,7 +419,7 @@ class DynamicTableMixin(BaseModel):
         # cast to DF
         if not isinstance(index, Iterable):
             index = [index]
-        index = pd.Index(data=index)
+        index = pd.Index(data=index, name="id")
         return pd.DataFrame(data, index=index)
 
     def _slice_range(
@@ -706,7 +706,7 @@ class AlignedDynamicTableMixin(BaseModel):
             ids = self.id[item]
             if not isinstance(ids, Iterable):
                 ids = pd.Series([ids])
-            ids = pd.DataFrame({"id": ids})
+            ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id"))
             tables = [ids]
             for category_name, category in self._categories.items():
                 table = category[item]

nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/hdmf_common_table.py

@@ -419,7 +419,7 @@ class DynamicTableMixin(BaseModel):
         # cast to DF
         if not isinstance(index, Iterable):
             index = [index]
-        index = pd.Index(data=index)
+        index = pd.Index(data=index, name="id")
         return pd.DataFrame(data, index=index)
 
     def _slice_range(
@@ -706,7 +706,7 @@ class AlignedDynamicTableMixin(BaseModel):
             ids = self.id[item]
             if not isinstance(ids, Iterable):
                 ids = pd.Series([ids])
-            ids = pd.DataFrame({"id": ids})
+            ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id"))
             tables = [ids]
             for category_name, category in self._categories.items():
                 table = category[item]