working thru tests for nwb file

This commit is contained in:
sneakers-the-rat 2024-09-11 21:04:41 -07:00
parent bb59c9d465
commit 91b2abf07e
Signed by untrusted user who does not match committer: jonny
GPG key ID: 6DCB96EF1E4D232D
16 changed files with 117 additions and 27 deletions

View file

@ -49,6 +49,10 @@ Remove monkeypatches/overrides once PRs are closed
Tests Tests
- [ ] Ensure schemas and pydantic modules in repos are up to date - [ ] Ensure schemas and pydantic modules in repos are up to date
Loading
- [ ] Top-level containers are still a little janky, eg. how `ProcessingModule` just accepts
extra args rather than properly abstracting `value` as a `__getitem__(self, key) -> T:`
## Docs TODOs ## Docs TODOs
```{todolist} ```{todolist}

View file

@ -71,7 +71,7 @@ adapter_parser = Sybil(
doctest_parser = Sybil( doctest_parser = Sybil(
parsers=[DocTestParser(optionflags=ELLIPSIS + NORMALIZE_WHITESPACE), PythonCodeBlockParser()], parsers=[DocTestParser(optionflags=ELLIPSIS + NORMALIZE_WHITESPACE), PythonCodeBlockParser()],
patterns=["*.py"], patterns=["providers/git.py"],
) )
pytest_collect_file = (adapter_parser + doctest_parser).pytest() pytest_collect_file = (adapter_parser + doctest_parser).pytest()

View file

@ -141,7 +141,7 @@ class DynamicTableMixin(BaseModel):
# cast to DF # cast to DF
if not isinstance(index, Iterable): if not isinstance(index, Iterable):
index = [index] index = [index]
index = pd.Index(data=index) index = pd.Index(data=index, name="id")
return pd.DataFrame(data, index=index) return pd.DataFrame(data, index=index)
def _slice_range( def _slice_range(
@ -627,7 +627,7 @@ class AlignedDynamicTableMixin(BaseModel):
ids = self.id[item] ids = self.id[item]
if not isinstance(ids, Iterable): if not isinstance(ids, Iterable):
ids = pd.Series([ids]) ids = pd.Series([ids])
ids = pd.DataFrame({"id": ids}) ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id"))
tables = [ids] tables = [ids]
for category_name, category in self._categories.items(): for category_name, category in self._categories.items():
table = category[item] table = category[item]

View file

@ -2,7 +2,16 @@
Placeholder test module to test reading from pynwb-generated NWB file Placeholder test module to test reading from pynwb-generated NWB file
""" """
import pytest
from datetime import datetime
from numpydantic.interface.hdf5 import H5Proxy
from nwb_linkml.io.hdf5 import HDF5IO from nwb_linkml.io.hdf5 import HDF5IO
from nwb_models.models import NWBFile
from pydantic import BaseModel
import numpy as np
import pandas as pd
from pynwb import NWBHDF5IO, NWBFile as PyNWBFile
def test_read_from_nwbfile(nwb_file): def test_read_from_nwbfile(nwb_file):
@ -15,6 +24,83 @@ def test_read_from_nwbfile(nwb_file):
res = HDF5IO(nwb_file).read() res = HDF5IO(nwb_file).read()
@pytest.fixture(scope="module")
def read_nwbfile(nwb_file) -> NWBFile:
res = HDF5IO(nwb_file).read()
return res
@pytest.fixture(scope="module")
def read_pynwb(nwb_file) -> PyNWBFile:
nwbf = NWBHDF5IO(nwb_file, "r")
res = nwbf.read()
yield res
nwbf.close()
def _compare_attrs(model: BaseModel, pymodel: object):
for field, value in model.model_dump().items():
if isinstance(value, (dict, H5Proxy)):
continue
if hasattr(pymodel, field):
pynwb_val = getattr(pymodel, field)
if isinstance(pynwb_val, list):
if isinstance(pynwb_val[0], datetime):
# need to normalize UTC numpy.datetime64 with datetime with tz
continue
assert all([val == pval for val, pval in zip(value, pynwb_val)])
else:
if not pynwb_val:
# pynwb instantiates some stuff as empty dicts where we use ``None``
assert bool(pynwb_val) == bool(value)
else:
assert value == pynwb_val
def test_nwbfile_base(read_nwbfile, read_pynwb):
"""
Base attributes on top-level nwbfile are correct
"""
_compare_attrs(read_nwbfile, read_pynwb)
def test_timeseries(read_nwbfile, read_pynwb):
py_acq = read_pynwb.get_acquisition("test_timeseries")
acq = read_nwbfile.acquisition["test_timeseries"]
_compare_attrs(acq, py_acq)
# data and timeseries should be equal
assert np.array_equal(acq.data[:], py_acq.data[:])
assert np.array_equal(acq.timestamps[:], py_acq.timestamps[:])
def test_position(read_nwbfile, read_pynwb):
trials = read_nwbfile.intervals.trials[:]
py_trials = read_pynwb.trials.to_dataframe()
pd.testing.assert_frame_equal(py_trials, trials)
spatial = read_nwbfile.processing["behavior"].Position.SpatialSeries
py_spatial = read_pynwb.processing["behavior"]["Position"]["SpatialSeries"]
_compare_attrs(spatial, py_spatial)
assert np.array_equal(spatial[:], py_spatial.data[:])
assert np.array_equal(spatial.timestamps[:], py_spatial.timestamps[:])
def test_ecephys(read_nwbfile, read_pynwb):
pass
def test_units(read_nwbfile, read_pynwb):
pass
def test_icephys(read_nwbfile, read_pynwb):
pass
def test_ca_imaging(read_nwbfile, read_pynwb):
pass
def test_read_from_yaml(nwb_file): def test_read_from_yaml(nwb_file):
""" """
Read data from a yaml-fied NWB file Read data from a yaml-fied NWB file

View file

@ -417,7 +417,7 @@ class DynamicTableMixin(BaseModel):
# cast to DF # cast to DF
if not isinstance(index, Iterable): if not isinstance(index, Iterable):
index = [index] index = [index]
index = pd.Index(data=index) index = pd.Index(data=index, name="id")
return pd.DataFrame(data, index=index) return pd.DataFrame(data, index=index)
def _slice_range( def _slice_range(
@ -704,7 +704,7 @@ class AlignedDynamicTableMixin(BaseModel):
ids = self.id[item] ids = self.id[item]
if not isinstance(ids, Iterable): if not isinstance(ids, Iterable):
ids = pd.Series([ids]) ids = pd.Series([ids])
ids = pd.DataFrame({"id": ids}) ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id"))
tables = [ids] tables = [ids]
for category_name, category in self._categories.items(): for category_name, category in self._categories.items():
table = category[item] table = category[item]

View file

@ -417,7 +417,7 @@ class DynamicTableMixin(BaseModel):
# cast to DF # cast to DF
if not isinstance(index, Iterable): if not isinstance(index, Iterable):
index = [index] index = [index]
index = pd.Index(data=index) index = pd.Index(data=index, name="id")
return pd.DataFrame(data, index=index) return pd.DataFrame(data, index=index)
def _slice_range( def _slice_range(
@ -704,7 +704,7 @@ class AlignedDynamicTableMixin(BaseModel):
ids = self.id[item] ids = self.id[item]
if not isinstance(ids, Iterable): if not isinstance(ids, Iterable):
ids = pd.Series([ids]) ids = pd.Series([ids])
ids = pd.DataFrame({"id": ids}) ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id"))
tables = [ids] tables = [ids]
for category_name, category in self._categories.items(): for category_name, category in self._categories.items():
table = category[item] table = category[item]

View file

@ -417,7 +417,7 @@ class DynamicTableMixin(BaseModel):
# cast to DF # cast to DF
if not isinstance(index, Iterable): if not isinstance(index, Iterable):
index = [index] index = [index]
index = pd.Index(data=index) index = pd.Index(data=index, name="id")
return pd.DataFrame(data, index=index) return pd.DataFrame(data, index=index)
def _slice_range( def _slice_range(
@ -704,7 +704,7 @@ class AlignedDynamicTableMixin(BaseModel):
ids = self.id[item] ids = self.id[item]
if not isinstance(ids, Iterable): if not isinstance(ids, Iterable):
ids = pd.Series([ids]) ids = pd.Series([ids])
ids = pd.DataFrame({"id": ids}) ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id"))
tables = [ids] tables = [ids]
for category_name, category in self._categories.items(): for category_name, category in self._categories.items():
table = category[item] table = category[item]

View file

@ -419,7 +419,7 @@ class DynamicTableMixin(BaseModel):
# cast to DF # cast to DF
if not isinstance(index, Iterable): if not isinstance(index, Iterable):
index = [index] index = [index]
index = pd.Index(data=index) index = pd.Index(data=index, name="id")
return pd.DataFrame(data, index=index) return pd.DataFrame(data, index=index)
def _slice_range( def _slice_range(
@ -706,7 +706,7 @@ class AlignedDynamicTableMixin(BaseModel):
ids = self.id[item] ids = self.id[item]
if not isinstance(ids, Iterable): if not isinstance(ids, Iterable):
ids = pd.Series([ids]) ids = pd.Series([ids])
ids = pd.DataFrame({"id": ids}) ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id"))
tables = [ids] tables = [ids]
for category_name, category in self._categories.items(): for category_name, category in self._categories.items():
table = category[item] table = category[item]

View file

@ -419,7 +419,7 @@ class DynamicTableMixin(BaseModel):
# cast to DF # cast to DF
if not isinstance(index, Iterable): if not isinstance(index, Iterable):
index = [index] index = [index]
index = pd.Index(data=index) index = pd.Index(data=index, name="id")
return pd.DataFrame(data, index=index) return pd.DataFrame(data, index=index)
def _slice_range( def _slice_range(
@ -706,7 +706,7 @@ class AlignedDynamicTableMixin(BaseModel):
ids = self.id[item] ids = self.id[item]
if not isinstance(ids, Iterable): if not isinstance(ids, Iterable):
ids = pd.Series([ids]) ids = pd.Series([ids])
ids = pd.DataFrame({"id": ids}) ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id"))
tables = [ids] tables = [ids]
for category_name, category in self._categories.items(): for category_name, category in self._categories.items():
table = category[item] table = category[item]

View file

@ -419,7 +419,7 @@ class DynamicTableMixin(BaseModel):
# cast to DF # cast to DF
if not isinstance(index, Iterable): if not isinstance(index, Iterable):
index = [index] index = [index]
index = pd.Index(data=index) index = pd.Index(data=index, name="id")
return pd.DataFrame(data, index=index) return pd.DataFrame(data, index=index)
def _slice_range( def _slice_range(
@ -706,7 +706,7 @@ class AlignedDynamicTableMixin(BaseModel):
ids = self.id[item] ids = self.id[item]
if not isinstance(ids, Iterable): if not isinstance(ids, Iterable):
ids = pd.Series([ids]) ids = pd.Series([ids])
ids = pd.DataFrame({"id": ids}) ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id"))
tables = [ids] tables = [ids]
for category_name, category in self._categories.items(): for category_name, category in self._categories.items():
table = category[item] table = category[item]

View file

@ -419,7 +419,7 @@ class DynamicTableMixin(BaseModel):
# cast to DF # cast to DF
if not isinstance(index, Iterable): if not isinstance(index, Iterable):
index = [index] index = [index]
index = pd.Index(data=index) index = pd.Index(data=index, name="id")
return pd.DataFrame(data, index=index) return pd.DataFrame(data, index=index)
def _slice_range( def _slice_range(
@ -706,7 +706,7 @@ class AlignedDynamicTableMixin(BaseModel):
ids = self.id[item] ids = self.id[item]
if not isinstance(ids, Iterable): if not isinstance(ids, Iterable):
ids = pd.Series([ids]) ids = pd.Series([ids])
ids = pd.DataFrame({"id": ids}) ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id"))
tables = [ids] tables = [ids]
for category_name, category in self._categories.items(): for category_name, category in self._categories.items():
table = category[item] table = category[item]

View file

@ -419,7 +419,7 @@ class DynamicTableMixin(BaseModel):
# cast to DF # cast to DF
if not isinstance(index, Iterable): if not isinstance(index, Iterable):
index = [index] index = [index]
index = pd.Index(data=index) index = pd.Index(data=index, name="id")
return pd.DataFrame(data, index=index) return pd.DataFrame(data, index=index)
def _slice_range( def _slice_range(
@ -706,7 +706,7 @@ class AlignedDynamicTableMixin(BaseModel):
ids = self.id[item] ids = self.id[item]
if not isinstance(ids, Iterable): if not isinstance(ids, Iterable):
ids = pd.Series([ids]) ids = pd.Series([ids])
ids = pd.DataFrame({"id": ids}) ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id"))
tables = [ids] tables = [ids]
for category_name, category in self._categories.items(): for category_name, category in self._categories.items():
table = category[item] table = category[item]

View file

@ -419,7 +419,7 @@ class DynamicTableMixin(BaseModel):
# cast to DF # cast to DF
if not isinstance(index, Iterable): if not isinstance(index, Iterable):
index = [index] index = [index]
index = pd.Index(data=index) index = pd.Index(data=index, name="id")
return pd.DataFrame(data, index=index) return pd.DataFrame(data, index=index)
def _slice_range( def _slice_range(
@ -706,7 +706,7 @@ class AlignedDynamicTableMixin(BaseModel):
ids = self.id[item] ids = self.id[item]
if not isinstance(ids, Iterable): if not isinstance(ids, Iterable):
ids = pd.Series([ids]) ids = pd.Series([ids])
ids = pd.DataFrame({"id": ids}) ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id"))
tables = [ids] tables = [ids]
for category_name, category in self._categories.items(): for category_name, category in self._categories.items():
table = category[item] table = category[item]

View file

@ -419,7 +419,7 @@ class DynamicTableMixin(BaseModel):
# cast to DF # cast to DF
if not isinstance(index, Iterable): if not isinstance(index, Iterable):
index = [index] index = [index]
index = pd.Index(data=index) index = pd.Index(data=index, name="id")
return pd.DataFrame(data, index=index) return pd.DataFrame(data, index=index)
def _slice_range( def _slice_range(
@ -706,7 +706,7 @@ class AlignedDynamicTableMixin(BaseModel):
ids = self.id[item] ids = self.id[item]
if not isinstance(ids, Iterable): if not isinstance(ids, Iterable):
ids = pd.Series([ids]) ids = pd.Series([ids])
ids = pd.DataFrame({"id": ids}) ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id"))
tables = [ids] tables = [ids]
for category_name, category in self._categories.items(): for category_name, category in self._categories.items():
table = category[item] table = category[item]

View file

@ -419,7 +419,7 @@ class DynamicTableMixin(BaseModel):
# cast to DF # cast to DF
if not isinstance(index, Iterable): if not isinstance(index, Iterable):
index = [index] index = [index]
index = pd.Index(data=index) index = pd.Index(data=index, name="id")
return pd.DataFrame(data, index=index) return pd.DataFrame(data, index=index)
def _slice_range( def _slice_range(
@ -706,7 +706,7 @@ class AlignedDynamicTableMixin(BaseModel):
ids = self.id[item] ids = self.id[item]
if not isinstance(ids, Iterable): if not isinstance(ids, Iterable):
ids = pd.Series([ids]) ids = pd.Series([ids])
ids = pd.DataFrame({"id": ids}) ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id"))
tables = [ids] tables = [ids]
for category_name, category in self._categories.items(): for category_name, category in self._categories.items():
table = category[item] table = category[item]

View file

@ -419,7 +419,7 @@ class DynamicTableMixin(BaseModel):
# cast to DF # cast to DF
if not isinstance(index, Iterable): if not isinstance(index, Iterable):
index = [index] index = [index]
index = pd.Index(data=index) index = pd.Index(data=index, name="id")
return pd.DataFrame(data, index=index) return pd.DataFrame(data, index=index)
def _slice_range( def _slice_range(
@ -706,7 +706,7 @@ class AlignedDynamicTableMixin(BaseModel):
ids = self.id[item] ids = self.id[item]
if not isinstance(ids, Iterable): if not isinstance(ids, Iterable):
ids = pd.Series([ids]) ids = pd.Series([ids])
ids = pd.DataFrame({"id": ids}) ids = pd.DataFrame({"id": ids}, index=pd.Index(data=ids, name="id"))
tables = [ids] tables = [ids]
for category_name, category in self._categories.items(): for category_name, category in self._categories.items():
table = category[item] table = category[item]