From a9909485a4b1511f87e4bd801550ccacc0031b6e Mon Sep 17 00:00:00 2001 From: sneakers-the-rat Date: Wed, 7 Aug 2024 02:03:04 -0700 Subject: [PATCH] my god it works but what have i done --- .../src/nwb_linkml/generators/pydantic.py | 2 + nwb_linkml/src/nwb_linkml/includes/hdmf.py | 80 ++++++++++--- .../hdmf_common/v1_1_0/hdmf_common_table.py | 65 ++++++++--- .../hdmf_common/v1_1_2/hdmf_common_table.py | 65 ++++++++--- .../hdmf_common/v1_1_3/hdmf_common_table.py | 65 ++++++++--- .../hdmf_common/v1_2_0/hdmf_common_table.py | 65 ++++++++--- .../hdmf_common/v1_2_1/hdmf_common_table.py | 65 ++++++++--- .../hdmf_common/v1_3_0/hdmf_common_table.py | 65 ++++++++--- .../hdmf_common/v1_4_0/hdmf_common_table.py | 65 ++++++++--- .../hdmf_common/v1_5_0/hdmf_common_table.py | 65 ++++++++--- .../hdmf_common/v1_5_1/hdmf_common_table.py | 65 ++++++++--- .../hdmf_common/v1_6_0/hdmf_common_table.py | 65 ++++++++--- .../hdmf_common/v1_7_0/hdmf_common_table.py | 65 ++++++++--- .../hdmf_common/v1_8_0/hdmf_common_table.py | 65 ++++++++--- nwb_linkml/tests/test_includes/test_hdmf.py | 105 ++++++++++++++---- .../test_providers/test_provider_schema.py | 3 +- 16 files changed, 725 insertions(+), 245 deletions(-) diff --git a/nwb_linkml/src/nwb_linkml/generators/pydantic.py b/nwb_linkml/src/nwb_linkml/generators/pydantic.py index f8c8033..e1f07af 100644 --- a/nwb_linkml/src/nwb_linkml/generators/pydantic.py +++ b/nwb_linkml/src/nwb_linkml/generators/pydantic.py @@ -250,6 +250,8 @@ class AfterGenerateClass: cls.cls.bases = ["VectorDataMixin"] elif cls.cls.name == "VectorIndex": cls.cls.bases = ["VectorIndexMixin"] + elif cls.cls.name == "DynamicTableRegion": + cls.cls.bases = ["DynamicTableRegionMixin", "VectorData"] return cls diff --git a/nwb_linkml/src/nwb_linkml/includes/hdmf.py b/nwb_linkml/src/nwb_linkml/includes/hdmf.py index d080d03..506f098 100644 --- a/nwb_linkml/src/nwb_linkml/includes/hdmf.py +++ b/nwb_linkml/src/nwb_linkml/includes/hdmf.py @@ -2,7 +2,18 @@ Special types for mimicking HDMF special case behavior """ -from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Tuple, Union, overload +from typing import ( + TYPE_CHECKING, + Any, + ClassVar, + Dict, + Iterable, + List, + Optional, + Tuple, + Union, + overload, +) import numpy as np from linkml.generators.pydanticgen.template import Import, Imports, ObjectImport @@ -69,7 +80,7 @@ class DynamicTableMixin(BaseModel): ]: ... @overload - def __getitem__(self, item: slice) -> DataFrame: ... + def __getitem__(self, item: Union[slice, "NDArray"]) -> DataFrame: ... def __getitem__( self, @@ -77,6 +88,7 @@ class DynamicTableMixin(BaseModel): str, int, slice, + "NDArray", Tuple[int, Union[int, str]], Tuple[Union[int, slice], ...], ], @@ -96,7 +108,7 @@ class DynamicTableMixin(BaseModel): """ if isinstance(item, str): return self._columns[item] - if isinstance(item, (int, slice)): + if isinstance(item, (int, slice, np.integer, np.ndarray)): return DataFrame.from_dict(self._slice_range(item)) elif isinstance(item, tuple): if len(item) != 2: @@ -107,7 +119,7 @@ class DynamicTableMixin(BaseModel): # all other cases are tuples of (rows, cols) rows, cols = item - if isinstance(cols, (int, slice)): + if isinstance(cols, (int, slice, np.integer)): cols = self.colnames[cols] if isinstance(rows, int) and isinstance(cols, str): @@ -120,7 +132,7 @@ class DynamicTableMixin(BaseModel): raise ValueError(f"Unsure how to get item with key {item}") def _slice_range( - self, rows: Union[int, slice], cols: Optional[Union[str, List[str]]] = None + self, rows: Union[int, slice, np.ndarray], cols: Optional[Union[str, List[str]]] = None ) -> Dict[str, Union[list, "NDArray", "VectorData"]]: if cols is None: cols = self.colnames @@ -128,7 +140,11 @@ class DynamicTableMixin(BaseModel): cols = [cols] data = {} for k in cols: - val = self._columns[k][rows] + if isinstance(rows, np.ndarray): + val = [self._columns[k][i] for i in rows] + else: + val = self._columns[k][rows] + if isinstance(val, BaseModel): # special case where pandas will unpack a pydantic model # into {n_fields} rows, rather than keeping it in a dict @@ -339,22 +355,22 @@ class VectorIndexMixin(BaseModel): """ Mimicking :func:`hdmf.common.table.VectorIndex.__getitem_helper` """ - start = 0 if arg == 0 else self.value[arg - 1] end = self.value[arg] return self.target.value[slice(start, end)] - def __getitem__(self, item: Union[int, slice]) -> Any: + def __getitem__(self, item: Union[int, slice, Iterable]) -> Any: if self.target is None: return self.value[item] - elif isinstance(self.target, VectorData): + else: if isinstance(item, int): return self._getitem_helper(item) + elif isinstance(item, (slice, Iterable)): + if isinstance(item, slice): + item = range(*item.indices(len(self.value))) + return [self._getitem_helper(i) for i in item] else: - idx = range(*item.indices(len(self.value))) - return [self._getitem_helper(i) for i in idx] - else: - raise AttributeError(f"Could not index with {item}") + raise AttributeError(f"Could not index with {item}") def __setitem__(self, key: Union[int, slice], value: Any) -> None: if self._index: @@ -387,13 +403,40 @@ class DynamicTableRegionMixin(BaseModel): Mixin to allow indexing references to regions of dynamictables """ - table: "DynamicTableMixin" + _index: Optional["VectorIndex"] = None - def __getitem__(self, item: Union[str, int, slice, Tuple[Union[str, int, slice], ...]]) -> Any: - return self.table[item] + table: "DynamicTableMixin" + value: Optional[NDArray] = None + + def __getitem__(self, item: Union[int, slice, Iterable]) -> Any: + """ + Use ``value`` to index the table. Works analogously to ``VectorIndex`` despite + this being a subclass of ``VectorData`` + """ + if self._index: + if isinstance(item, int): + # index returns an array of indices, + # and indexing table with an array returns a list of rows + return self.table[self._index[item]] + elif isinstance(item, slice): + # index returns a list of arrays of indices, + # so we index table with an array to construct + # a list of lists of rows + return [self.table[idx] for idx in self._index[item]] + else: + raise ValueError(f"Dont know how to index with {item}, need an int or a slice") + else: + if isinstance(item, int): + return self.table[self.value[item]] + elif isinstance(item, (slice, Iterable)): + if isinstance(item, slice): + item = range(*item.indices(len(self.value))) + return [self.table[self.value[i]] for i in item] + else: + raise ValueError(f"Dont know how to index with {item}, need an int or a slice") def __setitem__(self, key: Union[int, str, slice], value: Any) -> None: - self.table[key] = value + self.table[self.value[key]] = value DYNAMIC_TABLE_IMPORTS = Imports( @@ -405,8 +448,9 @@ DYNAMIC_TABLE_IMPORTS = Imports( module="typing", objects=[ ObjectImport(name="ClassVar"), - ObjectImport(name="overload"), + ObjectImport(name="Iterable"), ObjectImport(name="Tuple"), + ObjectImport(name="overload"), ], ), Import( diff --git a/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_1_0/hdmf_common_table.py b/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_1_0/hdmf_common_table.py index 8d3f9e9..d75a127 100644 --- a/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_1_0/hdmf_common_table.py +++ b/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_1_0/hdmf_common_table.py @@ -5,7 +5,7 @@ from enum import Enum import re import sys from pandas import DataFrame, Series -from typing import Any, ClassVar, List, Literal, Dict, Optional, Union, overload, Tuple +from typing import Any, ClassVar, List, Literal, Dict, Optional, Union, Iterable, Tuple, overload from numpydantic import NDArray, Shape from pydantic import ( BaseModel, @@ -128,22 +128,22 @@ class VectorIndexMixin(BaseModel): """ Mimicking :func:`hdmf.common.table.VectorIndex.__getitem_helper` """ - start = 0 if arg == 0 else self.value[arg - 1] end = self.value[arg] return self.target.value[slice(start, end)] - def __getitem__(self, item: Union[int, slice]) -> Any: + def __getitem__(self, item: Union[int, slice, Iterable]) -> Any: if self.target is None: return self.value[item] - elif isinstance(self.target, VectorData): + else: if isinstance(item, int): return self._getitem_helper(item) + elif isinstance(item, (slice, Iterable)): + if isinstance(item, slice): + item = range(*item.indices(len(self.value))) + return [self._getitem_helper(i) for i in item] else: - idx = range(*item.indices(len(self.value))) - return [self._getitem_helper(i) for i in idx] - else: - raise AttributeError(f"Could not index with {item}") + raise AttributeError(f"Could not index with {item}") def __setitem__(self, key: Union[int, slice], value: Any) -> None: if self._index: @@ -176,13 +176,40 @@ class DynamicTableRegionMixin(BaseModel): Mixin to allow indexing references to regions of dynamictables """ - table: "DynamicTableMixin" + _index: Optional["VectorIndex"] = None - def __getitem__(self, item: Union[str, int, slice, Tuple[Union[str, int, slice], ...]]) -> Any: - return self.table[item] + table: "DynamicTableMixin" + value: Optional[NDArray] = None + + def __getitem__(self, item: Union[int, slice, Iterable]) -> Any: + """ + Use ``value`` to index the table. Works analogously to ``VectorIndex`` despite + this being a subclass of ``VectorData`` + """ + if self._index: + if isinstance(item, int): + # index returns an array of indices, + # and indexing table with an array returns a list of rows + return self.table[self._index[item]] + elif isinstance(item, slice): + # index returns a list of arrays of indices, + # so we index table with an array to construct + # a list of lists of rows + return [self.table[idx] for idx in self._index[item]] + else: + raise ValueError(f"Dont know how to index with {item}, need an int or a slice") + else: + if isinstance(item, int): + return self.table[self.value[item]] + elif isinstance(item, (slice, Iterable)): + if isinstance(item, slice): + item = range(*item.indices(len(self.value))) + return [self.table[self.value[i]] for i in item] + else: + raise ValueError(f"Dont know how to index with {item}, need an int or a slice") def __setitem__(self, key: Union[int, str, slice], value: Any) -> None: - self.table[key] = value + self.table[self.value[key]] = value class DynamicTableMixin(BaseModel): @@ -258,7 +285,7 @@ class DynamicTableMixin(BaseModel): """ if isinstance(item, str): return self._columns[item] - if isinstance(item, (int, slice)): + if isinstance(item, (int, slice, np.integer, np.ndarray)): return DataFrame.from_dict(self._slice_range(item)) elif isinstance(item, tuple): if len(item) != 2: @@ -269,7 +296,7 @@ class DynamicTableMixin(BaseModel): # all other cases are tuples of (rows, cols) rows, cols = item - if isinstance(cols, (int, slice)): + if isinstance(cols, (int, slice, np.integer)): cols = self.colnames[cols] if isinstance(rows, int) and isinstance(cols, str): @@ -282,7 +309,7 @@ class DynamicTableMixin(BaseModel): raise ValueError(f"Unsure how to get item with key {item}") def _slice_range( - self, rows: Union[int, slice], cols: Optional[Union[str, List[str]]] = None + self, rows: Union[int, slice, np.ndarray], cols: Optional[Union[str, List[str]]] = None ) -> Dict[str, Union[list, "NDArray", "VectorData"]]: if cols is None: cols = self.colnames @@ -290,7 +317,11 @@ class DynamicTableMixin(BaseModel): cols = [cols] data = {} for k in cols: - val = self._columns[k][rows] + if isinstance(rows, np.ndarray): + val = [self._columns[k][i] for i in rows] + else: + val = self._columns[k][rows] + if isinstance(val, BaseModel): # special case where pandas will unpack a pydantic model # into {n_fields} rows, rather than keeping it in a dict @@ -515,7 +546,7 @@ class ElementIdentifiers(Data): ) -class DynamicTableRegion(VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ diff --git a/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_1_2/hdmf_common_table.py b/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_1_2/hdmf_common_table.py index 2be1fbe..3882294 100644 --- a/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_1_2/hdmf_common_table.py +++ b/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_1_2/hdmf_common_table.py @@ -5,7 +5,7 @@ from enum import Enum import re import sys from pandas import DataFrame, Series -from typing import Any, ClassVar, List, Literal, Dict, Optional, Union, overload, Tuple +from typing import Any, ClassVar, List, Literal, Dict, Optional, Union, Iterable, Tuple, overload from numpydantic import NDArray, Shape from pydantic import ( BaseModel, @@ -128,22 +128,22 @@ class VectorIndexMixin(BaseModel): """ Mimicking :func:`hdmf.common.table.VectorIndex.__getitem_helper` """ - start = 0 if arg == 0 else self.value[arg - 1] end = self.value[arg] return self.target.value[slice(start, end)] - def __getitem__(self, item: Union[int, slice]) -> Any: + def __getitem__(self, item: Union[int, slice, Iterable]) -> Any: if self.target is None: return self.value[item] - elif isinstance(self.target, VectorData): + else: if isinstance(item, int): return self._getitem_helper(item) + elif isinstance(item, (slice, Iterable)): + if isinstance(item, slice): + item = range(*item.indices(len(self.value))) + return [self._getitem_helper(i) for i in item] else: - idx = range(*item.indices(len(self.value))) - return [self._getitem_helper(i) for i in idx] - else: - raise AttributeError(f"Could not index with {item}") + raise AttributeError(f"Could not index with {item}") def __setitem__(self, key: Union[int, slice], value: Any) -> None: if self._index: @@ -176,13 +176,40 @@ class DynamicTableRegionMixin(BaseModel): Mixin to allow indexing references to regions of dynamictables """ - table: "DynamicTableMixin" + _index: Optional["VectorIndex"] = None - def __getitem__(self, item: Union[str, int, slice, Tuple[Union[str, int, slice], ...]]) -> Any: - return self.table[item] + table: "DynamicTableMixin" + value: Optional[NDArray] = None + + def __getitem__(self, item: Union[int, slice, Iterable]) -> Any: + """ + Use ``value`` to index the table. Works analogously to ``VectorIndex`` despite + this being a subclass of ``VectorData`` + """ + if self._index: + if isinstance(item, int): + # index returns an array of indices, + # and indexing table with an array returns a list of rows + return self.table[self._index[item]] + elif isinstance(item, slice): + # index returns a list of arrays of indices, + # so we index table with an array to construct + # a list of lists of rows + return [self.table[idx] for idx in self._index[item]] + else: + raise ValueError(f"Dont know how to index with {item}, need an int or a slice") + else: + if isinstance(item, int): + return self.table[self.value[item]] + elif isinstance(item, (slice, Iterable)): + if isinstance(item, slice): + item = range(*item.indices(len(self.value))) + return [self.table[self.value[i]] for i in item] + else: + raise ValueError(f"Dont know how to index with {item}, need an int or a slice") def __setitem__(self, key: Union[int, str, slice], value: Any) -> None: - self.table[key] = value + self.table[self.value[key]] = value class DynamicTableMixin(BaseModel): @@ -258,7 +285,7 @@ class DynamicTableMixin(BaseModel): """ if isinstance(item, str): return self._columns[item] - if isinstance(item, (int, slice)): + if isinstance(item, (int, slice, np.integer, np.ndarray)): return DataFrame.from_dict(self._slice_range(item)) elif isinstance(item, tuple): if len(item) != 2: @@ -269,7 +296,7 @@ class DynamicTableMixin(BaseModel): # all other cases are tuples of (rows, cols) rows, cols = item - if isinstance(cols, (int, slice)): + if isinstance(cols, (int, slice, np.integer)): cols = self.colnames[cols] if isinstance(rows, int) and isinstance(cols, str): @@ -282,7 +309,7 @@ class DynamicTableMixin(BaseModel): raise ValueError(f"Unsure how to get item with key {item}") def _slice_range( - self, rows: Union[int, slice], cols: Optional[Union[str, List[str]]] = None + self, rows: Union[int, slice, np.ndarray], cols: Optional[Union[str, List[str]]] = None ) -> Dict[str, Union[list, "NDArray", "VectorData"]]: if cols is None: cols = self.colnames @@ -290,7 +317,11 @@ class DynamicTableMixin(BaseModel): cols = [cols] data = {} for k in cols: - val = self._columns[k][rows] + if isinstance(rows, np.ndarray): + val = [self._columns[k][i] for i in rows] + else: + val = self._columns[k][rows] + if isinstance(val, BaseModel): # special case where pandas will unpack a pydantic model # into {n_fields} rows, rather than keeping it in a dict @@ -515,7 +546,7 @@ class ElementIdentifiers(Data): ) -class DynamicTableRegion(VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ diff --git a/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_1_3/hdmf_common_table.py b/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_1_3/hdmf_common_table.py index 6cf1bf2..8df75da 100644 --- a/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_1_3/hdmf_common_table.py +++ b/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_1_3/hdmf_common_table.py @@ -5,7 +5,7 @@ from enum import Enum import re import sys from pandas import DataFrame, Series -from typing import Any, ClassVar, List, Literal, Dict, Optional, Union, overload, Tuple +from typing import Any, ClassVar, List, Literal, Dict, Optional, Union, Iterable, Tuple, overload from numpydantic import NDArray, Shape from pydantic import ( BaseModel, @@ -128,22 +128,22 @@ class VectorIndexMixin(BaseModel): """ Mimicking :func:`hdmf.common.table.VectorIndex.__getitem_helper` """ - start = 0 if arg == 0 else self.value[arg - 1] end = self.value[arg] return self.target.value[slice(start, end)] - def __getitem__(self, item: Union[int, slice]) -> Any: + def __getitem__(self, item: Union[int, slice, Iterable]) -> Any: if self.target is None: return self.value[item] - elif isinstance(self.target, VectorData): + else: if isinstance(item, int): return self._getitem_helper(item) + elif isinstance(item, (slice, Iterable)): + if isinstance(item, slice): + item = range(*item.indices(len(self.value))) + return [self._getitem_helper(i) for i in item] else: - idx = range(*item.indices(len(self.value))) - return [self._getitem_helper(i) for i in idx] - else: - raise AttributeError(f"Could not index with {item}") + raise AttributeError(f"Could not index with {item}") def __setitem__(self, key: Union[int, slice], value: Any) -> None: if self._index: @@ -176,13 +176,40 @@ class DynamicTableRegionMixin(BaseModel): Mixin to allow indexing references to regions of dynamictables """ - table: "DynamicTableMixin" + _index: Optional["VectorIndex"] = None - def __getitem__(self, item: Union[str, int, slice, Tuple[Union[str, int, slice], ...]]) -> Any: - return self.table[item] + table: "DynamicTableMixin" + value: Optional[NDArray] = None + + def __getitem__(self, item: Union[int, slice, Iterable]) -> Any: + """ + Use ``value`` to index the table. Works analogously to ``VectorIndex`` despite + this being a subclass of ``VectorData`` + """ + if self._index: + if isinstance(item, int): + # index returns an array of indices, + # and indexing table with an array returns a list of rows + return self.table[self._index[item]] + elif isinstance(item, slice): + # index returns a list of arrays of indices, + # so we index table with an array to construct + # a list of lists of rows + return [self.table[idx] for idx in self._index[item]] + else: + raise ValueError(f"Dont know how to index with {item}, need an int or a slice") + else: + if isinstance(item, int): + return self.table[self.value[item]] + elif isinstance(item, (slice, Iterable)): + if isinstance(item, slice): + item = range(*item.indices(len(self.value))) + return [self.table[self.value[i]] for i in item] + else: + raise ValueError(f"Dont know how to index with {item}, need an int or a slice") def __setitem__(self, key: Union[int, str, slice], value: Any) -> None: - self.table[key] = value + self.table[self.value[key]] = value class DynamicTableMixin(BaseModel): @@ -258,7 +285,7 @@ class DynamicTableMixin(BaseModel): """ if isinstance(item, str): return self._columns[item] - if isinstance(item, (int, slice)): + if isinstance(item, (int, slice, np.integer, np.ndarray)): return DataFrame.from_dict(self._slice_range(item)) elif isinstance(item, tuple): if len(item) != 2: @@ -269,7 +296,7 @@ class DynamicTableMixin(BaseModel): # all other cases are tuples of (rows, cols) rows, cols = item - if isinstance(cols, (int, slice)): + if isinstance(cols, (int, slice, np.integer)): cols = self.colnames[cols] if isinstance(rows, int) and isinstance(cols, str): @@ -282,7 +309,7 @@ class DynamicTableMixin(BaseModel): raise ValueError(f"Unsure how to get item with key {item}") def _slice_range( - self, rows: Union[int, slice], cols: Optional[Union[str, List[str]]] = None + self, rows: Union[int, slice, np.ndarray], cols: Optional[Union[str, List[str]]] = None ) -> Dict[str, Union[list, "NDArray", "VectorData"]]: if cols is None: cols = self.colnames @@ -290,7 +317,11 @@ class DynamicTableMixin(BaseModel): cols = [cols] data = {} for k in cols: - val = self._columns[k][rows] + if isinstance(rows, np.ndarray): + val = [self._columns[k][i] for i in rows] + else: + val = self._columns[k][rows] + if isinstance(val, BaseModel): # special case where pandas will unpack a pydantic model # into {n_fields} rows, rather than keeping it in a dict @@ -526,7 +557,7 @@ class ElementIdentifiers(Data): ) -class DynamicTableRegion(VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ diff --git a/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_2_0/hdmf_common_table.py b/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_2_0/hdmf_common_table.py index 8e79364..0823281 100644 --- a/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_2_0/hdmf_common_table.py +++ b/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_2_0/hdmf_common_table.py @@ -6,7 +6,7 @@ import re import sys from ...hdmf_common.v1_2_0.hdmf_common_base import Data, Container from pandas import DataFrame, Series -from typing import Any, ClassVar, List, Literal, Dict, Optional, Union, overload, Tuple +from typing import Any, ClassVar, List, Literal, Dict, Optional, Union, Iterable, Tuple, overload from numpydantic import NDArray, Shape from pydantic import ( BaseModel, @@ -129,22 +129,22 @@ class VectorIndexMixin(BaseModel): """ Mimicking :func:`hdmf.common.table.VectorIndex.__getitem_helper` """ - start = 0 if arg == 0 else self.value[arg - 1] end = self.value[arg] return self.target.value[slice(start, end)] - def __getitem__(self, item: Union[int, slice]) -> Any: + def __getitem__(self, item: Union[int, slice, Iterable]) -> Any: if self.target is None: return self.value[item] - elif isinstance(self.target, VectorData): + else: if isinstance(item, int): return self._getitem_helper(item) + elif isinstance(item, (slice, Iterable)): + if isinstance(item, slice): + item = range(*item.indices(len(self.value))) + return [self._getitem_helper(i) for i in item] else: - idx = range(*item.indices(len(self.value))) - return [self._getitem_helper(i) for i in idx] - else: - raise AttributeError(f"Could not index with {item}") + raise AttributeError(f"Could not index with {item}") def __setitem__(self, key: Union[int, slice], value: Any) -> None: if self._index: @@ -177,13 +177,40 @@ class DynamicTableRegionMixin(BaseModel): Mixin to allow indexing references to regions of dynamictables """ - table: "DynamicTableMixin" + _index: Optional["VectorIndex"] = None - def __getitem__(self, item: Union[str, int, slice, Tuple[Union[str, int, slice], ...]]) -> Any: - return self.table[item] + table: "DynamicTableMixin" + value: Optional[NDArray] = None + + def __getitem__(self, item: Union[int, slice, Iterable]) -> Any: + """ + Use ``value`` to index the table. Works analogously to ``VectorIndex`` despite + this being a subclass of ``VectorData`` + """ + if self._index: + if isinstance(item, int): + # index returns an array of indices, + # and indexing table with an array returns a list of rows + return self.table[self._index[item]] + elif isinstance(item, slice): + # index returns a list of arrays of indices, + # so we index table with an array to construct + # a list of lists of rows + return [self.table[idx] for idx in self._index[item]] + else: + raise ValueError(f"Dont know how to index with {item}, need an int or a slice") + else: + if isinstance(item, int): + return self.table[self.value[item]] + elif isinstance(item, (slice, Iterable)): + if isinstance(item, slice): + item = range(*item.indices(len(self.value))) + return [self.table[self.value[i]] for i in item] + else: + raise ValueError(f"Dont know how to index with {item}, need an int or a slice") def __setitem__(self, key: Union[int, str, slice], value: Any) -> None: - self.table[key] = value + self.table[self.value[key]] = value class DynamicTableMixin(BaseModel): @@ -259,7 +286,7 @@ class DynamicTableMixin(BaseModel): """ if isinstance(item, str): return self._columns[item] - if isinstance(item, (int, slice)): + if isinstance(item, (int, slice, np.integer, np.ndarray)): return DataFrame.from_dict(self._slice_range(item)) elif isinstance(item, tuple): if len(item) != 2: @@ -270,7 +297,7 @@ class DynamicTableMixin(BaseModel): # all other cases are tuples of (rows, cols) rows, cols = item - if isinstance(cols, (int, slice)): + if isinstance(cols, (int, slice, np.integer)): cols = self.colnames[cols] if isinstance(rows, int) and isinstance(cols, str): @@ -283,7 +310,7 @@ class DynamicTableMixin(BaseModel): raise ValueError(f"Unsure how to get item with key {item}") def _slice_range( - self, rows: Union[int, slice], cols: Optional[Union[str, List[str]]] = None + self, rows: Union[int, slice, np.ndarray], cols: Optional[Union[str, List[str]]] = None ) -> Dict[str, Union[list, "NDArray", "VectorData"]]: if cols is None: cols = self.colnames @@ -291,7 +318,11 @@ class DynamicTableMixin(BaseModel): cols = [cols] data = {} for k in cols: - val = self._columns[k][rows] + if isinstance(rows, np.ndarray): + val = [self._columns[k][i] for i in rows] + else: + val = self._columns[k][rows] + if isinstance(val, BaseModel): # special case where pandas will unpack a pydantic model # into {n_fields} rows, rather than keeping it in a dict @@ -506,7 +537,7 @@ class ElementIdentifiers(Data): ) -class DynamicTableRegion(VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ diff --git a/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_2_1/hdmf_common_table.py b/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_2_1/hdmf_common_table.py index b78ba5c..88405cc 100644 --- a/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_2_1/hdmf_common_table.py +++ b/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_2_1/hdmf_common_table.py @@ -6,7 +6,7 @@ import re import sys from ...hdmf_common.v1_2_1.hdmf_common_base import Data, Container from pandas import DataFrame, Series -from typing import Any, ClassVar, List, Literal, Dict, Optional, Union, overload, Tuple +from typing import Any, ClassVar, List, Literal, Dict, Optional, Union, Iterable, Tuple, overload from numpydantic import NDArray, Shape from pydantic import ( BaseModel, @@ -129,22 +129,22 @@ class VectorIndexMixin(BaseModel): """ Mimicking :func:`hdmf.common.table.VectorIndex.__getitem_helper` """ - start = 0 if arg == 0 else self.value[arg - 1] end = self.value[arg] return self.target.value[slice(start, end)] - def __getitem__(self, item: Union[int, slice]) -> Any: + def __getitem__(self, item: Union[int, slice, Iterable]) -> Any: if self.target is None: return self.value[item] - elif isinstance(self.target, VectorData): + else: if isinstance(item, int): return self._getitem_helper(item) + elif isinstance(item, (slice, Iterable)): + if isinstance(item, slice): + item = range(*item.indices(len(self.value))) + return [self._getitem_helper(i) for i in item] else: - idx = range(*item.indices(len(self.value))) - return [self._getitem_helper(i) for i in idx] - else: - raise AttributeError(f"Could not index with {item}") + raise AttributeError(f"Could not index with {item}") def __setitem__(self, key: Union[int, slice], value: Any) -> None: if self._index: @@ -177,13 +177,40 @@ class DynamicTableRegionMixin(BaseModel): Mixin to allow indexing references to regions of dynamictables """ - table: "DynamicTableMixin" + _index: Optional["VectorIndex"] = None - def __getitem__(self, item: Union[str, int, slice, Tuple[Union[str, int, slice], ...]]) -> Any: - return self.table[item] + table: "DynamicTableMixin" + value: Optional[NDArray] = None + + def __getitem__(self, item: Union[int, slice, Iterable]) -> Any: + """ + Use ``value`` to index the table. Works analogously to ``VectorIndex`` despite + this being a subclass of ``VectorData`` + """ + if self._index: + if isinstance(item, int): + # index returns an array of indices, + # and indexing table with an array returns a list of rows + return self.table[self._index[item]] + elif isinstance(item, slice): + # index returns a list of arrays of indices, + # so we index table with an array to construct + # a list of lists of rows + return [self.table[idx] for idx in self._index[item]] + else: + raise ValueError(f"Dont know how to index with {item}, need an int or a slice") + else: + if isinstance(item, int): + return self.table[self.value[item]] + elif isinstance(item, (slice, Iterable)): + if isinstance(item, slice): + item = range(*item.indices(len(self.value))) + return [self.table[self.value[i]] for i in item] + else: + raise ValueError(f"Dont know how to index with {item}, need an int or a slice") def __setitem__(self, key: Union[int, str, slice], value: Any) -> None: - self.table[key] = value + self.table[self.value[key]] = value class DynamicTableMixin(BaseModel): @@ -259,7 +286,7 @@ class DynamicTableMixin(BaseModel): """ if isinstance(item, str): return self._columns[item] - if isinstance(item, (int, slice)): + if isinstance(item, (int, slice, np.integer, np.ndarray)): return DataFrame.from_dict(self._slice_range(item)) elif isinstance(item, tuple): if len(item) != 2: @@ -270,7 +297,7 @@ class DynamicTableMixin(BaseModel): # all other cases are tuples of (rows, cols) rows, cols = item - if isinstance(cols, (int, slice)): + if isinstance(cols, (int, slice, np.integer)): cols = self.colnames[cols] if isinstance(rows, int) and isinstance(cols, str): @@ -283,7 +310,7 @@ class DynamicTableMixin(BaseModel): raise ValueError(f"Unsure how to get item with key {item}") def _slice_range( - self, rows: Union[int, slice], cols: Optional[Union[str, List[str]]] = None + self, rows: Union[int, slice, np.ndarray], cols: Optional[Union[str, List[str]]] = None ) -> Dict[str, Union[list, "NDArray", "VectorData"]]: if cols is None: cols = self.colnames @@ -291,7 +318,11 @@ class DynamicTableMixin(BaseModel): cols = [cols] data = {} for k in cols: - val = self._columns[k][rows] + if isinstance(rows, np.ndarray): + val = [self._columns[k][i] for i in rows] + else: + val = self._columns[k][rows] + if isinstance(val, BaseModel): # special case where pandas will unpack a pydantic model # into {n_fields} rows, rather than keeping it in a dict @@ -506,7 +537,7 @@ class ElementIdentifiers(Data): ) -class DynamicTableRegion(VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ diff --git a/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_3_0/hdmf_common_table.py b/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_3_0/hdmf_common_table.py index b17a853..545f0e9 100644 --- a/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_3_0/hdmf_common_table.py +++ b/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_3_0/hdmf_common_table.py @@ -6,7 +6,7 @@ import re import sys from ...hdmf_common.v1_3_0.hdmf_common_base import Data, Container from pandas import DataFrame, Series -from typing import Any, ClassVar, List, Literal, Dict, Optional, Union, overload, Tuple +from typing import Any, ClassVar, List, Literal, Dict, Optional, Union, Iterable, Tuple, overload from numpydantic import NDArray, Shape from pydantic import ( BaseModel, @@ -129,22 +129,22 @@ class VectorIndexMixin(BaseModel): """ Mimicking :func:`hdmf.common.table.VectorIndex.__getitem_helper` """ - start = 0 if arg == 0 else self.value[arg - 1] end = self.value[arg] return self.target.value[slice(start, end)] - def __getitem__(self, item: Union[int, slice]) -> Any: + def __getitem__(self, item: Union[int, slice, Iterable]) -> Any: if self.target is None: return self.value[item] - elif isinstance(self.target, VectorData): + else: if isinstance(item, int): return self._getitem_helper(item) + elif isinstance(item, (slice, Iterable)): + if isinstance(item, slice): + item = range(*item.indices(len(self.value))) + return [self._getitem_helper(i) for i in item] else: - idx = range(*item.indices(len(self.value))) - return [self._getitem_helper(i) for i in idx] - else: - raise AttributeError(f"Could not index with {item}") + raise AttributeError(f"Could not index with {item}") def __setitem__(self, key: Union[int, slice], value: Any) -> None: if self._index: @@ -177,13 +177,40 @@ class DynamicTableRegionMixin(BaseModel): Mixin to allow indexing references to regions of dynamictables """ - table: "DynamicTableMixin" + _index: Optional["VectorIndex"] = None - def __getitem__(self, item: Union[str, int, slice, Tuple[Union[str, int, slice], ...]]) -> Any: - return self.table[item] + table: "DynamicTableMixin" + value: Optional[NDArray] = None + + def __getitem__(self, item: Union[int, slice, Iterable]) -> Any: + """ + Use ``value`` to index the table. Works analogously to ``VectorIndex`` despite + this being a subclass of ``VectorData`` + """ + if self._index: + if isinstance(item, int): + # index returns an array of indices, + # and indexing table with an array returns a list of rows + return self.table[self._index[item]] + elif isinstance(item, slice): + # index returns a list of arrays of indices, + # so we index table with an array to construct + # a list of lists of rows + return [self.table[idx] for idx in self._index[item]] + else: + raise ValueError(f"Dont know how to index with {item}, need an int or a slice") + else: + if isinstance(item, int): + return self.table[self.value[item]] + elif isinstance(item, (slice, Iterable)): + if isinstance(item, slice): + item = range(*item.indices(len(self.value))) + return [self.table[self.value[i]] for i in item] + else: + raise ValueError(f"Dont know how to index with {item}, need an int or a slice") def __setitem__(self, key: Union[int, str, slice], value: Any) -> None: - self.table[key] = value + self.table[self.value[key]] = value class DynamicTableMixin(BaseModel): @@ -259,7 +286,7 @@ class DynamicTableMixin(BaseModel): """ if isinstance(item, str): return self._columns[item] - if isinstance(item, (int, slice)): + if isinstance(item, (int, slice, np.integer, np.ndarray)): return DataFrame.from_dict(self._slice_range(item)) elif isinstance(item, tuple): if len(item) != 2: @@ -270,7 +297,7 @@ class DynamicTableMixin(BaseModel): # all other cases are tuples of (rows, cols) rows, cols = item - if isinstance(cols, (int, slice)): + if isinstance(cols, (int, slice, np.integer)): cols = self.colnames[cols] if isinstance(rows, int) and isinstance(cols, str): @@ -283,7 +310,7 @@ class DynamicTableMixin(BaseModel): raise ValueError(f"Unsure how to get item with key {item}") def _slice_range( - self, rows: Union[int, slice], cols: Optional[Union[str, List[str]]] = None + self, rows: Union[int, slice, np.ndarray], cols: Optional[Union[str, List[str]]] = None ) -> Dict[str, Union[list, "NDArray", "VectorData"]]: if cols is None: cols = self.colnames @@ -291,7 +318,11 @@ class DynamicTableMixin(BaseModel): cols = [cols] data = {} for k in cols: - val = self._columns[k][rows] + if isinstance(rows, np.ndarray): + val = [self._columns[k][i] for i in rows] + else: + val = self._columns[k][rows] + if isinstance(val, BaseModel): # special case where pandas will unpack a pydantic model # into {n_fields} rows, rather than keeping it in a dict @@ -506,7 +537,7 @@ class ElementIdentifiers(Data): ) -class DynamicTableRegion(VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ diff --git a/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_4_0/hdmf_common_table.py b/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_4_0/hdmf_common_table.py index 536d5af..c3fb548 100644 --- a/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_4_0/hdmf_common_table.py +++ b/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_4_0/hdmf_common_table.py @@ -6,7 +6,7 @@ import re import sys from ...hdmf_common.v1_4_0.hdmf_common_base import Data, Container from pandas import DataFrame, Series -from typing import Any, ClassVar, List, Literal, Dict, Optional, Union, overload, Tuple +from typing import Any, ClassVar, List, Literal, Dict, Optional, Union, Iterable, Tuple, overload from numpydantic import NDArray, Shape from pydantic import ( BaseModel, @@ -129,22 +129,22 @@ class VectorIndexMixin(BaseModel): """ Mimicking :func:`hdmf.common.table.VectorIndex.__getitem_helper` """ - start = 0 if arg == 0 else self.value[arg - 1] end = self.value[arg] return self.target.value[slice(start, end)] - def __getitem__(self, item: Union[int, slice]) -> Any: + def __getitem__(self, item: Union[int, slice, Iterable]) -> Any: if self.target is None: return self.value[item] - elif isinstance(self.target, VectorData): + else: if isinstance(item, int): return self._getitem_helper(item) + elif isinstance(item, (slice, Iterable)): + if isinstance(item, slice): + item = range(*item.indices(len(self.value))) + return [self._getitem_helper(i) for i in item] else: - idx = range(*item.indices(len(self.value))) - return [self._getitem_helper(i) for i in idx] - else: - raise AttributeError(f"Could not index with {item}") + raise AttributeError(f"Could not index with {item}") def __setitem__(self, key: Union[int, slice], value: Any) -> None: if self._index: @@ -177,13 +177,40 @@ class DynamicTableRegionMixin(BaseModel): Mixin to allow indexing references to regions of dynamictables """ - table: "DynamicTableMixin" + _index: Optional["VectorIndex"] = None - def __getitem__(self, item: Union[str, int, slice, Tuple[Union[str, int, slice], ...]]) -> Any: - return self.table[item] + table: "DynamicTableMixin" + value: Optional[NDArray] = None + + def __getitem__(self, item: Union[int, slice, Iterable]) -> Any: + """ + Use ``value`` to index the table. Works analogously to ``VectorIndex`` despite + this being a subclass of ``VectorData`` + """ + if self._index: + if isinstance(item, int): + # index returns an array of indices, + # and indexing table with an array returns a list of rows + return self.table[self._index[item]] + elif isinstance(item, slice): + # index returns a list of arrays of indices, + # so we index table with an array to construct + # a list of lists of rows + return [self.table[idx] for idx in self._index[item]] + else: + raise ValueError(f"Dont know how to index with {item}, need an int or a slice") + else: + if isinstance(item, int): + return self.table[self.value[item]] + elif isinstance(item, (slice, Iterable)): + if isinstance(item, slice): + item = range(*item.indices(len(self.value))) + return [self.table[self.value[i]] for i in item] + else: + raise ValueError(f"Dont know how to index with {item}, need an int or a slice") def __setitem__(self, key: Union[int, str, slice], value: Any) -> None: - self.table[key] = value + self.table[self.value[key]] = value class DynamicTableMixin(BaseModel): @@ -259,7 +286,7 @@ class DynamicTableMixin(BaseModel): """ if isinstance(item, str): return self._columns[item] - if isinstance(item, (int, slice)): + if isinstance(item, (int, slice, np.integer, np.ndarray)): return DataFrame.from_dict(self._slice_range(item)) elif isinstance(item, tuple): if len(item) != 2: @@ -270,7 +297,7 @@ class DynamicTableMixin(BaseModel): # all other cases are tuples of (rows, cols) rows, cols = item - if isinstance(cols, (int, slice)): + if isinstance(cols, (int, slice, np.integer)): cols = self.colnames[cols] if isinstance(rows, int) and isinstance(cols, str): @@ -283,7 +310,7 @@ class DynamicTableMixin(BaseModel): raise ValueError(f"Unsure how to get item with key {item}") def _slice_range( - self, rows: Union[int, slice], cols: Optional[Union[str, List[str]]] = None + self, rows: Union[int, slice, np.ndarray], cols: Optional[Union[str, List[str]]] = None ) -> Dict[str, Union[list, "NDArray", "VectorData"]]: if cols is None: cols = self.colnames @@ -291,7 +318,11 @@ class DynamicTableMixin(BaseModel): cols = [cols] data = {} for k in cols: - val = self._columns[k][rows] + if isinstance(rows, np.ndarray): + val = [self._columns[k][i] for i in rows] + else: + val = self._columns[k][rows] + if isinstance(val, BaseModel): # special case where pandas will unpack a pydantic model # into {n_fields} rows, rather than keeping it in a dict @@ -506,7 +537,7 @@ class ElementIdentifiers(Data): ) -class DynamicTableRegion(VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ diff --git a/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_5_0/hdmf_common_table.py b/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_5_0/hdmf_common_table.py index 80d1d6d..5dda9ab 100644 --- a/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_5_0/hdmf_common_table.py +++ b/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_5_0/hdmf_common_table.py @@ -6,7 +6,7 @@ import re import sys from ...hdmf_common.v1_5_0.hdmf_common_base import Data, Container from pandas import DataFrame, Series -from typing import Any, ClassVar, List, Literal, Dict, Optional, Union, overload, Tuple +from typing import Any, ClassVar, List, Literal, Dict, Optional, Union, Iterable, Tuple, overload from pydantic import ( BaseModel, ConfigDict, @@ -129,22 +129,22 @@ class VectorIndexMixin(BaseModel): """ Mimicking :func:`hdmf.common.table.VectorIndex.__getitem_helper` """ - start = 0 if arg == 0 else self.value[arg - 1] end = self.value[arg] return self.target.value[slice(start, end)] - def __getitem__(self, item: Union[int, slice]) -> Any: + def __getitem__(self, item: Union[int, slice, Iterable]) -> Any: if self.target is None: return self.value[item] - elif isinstance(self.target, VectorData): + else: if isinstance(item, int): return self._getitem_helper(item) + elif isinstance(item, (slice, Iterable)): + if isinstance(item, slice): + item = range(*item.indices(len(self.value))) + return [self._getitem_helper(i) for i in item] else: - idx = range(*item.indices(len(self.value))) - return [self._getitem_helper(i) for i in idx] - else: - raise AttributeError(f"Could not index with {item}") + raise AttributeError(f"Could not index with {item}") def __setitem__(self, key: Union[int, slice], value: Any) -> None: if self._index: @@ -177,13 +177,40 @@ class DynamicTableRegionMixin(BaseModel): Mixin to allow indexing references to regions of dynamictables """ - table: "DynamicTableMixin" + _index: Optional["VectorIndex"] = None - def __getitem__(self, item: Union[str, int, slice, Tuple[Union[str, int, slice], ...]]) -> Any: - return self.table[item] + table: "DynamicTableMixin" + value: Optional[NDArray] = None + + def __getitem__(self, item: Union[int, slice, Iterable]) -> Any: + """ + Use ``value`` to index the table. Works analogously to ``VectorIndex`` despite + this being a subclass of ``VectorData`` + """ + if self._index: + if isinstance(item, int): + # index returns an array of indices, + # and indexing table with an array returns a list of rows + return self.table[self._index[item]] + elif isinstance(item, slice): + # index returns a list of arrays of indices, + # so we index table with an array to construct + # a list of lists of rows + return [self.table[idx] for idx in self._index[item]] + else: + raise ValueError(f"Dont know how to index with {item}, need an int or a slice") + else: + if isinstance(item, int): + return self.table[self.value[item]] + elif isinstance(item, (slice, Iterable)): + if isinstance(item, slice): + item = range(*item.indices(len(self.value))) + return [self.table[self.value[i]] for i in item] + else: + raise ValueError(f"Dont know how to index with {item}, need an int or a slice") def __setitem__(self, key: Union[int, str, slice], value: Any) -> None: - self.table[key] = value + self.table[self.value[key]] = value class DynamicTableMixin(BaseModel): @@ -259,7 +286,7 @@ class DynamicTableMixin(BaseModel): """ if isinstance(item, str): return self._columns[item] - if isinstance(item, (int, slice)): + if isinstance(item, (int, slice, np.integer, np.ndarray)): return DataFrame.from_dict(self._slice_range(item)) elif isinstance(item, tuple): if len(item) != 2: @@ -270,7 +297,7 @@ class DynamicTableMixin(BaseModel): # all other cases are tuples of (rows, cols) rows, cols = item - if isinstance(cols, (int, slice)): + if isinstance(cols, (int, slice, np.integer)): cols = self.colnames[cols] if isinstance(rows, int) and isinstance(cols, str): @@ -283,7 +310,7 @@ class DynamicTableMixin(BaseModel): raise ValueError(f"Unsure how to get item with key {item}") def _slice_range( - self, rows: Union[int, slice], cols: Optional[Union[str, List[str]]] = None + self, rows: Union[int, slice, np.ndarray], cols: Optional[Union[str, List[str]]] = None ) -> Dict[str, Union[list, "NDArray", "VectorData"]]: if cols is None: cols = self.colnames @@ -291,7 +318,11 @@ class DynamicTableMixin(BaseModel): cols = [cols] data = {} for k in cols: - val = self._columns[k][rows] + if isinstance(rows, np.ndarray): + val = [self._columns[k][i] for i in rows] + else: + val = self._columns[k][rows] + if isinstance(val, BaseModel): # special case where pandas will unpack a pydantic model # into {n_fields} rows, rather than keeping it in a dict @@ -506,7 +537,7 @@ class ElementIdentifiers(Data): ) -class DynamicTableRegion(VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ diff --git a/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_5_1/hdmf_common_table.py b/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_5_1/hdmf_common_table.py index 7ddc7ec..910b294 100644 --- a/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_5_1/hdmf_common_table.py +++ b/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_5_1/hdmf_common_table.py @@ -6,7 +6,7 @@ import re import sys from ...hdmf_common.v1_5_1.hdmf_common_base import Data, Container from pandas import DataFrame, Series -from typing import Any, ClassVar, List, Literal, Dict, Optional, Union, overload, Tuple +from typing import Any, ClassVar, List, Literal, Dict, Optional, Union, Iterable, Tuple, overload from pydantic import ( BaseModel, ConfigDict, @@ -129,22 +129,22 @@ class VectorIndexMixin(BaseModel): """ Mimicking :func:`hdmf.common.table.VectorIndex.__getitem_helper` """ - start = 0 if arg == 0 else self.value[arg - 1] end = self.value[arg] return self.target.value[slice(start, end)] - def __getitem__(self, item: Union[int, slice]) -> Any: + def __getitem__(self, item: Union[int, slice, Iterable]) -> Any: if self.target is None: return self.value[item] - elif isinstance(self.target, VectorData): + else: if isinstance(item, int): return self._getitem_helper(item) + elif isinstance(item, (slice, Iterable)): + if isinstance(item, slice): + item = range(*item.indices(len(self.value))) + return [self._getitem_helper(i) for i in item] else: - idx = range(*item.indices(len(self.value))) - return [self._getitem_helper(i) for i in idx] - else: - raise AttributeError(f"Could not index with {item}") + raise AttributeError(f"Could not index with {item}") def __setitem__(self, key: Union[int, slice], value: Any) -> None: if self._index: @@ -177,13 +177,40 @@ class DynamicTableRegionMixin(BaseModel): Mixin to allow indexing references to regions of dynamictables """ - table: "DynamicTableMixin" + _index: Optional["VectorIndex"] = None - def __getitem__(self, item: Union[str, int, slice, Tuple[Union[str, int, slice], ...]]) -> Any: - return self.table[item] + table: "DynamicTableMixin" + value: Optional[NDArray] = None + + def __getitem__(self, item: Union[int, slice, Iterable]) -> Any: + """ + Use ``value`` to index the table. Works analogously to ``VectorIndex`` despite + this being a subclass of ``VectorData`` + """ + if self._index: + if isinstance(item, int): + # index returns an array of indices, + # and indexing table with an array returns a list of rows + return self.table[self._index[item]] + elif isinstance(item, slice): + # index returns a list of arrays of indices, + # so we index table with an array to construct + # a list of lists of rows + return [self.table[idx] for idx in self._index[item]] + else: + raise ValueError(f"Dont know how to index with {item}, need an int or a slice") + else: + if isinstance(item, int): + return self.table[self.value[item]] + elif isinstance(item, (slice, Iterable)): + if isinstance(item, slice): + item = range(*item.indices(len(self.value))) + return [self.table[self.value[i]] for i in item] + else: + raise ValueError(f"Dont know how to index with {item}, need an int or a slice") def __setitem__(self, key: Union[int, str, slice], value: Any) -> None: - self.table[key] = value + self.table[self.value[key]] = value class DynamicTableMixin(BaseModel): @@ -259,7 +286,7 @@ class DynamicTableMixin(BaseModel): """ if isinstance(item, str): return self._columns[item] - if isinstance(item, (int, slice)): + if isinstance(item, (int, slice, np.integer, np.ndarray)): return DataFrame.from_dict(self._slice_range(item)) elif isinstance(item, tuple): if len(item) != 2: @@ -270,7 +297,7 @@ class DynamicTableMixin(BaseModel): # all other cases are tuples of (rows, cols) rows, cols = item - if isinstance(cols, (int, slice)): + if isinstance(cols, (int, slice, np.integer)): cols = self.colnames[cols] if isinstance(rows, int) and isinstance(cols, str): @@ -283,7 +310,7 @@ class DynamicTableMixin(BaseModel): raise ValueError(f"Unsure how to get item with key {item}") def _slice_range( - self, rows: Union[int, slice], cols: Optional[Union[str, List[str]]] = None + self, rows: Union[int, slice, np.ndarray], cols: Optional[Union[str, List[str]]] = None ) -> Dict[str, Union[list, "NDArray", "VectorData"]]: if cols is None: cols = self.colnames @@ -291,7 +318,11 @@ class DynamicTableMixin(BaseModel): cols = [cols] data = {} for k in cols: - val = self._columns[k][rows] + if isinstance(rows, np.ndarray): + val = [self._columns[k][i] for i in rows] + else: + val = self._columns[k][rows] + if isinstance(val, BaseModel): # special case where pandas will unpack a pydantic model # into {n_fields} rows, rather than keeping it in a dict @@ -506,7 +537,7 @@ class ElementIdentifiers(Data): ) -class DynamicTableRegion(VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ diff --git a/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_6_0/hdmf_common_table.py b/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_6_0/hdmf_common_table.py index a8315f3..3df1e78 100644 --- a/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_6_0/hdmf_common_table.py +++ b/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_6_0/hdmf_common_table.py @@ -6,7 +6,7 @@ import re import sys from ...hdmf_common.v1_6_0.hdmf_common_base import Data, Container from pandas import DataFrame, Series -from typing import Any, ClassVar, List, Literal, Dict, Optional, Union, overload, Tuple +from typing import Any, ClassVar, List, Literal, Dict, Optional, Union, Iterable, Tuple, overload from pydantic import ( BaseModel, ConfigDict, @@ -129,22 +129,22 @@ class VectorIndexMixin(BaseModel): """ Mimicking :func:`hdmf.common.table.VectorIndex.__getitem_helper` """ - start = 0 if arg == 0 else self.value[arg - 1] end = self.value[arg] return self.target.value[slice(start, end)] - def __getitem__(self, item: Union[int, slice]) -> Any: + def __getitem__(self, item: Union[int, slice, Iterable]) -> Any: if self.target is None: return self.value[item] - elif isinstance(self.target, VectorData): + else: if isinstance(item, int): return self._getitem_helper(item) + elif isinstance(item, (slice, Iterable)): + if isinstance(item, slice): + item = range(*item.indices(len(self.value))) + return [self._getitem_helper(i) for i in item] else: - idx = range(*item.indices(len(self.value))) - return [self._getitem_helper(i) for i in idx] - else: - raise AttributeError(f"Could not index with {item}") + raise AttributeError(f"Could not index with {item}") def __setitem__(self, key: Union[int, slice], value: Any) -> None: if self._index: @@ -177,13 +177,40 @@ class DynamicTableRegionMixin(BaseModel): Mixin to allow indexing references to regions of dynamictables """ - table: "DynamicTableMixin" + _index: Optional["VectorIndex"] = None - def __getitem__(self, item: Union[str, int, slice, Tuple[Union[str, int, slice], ...]]) -> Any: - return self.table[item] + table: "DynamicTableMixin" + value: Optional[NDArray] = None + + def __getitem__(self, item: Union[int, slice, Iterable]) -> Any: + """ + Use ``value`` to index the table. Works analogously to ``VectorIndex`` despite + this being a subclass of ``VectorData`` + """ + if self._index: + if isinstance(item, int): + # index returns an array of indices, + # and indexing table with an array returns a list of rows + return self.table[self._index[item]] + elif isinstance(item, slice): + # index returns a list of arrays of indices, + # so we index table with an array to construct + # a list of lists of rows + return [self.table[idx] for idx in self._index[item]] + else: + raise ValueError(f"Dont know how to index with {item}, need an int or a slice") + else: + if isinstance(item, int): + return self.table[self.value[item]] + elif isinstance(item, (slice, Iterable)): + if isinstance(item, slice): + item = range(*item.indices(len(self.value))) + return [self.table[self.value[i]] for i in item] + else: + raise ValueError(f"Dont know how to index with {item}, need an int or a slice") def __setitem__(self, key: Union[int, str, slice], value: Any) -> None: - self.table[key] = value + self.table[self.value[key]] = value class DynamicTableMixin(BaseModel): @@ -259,7 +286,7 @@ class DynamicTableMixin(BaseModel): """ if isinstance(item, str): return self._columns[item] - if isinstance(item, (int, slice)): + if isinstance(item, (int, slice, np.integer, np.ndarray)): return DataFrame.from_dict(self._slice_range(item)) elif isinstance(item, tuple): if len(item) != 2: @@ -270,7 +297,7 @@ class DynamicTableMixin(BaseModel): # all other cases are tuples of (rows, cols) rows, cols = item - if isinstance(cols, (int, slice)): + if isinstance(cols, (int, slice, np.integer)): cols = self.colnames[cols] if isinstance(rows, int) and isinstance(cols, str): @@ -283,7 +310,7 @@ class DynamicTableMixin(BaseModel): raise ValueError(f"Unsure how to get item with key {item}") def _slice_range( - self, rows: Union[int, slice], cols: Optional[Union[str, List[str]]] = None + self, rows: Union[int, slice, np.ndarray], cols: Optional[Union[str, List[str]]] = None ) -> Dict[str, Union[list, "NDArray", "VectorData"]]: if cols is None: cols = self.colnames @@ -291,7 +318,11 @@ class DynamicTableMixin(BaseModel): cols = [cols] data = {} for k in cols: - val = self._columns[k][rows] + if isinstance(rows, np.ndarray): + val = [self._columns[k][i] for i in rows] + else: + val = self._columns[k][rows] + if isinstance(val, BaseModel): # special case where pandas will unpack a pydantic model # into {n_fields} rows, rather than keeping it in a dict @@ -506,7 +537,7 @@ class ElementIdentifiers(Data): ) -class DynamicTableRegion(VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ diff --git a/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_7_0/hdmf_common_table.py b/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_7_0/hdmf_common_table.py index 91e25e7..3e438ce 100644 --- a/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_7_0/hdmf_common_table.py +++ b/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_7_0/hdmf_common_table.py @@ -6,7 +6,7 @@ import re import sys from ...hdmf_common.v1_7_0.hdmf_common_base import Data, Container from pandas import DataFrame, Series -from typing import Any, ClassVar, List, Literal, Dict, Optional, Union, overload, Tuple +from typing import Any, ClassVar, List, Literal, Dict, Optional, Union, Iterable, Tuple, overload from pydantic import ( BaseModel, ConfigDict, @@ -129,22 +129,22 @@ class VectorIndexMixin(BaseModel): """ Mimicking :func:`hdmf.common.table.VectorIndex.__getitem_helper` """ - start = 0 if arg == 0 else self.value[arg - 1] end = self.value[arg] return self.target.value[slice(start, end)] - def __getitem__(self, item: Union[int, slice]) -> Any: + def __getitem__(self, item: Union[int, slice, Iterable]) -> Any: if self.target is None: return self.value[item] - elif isinstance(self.target, VectorData): + else: if isinstance(item, int): return self._getitem_helper(item) + elif isinstance(item, (slice, Iterable)): + if isinstance(item, slice): + item = range(*item.indices(len(self.value))) + return [self._getitem_helper(i) for i in item] else: - idx = range(*item.indices(len(self.value))) - return [self._getitem_helper(i) for i in idx] - else: - raise AttributeError(f"Could not index with {item}") + raise AttributeError(f"Could not index with {item}") def __setitem__(self, key: Union[int, slice], value: Any) -> None: if self._index: @@ -177,13 +177,40 @@ class DynamicTableRegionMixin(BaseModel): Mixin to allow indexing references to regions of dynamictables """ - table: "DynamicTableMixin" + _index: Optional["VectorIndex"] = None - def __getitem__(self, item: Union[str, int, slice, Tuple[Union[str, int, slice], ...]]) -> Any: - return self.table[item] + table: "DynamicTableMixin" + value: Optional[NDArray] = None + + def __getitem__(self, item: Union[int, slice, Iterable]) -> Any: + """ + Use ``value`` to index the table. Works analogously to ``VectorIndex`` despite + this being a subclass of ``VectorData`` + """ + if self._index: + if isinstance(item, int): + # index returns an array of indices, + # and indexing table with an array returns a list of rows + return self.table[self._index[item]] + elif isinstance(item, slice): + # index returns a list of arrays of indices, + # so we index table with an array to construct + # a list of lists of rows + return [self.table[idx] for idx in self._index[item]] + else: + raise ValueError(f"Dont know how to index with {item}, need an int or a slice") + else: + if isinstance(item, int): + return self.table[self.value[item]] + elif isinstance(item, (slice, Iterable)): + if isinstance(item, slice): + item = range(*item.indices(len(self.value))) + return [self.table[self.value[i]] for i in item] + else: + raise ValueError(f"Dont know how to index with {item}, need an int or a slice") def __setitem__(self, key: Union[int, str, slice], value: Any) -> None: - self.table[key] = value + self.table[self.value[key]] = value class DynamicTableMixin(BaseModel): @@ -259,7 +286,7 @@ class DynamicTableMixin(BaseModel): """ if isinstance(item, str): return self._columns[item] - if isinstance(item, (int, slice)): + if isinstance(item, (int, slice, np.integer, np.ndarray)): return DataFrame.from_dict(self._slice_range(item)) elif isinstance(item, tuple): if len(item) != 2: @@ -270,7 +297,7 @@ class DynamicTableMixin(BaseModel): # all other cases are tuples of (rows, cols) rows, cols = item - if isinstance(cols, (int, slice)): + if isinstance(cols, (int, slice, np.integer)): cols = self.colnames[cols] if isinstance(rows, int) and isinstance(cols, str): @@ -283,7 +310,7 @@ class DynamicTableMixin(BaseModel): raise ValueError(f"Unsure how to get item with key {item}") def _slice_range( - self, rows: Union[int, slice], cols: Optional[Union[str, List[str]]] = None + self, rows: Union[int, slice, np.ndarray], cols: Optional[Union[str, List[str]]] = None ) -> Dict[str, Union[list, "NDArray", "VectorData"]]: if cols is None: cols = self.colnames @@ -291,7 +318,11 @@ class DynamicTableMixin(BaseModel): cols = [cols] data = {} for k in cols: - val = self._columns[k][rows] + if isinstance(rows, np.ndarray): + val = [self._columns[k][i] for i in rows] + else: + val = self._columns[k][rows] + if isinstance(val, BaseModel): # special case where pandas will unpack a pydantic model # into {n_fields} rows, rather than keeping it in a dict @@ -506,7 +537,7 @@ class ElementIdentifiers(Data): ) -class DynamicTableRegion(VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ diff --git a/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_8_0/hdmf_common_table.py b/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_8_0/hdmf_common_table.py index 8e9d681..c016f61 100644 --- a/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_8_0/hdmf_common_table.py +++ b/nwb_linkml/src/nwb_linkml/models/pydantic/hdmf_common/v1_8_0/hdmf_common_table.py @@ -6,7 +6,7 @@ import re import sys from ...hdmf_common.v1_8_0.hdmf_common_base import Data, Container from pandas import DataFrame, Series -from typing import Any, ClassVar, List, Literal, Dict, Optional, Union, overload, Tuple +from typing import Any, ClassVar, List, Literal, Dict, Optional, Union, Iterable, Tuple, overload from pydantic import ( BaseModel, ConfigDict, @@ -129,22 +129,22 @@ class VectorIndexMixin(BaseModel): """ Mimicking :func:`hdmf.common.table.VectorIndex.__getitem_helper` """ - start = 0 if arg == 0 else self.value[arg - 1] end = self.value[arg] return self.target.value[slice(start, end)] - def __getitem__(self, item: Union[int, slice]) -> Any: + def __getitem__(self, item: Union[int, slice, Iterable]) -> Any: if self.target is None: return self.value[item] - elif isinstance(self.target, VectorData): + else: if isinstance(item, int): return self._getitem_helper(item) + elif isinstance(item, (slice, Iterable)): + if isinstance(item, slice): + item = range(*item.indices(len(self.value))) + return [self._getitem_helper(i) for i in item] else: - idx = range(*item.indices(len(self.value))) - return [self._getitem_helper(i) for i in idx] - else: - raise AttributeError(f"Could not index with {item}") + raise AttributeError(f"Could not index with {item}") def __setitem__(self, key: Union[int, slice], value: Any) -> None: if self._index: @@ -177,13 +177,40 @@ class DynamicTableRegionMixin(BaseModel): Mixin to allow indexing references to regions of dynamictables """ - table: "DynamicTableMixin" + _index: Optional["VectorIndex"] = None - def __getitem__(self, item: Union[str, int, slice, Tuple[Union[str, int, slice], ...]]) -> Any: - return self.table[item] + table: "DynamicTableMixin" + value: Optional[NDArray] = None + + def __getitem__(self, item: Union[int, slice, Iterable]) -> Any: + """ + Use ``value`` to index the table. Works analogously to ``VectorIndex`` despite + this being a subclass of ``VectorData`` + """ + if self._index: + if isinstance(item, int): + # index returns an array of indices, + # and indexing table with an array returns a list of rows + return self.table[self._index[item]] + elif isinstance(item, slice): + # index returns a list of arrays of indices, + # so we index table with an array to construct + # a list of lists of rows + return [self.table[idx] for idx in self._index[item]] + else: + raise ValueError(f"Dont know how to index with {item}, need an int or a slice") + else: + if isinstance(item, int): + return self.table[self.value[item]] + elif isinstance(item, (slice, Iterable)): + if isinstance(item, slice): + item = range(*item.indices(len(self.value))) + return [self.table[self.value[i]] for i in item] + else: + raise ValueError(f"Dont know how to index with {item}, need an int or a slice") def __setitem__(self, key: Union[int, str, slice], value: Any) -> None: - self.table[key] = value + self.table[self.value[key]] = value class DynamicTableMixin(BaseModel): @@ -259,7 +286,7 @@ class DynamicTableMixin(BaseModel): """ if isinstance(item, str): return self._columns[item] - if isinstance(item, (int, slice)): + if isinstance(item, (int, slice, np.integer, np.ndarray)): return DataFrame.from_dict(self._slice_range(item)) elif isinstance(item, tuple): if len(item) != 2: @@ -270,7 +297,7 @@ class DynamicTableMixin(BaseModel): # all other cases are tuples of (rows, cols) rows, cols = item - if isinstance(cols, (int, slice)): + if isinstance(cols, (int, slice, np.integer)): cols = self.colnames[cols] if isinstance(rows, int) and isinstance(cols, str): @@ -283,7 +310,7 @@ class DynamicTableMixin(BaseModel): raise ValueError(f"Unsure how to get item with key {item}") def _slice_range( - self, rows: Union[int, slice], cols: Optional[Union[str, List[str]]] = None + self, rows: Union[int, slice, np.ndarray], cols: Optional[Union[str, List[str]]] = None ) -> Dict[str, Union[list, "NDArray", "VectorData"]]: if cols is None: cols = self.colnames @@ -291,7 +318,11 @@ class DynamicTableMixin(BaseModel): cols = [cols] data = {} for k in cols: - val = self._columns[k][rows] + if isinstance(rows, np.ndarray): + val = [self._columns[k][i] for i in rows] + else: + val = self._columns[k][rows] + if isinstance(val, BaseModel): # special case where pandas will unpack a pydantic model # into {n_fields} rows, rather than keeping it in a dict @@ -506,7 +537,7 @@ class ElementIdentifiers(Data): ) -class DynamicTableRegion(VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ diff --git a/nwb_linkml/tests/test_includes/test_hdmf.py b/nwb_linkml/tests/test_includes/test_hdmf.py index 8e1bd64..87e1c88 100644 --- a/nwb_linkml/tests/test_includes/test_hdmf.py +++ b/nwb_linkml/tests/test_includes/test_hdmf.py @@ -6,11 +6,13 @@ import pytest # FIXME: Make this just be the output of the provider by patching into import machinery from nwb_linkml.models.pydantic.core.v2_7_0.namespace import ( Device, + DynamicTable, DynamicTableRegion, ElectricalSeries, ElectrodeGroup, ExtracellularEphysElectrodes, Units, + VectorIndex, ) @@ -49,7 +51,10 @@ def electrical_series() -> Tuple["ElectricalSeries", "ExtracellularEphysElectrod electrical_series = ElectricalSeries( name="my recording!", electrodes=DynamicTableRegion( - table=electrodes, value=np.arange(0, n_electrodes), name="electrodes", description="hey" + table=electrodes, + value=np.arange(n_electrodes - 1, -1, step=-1), + name="electrodes", + description="hey", ), timestamps=timestamps, data=data, @@ -57,16 +62,7 @@ def electrical_series() -> Tuple["ElectricalSeries", "ExtracellularEphysElectrod return electrical_series, electrodes -@pytest.fixture(params=[True, False]) -def units(request) -> Tuple[Units, list[np.ndarray], np.ndarray]: - """ - Test case for units - - Parameterized by extra_column because pandas likes to pivot dataframes - to long when there is only one column and it's not len() == 1 - """ - - n_units = 24 +def _ragged_array(n_units: int) -> tuple[list[np.ndarray], np.ndarray]: generator = np.random.default_rng() spike_times = [ np.full(shape=generator.integers(10, 50), fill_value=i, dtype=float) for i in range(n_units) @@ -78,6 +74,18 @@ def units(request) -> Tuple[Units, list[np.ndarray], np.ndarray]: else: spike_idx.append(len(spike_times[i]) + spike_idx[i - 1]) spike_idx = np.array(spike_idx) + return spike_times, spike_idx + + +@pytest.fixture(params=[True, False]) +def units(request) -> Tuple[Units, list[np.ndarray], np.ndarray]: + """ + Test case for units + + Parameterized by extra_column because pandas likes to pivot dataframes + to long when there is only one column and it's not len() == 1 + """ + spike_times, spike_idx = _ragged_array(24) spike_times_flat = np.concatenate(spike_times) @@ -87,7 +95,7 @@ def units(request) -> Tuple[Units, list[np.ndarray], np.ndarray]: "spike_times_index": spike_idx, } if request.param: - kwargs["extra_column"] = ["hey!"] * n_units + kwargs["extra_column"] = ["hey!"] * 24 units = Units(**kwargs) return units, spike_times, spike_idx @@ -142,20 +150,75 @@ def test_dynamictable_indexing(electrical_series): assert subsection.dtypes.values.tolist() == dtypes[0:3] -def test_dynamictable_region(electrical_series): +def test_dynamictable_region_basic(electrical_series): """ - Dynamictableregion should - Args: - electrical_series: - - Returns: - + DynamicTableRegion should be able to refer to a row or rows of another table + itself as a column within a table """ series, electrodes = electrical_series - + row = series.electrodes[0] + # check that we correctly got the 4th row instead of the 0th row, + # since the indexed table was constructed with inverted indexes because it's a test, ya dummy. + # we will only vaguely check the basic functionality here bc + # a) the indexing behavior of the indexed objects is tested above, and + # b) every other object in the chain is strictly validated, + # so we assume if we got a right shaped df that it is the correct one. + # feel free to @ me when i am wrong about this + assert row.id == 4 + assert row.shape == (1, 7) + # and we should still be preserving the model that is the contents of the cell of this row + # so this is a dataframe row with a column "group" that contains an array of ElectrodeGroup + # objects and that's as far as we are going to chase the recursion in this basic indexing test + # ElectrodeGroup is strictly validating so an instance check is all we need. + assert isinstance(row.group.values[0], ElectrodeGroup) + + # getting a list of table rows is actually correct behavior here because + # this list of table rows is actually the cell of another table + rows = series.electrodes[0:3] + assert all([row.id == idx for row, idx in zip(rows, [4, 3, 2])]) -def test_dynamictable_ragged_arrays(units): +def test_dynamictable_region_ragged(): + """ + Dynamictables can also have indexes so that they are ragged arrays of column rows + """ + spike_times, spike_idx = _ragged_array(24) + spike_times_flat = np.concatenate(spike_times) + + # construct a secondary index that selects overlapping segments of the first table + value = np.array([0, 1, 2, 1, 2, 3, 2, 3, 4]) + idx = np.array([3, 6, 9]) + + table = DynamicTable( + name="table", + description="a table what else would it be", + id=np.arange(len(spike_idx)), + timeseries=spike_times, + timeseries_index=spike_idx, + ) + region = DynamicTableRegion( + name="dynamictableregion", + description="this field should be optional", + table=table, + value=value, + ) + index = VectorIndex(name="index", description="hgggggggjjjj", target=region, value=idx) + region._index = index + rows = region[1] + # i guess this is right? + # the region should be a set of three rows of the table, with a ragged array column timeseries + # like... + # + # id timeseries + # 0 1 [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ... + # 1 2 [2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, ... + # 2 3 [3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, ... + assert rows.shape(3, 2) + assert all(rows.id == [1, 2, 3]) + assert all([all(row[1].timeseries == i) for i, row in zip([1, 2, 3], rows.iterrows())]) + + +def test_dynamictable_ragged(units): """ Should be able to index ragged arrays using an implicit _index column diff --git a/nwb_linkml/tests/test_providers/test_provider_schema.py b/nwb_linkml/tests/test_providers/test_provider_schema.py index 9da6296..a455e29 100644 --- a/nwb_linkml/tests/test_providers/test_provider_schema.py +++ b/nwb_linkml/tests/test_providers/test_provider_schema.py @@ -3,10 +3,9 @@ import sys from pathlib import Path from typing import Optional +import numpy as np import pytest from numpydantic import NDArray, Shape -import numpy as np - import nwb_linkml from nwb_linkml.maps.naming import version_module_case