From ccc09de40071eb996557edb5a3b6072862cc7cfc Mon Sep 17 00:00:00 2001 From: sneakers-the-rat Date: Tue, 5 Sep 2023 19:25:20 -0700 Subject: [PATCH] refactor nwb_linkml to src layout --- .github/workflows/tests.yml | 2 + hdf5_linkml/tests/test_scratch.py | 2 + nwb_linkml/__init__.py | 1 - nwb_linkml/adapters/__init__.py | 4 -- nwb_linkml/maps/__init__.py | 3 - poetry.lock => nwb_linkml/poetry.lock | 0 pyproject.toml => nwb_linkml/pyproject.toml | 4 +- nwb_linkml/src/nwb_linkml/__init__.py | 1 + .../src/nwb_linkml/adapters/__init__.py | 1 + .../{ => src/nwb_linkml}/adapters/adapter.py | 0 .../{ => src/nwb_linkml}/adapters/classes.py | 4 +- .../{ => src/nwb_linkml}/adapters/dataset.py | 2 +- .../{ => src/nwb_linkml}/adapters/group.py | 2 +- .../nwb_linkml}/adapters/namespaces.py | 0 .../{ => src/nwb_linkml}/adapters/schema.py | 3 +- .../nwb_linkml}/generators/__init__.py | 0 .../nwb_linkml}/generators/pydantic.py | 2 +- .../src/nwb_linkml/io}/__init__.py | 0 nwb_linkml/src/nwb_linkml/io/hdf5.py | 61 +++++++++++++++++++ nwb_linkml/{ => src/nwb_linkml/io}/io.py | 10 ++- .../{ => src/nwb_linkml}/lang_elements.py | 2 +- nwb_linkml/{ => src/nwb_linkml}/map.py | 1 - nwb_linkml/src/nwb_linkml/maps/__init__.py | 3 + nwb_linkml/{ => src/nwb_linkml}/maps/dtype.py | 0 .../{ => src/nwb_linkml}/maps/preload.py | 2 +- .../{ => src/nwb_linkml}/maps/quantity.py | 0 .../{ => src/nwb_linkml}/models/core.py | 0 .../nwb_linkml}/models/core_nwb_base.py | 0 .../nwb_linkml}/models/core_nwb_behavior.py | 0 .../nwb_linkml}/models/core_nwb_device.py | 0 .../nwb_linkml}/models/core_nwb_ecephys.py | 0 .../nwb_linkml}/models/core_nwb_epoch.py | 0 .../nwb_linkml}/models/core_nwb_file.py | 0 .../nwb_linkml}/models/core_nwb_icephys.py | 0 .../nwb_linkml}/models/core_nwb_image.py | 0 .../nwb_linkml}/models/core_nwb_misc.py | 0 .../nwb_linkml}/models/core_nwb_ogen.py | 0 .../nwb_linkml}/models/core_nwb_ophys.py | 0 .../nwb_linkml}/models/core_nwb_retinotopy.py | 0 .../nwb_linkml}/models/hdmf_common.py | 0 .../nwb_linkml}/models/hdmf_common_base.py | 0 .../nwb_linkml}/models/hdmf_common_sparse.py | 0 .../nwb_linkml}/models/hdmf_common_table.py | 0 .../nwb_linkml}/models/hdmf_experimental.py | 0 .../models/hdmf_experimental_experimental.py | 0 .../models/hdmf_experimental_resources.py | 0 .../nwb_linkml}/models/nwb_language.py | 0 nwb_linkml/{ => src/nwb_linkml}/namespaces.py | 0 nwb_linkml/{ => src/nwb_linkml}/plot.py | 4 +- .../nwb_linkml}/schema/core.nwb.base.yaml | 0 .../nwb_linkml}/schema/core.nwb.behavior.yaml | 0 .../nwb_linkml}/schema/core.nwb.device.yaml | 0 .../nwb_linkml}/schema/core.nwb.ecephys.yaml | 0 .../nwb_linkml}/schema/core.nwb.epoch.yaml | 0 .../nwb_linkml}/schema/core.nwb.file.yaml | 0 .../nwb_linkml}/schema/core.nwb.icephys.yaml | 0 .../nwb_linkml}/schema/core.nwb.image.yaml | 0 .../nwb_linkml}/schema/core.nwb.misc.yaml | 0 .../nwb_linkml}/schema/core.nwb.ogen.yaml | 0 .../nwb_linkml}/schema/core.nwb.ophys.yaml | 0 .../schema/core.nwb.retinotopy.yaml | 0 .../{ => src/nwb_linkml}/schema/core.yaml | 0 .../nwb_linkml}/schema/hdmf-common.base.yaml | 0 .../schema/hdmf-common.sparse.yaml | 0 .../nwb_linkml}/schema/hdmf-common.table.yaml | 0 .../nwb_linkml}/schema/hdmf-common.yaml | 0 .../hdmf-experimental.experimental.yaml | 0 .../schema/hdmf-experimental.resources.yaml | 0 .../nwb_linkml}/schema/hdmf-experimental.yaml | 0 .../nwb_linkml}/schema/nwb.language.yaml | 0 nwb_linkml/{ => src/nwb_linkml}/translate.py | 10 +-- nwb_linkml/tests/__init__.py | 0 {tests => nwb_linkml/tests}/fixtures.py | 6 +- nwb_linkml/tests/test_adapters/__init__.py | 0 .../tests/test_adapters}/test_adapter.py | 0 .../test_adapters}/test_adapter_dataset.py | 0 .../test_adapters}/test_adapter_group.py | 0 .../test_adapters}/test_adapter_namespaces.py | 0 .../test_adapters}/test_adapter_schema.py | 0 {tests => nwb_linkml/tests}/test_generate.py | 0 {tests => nwb_linkml/tests}/test_translate.py | 1 + pytest.ini | 2 +- scripts/generate_core.py | 7 ++- 83 files changed, 95 insertions(+), 45 deletions(-) create mode 100644 hdf5_linkml/tests/test_scratch.py delete mode 100644 nwb_linkml/__init__.py delete mode 100644 nwb_linkml/adapters/__init__.py delete mode 100644 nwb_linkml/maps/__init__.py rename poetry.lock => nwb_linkml/poetry.lock (100%) rename pyproject.toml => nwb_linkml/pyproject.toml (91%) create mode 100644 nwb_linkml/src/nwb_linkml/__init__.py create mode 100644 nwb_linkml/src/nwb_linkml/adapters/__init__.py rename nwb_linkml/{ => src/nwb_linkml}/adapters/adapter.py (100%) rename nwb_linkml/{ => src/nwb_linkml}/adapters/classes.py (98%) rename nwb_linkml/{ => src/nwb_linkml}/adapters/dataset.py (99%) rename nwb_linkml/{ => src/nwb_linkml}/adapters/group.py (99%) rename nwb_linkml/{ => src/nwb_linkml}/adapters/namespaces.py (100%) rename nwb_linkml/{ => src/nwb_linkml}/adapters/schema.py (98%) rename nwb_linkml/{ => src/nwb_linkml}/generators/__init__.py (100%) rename nwb_linkml/{ => src/nwb_linkml}/generators/pydantic.py (99%) rename {tests => nwb_linkml/src/nwb_linkml/io}/__init__.py (100%) create mode 100644 nwb_linkml/src/nwb_linkml/io/hdf5.py rename nwb_linkml/{ => src/nwb_linkml/io}/io.py (93%) rename nwb_linkml/{ => src/nwb_linkml}/lang_elements.py (98%) rename nwb_linkml/{ => src/nwb_linkml}/map.py (96%) create mode 100644 nwb_linkml/src/nwb_linkml/maps/__init__.py rename nwb_linkml/{ => src/nwb_linkml}/maps/dtype.py (100%) rename nwb_linkml/{ => src/nwb_linkml}/maps/preload.py (87%) rename nwb_linkml/{ => src/nwb_linkml}/maps/quantity.py (100%) rename nwb_linkml/{ => src/nwb_linkml}/models/core.py (100%) rename nwb_linkml/{ => src/nwb_linkml}/models/core_nwb_base.py (100%) rename nwb_linkml/{ => src/nwb_linkml}/models/core_nwb_behavior.py (100%) rename nwb_linkml/{ => src/nwb_linkml}/models/core_nwb_device.py (100%) rename nwb_linkml/{ => src/nwb_linkml}/models/core_nwb_ecephys.py (100%) rename nwb_linkml/{ => src/nwb_linkml}/models/core_nwb_epoch.py (100%) rename nwb_linkml/{ => src/nwb_linkml}/models/core_nwb_file.py (100%) rename nwb_linkml/{ => src/nwb_linkml}/models/core_nwb_icephys.py (100%) rename nwb_linkml/{ => src/nwb_linkml}/models/core_nwb_image.py (100%) rename nwb_linkml/{ => src/nwb_linkml}/models/core_nwb_misc.py (100%) rename nwb_linkml/{ => src/nwb_linkml}/models/core_nwb_ogen.py (100%) rename nwb_linkml/{ => src/nwb_linkml}/models/core_nwb_ophys.py (100%) rename nwb_linkml/{ => src/nwb_linkml}/models/core_nwb_retinotopy.py (100%) rename nwb_linkml/{ => src/nwb_linkml}/models/hdmf_common.py (100%) rename nwb_linkml/{ => src/nwb_linkml}/models/hdmf_common_base.py (100%) rename nwb_linkml/{ => src/nwb_linkml}/models/hdmf_common_sparse.py (100%) rename nwb_linkml/{ => src/nwb_linkml}/models/hdmf_common_table.py (100%) rename nwb_linkml/{ => src/nwb_linkml}/models/hdmf_experimental.py (100%) rename nwb_linkml/{ => src/nwb_linkml}/models/hdmf_experimental_experimental.py (100%) rename nwb_linkml/{ => src/nwb_linkml}/models/hdmf_experimental_resources.py (100%) rename nwb_linkml/{ => src/nwb_linkml}/models/nwb_language.py (100%) rename nwb_linkml/{ => src/nwb_linkml}/namespaces.py (100%) rename nwb_linkml/{ => src/nwb_linkml}/plot.py (97%) rename nwb_linkml/{ => src/nwb_linkml}/schema/core.nwb.base.yaml (100%) rename nwb_linkml/{ => src/nwb_linkml}/schema/core.nwb.behavior.yaml (100%) rename nwb_linkml/{ => src/nwb_linkml}/schema/core.nwb.device.yaml (100%) rename nwb_linkml/{ => src/nwb_linkml}/schema/core.nwb.ecephys.yaml (100%) rename nwb_linkml/{ => src/nwb_linkml}/schema/core.nwb.epoch.yaml (100%) rename nwb_linkml/{ => src/nwb_linkml}/schema/core.nwb.file.yaml (100%) rename nwb_linkml/{ => src/nwb_linkml}/schema/core.nwb.icephys.yaml (100%) rename nwb_linkml/{ => src/nwb_linkml}/schema/core.nwb.image.yaml (100%) rename nwb_linkml/{ => src/nwb_linkml}/schema/core.nwb.misc.yaml (100%) rename nwb_linkml/{ => src/nwb_linkml}/schema/core.nwb.ogen.yaml (100%) rename nwb_linkml/{ => src/nwb_linkml}/schema/core.nwb.ophys.yaml (100%) rename nwb_linkml/{ => src/nwb_linkml}/schema/core.nwb.retinotopy.yaml (100%) rename nwb_linkml/{ => src/nwb_linkml}/schema/core.yaml (100%) rename nwb_linkml/{ => src/nwb_linkml}/schema/hdmf-common.base.yaml (100%) rename nwb_linkml/{ => src/nwb_linkml}/schema/hdmf-common.sparse.yaml (100%) rename nwb_linkml/{ => src/nwb_linkml}/schema/hdmf-common.table.yaml (100%) rename nwb_linkml/{ => src/nwb_linkml}/schema/hdmf-common.yaml (100%) rename nwb_linkml/{ => src/nwb_linkml}/schema/hdmf-experimental.experimental.yaml (100%) rename nwb_linkml/{ => src/nwb_linkml}/schema/hdmf-experimental.resources.yaml (100%) rename nwb_linkml/{ => src/nwb_linkml}/schema/hdmf-experimental.yaml (100%) rename nwb_linkml/{ => src/nwb_linkml}/schema/nwb.language.yaml (100%) rename nwb_linkml/{ => src/nwb_linkml}/translate.py (94%) create mode 100644 nwb_linkml/tests/__init__.py rename {tests => nwb_linkml/tests}/fixtures.py (75%) create mode 100644 nwb_linkml/tests/test_adapters/__init__.py rename {tests => nwb_linkml/tests/test_adapters}/test_adapter.py (100%) rename {tests => nwb_linkml/tests/test_adapters}/test_adapter_dataset.py (100%) rename {tests => nwb_linkml/tests/test_adapters}/test_adapter_group.py (100%) rename {tests => nwb_linkml/tests/test_adapters}/test_adapter_namespaces.py (100%) rename {tests => nwb_linkml/tests/test_adapters}/test_adapter_schema.py (100%) rename {tests => nwb_linkml/tests}/test_generate.py (100%) rename {tests => nwb_linkml/tests}/test_translate.py (99%) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8f889ac..678619d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,8 +21,10 @@ jobs: - name: Install dependencies run: pip install .[tests] pytest-md pytest-emoji + working-directory: nwb_linkml - uses: pavelzw/pytest-action@v2 + working-directory: nwb_linkml with: emoji: true verbose: true diff --git a/hdf5_linkml/tests/test_scratch.py b/hdf5_linkml/tests/test_scratch.py new file mode 100644 index 0000000..139597f --- /dev/null +++ b/hdf5_linkml/tests/test_scratch.py @@ -0,0 +1,2 @@ + + diff --git a/nwb_linkml/__init__.py b/nwb_linkml/__init__.py deleted file mode 100644 index 2ea95fc..0000000 --- a/nwb_linkml/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from nwb_linkml.maps import preload \ No newline at end of file diff --git a/nwb_linkml/adapters/__init__.py b/nwb_linkml/adapters/__init__.py deleted file mode 100644 index 7647640..0000000 --- a/nwb_linkml/adapters/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from nwb_linkml.adapters.adapter import Adapter -from nwb_linkml.adapters.namespaces import NamespacesAdapter -from nwb_linkml.adapters.schema import SchemaAdapter -from nwb_linkml.adapters.classes import ClassAdapter \ No newline at end of file diff --git a/nwb_linkml/maps/__init__.py b/nwb_linkml/maps/__init__.py deleted file mode 100644 index 6078f74..0000000 --- a/nwb_linkml/maps/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# Import everything so it's defined, but shoudlnt' necessarily be used from here -from nwb_linkml.maps.preload import MAP_HDMF_DATATYPE_DEF, MAP_HDMF_DATATYPE_INC -from nwb_linkml.maps.quantity import QUANTITY_MAP \ No newline at end of file diff --git a/poetry.lock b/nwb_linkml/poetry.lock similarity index 100% rename from poetry.lock rename to nwb_linkml/poetry.lock diff --git a/pyproject.toml b/nwb_linkml/pyproject.toml similarity index 91% rename from pyproject.toml rename to nwb_linkml/pyproject.toml index 35f5fb9..378ad53 100644 --- a/pyproject.toml +++ b/nwb_linkml/pyproject.toml @@ -6,8 +6,8 @@ authors = ["sneakers-the-rat "] license = "GPL-3.0" readme = "README.md" packages = [ - {include = "nwb_linkml"}, - {include = "nwb_schema_language", from="nwb_schema_language/src"} + {include = "nwb_linkml", from="src"}, + {include = "nwb_schema_language", from="../nwb_schema_language/src"} ] [tool.poetry.dependencies] diff --git a/nwb_linkml/src/nwb_linkml/__init__.py b/nwb_linkml/src/nwb_linkml/__init__.py new file mode 100644 index 0000000..7dc898d --- /dev/null +++ b/nwb_linkml/src/nwb_linkml/__init__.py @@ -0,0 +1 @@ +from nwb_linkml.src.nwb_linkml.maps import preload \ No newline at end of file diff --git a/nwb_linkml/src/nwb_linkml/adapters/__init__.py b/nwb_linkml/src/nwb_linkml/adapters/__init__.py new file mode 100644 index 0000000..03a252b --- /dev/null +++ b/nwb_linkml/src/nwb_linkml/adapters/__init__.py @@ -0,0 +1 @@ +from nwb_linkml.src.nwb_linkml.adapters.namespaces import NamespacesAdapter diff --git a/nwb_linkml/adapters/adapter.py b/nwb_linkml/src/nwb_linkml/adapters/adapter.py similarity index 100% rename from nwb_linkml/adapters/adapter.py rename to nwb_linkml/src/nwb_linkml/adapters/adapter.py diff --git a/nwb_linkml/adapters/classes.py b/nwb_linkml/src/nwb_linkml/adapters/classes.py similarity index 98% rename from nwb_linkml/adapters/classes.py rename to nwb_linkml/src/nwb_linkml/adapters/classes.py index 64d2615..18d2350 100644 --- a/nwb_linkml/adapters/classes.py +++ b/nwb_linkml/src/nwb_linkml/adapters/classes.py @@ -1,15 +1,13 @@ """ Adapters to linkML classes """ -import pdb import re from abc import abstractmethod from typing import List, Optional from nwb_schema_language import Dataset, Group, ReferenceDtype, CompoundDtype, DTypeType from nwb_linkml.adapters.adapter import Adapter, BuildResult from linkml_runtime.linkml_model import ClassDefinition, SlotDefinition -from nwb_linkml.maps import QUANTITY_MAP -from nwb_linkml.lang_elements import Arraylike +from nwb_linkml.src.nwb_linkml.maps import QUANTITY_MAP CAMEL_TO_SNAKE = re.compile('((?<=[a-z0-9])[A-Z]|(?!^)[A-Z](?=[a-z]))') """ diff --git a/nwb_linkml/adapters/dataset.py b/nwb_linkml/src/nwb_linkml/adapters/dataset.py similarity index 99% rename from nwb_linkml/adapters/dataset.py rename to nwb_linkml/src/nwb_linkml/adapters/dataset.py index f3a5268..f328948 100644 --- a/nwb_linkml/adapters/dataset.py +++ b/nwb_linkml/src/nwb_linkml/adapters/dataset.py @@ -11,7 +11,7 @@ from pydantic import PrivateAttr from nwb_schema_language import Dataset, ReferenceDtype, CompoundDtype, DTypeType from nwb_linkml.adapters.classes import ClassAdapter, camel_to_snake from nwb_linkml.adapters.adapter import BuildResult -from nwb_linkml.maps import QUANTITY_MAP +from nwb_linkml.src.nwb_linkml.maps import QUANTITY_MAP class DatasetAdapter(ClassAdapter): cls: Dataset diff --git a/nwb_linkml/adapters/group.py b/nwb_linkml/src/nwb_linkml/adapters/group.py similarity index 99% rename from nwb_linkml/adapters/group.py rename to nwb_linkml/src/nwb_linkml/adapters/group.py index 766dea6..7b84b66 100644 --- a/nwb_linkml/adapters/group.py +++ b/nwb_linkml/src/nwb_linkml/adapters/group.py @@ -9,7 +9,7 @@ from nwb_schema_language import Dataset, Group, ReferenceDtype, CompoundDtype, D from nwb_linkml.adapters.classes import ClassAdapter, camel_to_snake from nwb_linkml.adapters.dataset import DatasetAdapter from nwb_linkml.adapters.adapter import BuildResult -from nwb_linkml.maps import QUANTITY_MAP +from nwb_linkml.src.nwb_linkml.maps import QUANTITY_MAP class GroupAdapter(ClassAdapter): cls: Group diff --git a/nwb_linkml/adapters/namespaces.py b/nwb_linkml/src/nwb_linkml/adapters/namespaces.py similarity index 100% rename from nwb_linkml/adapters/namespaces.py rename to nwb_linkml/src/nwb_linkml/adapters/namespaces.py diff --git a/nwb_linkml/adapters/schema.py b/nwb_linkml/src/nwb_linkml/adapters/schema.py similarity index 98% rename from nwb_linkml/adapters/schema.py rename to nwb_linkml/src/nwb_linkml/adapters/schema.py index 277faba..4559624 100644 --- a/nwb_linkml/adapters/schema.py +++ b/nwb_linkml/src/nwb_linkml/adapters/schema.py @@ -2,7 +2,6 @@ Since NWB doesn't necessarily have a term for a single nwb schema file, we're going to call them "schema" objects """ -import pdb from typing import Optional, List, TYPE_CHECKING, Type from pathlib import Path from pydantic import Field, PrivateAttr @@ -11,7 +10,7 @@ from nwb_linkml.adapters.adapter import Adapter, BuildResult from nwb_linkml.adapters.dataset import DatasetAdapter from nwb_linkml.adapters.group import GroupAdapter if TYPE_CHECKING: - from nwb_linkml.adapters.namespaces import NamespacesAdapter + pass from nwb_schema_language import Group, Dataset from typing import NamedTuple diff --git a/nwb_linkml/generators/__init__.py b/nwb_linkml/src/nwb_linkml/generators/__init__.py similarity index 100% rename from nwb_linkml/generators/__init__.py rename to nwb_linkml/src/nwb_linkml/generators/__init__.py diff --git a/nwb_linkml/generators/pydantic.py b/nwb_linkml/src/nwb_linkml/generators/pydantic.py similarity index 99% rename from nwb_linkml/generators/pydantic.py rename to nwb_linkml/src/nwb_linkml/generators/pydantic.py index 5ab9365..32bcd3d 100644 --- a/nwb_linkml/generators/pydantic.py +++ b/nwb_linkml/src/nwb_linkml/generators/pydantic.py @@ -25,7 +25,7 @@ from types import ModuleType from copy import deepcopy import warnings -from nwb_linkml.maps.dtype import flat_to_npytyping +from nwb_linkml.src.nwb_linkml.maps import flat_to_npytyping from linkml.generators import PydanticGenerator from linkml_runtime.linkml_model.meta import ( Annotation, diff --git a/tests/__init__.py b/nwb_linkml/src/nwb_linkml/io/__init__.py similarity index 100% rename from tests/__init__.py rename to nwb_linkml/src/nwb_linkml/io/__init__.py diff --git a/nwb_linkml/src/nwb_linkml/io/hdf5.py b/nwb_linkml/src/nwb_linkml/io/hdf5.py new file mode 100644 index 0000000..e5c2e33 --- /dev/null +++ b/nwb_linkml/src/nwb_linkml/io/hdf5.py @@ -0,0 +1,61 @@ +""" +This is a sandbox file that should be split out to its own pydantic-hdf5 package, but just experimenting here to get our bearings +""" +from typing import Optional, List, Dict +from pathlib import Path +from types import ModuleType + +import h5py + +from nwb_linkml.translate import generate_from_nwbfile + +class HDF5IO(): + + def __init__(self, path:Path): + self.path = Path(path) + self._modules: Dict[str, ModuleType] = {} + + @property + def modules(self) -> Dict[str, ModuleType]: + if len(self._modules) == 0: + self._modules = generate_from_nwbfile(self.path) + return self._modules + + def process_group(self, group:h5py.Group|h5py.File) -> dict | list: + attrs = dict(group.attrs) + + # how to process the group? + # -------------------------------------------------- + # list-like + # -------------------------------------------------- + # a list of data classes + if 'neurodata_type' not in attrs and \ + all([isinstance(v, h5py.Group) for v in group.values()]) and \ + all(['neurodata_type' in v.attrs for v in group.values()]): + + return [self.process_group(v) for v in group.values()] + + # -------------------------------------------------- + # dict-like + # -------------------------------------------------- + + res = {} + + + for key, val in group.items(): + if isinstance(val, h5py.Group): + res[key] = self.process_group(val) + elif isinstance(val, h5py.Dataset): + res[key] = self.process_dataset(val) + return res + + def process_dataset(self, data: h5py.Dataset) -> dict | list: + if len(data.shape) == 1: + return list(data[:]) + + +if __name__ == "__main__": + NWBFILE = Path('/Users/jonny/Dropbox/lab/p2p_ld/data/nwb/sub-738651046_ses-760693773.nwb') + h5f = HDF5IO(NWBFILE) + + diff --git a/nwb_linkml/io.py b/nwb_linkml/src/nwb_linkml/io/io.py similarity index 93% rename from nwb_linkml/io.py rename to nwb_linkml/src/nwb_linkml/io/io.py index c149853..fd04470 100644 --- a/nwb_linkml/io.py +++ b/nwb_linkml/src/nwb_linkml/io/io.py @@ -1,9 +1,8 @@ """ Loading/saving NWB Schema yaml files """ -import pdb from pathlib import Path -from typing import TypedDict, List, Dict, Optional +from typing import Optional from pprint import pprint import warnings @@ -12,9 +11,8 @@ import yaml from nwb_schema_language import Namespaces, Group, Dataset from nwb_linkml.namespaces import NamespaceRepo, NWB_CORE_REPO, HDMF_COMMON_REPO -from nwb_linkml.maps import preload -from nwb_linkml.map import PHASES, Map -from nwb_linkml.adapters.namespaces import NamespacesAdapter +from nwb_linkml.src.nwb_linkml.map import PHASES, Map +from nwb_linkml.src.nwb_linkml.adapters.namespaces import NamespacesAdapter from nwb_linkml.adapters.schema import SchemaAdapter @@ -70,7 +68,7 @@ def load_schema_file(path:Path, yaml:Optional[dict] = None) -> SchemaAdapter: ) return schema -def load_namespace_schema(namespace: Namespaces, path:Path=Path('.')) -> NamespacesAdapter: +def load_namespace_schema(namespace: Namespaces, path:Path=Path('..')) -> NamespacesAdapter: """ Load all schema referenced by a namespace file diff --git a/nwb_linkml/lang_elements.py b/nwb_linkml/src/nwb_linkml/lang_elements.py similarity index 98% rename from nwb_linkml/lang_elements.py rename to nwb_linkml/src/nwb_linkml/lang_elements.py index 8ee539e..bd3e7cf 100644 --- a/nwb_linkml/lang_elements.py +++ b/nwb_linkml/src/nwb_linkml/lang_elements.py @@ -12,7 +12,7 @@ from linkml_runtime.linkml_model import \ TypeDefinition,\ Prefix,\ PermissibleValue -from nwb_linkml.maps.dtype import flat_to_linkml +from nwb_linkml.src.nwb_linkml.maps import flat_to_linkml FlatDType = EnumDefinition( diff --git a/nwb_linkml/map.py b/nwb_linkml/src/nwb_linkml/map.py similarity index 96% rename from nwb_linkml/map.py rename to nwb_linkml/src/nwb_linkml/map.py index 244be5e..5f671c4 100644 --- a/nwb_linkml/map.py +++ b/nwb_linkml/src/nwb_linkml/map.py @@ -66,7 +66,6 @@ class KeyMap(Map): def apply_preload(ns_dict) -> dict: - from nwb_linkml.maps import preload maps = [m for m in Map.instances if m.phase == PHASES.postload] for amap in maps: ns_dict = amap.apply(ns_dict) diff --git a/nwb_linkml/src/nwb_linkml/maps/__init__.py b/nwb_linkml/src/nwb_linkml/maps/__init__.py new file mode 100644 index 0000000..f5b3780 --- /dev/null +++ b/nwb_linkml/src/nwb_linkml/maps/__init__.py @@ -0,0 +1,3 @@ +# Import everything so it's defined, but shoudlnt' necessarily be used from here +from nwb_linkml.src.nwb_linkml.maps import MAP_HDMF_DATATYPE_DEF, MAP_HDMF_DATATYPE_INC +from nwb_linkml.src.nwb_linkml.maps import QUANTITY_MAP \ No newline at end of file diff --git a/nwb_linkml/maps/dtype.py b/nwb_linkml/src/nwb_linkml/maps/dtype.py similarity index 100% rename from nwb_linkml/maps/dtype.py rename to nwb_linkml/src/nwb_linkml/maps/dtype.py diff --git a/nwb_linkml/maps/preload.py b/nwb_linkml/src/nwb_linkml/maps/preload.py similarity index 87% rename from nwb_linkml/maps/preload.py rename to nwb_linkml/src/nwb_linkml/maps/preload.py index 9fdd849..b166055 100644 --- a/nwb_linkml/maps/preload.py +++ b/nwb_linkml/src/nwb_linkml/maps/preload.py @@ -2,7 +2,7 @@ Maps to change the loaded .yaml from nwb schema before it's given to the nwb_schema_language models """ -from nwb_linkml.map import KeyMap, SCOPE_TYPES, PHASES +from nwb_linkml.src.nwb_linkml.map import KeyMap, SCOPE_TYPES, PHASES MAP_HDMF_DATATYPE_DEF = KeyMap( source="\'data_type_def\'", diff --git a/nwb_linkml/maps/quantity.py b/nwb_linkml/src/nwb_linkml/maps/quantity.py similarity index 100% rename from nwb_linkml/maps/quantity.py rename to nwb_linkml/src/nwb_linkml/maps/quantity.py diff --git a/nwb_linkml/models/core.py b/nwb_linkml/src/nwb_linkml/models/core.py similarity index 100% rename from nwb_linkml/models/core.py rename to nwb_linkml/src/nwb_linkml/models/core.py diff --git a/nwb_linkml/models/core_nwb_base.py b/nwb_linkml/src/nwb_linkml/models/core_nwb_base.py similarity index 100% rename from nwb_linkml/models/core_nwb_base.py rename to nwb_linkml/src/nwb_linkml/models/core_nwb_base.py diff --git a/nwb_linkml/models/core_nwb_behavior.py b/nwb_linkml/src/nwb_linkml/models/core_nwb_behavior.py similarity index 100% rename from nwb_linkml/models/core_nwb_behavior.py rename to nwb_linkml/src/nwb_linkml/models/core_nwb_behavior.py diff --git a/nwb_linkml/models/core_nwb_device.py b/nwb_linkml/src/nwb_linkml/models/core_nwb_device.py similarity index 100% rename from nwb_linkml/models/core_nwb_device.py rename to nwb_linkml/src/nwb_linkml/models/core_nwb_device.py diff --git a/nwb_linkml/models/core_nwb_ecephys.py b/nwb_linkml/src/nwb_linkml/models/core_nwb_ecephys.py similarity index 100% rename from nwb_linkml/models/core_nwb_ecephys.py rename to nwb_linkml/src/nwb_linkml/models/core_nwb_ecephys.py diff --git a/nwb_linkml/models/core_nwb_epoch.py b/nwb_linkml/src/nwb_linkml/models/core_nwb_epoch.py similarity index 100% rename from nwb_linkml/models/core_nwb_epoch.py rename to nwb_linkml/src/nwb_linkml/models/core_nwb_epoch.py diff --git a/nwb_linkml/models/core_nwb_file.py b/nwb_linkml/src/nwb_linkml/models/core_nwb_file.py similarity index 100% rename from nwb_linkml/models/core_nwb_file.py rename to nwb_linkml/src/nwb_linkml/models/core_nwb_file.py diff --git a/nwb_linkml/models/core_nwb_icephys.py b/nwb_linkml/src/nwb_linkml/models/core_nwb_icephys.py similarity index 100% rename from nwb_linkml/models/core_nwb_icephys.py rename to nwb_linkml/src/nwb_linkml/models/core_nwb_icephys.py diff --git a/nwb_linkml/models/core_nwb_image.py b/nwb_linkml/src/nwb_linkml/models/core_nwb_image.py similarity index 100% rename from nwb_linkml/models/core_nwb_image.py rename to nwb_linkml/src/nwb_linkml/models/core_nwb_image.py diff --git a/nwb_linkml/models/core_nwb_misc.py b/nwb_linkml/src/nwb_linkml/models/core_nwb_misc.py similarity index 100% rename from nwb_linkml/models/core_nwb_misc.py rename to nwb_linkml/src/nwb_linkml/models/core_nwb_misc.py diff --git a/nwb_linkml/models/core_nwb_ogen.py b/nwb_linkml/src/nwb_linkml/models/core_nwb_ogen.py similarity index 100% rename from nwb_linkml/models/core_nwb_ogen.py rename to nwb_linkml/src/nwb_linkml/models/core_nwb_ogen.py diff --git a/nwb_linkml/models/core_nwb_ophys.py b/nwb_linkml/src/nwb_linkml/models/core_nwb_ophys.py similarity index 100% rename from nwb_linkml/models/core_nwb_ophys.py rename to nwb_linkml/src/nwb_linkml/models/core_nwb_ophys.py diff --git a/nwb_linkml/models/core_nwb_retinotopy.py b/nwb_linkml/src/nwb_linkml/models/core_nwb_retinotopy.py similarity index 100% rename from nwb_linkml/models/core_nwb_retinotopy.py rename to nwb_linkml/src/nwb_linkml/models/core_nwb_retinotopy.py diff --git a/nwb_linkml/models/hdmf_common.py b/nwb_linkml/src/nwb_linkml/models/hdmf_common.py similarity index 100% rename from nwb_linkml/models/hdmf_common.py rename to nwb_linkml/src/nwb_linkml/models/hdmf_common.py diff --git a/nwb_linkml/models/hdmf_common_base.py b/nwb_linkml/src/nwb_linkml/models/hdmf_common_base.py similarity index 100% rename from nwb_linkml/models/hdmf_common_base.py rename to nwb_linkml/src/nwb_linkml/models/hdmf_common_base.py diff --git a/nwb_linkml/models/hdmf_common_sparse.py b/nwb_linkml/src/nwb_linkml/models/hdmf_common_sparse.py similarity index 100% rename from nwb_linkml/models/hdmf_common_sparse.py rename to nwb_linkml/src/nwb_linkml/models/hdmf_common_sparse.py diff --git a/nwb_linkml/models/hdmf_common_table.py b/nwb_linkml/src/nwb_linkml/models/hdmf_common_table.py similarity index 100% rename from nwb_linkml/models/hdmf_common_table.py rename to nwb_linkml/src/nwb_linkml/models/hdmf_common_table.py diff --git a/nwb_linkml/models/hdmf_experimental.py b/nwb_linkml/src/nwb_linkml/models/hdmf_experimental.py similarity index 100% rename from nwb_linkml/models/hdmf_experimental.py rename to nwb_linkml/src/nwb_linkml/models/hdmf_experimental.py diff --git a/nwb_linkml/models/hdmf_experimental_experimental.py b/nwb_linkml/src/nwb_linkml/models/hdmf_experimental_experimental.py similarity index 100% rename from nwb_linkml/models/hdmf_experimental_experimental.py rename to nwb_linkml/src/nwb_linkml/models/hdmf_experimental_experimental.py diff --git a/nwb_linkml/models/hdmf_experimental_resources.py b/nwb_linkml/src/nwb_linkml/models/hdmf_experimental_resources.py similarity index 100% rename from nwb_linkml/models/hdmf_experimental_resources.py rename to nwb_linkml/src/nwb_linkml/models/hdmf_experimental_resources.py diff --git a/nwb_linkml/models/nwb_language.py b/nwb_linkml/src/nwb_linkml/models/nwb_language.py similarity index 100% rename from nwb_linkml/models/nwb_language.py rename to nwb_linkml/src/nwb_linkml/models/nwb_language.py diff --git a/nwb_linkml/namespaces.py b/nwb_linkml/src/nwb_linkml/namespaces.py similarity index 100% rename from nwb_linkml/namespaces.py rename to nwb_linkml/src/nwb_linkml/namespaces.py diff --git a/nwb_linkml/plot.py b/nwb_linkml/src/nwb_linkml/plot.py similarity index 97% rename from nwb_linkml/plot.py rename to nwb_linkml/src/nwb_linkml/plot.py index cd451d2..f52f874 100644 --- a/nwb_linkml/plot.py +++ b/nwb_linkml/src/nwb_linkml/plot.py @@ -1,7 +1,7 @@ """ Various visualization routines, mostly to help development for now """ -from typing import TYPE_CHECKING, Optional, List, TypedDict, Union +from typing import TYPE_CHECKING, Optional, List, TypedDict from rich import print import random @@ -10,7 +10,7 @@ import dash_cytoscape as cyto cyto.load_extra_layouts() from nwb_schema_language import Namespace, Group, Dataset -from nwb_linkml.io import load_nwb_core +from nwb_linkml.src.nwb_linkml.io import load_nwb_core if TYPE_CHECKING: from nwb_linkml.adapters import NamespacesAdapter diff --git a/nwb_linkml/schema/core.nwb.base.yaml b/nwb_linkml/src/nwb_linkml/schema/core.nwb.base.yaml similarity index 100% rename from nwb_linkml/schema/core.nwb.base.yaml rename to nwb_linkml/src/nwb_linkml/schema/core.nwb.base.yaml diff --git a/nwb_linkml/schema/core.nwb.behavior.yaml b/nwb_linkml/src/nwb_linkml/schema/core.nwb.behavior.yaml similarity index 100% rename from nwb_linkml/schema/core.nwb.behavior.yaml rename to nwb_linkml/src/nwb_linkml/schema/core.nwb.behavior.yaml diff --git a/nwb_linkml/schema/core.nwb.device.yaml b/nwb_linkml/src/nwb_linkml/schema/core.nwb.device.yaml similarity index 100% rename from nwb_linkml/schema/core.nwb.device.yaml rename to nwb_linkml/src/nwb_linkml/schema/core.nwb.device.yaml diff --git a/nwb_linkml/schema/core.nwb.ecephys.yaml b/nwb_linkml/src/nwb_linkml/schema/core.nwb.ecephys.yaml similarity index 100% rename from nwb_linkml/schema/core.nwb.ecephys.yaml rename to nwb_linkml/src/nwb_linkml/schema/core.nwb.ecephys.yaml diff --git a/nwb_linkml/schema/core.nwb.epoch.yaml b/nwb_linkml/src/nwb_linkml/schema/core.nwb.epoch.yaml similarity index 100% rename from nwb_linkml/schema/core.nwb.epoch.yaml rename to nwb_linkml/src/nwb_linkml/schema/core.nwb.epoch.yaml diff --git a/nwb_linkml/schema/core.nwb.file.yaml b/nwb_linkml/src/nwb_linkml/schema/core.nwb.file.yaml similarity index 100% rename from nwb_linkml/schema/core.nwb.file.yaml rename to nwb_linkml/src/nwb_linkml/schema/core.nwb.file.yaml diff --git a/nwb_linkml/schema/core.nwb.icephys.yaml b/nwb_linkml/src/nwb_linkml/schema/core.nwb.icephys.yaml similarity index 100% rename from nwb_linkml/schema/core.nwb.icephys.yaml rename to nwb_linkml/src/nwb_linkml/schema/core.nwb.icephys.yaml diff --git a/nwb_linkml/schema/core.nwb.image.yaml b/nwb_linkml/src/nwb_linkml/schema/core.nwb.image.yaml similarity index 100% rename from nwb_linkml/schema/core.nwb.image.yaml rename to nwb_linkml/src/nwb_linkml/schema/core.nwb.image.yaml diff --git a/nwb_linkml/schema/core.nwb.misc.yaml b/nwb_linkml/src/nwb_linkml/schema/core.nwb.misc.yaml similarity index 100% rename from nwb_linkml/schema/core.nwb.misc.yaml rename to nwb_linkml/src/nwb_linkml/schema/core.nwb.misc.yaml diff --git a/nwb_linkml/schema/core.nwb.ogen.yaml b/nwb_linkml/src/nwb_linkml/schema/core.nwb.ogen.yaml similarity index 100% rename from nwb_linkml/schema/core.nwb.ogen.yaml rename to nwb_linkml/src/nwb_linkml/schema/core.nwb.ogen.yaml diff --git a/nwb_linkml/schema/core.nwb.ophys.yaml b/nwb_linkml/src/nwb_linkml/schema/core.nwb.ophys.yaml similarity index 100% rename from nwb_linkml/schema/core.nwb.ophys.yaml rename to nwb_linkml/src/nwb_linkml/schema/core.nwb.ophys.yaml diff --git a/nwb_linkml/schema/core.nwb.retinotopy.yaml b/nwb_linkml/src/nwb_linkml/schema/core.nwb.retinotopy.yaml similarity index 100% rename from nwb_linkml/schema/core.nwb.retinotopy.yaml rename to nwb_linkml/src/nwb_linkml/schema/core.nwb.retinotopy.yaml diff --git a/nwb_linkml/schema/core.yaml b/nwb_linkml/src/nwb_linkml/schema/core.yaml similarity index 100% rename from nwb_linkml/schema/core.yaml rename to nwb_linkml/src/nwb_linkml/schema/core.yaml diff --git a/nwb_linkml/schema/hdmf-common.base.yaml b/nwb_linkml/src/nwb_linkml/schema/hdmf-common.base.yaml similarity index 100% rename from nwb_linkml/schema/hdmf-common.base.yaml rename to nwb_linkml/src/nwb_linkml/schema/hdmf-common.base.yaml diff --git a/nwb_linkml/schema/hdmf-common.sparse.yaml b/nwb_linkml/src/nwb_linkml/schema/hdmf-common.sparse.yaml similarity index 100% rename from nwb_linkml/schema/hdmf-common.sparse.yaml rename to nwb_linkml/src/nwb_linkml/schema/hdmf-common.sparse.yaml diff --git a/nwb_linkml/schema/hdmf-common.table.yaml b/nwb_linkml/src/nwb_linkml/schema/hdmf-common.table.yaml similarity index 100% rename from nwb_linkml/schema/hdmf-common.table.yaml rename to nwb_linkml/src/nwb_linkml/schema/hdmf-common.table.yaml diff --git a/nwb_linkml/schema/hdmf-common.yaml b/nwb_linkml/src/nwb_linkml/schema/hdmf-common.yaml similarity index 100% rename from nwb_linkml/schema/hdmf-common.yaml rename to nwb_linkml/src/nwb_linkml/schema/hdmf-common.yaml diff --git a/nwb_linkml/schema/hdmf-experimental.experimental.yaml b/nwb_linkml/src/nwb_linkml/schema/hdmf-experimental.experimental.yaml similarity index 100% rename from nwb_linkml/schema/hdmf-experimental.experimental.yaml rename to nwb_linkml/src/nwb_linkml/schema/hdmf-experimental.experimental.yaml diff --git a/nwb_linkml/schema/hdmf-experimental.resources.yaml b/nwb_linkml/src/nwb_linkml/schema/hdmf-experimental.resources.yaml similarity index 100% rename from nwb_linkml/schema/hdmf-experimental.resources.yaml rename to nwb_linkml/src/nwb_linkml/schema/hdmf-experimental.resources.yaml diff --git a/nwb_linkml/schema/hdmf-experimental.yaml b/nwb_linkml/src/nwb_linkml/schema/hdmf-experimental.yaml similarity index 100% rename from nwb_linkml/schema/hdmf-experimental.yaml rename to nwb_linkml/src/nwb_linkml/schema/hdmf-experimental.yaml diff --git a/nwb_linkml/schema/nwb.language.yaml b/nwb_linkml/src/nwb_linkml/schema/nwb.language.yaml similarity index 100% rename from nwb_linkml/schema/nwb.language.yaml rename to nwb_linkml/src/nwb_linkml/schema/nwb.language.yaml diff --git a/nwb_linkml/translate.py b/nwb_linkml/src/nwb_linkml/translate.py similarity index 94% rename from nwb_linkml/translate.py rename to nwb_linkml/src/nwb_linkml/translate.py index 0cfe8e4..b6258c9 100644 --- a/nwb_linkml/translate.py +++ b/nwb_linkml/src/nwb_linkml/translate.py @@ -1,7 +1,6 @@ """ Convenience functions for translating NWB schema """ -import pdb import tempfile from typing import List, Optional, Dict from types import ModuleType @@ -10,17 +9,12 @@ import json import h5py -from linkml_runtime import SchemaView -from linkml_runtime.linkml_model import SchemaDefinition from linkml_runtime.dumpers import yaml_dumper -from linkml_runtime.utils.compile_python import compile_python - - from nwb_schema_language import Namespaces -from nwb_linkml.io import load_schema_file +from nwb_linkml.src.nwb_linkml.io import load_schema_file from nwb_linkml.generators.pydantic import NWBPydanticGenerator -from nwb_linkml.map import apply_preload +from nwb_linkml.src.nwb_linkml.map import apply_preload from nwb_linkml.adapters import SchemaAdapter, NamespacesAdapter def make_namespace_adapter(schema: dict) -> NamespacesAdapter: diff --git a/nwb_linkml/tests/__init__.py b/nwb_linkml/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures.py b/nwb_linkml/tests/fixtures.py similarity index 75% rename from tests/fixtures.py rename to nwb_linkml/tests/fixtures.py index 363a71f..bdc18f5 100644 --- a/tests/fixtures.py +++ b/nwb_linkml/tests/fixtures.py @@ -1,9 +1,7 @@ import pytest -from typing import Dict - -from nwb_linkml import io -from nwb_linkml.adapters.namespaces import NamespacesAdapter +from nwb_linkml.src.nwb_linkml import io +from nwb_linkml.src.nwb_linkml.adapters.namespaces import NamespacesAdapter import shutil from pathlib import Path diff --git a/nwb_linkml/tests/test_adapters/__init__.py b/nwb_linkml/tests/test_adapters/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_adapter.py b/nwb_linkml/tests/test_adapters/test_adapter.py similarity index 100% rename from tests/test_adapter.py rename to nwb_linkml/tests/test_adapters/test_adapter.py diff --git a/tests/test_adapter_dataset.py b/nwb_linkml/tests/test_adapters/test_adapter_dataset.py similarity index 100% rename from tests/test_adapter_dataset.py rename to nwb_linkml/tests/test_adapters/test_adapter_dataset.py diff --git a/tests/test_adapter_group.py b/nwb_linkml/tests/test_adapters/test_adapter_group.py similarity index 100% rename from tests/test_adapter_group.py rename to nwb_linkml/tests/test_adapters/test_adapter_group.py diff --git a/tests/test_adapter_namespaces.py b/nwb_linkml/tests/test_adapters/test_adapter_namespaces.py similarity index 100% rename from tests/test_adapter_namespaces.py rename to nwb_linkml/tests/test_adapters/test_adapter_namespaces.py diff --git a/tests/test_adapter_schema.py b/nwb_linkml/tests/test_adapters/test_adapter_schema.py similarity index 100% rename from tests/test_adapter_schema.py rename to nwb_linkml/tests/test_adapters/test_adapter_schema.py diff --git a/tests/test_generate.py b/nwb_linkml/tests/test_generate.py similarity index 100% rename from tests/test_generate.py rename to nwb_linkml/tests/test_generate.py diff --git a/tests/test_translate.py b/nwb_linkml/tests/test_translate.py similarity index 99% rename from tests/test_translate.py rename to nwb_linkml/tests/test_translate.py index f403555..05f4013 100644 --- a/tests/test_translate.py +++ b/nwb_linkml/tests/test_translate.py @@ -13,3 +13,4 @@ def test_generate_pydantic(): #pydantic_module = generate_from_nwbfile(NWBFILE) #pdb.set_trace() + diff --git a/pytest.ini b/pytest.ini index 133ec0e..36c65df 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,4 @@ [pytest] testpaths = - tests \ No newline at end of file + nwb_linkml/tests \ No newline at end of file diff --git a/scripts/generate_core.py b/scripts/generate_core.py index 03367e1..d850591 100644 --- a/scripts/generate_core.py +++ b/scripts/generate_core.py @@ -3,7 +3,8 @@ from pathlib import Path from linkml_runtime.dumpers import yaml_dumper from nwb_linkml.generators.pydantic import NWBPydanticGenerator -from nwb_linkml import io +from nwb_linkml.src.nwb_linkml import io + def generate_core_yaml(output_path:Path): core = io.load_nwb_core() @@ -34,13 +35,13 @@ def parser() -> ArgumentParser: '--yaml', help="directory to export linkML schema to", type=Path, - default=Path(__file__).parent.parent / 'nwb_linkml' / 'schema' + default=Path(__file__).parent.parent / 'nwb_linkml' / 'src' / 'nwb_linkml' / 'schema' ) parser.add_argument( '--pydantic', help="directory to export pydantic models", type=Path, - default=Path(__file__).parent.parent / 'nwb_linkml' / 'models' + default=Path(__file__).parent.parent / 'nwb_linkml' / 'src' / 'nwb_linkml' / 'models' ) return parser