refactor nwb_linkml to src layout

.github/workflows/tests.yml

@@ -21,8 +21,10 @@ jobs:
 
       - name: Install dependencies
         run: pip install .[tests] pytest-md pytest-emoji
+        working-directory: nwb_linkml
 
       - uses: pavelzw/pytest-action@v2
+        working-directory: nwb_linkml
         with:
           emoji: true
           verbose: true

hdf5_linkml/tests/test_scratch.py

@@ -0,0 +1,2 @@
+
+

nwb_linkml/__init__.py

@@ -1 +0,0 @@
-from nwb_linkml.maps import preload

nwb_linkml/adapters/__init__.py

@@ -1,4 +0,0 @@
-from nwb_linkml.adapters.adapter import Adapter
-from nwb_linkml.adapters.namespaces import NamespacesAdapter
-from nwb_linkml.adapters.schema import SchemaAdapter
-from nwb_linkml.adapters.classes import ClassAdapter

nwb_linkml/maps/__init__.py

@@ -1,3 +0,0 @@
-# Import everything so it's defined, but shoudlnt' necessarily be used from here
-from nwb_linkml.maps.preload import MAP_HDMF_DATATYPE_DEF, MAP_HDMF_DATATYPE_INC
-from nwb_linkml.maps.quantity import QUANTITY_MAP

nwb_linkml/pyproject.toml

@@ -6,8 +6,8 @@ authors = ["sneakers-the-rat <JLSaunders987@gmail.com>"]
 license = "GPL-3.0"
 readme = "README.md"
 packages = [
-    {include = "nwb_linkml"},
-    {include = "nwb_schema_language", from="nwb_schema_language/src"}
+    {include = "nwb_linkml", from="src"},
+    {include = "nwb_schema_language", from="../nwb_schema_language/src"}
 ]
 
 [tool.poetry.dependencies]

nwb_linkml/src/nwb_linkml/__init__.py

@@ -0,0 +1 @@
+from nwb_linkml.src.nwb_linkml.maps import preload

nwb_linkml/src/nwb_linkml/adapters/__init__.py

@@ -0,0 +1 @@
+from nwb_linkml.src.nwb_linkml.adapters.namespaces import NamespacesAdapter

nwb_linkml/src/nwb_linkml/adapters/classes.py

@@ -1,15 +1,13 @@
 """
 Adapters to linkML classes
 """
-import pdb
 import re
 from abc import abstractmethod
 from typing import List, Optional
 from nwb_schema_language import Dataset, Group, ReferenceDtype, CompoundDtype, DTypeType
 from nwb_linkml.adapters.adapter import Adapter, BuildResult
 from linkml_runtime.linkml_model import ClassDefinition, SlotDefinition
-from nwb_linkml.maps import QUANTITY_MAP
-from nwb_linkml.lang_elements import Arraylike
+from nwb_linkml.src.nwb_linkml.maps import QUANTITY_MAP
 
 CAMEL_TO_SNAKE = re.compile('((?<=[a-z0-9])[A-Z]|(?!^)[A-Z](?=[a-z]))')
 """

nwb_linkml/src/nwb_linkml/adapters/dataset.py

@@ -11,7 +11,7 @@ from pydantic import PrivateAttr
 from nwb_schema_language import Dataset, ReferenceDtype, CompoundDtype, DTypeType
 from nwb_linkml.adapters.classes import ClassAdapter, camel_to_snake
 from nwb_linkml.adapters.adapter import BuildResult
-from nwb_linkml.maps import QUANTITY_MAP
+from nwb_linkml.src.nwb_linkml.maps import QUANTITY_MAP
 
 class DatasetAdapter(ClassAdapter):
     cls: Dataset

nwb_linkml/src/nwb_linkml/adapters/group.py

@@ -9,7 +9,7 @@ from nwb_schema_language import Dataset, Group, ReferenceDtype, CompoundDtype, D
 from nwb_linkml.adapters.classes import ClassAdapter, camel_to_snake
 from nwb_linkml.adapters.dataset import DatasetAdapter
 from nwb_linkml.adapters.adapter import BuildResult
-from nwb_linkml.maps import QUANTITY_MAP
+from nwb_linkml.src.nwb_linkml.maps import QUANTITY_MAP
 
 class GroupAdapter(ClassAdapter):
     cls: Group

nwb_linkml/src/nwb_linkml/adapters/schema.py

@@ -2,7 +2,6 @@
 Since NWB doesn't necessarily have a term for a single nwb schema file, we're going
 to call them "schema" objects
 """
-import pdb
 from typing import Optional, List, TYPE_CHECKING, Type
 from pathlib import Path
 from pydantic import Field, PrivateAttr
@@ -11,7 +10,7 @@ from nwb_linkml.adapters.adapter import Adapter, BuildResult
 from nwb_linkml.adapters.dataset import DatasetAdapter
 from nwb_linkml.adapters.group import GroupAdapter
 if TYPE_CHECKING:
-    from nwb_linkml.adapters.namespaces import NamespacesAdapter
+    pass
 
 from nwb_schema_language import Group, Dataset
 from typing import NamedTuple

nwb_linkml/src/nwb_linkml/generators/pydantic.py

@@ -25,7 +25,7 @@ from types import ModuleType
 from copy import deepcopy
 import warnings
 
-from nwb_linkml.maps.dtype import flat_to_npytyping
+from nwb_linkml.src.nwb_linkml.maps import flat_to_npytyping
 from linkml.generators import PydanticGenerator
 from linkml_runtime.linkml_model.meta import (
     Annotation,

nwb_linkml/src/nwb_linkml/io/hdf5.py

@@ -0,0 +1,61 @@
+"""
+This is a sandbox file that should be split out to its own pydantic-hdf5 package, but just experimenting here to get our bearings
+"""
+from typing import Optional, List, Dict
+from pathlib import Path
+from types import ModuleType
+
+import h5py
+
+from nwb_linkml.translate import generate_from_nwbfile
+
+class HDF5IO():
+
+    def __init__(self, path:Path):
+        self.path = Path(path)
+        self._modules: Dict[str, ModuleType] = {}
+
+    @property
+    def modules(self) -> Dict[str, ModuleType]:
+        if len(self._modules) == 0:
+            self._modules = generate_from_nwbfile(self.path)
+        return self._modules
+
+    def process_group(self, group:h5py.Group|h5py.File) -> dict | list:
+        attrs = dict(group.attrs)
+
+        # how to process the group?
+        # --------------------------------------------------
+        # list-like
+        # --------------------------------------------------
+        # a list of data classes
+        if 'neurodata_type' not in attrs and \
+            all([isinstance(v, h5py.Group) for v in group.values()]) and \
+            all(['neurodata_type' in v.attrs for v in group.values()]):
+
+            return [self.process_group(v) for v in group.values()]
+
+        # --------------------------------------------------
+        # dict-like
+        # --------------------------------------------------
+
+        res = {}
+
+
+        for key, val in group.items():
+            if isinstance(val, h5py.Group):
+                res[key] = self.process_group(val)
+            elif isinstance(val, h5py.Dataset):
+                res[key] = self.process_dataset(val)
+        return res
+
+    def process_dataset(self, data: h5py.Dataset) -> dict | list:
+        if len(data.shape) == 1:
+            return list(data[:])
+
+
+if __name__ == "__main__":
+    NWBFILE = Path('/Users/jonny/Dropbox/lab/p2p_ld/data/nwb/sub-738651046_ses-760693773.nwb')
+    h5f = HDF5IO(NWBFILE)
+
+

nwb_linkml/src/nwb_linkml/io/io.py

@@ -1,9 +1,8 @@
 """
 Loading/saving NWB Schema yaml files
 """
-import pdb
 from pathlib import Path
-from typing import TypedDict, List, Dict, Optional
+from typing import Optional
 from pprint import pprint
 import warnings
 
@@ -12,9 +11,8 @@ import yaml
 
 from nwb_schema_language import Namespaces,  Group, Dataset
 from nwb_linkml.namespaces import NamespaceRepo, NWB_CORE_REPO, HDMF_COMMON_REPO
-from nwb_linkml.maps import preload
-from nwb_linkml.map import PHASES, Map
-from nwb_linkml.adapters.namespaces import NamespacesAdapter
+from nwb_linkml.src.nwb_linkml.map import PHASES, Map
+from nwb_linkml.src.nwb_linkml.adapters.namespaces import NamespacesAdapter
 from nwb_linkml.adapters.schema import SchemaAdapter
 
 
@@ -70,7 +68,7 @@ def load_schema_file(path:Path, yaml:Optional[dict] = None) -> SchemaAdapter:
     )
     return schema
 
-def load_namespace_schema(namespace: Namespaces, path:Path=Path('.')) -> NamespacesAdapter:
+def load_namespace_schema(namespace: Namespaces, path:Path=Path('..')) -> NamespacesAdapter:
     """
     Load all schema referenced by a namespace file
 

nwb_linkml/src/nwb_linkml/lang_elements.py

@@ -12,7 +12,7 @@ from linkml_runtime.linkml_model import \
     TypeDefinition,\
     Prefix,\
     PermissibleValue
-from nwb_linkml.maps.dtype import flat_to_linkml
+from nwb_linkml.src.nwb_linkml.maps import flat_to_linkml
 
 
 FlatDType = EnumDefinition(

nwb_linkml/src/nwb_linkml/map.py

@@ -66,7 +66,6 @@ class KeyMap(Map):
 
 
 def apply_preload(ns_dict) -> dict:
-    from nwb_linkml.maps import preload
     maps = [m for m in Map.instances if m.phase == PHASES.postload]
     for amap in maps:
         ns_dict = amap.apply(ns_dict)

nwb_linkml/src/nwb_linkml/maps/__init__.py

@@ -0,0 +1,3 @@
+# Import everything so it's defined, but shoudlnt' necessarily be used from here
+from nwb_linkml.src.nwb_linkml.maps import MAP_HDMF_DATATYPE_DEF, MAP_HDMF_DATATYPE_INC
+from nwb_linkml.src.nwb_linkml.maps import QUANTITY_MAP

nwb_linkml/src/nwb_linkml/maps/preload.py

@@ -2,7 +2,7 @@
 Maps to change the loaded .yaml from nwb schema before it's given to the nwb_schema_language models
 """
 
-from nwb_linkml.map import KeyMap, SCOPE_TYPES, PHASES
+from nwb_linkml.src.nwb_linkml.map import KeyMap, SCOPE_TYPES, PHASES
 
 MAP_HDMF_DATATYPE_DEF = KeyMap(
     source="\'data_type_def\'",

nwb_linkml/src/nwb_linkml/plot.py

@@ -1,7 +1,7 @@
 """
 Various visualization routines, mostly to help development for now
 """
-from typing import TYPE_CHECKING, Optional, List, TypedDict, Union
+from typing import TYPE_CHECKING, Optional, List, TypedDict
 from rich import print
 import random
 
@@ -10,7 +10,7 @@ import dash_cytoscape as cyto
 cyto.load_extra_layouts()
 
 from nwb_schema_language import Namespace, Group, Dataset
-from nwb_linkml.io import load_nwb_core
+from nwb_linkml.src.nwb_linkml.io import load_nwb_core
 
 if TYPE_CHECKING:
     from nwb_linkml.adapters import NamespacesAdapter

nwb_linkml/src/nwb_linkml/translate.py

@@ -1,7 +1,6 @@
 """
 Convenience functions for translating NWB schema
 """
-import pdb
 import tempfile
 from typing import List, Optional, Dict
 from types import ModuleType
@@ -10,17 +9,12 @@ import json
 
 
 import h5py
-from linkml_runtime import SchemaView
-from linkml_runtime.linkml_model import SchemaDefinition
 from linkml_runtime.dumpers import yaml_dumper
-from linkml_runtime.utils.compile_python import compile_python
-
-
 
 from nwb_schema_language import Namespaces
-from nwb_linkml.io import load_schema_file
+from nwb_linkml.src.nwb_linkml.io import load_schema_file
 from nwb_linkml.generators.pydantic import NWBPydanticGenerator
-from nwb_linkml.map import apply_preload
+from nwb_linkml.src.nwb_linkml.map import apply_preload
 from nwb_linkml.adapters import SchemaAdapter, NamespacesAdapter
 
 def make_namespace_adapter(schema: dict) -> NamespacesAdapter:

nwb_linkml/tests/fixtures.py

@@ -1,9 +1,7 @@
 import pytest
-from typing import Dict
 
-
-from nwb_linkml import io
-from nwb_linkml.adapters.namespaces import NamespacesAdapter
+from nwb_linkml.src.nwb_linkml import io
+from nwb_linkml.src.nwb_linkml.adapters.namespaces import NamespacesAdapter
 import shutil
 from pathlib import Path
 

nwb_linkml/tests/test_translate.py

@@ -13,3 +13,4 @@ def test_generate_pydantic():
     #pydantic_module = generate_from_nwbfile(NWBFILE)
 
     #pdb.set_trace()
+

pytest.ini

@@ -1,4 +1,4 @@
 [pytest]
 
 testpaths =
-    tests
+    nwb_linkml/tests

scripts/generate_core.py

@@ -3,7 +3,8 @@ from pathlib import Path
 from linkml_runtime.dumpers import yaml_dumper
 from nwb_linkml.generators.pydantic import NWBPydanticGenerator
 
-from nwb_linkml import io
+from nwb_linkml.src.nwb_linkml import io
+
 
 def generate_core_yaml(output_path:Path):
     core = io.load_nwb_core()
@@ -34,13 +35,13 @@ def parser() -> ArgumentParser:
         '--yaml',
         help="directory to export linkML schema to",
         type=Path,
-        default=Path(__file__).parent.parent / 'nwb_linkml' / 'schema'
+        default=Path(__file__).parent.parent / 'nwb_linkml' / 'src' / 'nwb_linkml' / 'schema'
     )
     parser.add_argument(
         '--pydantic',
         help="directory to export pydantic models",
         type=Path,
-        default=Path(__file__).parent.parent / 'nwb_linkml' / 'models'
+        default=Path(__file__).parent.parent / 'nwb_linkml' / 'src' / 'nwb_linkml' / 'models'
     )
     return parser