diff --git a/nwb_linkml/src/nwb_linkml/config.py b/nwb_linkml/src/nwb_linkml/config.py index 0d55020..83c8298 100644 --- a/nwb_linkml/src/nwb_linkml/config.py +++ b/nwb_linkml/src/nwb_linkml/config.py @@ -33,6 +33,12 @@ class Config(BaseSettings): """Directory to store generated pydantic models""" return self.cache_dir / 'pydantic' + @computed_field + @property + def git_dir(self) -> Path: + """Directory for :class:`nwb_linkml.providers.git.GitRepo` to clone to""" + return self.cache_dir / 'git' + @field_validator('cache_dir', mode='before') @classmethod @@ -46,5 +52,6 @@ class Config(BaseSettings): self.cache_dir.mkdir(exist_ok=True) self.linkml_dir.mkdir(exist_ok=True) self.pydantic_dir.mkdir(exist_ok=True) + self.git_dir.mkdir(exist_ok=True) diff --git a/nwb_linkml/src/nwb_linkml/io/schema.py b/nwb_linkml/src/nwb_linkml/io/schema.py index b9d434d..b8ebf4a 100644 --- a/nwb_linkml/src/nwb_linkml/io/schema.py +++ b/nwb_linkml/src/nwb_linkml/io/schema.py @@ -87,7 +87,7 @@ def load_namespace_adapter( namespaces = _load_namespaces(path) elif isinstance(namespace, NamespaceRepo): path = namespace.provide_from_git(commit=version) - namespaces = _load_namespaces(namespace) + namespaces = _load_namespaces(path) elif isinstance(namespace, Namespaces): namespaces = namespace diff --git a/nwb_linkml/src/nwb_linkml/providers/git.py b/nwb_linkml/src/nwb_linkml/providers/git.py index 4cc3490..a542ddb 100644 --- a/nwb_linkml/src/nwb_linkml/providers/git.py +++ b/nwb_linkml/src/nwb_linkml/providers/git.py @@ -11,6 +11,8 @@ import shutil from pydantic import BaseModel, HttpUrl, FilePath, DirectoryPath, Field +from nwb_linkml.config import Config + class NamespaceRepo(BaseModel): """ Definition of one NWB namespaces file to import from a git repository @@ -55,11 +57,24 @@ class GitRepo: """ Manage a temporary git repository that provides the NWB yaml files """ - def __init__(self, namespace:NamespaceRepo, commit:str|None=None): - self._temp_directory = None + def __init__( + self, + namespace:NamespaceRepo, + commit:str|None=None, + path: Optional[Path] = None + ): + """ + Args: + namespace (:class:`.NamespaceRepo`): The namespace repository to clone! + commit (str): A specific commit or tag to check out + path (:class:`pathlib.Path`): A directory to clone to - if ``None``, use :attr:`~.Config.git_dir` / :attr:`.NamespaceRepo.name` + """ + self._temp_directory = path self.namespace = namespace self._commit = commit + + def _git_call(self, *args) -> subprocess.CompletedProcess: res = subprocess.run( ['git', '-C', self.temp_directory, *args], @@ -75,7 +90,8 @@ class GitRepo: Temporary directory where this repository will be cloned to """ if self._temp_directory is None: - self._temp_directory = Path(tempfile.gettempdir()) / f'nwb_linkml__{self.namespace.name}' + git_dir = Config().git_dir + self._temp_directory = git_dir / self.namespace.name if not self._temp_directory.exists(): self._temp_directory.mkdir(parents=True) @@ -222,12 +238,20 @@ class GitRepo: # otherwise we're good return True - def cleanup(self): + def cleanup(self, force: bool=False): """ Delete contents of temporary directory + + If the temp_directory is outside the system temporary directory or + + Args: + force (bool): If ``True``, remove git directory no matter where it is """ - if not str(self.temp_directory).startswith(tempfile.gettempdir()): - warnings.warn('Temp directory is outside of the system temp dir, not deleting in case this has been changed by mistake') + if not force and not ( + str(self.temp_directory).startswith(tempfile.gettempdir()) or + str(self.temp_directory).startswith(str(Config().git_dir)) + ): + warnings.warn('Temp directory is outside of the system temp dir or git directory set by environmental variables, not deleting in case this has been changed by mistake') self._temp_directory = None return diff --git a/nwb_linkml/tests/conftest.py b/nwb_linkml/tests/conftest.py new file mode 100644 index 0000000..7e2d2e1 --- /dev/null +++ b/nwb_linkml/tests/conftest.py @@ -0,0 +1,8 @@ +import os +import pytest + +from .fixtures import tmp_output_dir + +@pytest.fixture(autouse=True, scope='session') +def set_config_vars(tmp_output_dir): + os.environ['NWB_LINKML_CACHE_DIR'] = str(tmp_output_dir) diff --git a/nwb_linkml/tests/fixtures.py b/nwb_linkml/tests/fixtures.py index 5c529d8..55d6dc9 100644 --- a/nwb_linkml/tests/fixtures.py +++ b/nwb_linkml/tests/fixtures.py @@ -45,9 +45,6 @@ def tmp_output_dir_mod(tmp_output_dir) -> Path: subpath.mkdir() return subpath -@pytest.fixture(autouse=True, scope='session') -def set_config_vars(tmp_output_dir): - os.environ['NWB_LINKML_CACHE_DIR'] = str(tmp_output_dir) @@ -59,6 +56,9 @@ def set_config_vars(tmp_output_dir): def nwb_core_fixture(request) -> NamespacesAdapter: nwb_core = io.load_nwb_core(**request.param) nwb_core.populate_imports() + assert request.param['core_version'] in nwb_core.versions['core'] # 2.6.0 is actually 2.6.0-alpha + assert nwb_core.versions['hdmf-common'] == request.param['hdmf_version'] + return nwb_core diff --git a/nwb_linkml/tests/test_adapters/test_adapter.py b/nwb_linkml/tests/test_adapters/test_adapter.py index d5e6307..6e09c24 100644 --- a/nwb_linkml/tests/test_adapters/test_adapter.py +++ b/nwb_linkml/tests/test_adapters/test_adapter.py @@ -16,14 +16,19 @@ def test_walk(nwb_core_fixture): Not sure exactly what should be tested here, for now just testing that we get an expected value """ everything = nwb_core_fixture.walk(nwb_core_fixture) - assert len(list(everything)) == 9959 + + # number of items obviously changes based on what version we're talking about + # this is configured as a test matrix on the nwb_core_fixture, currently only testing the latest version + if nwb_core_fixture.versions['core'] == '2.6.0-alpha' and nwb_core_fixture.versions['hdmf-common'] == '1.5.0': + + assert len(list(everything)) == 9908 @pytest.mark.parametrize( ['walk_class', 'known_number'], [ - (Dataset, 211), + (Dataset, 210), (Group, 144), - ((Dataset, Group), 355), + ((Dataset, Group), 354), (Schema, 19) ] ) @@ -48,7 +53,7 @@ def test_walk_field_values(nwb_core_fixture): text_models = list(nwb_core_fixture.walk_field_values(nwb_core_fixture, 'dtype', value='text')) assert all([d.dtype == 'text' for d in text_models]) # 135 known value from regex search - assert len(text_models) == len([d for d in dtype_models if d.dtype == 'text']) == 135 + assert len(text_models) == len([d for d in dtype_models if d.dtype == 'text']) == 134 def test_build_result(linkml_schema_bare): diff --git a/nwb_linkml/tests/test_config.py b/nwb_linkml/tests/test_config.py new file mode 100644 index 0000000..088270f --- /dev/null +++ b/nwb_linkml/tests/test_config.py @@ -0,0 +1,37 @@ +import pytest +import tempfile +from pathlib import Path +import os +import shutil + +from nwb_linkml.config import Config + + +def test_config_dir(): + """Ensure that the temporary directory is the same across multiple instantiations of the singleton-like config object""" + c1 = Config() + c2 = Config() + assert c1.cache_dir == c2.cache_dir + +def test_config_env(): + """ + Base cache dir can be overridden by environmental variable + """ + + orig_env = os.environ['NWB_LINKML_CACHE_DIR'] + + new_temp = Path(tempfile.gettempdir()) / 'test_tmp_dir' + new_temp.mkdir() + + try: + assert Path(orig_env) != new_temp + + os.environ['NWB_LINKML_CACHE_DIR'] = str(new_temp) + + conf = Config() + + assert conf.cache_dir == new_temp + finally: + shutil.rmtree(new_temp) + os.environ['NWB_LINKML_CACHE_DIR'] = orig_env + diff --git a/nwb_linkml/tests/test_io/test_io_hdf5.py b/nwb_linkml/tests/test_io/test_io_hdf5.py index 47ac3b7..fe21bbf 100644 --- a/nwb_linkml/tests/test_io/test_io_hdf5.py +++ b/nwb_linkml/tests/test_io/test_io_hdf5.py @@ -5,7 +5,7 @@ import pytest from pathlib import Path import numpy as np -from ..fixtures import tmp_output_dir, set_config_vars, data_dir +from ..fixtures import tmp_output_dir, data_dir from nwb_linkml.io.hdf5 import HDF5IO from nwb_linkml.io.hdf5 import truncate_file diff --git a/nwb_linkml/tests/test_providers/test_provider_schema.py b/nwb_linkml/tests/test_providers/test_provider_schema.py index afe0159..2d8ae5c 100644 --- a/nwb_linkml/tests/test_providers/test_provider_schema.py +++ b/nwb_linkml/tests/test_providers/test_provider_schema.py @@ -6,7 +6,7 @@ import warnings from pathlib import Path from typing import Optional, Union, List -from ..fixtures import tmp_output_dir, set_config_vars +from ..fixtures import tmp_output_dir import pytest