requests caching, python 10

.gitignore

@@ -164,4 +164,5 @@ __tmp__
 prof
 jupyter_execute
 
-.pdm-python
+.pdm-python
+.venv*

nwb_linkml/pdm.lock

@@ -2,10 +2,10 @@
 # It is not intended for manual editing.
 
 [metadata]
-groups = ["default", "dev"]
+groups = ["default", "dev", "tests"]
 strategy = ["cross_platform", "inherit_metadata"]
 lock_version = "4.4.2"
-content_hash = "sha256:94d398adc70609c1254f79f19c1b8ee5feb0adfc6c38b58415f7f506b94e8782"
+content_hash = "sha256:8cb98f940354e71443df87fd1702300e1c793627b0a016a6233811a640f47c18"
 
 [[package]]
 name = "annotated-types"
@@ -47,7 +47,7 @@ name = "attrs"
 version = "23.2.0"
 requires_python = ">=3.7"
 summary = "Classes Without Boilerplate"
-groups = ["default", "dev"]
+groups = ["default", "dev", "tests"]
 files = [
     {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"},
     {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"},
@@ -123,12 +123,28 @@ files = [
     {file = "blosc2-2.7.0.tar.gz", hash = "sha256:9b982c1d40560eefb4a01d67c57e786d39a5ee9696f3deadd32ebf5f8885eb2a"},
 ]
 
+[[package]]
+name = "cattrs"
+version = "23.2.3"
+requires_python = ">=3.8"
+summary = "Composable complex class support for attrs and dataclasses."
+groups = ["dev", "tests"]
+dependencies = [
+    "attrs>=23.1.0",
+    "exceptiongroup>=1.1.1; python_version < \"3.11\"",
+    "typing-extensions!=4.6.3,>=4.1.0; python_version < \"3.11\"",
+]
+files = [
+    {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"},
+    {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"},
+]
+
 [[package]]
 name = "certifi"
 version = "2024.6.2"
 requires_python = ">=3.6"
 summary = "Python package for providing Mozilla's CA Bundle."
-groups = ["default", "dev"]
+groups = ["default", "dev", "tests"]
 files = [
     {file = "certifi-2024.6.2-py3-none-any.whl", hash = "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56"},
     {file = "certifi-2024.6.2.tar.gz", hash = "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516"},
@@ -162,7 +178,7 @@ name = "charset-normalizer"
 version = "3.3.2"
 requires_python = ">=3.7.0"
 summary = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
-groups = ["default", "dev"]
+groups = ["default", "dev", "tests"]
 files = [
     {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"},
     {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"},
@@ -243,7 +259,7 @@ name = "colorama"
 version = "0.4.6"
 requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
 summary = "Cross-platform colored terminal text."
-groups = ["default", "dev"]
+groups = ["default", "dev", "tests"]
 files = [
     {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"},
     {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"},
@@ -254,7 +270,7 @@ name = "coverage"
 version = "6.5.0"
 requires_python = ">=3.7"
 summary = "Code coverage measurement for Python"
-groups = ["dev"]
+groups = ["dev", "tests"]
 files = [
     {file = "coverage-6.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef8674b0ee8cc11e2d574e3e2998aea5df5ab242e012286824ea3c6970580e53"},
     {file = "coverage-6.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:784f53ebc9f3fd0e2a3f6a78b2be1bd1f5575d7863e10c6e12504f240fd06660"},
@@ -285,7 +301,7 @@ version = "6.5.0"
 extras = ["toml"]
 requires_python = ">=3.7"
 summary = "Code coverage measurement for Python"
-groups = ["dev"]
+groups = ["dev", "tests"]
 dependencies = [
     "coverage==6.5.0",
     "tomli; python_full_version <= \"3.11.0a6\"",
@@ -319,7 +335,7 @@ name = "coveralls"
 version = "3.3.1"
 requires_python = ">= 3.5"
 summary = "Show coverage stats online via coveralls.io"
-groups = ["dev"]
+groups = ["dev", "tests"]
 dependencies = [
     "coverage!=6.0.*,!=6.1,!=6.1.1,<7.0,>=4.1",
     "docopt>=0.6.1",
@@ -385,7 +401,7 @@ files = [
 name = "docopt"
 version = "0.6.2"
 summary = "Pythonic argument parser, that will make you smile"
-groups = ["dev"]
+groups = ["dev", "tests"]
 files = [
     {file = "docopt-0.6.2.tar.gz", hash = "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"},
 ]
@@ -406,7 +422,7 @@ name = "exceptiongroup"
 version = "1.2.1"
 requires_python = ">=3.7"
 summary = "Backport of PEP 654 (exception groups)"
-groups = ["default", "dev"]
+groups = ["default", "dev", "tests"]
 marker = "python_version < \"3.11\""
 files = [
     {file = "exceptiongroup-1.2.1-py3-none-any.whl", hash = "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad"},
@@ -440,7 +456,7 @@ name = "future-fstrings"
 version = "1.2.0"
 requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 summary = "A backport of fstrings to python<3.6"
-groups = ["dev"]
+groups = ["dev", "tests"]
 files = [
     {file = "future_fstrings-1.2.0-py2.py3-none-any.whl", hash = "sha256:90e49598b553d8746c4dc7d9442e0359d038c3039d802c91c0a55505da318c63"},
     {file = "future_fstrings-1.2.0.tar.gz", hash = "sha256:6cf41cbe97c398ab5a81168ce0dbb8ad95862d3caf23c21e4430627b90844089"},
@@ -451,7 +467,7 @@ name = "gprof2dot"
 version = "2024.6.6"
 requires_python = ">=3.8"
 summary = "Generate a dot graph from the output of several profilers."
-groups = ["dev"]
+groups = ["dev", "tests"]
 files = [
     {file = "gprof2dot-2024.6.6-py2.py3-none-any.whl", hash = "sha256:45b14ad7ce64e299c8f526881007b9eb2c6b75505d5613e96e66ee4d5ab33696"},
     {file = "gprof2dot-2024.6.6.tar.gz", hash = "sha256:fa1420c60025a9eb7734f65225b4da02a10fc6dd741b37fa129bc6b41951e5ab"},
@@ -547,7 +563,7 @@ name = "idna"
 version = "3.7"
 requires_python = ">=3.5"
 summary = "Internationalized Domain Names in Applications (IDNA)"
-groups = ["default", "dev"]
+groups = ["default", "dev", "tests"]
 files = [
     {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"},
     {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"},
@@ -573,7 +589,7 @@ name = "iniconfig"
 version = "2.0.0"
 requires_python = ">=3.7"
 summary = "brain-dead simple config-ini parsing"
-groups = ["default", "dev"]
+groups = ["default", "dev", "tests"]
 files = [
     {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"},
     {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"},
@@ -978,7 +994,7 @@ name = "networkx"
 version = "3.3"
 requires_python = ">=3.10"
 summary = "Python package for creating and manipulating graphs and networks"
-groups = ["dev"]
+groups = ["dev", "tests"]
 files = [
     {file = "networkx-3.3-py3-none-any.whl", hash = "sha256:28575580c6ebdaf4505b22c6256a2b9de86b316dc63ba9e93abde3d78dfdbcf2"},
     {file = "networkx-3.3.tar.gz", hash = "sha256:0c127d8b2f4865f59ae9cb8aafcd60b5c70f3241ebd66f7defad7c4ab90126c9"},
@@ -1103,7 +1119,7 @@ name = "packaging"
 version = "24.1"
 requires_python = ">=3.8"
 summary = "Core utilities for Python packages"
-groups = ["default", "dev"]
+groups = ["default", "dev", "tests"]
 files = [
     {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"},
     {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"},
@@ -1150,7 +1166,7 @@ name = "platformdirs"
 version = "4.2.2"
 requires_python = ">=3.8"
 summary = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`."
-groups = ["dev"]
+groups = ["dev", "tests"]
 files = [
     {file = "platformdirs-4.2.2-py3-none-any.whl", hash = "sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee"},
     {file = "platformdirs-4.2.2.tar.gz", hash = "sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3"},
@@ -1161,7 +1177,7 @@ name = "pluggy"
 version = "1.5.0"
 requires_python = ">=3.8"
 summary = "plugin and hook calling mechanisms for python"
-groups = ["default", "dev"]
+groups = ["default", "dev", "tests"]
 files = [
     {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"},
     {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"},
@@ -1397,7 +1413,7 @@ name = "pytest"
 version = "7.4.4"
 requires_python = ">=3.7"
 summary = "pytest: simple powerful testing with Python"
-groups = ["default", "dev"]
+groups = ["default", "dev", "tests"]
 dependencies = [
     "colorama; sys_platform == \"win32\"",
     "exceptiongroup>=1.0.0rc8; python_version < \"3.11\"",
@@ -1416,7 +1432,7 @@ name = "pytest-cov"
 version = "4.1.0"
 requires_python = ">=3.7"
 summary = "Pytest plugin for measuring coverage."
-groups = ["dev"]
+groups = ["dev", "tests"]
 dependencies = [
     "coverage[toml]>=5.2.1",
     "pytest>=4.6",
@@ -1430,7 +1446,7 @@ files = [
 name = "pytest-depends"
 version = "1.0.1"
 summary = "Tests that depend on other tests"
-groups = ["dev"]
+groups = ["dev", "tests"]
 dependencies = [
     "colorama",
     "future-fstrings",
@@ -1459,7 +1475,7 @@ name = "pytest-md"
 version = "0.2.0"
 requires_python = ">=3.6"
 summary = "Plugin for generating Markdown reports for pytest results"
-groups = ["dev"]
+groups = ["dev", "tests"]
 dependencies = [
     "pytest>=4.2.1",
 ]
@@ -1472,7 +1488,7 @@ files = [
 name = "pytest-profiling"
 version = "1.7.0"
 summary = "Profiling plugin for py.test"
-groups = ["dev"]
+groups = ["dev", "tests"]
 dependencies = [
     "gprof2dot",
     "pytest",
@@ -1617,7 +1633,7 @@ name = "requests"
 version = "2.32.3"
 requires_python = ">=3.8"
 summary = "Python HTTP for Humans."
-groups = ["default", "dev"]
+groups = ["default", "dev", "tests"]
 dependencies = [
     "certifi>=2017.4.17",
     "charset-normalizer<4,>=2",
@@ -1629,6 +1645,25 @@ files = [
     {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"},
 ]
 
+[[package]]
+name = "requests-cache"
+version = "1.2.1"
+requires_python = ">=3.8"
+summary = "A persistent cache for python requests"
+groups = ["dev", "tests"]
+dependencies = [
+    "attrs>=21.2",
+    "cattrs>=22.2",
+    "platformdirs>=2.5",
+    "requests>=2.22",
+    "url-normalize>=1.4",
+    "urllib3>=1.25.5",
+]
+files = [
+    {file = "requests_cache-1.2.1-py3-none-any.whl", hash = "sha256:1285151cddf5331067baa82598afe2d47c7495a1334bfe7a7d329b43e9fd3603"},
+    {file = "requests_cache-1.2.1.tar.gz", hash = "sha256:68abc986fdc5b8d0911318fbb5f7c80eebcd4d01bfacc6685ecf8876052511d1"},
+]
+
 [[package]]
 name = "rfc3339-validator"
 version = "0.1.4"
@@ -1844,7 +1879,7 @@ name = "six"
 version = "1.16.0"
 requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
 summary = "Python 2 and 3 compatibility utilities"
-groups = ["default", "dev"]
+groups = ["default", "dev", "tests"]
 files = [
     {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"},
     {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
@@ -1934,7 +1969,7 @@ name = "sybil"
 version = "5.0.3"
 requires_python = ">=3.7"
 summary = "Automated testing for the examples in your code and documentation."
-groups = ["dev"]
+groups = ["dev", "tests"]
 files = [
     {file = "sybil-5.0.3-py3-none-any.whl", hash = "sha256:6f3c30822169895c4fb34c8366bdb132cf62bb68fb1d03d2ebb05282eab08c95"},
     {file = "sybil-5.0.3.tar.gz", hash = "sha256:20dfe3a35a8d1ffcb4311434d1abf38c030c91064d75ff6b56ddd1060e08e758"},
@@ -1945,7 +1980,7 @@ name = "tomli"
 version = "2.0.1"
 requires_python = ">=3.7"
 summary = "A lil' TOML parser"
-groups = ["default", "dev"]
+groups = ["default", "dev", "tests"]
 marker = "python_version < \"3.11\""
 files = [
     {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"},
@@ -1993,7 +2028,7 @@ name = "typing-extensions"
 version = "4.12.2"
 requires_python = ">=3.8"
 summary = "Backported and Experimental Type Hints for Python 3.8+"
-groups = ["default", "dev"]
+groups = ["default", "dev", "tests"]
 files = [
     {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"},
     {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"},
@@ -2010,12 +2045,26 @@ files = [
     {file = "uri_template-1.3.0-py3-none-any.whl", hash = "sha256:a44a133ea12d44a0c0f06d7d42a52d71282e77e2f937d8abd5655b8d56fc1363"},
 ]
 
+[[package]]
+name = "url-normalize"
+version = "1.4.3"
+requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"
+summary = "URL normalization for Python"
+groups = ["dev", "tests"]
+dependencies = [
+    "six",
+]
+files = [
+    {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"},
+    {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"},
+]
+
 [[package]]
 name = "urllib3"
 version = "2.2.2"
 requires_python = ">=3.8"
 summary = "HTTP library with thread-safe connection pooling, file post, and more."
-groups = ["default", "dev"]
+groups = ["default", "dev", "tests"]
 files = [
     {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"},
     {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"},

nwb_linkml/pyproject.toml

@@ -43,6 +43,7 @@ tests = [
     "coveralls<4.0.0,>=3.3.1",
     "pytest-profiling<2.0.0,>=1.7.0",
     "sybil<6.0.0,>=5.0.3",
+    "requests-cache>=1.2.1",
 ]
 dev = [
     "nwb-linkml[tests]",

nwb_linkml/src/nwb_linkml/io/hdf5.py

@@ -24,10 +24,11 @@ import json
 import os
 import shutil
 import subprocess
+import sys
 import warnings
 from pathlib import Path
 from types import ModuleType
-from typing import TYPE_CHECKING, Dict, List, Never, Optional, Union, overload
+from typing import TYPE_CHECKING, Dict, List, Optional, Union, overload
 
 import h5py
 import numpy as np
@@ -41,6 +42,11 @@ if TYPE_CHECKING:
     from nwb_linkml.models import NWBFile
 from nwb_linkml.providers.schema import SchemaProvider
 
+if sys.version_info.minor >= 11:
+    from typing import Never
+else:
+    from typing_extensions import Never
+
 
 class HDF5IO:
     """

nwb_linkml/src/nwb_linkml/maps/hdf5.py

@@ -12,8 +12,8 @@ so we will make our own mapping class here and re-evaluate whether they should b
 import contextlib
 import datetime
 import inspect
+import sys
 from abc import abstractmethod
-from enum import StrEnum
 from pathlib import Path
 from typing import Dict, List, Literal, Optional, Tuple, Type, Union
 
@@ -27,6 +27,14 @@ from nwb_linkml.providers.schema import SchemaProvider
 from nwb_linkml.types.hdf5 import HDF5_Path
 from nwb_linkml.types.ndarray import NDArrayProxy
 
+if sys.version_info.minor >= 11:
+    from enum import StrEnum
+else:
+    from enum import Enum
+
+    class StrEnum(str, Enum):
+        """StrEnum-ish class for python 3.10"""
+
 
 class ReadPhases(StrEnum):
     plan = "plan"

nwb_linkml/tests/conftest.py

@@ -1,7 +1,9 @@
 import os
 from doctest import ELLIPSIS, NORMALIZE_WHITESPACE
+from pathlib import Path
 
 import pytest
+import requests_cache
 from sybil import Sybil
 from sybil.parsers.rest import DocTestParser, PythonCodeBlockParser
 
@@ -16,6 +18,39 @@ pytest_collect_file = Sybil(
 ).pytest()
 
 
+def pytest_addoption(parser):
+    parser.addoption(
+        "--with-output",
+        action="store_true",
+        help="dump output in compliance test for richer debugging information",
+    )
+    parser.addoption(
+        "--without-cache", action="store_true", help="Don't use a sqlite cache for network requests"
+    )
+
+
 @pytest.fixture(autouse=True, scope="session")
 def set_config_vars(tmp_output_dir):
     os.environ["NWB_LINKML_CACHE_DIR"] = str(tmp_output_dir)
+
+
+@pytest.fixture(scope="session", autouse=True)
+def patch_requests_cache(pytestconfig):
+    """
+    Cache network requests - for each unique network request, store it in
+    an sqlite cache. only do unique requests once per session.
+    """
+    if pytestconfig.getoption("--without-cache"):
+        yield
+        return
+    cache_file = Path(__file__).parent / "output" / "requests-cache.sqlite"
+    requests_cache.install_cache(
+        str(cache_file),
+        backend="sqlite",
+        urls_expire_after={"localhost": requests_cache.DO_NOT_CACHE},
+    )
+    requests_cache.clear()
+    yield
+    # delete cache file unless we have requested it to persist for inspection
+    if not pytestconfig.getoption("--with-output"):
+        cache_file.unlink(missing_ok=True)

nwb_linkml/tests/test_io/test_io_hdf5.py

@@ -7,7 +7,7 @@ import pytest
 from nwb_linkml.io.hdf5 import HDF5IO, truncate_file
 
 
-@pytest.mark.xfail()
+@pytest.mark.skip()
 @pytest.mark.parametrize("dset", ["aibs.nwb", "aibs_ecephys.nwb"])
 def test_hdf_read(data_dir, dset):
     NWBFILE = data_dir / dset