requests caching, python 10

This commit is contained in:
sneakers-the-rat 2024-07-02 00:34:01 -07:00
parent 38c7e9ebf8
commit 2e40506443
Signed by untrusted user who does not match committer: jonny
GPG key ID: 6DCB96EF1E4D232D
7 changed files with 134 additions and 34 deletions

1
.gitignore vendored
View file

@ -165,3 +165,4 @@ prof
jupyter_execute
.pdm-python
.venv*

View file

@ -2,10 +2,10 @@
# It is not intended for manual editing.
[metadata]
groups = ["default", "dev"]
groups = ["default", "dev", "tests"]
strategy = ["cross_platform", "inherit_metadata"]
lock_version = "4.4.2"
content_hash = "sha256:94d398adc70609c1254f79f19c1b8ee5feb0adfc6c38b58415f7f506b94e8782"
content_hash = "sha256:8cb98f940354e71443df87fd1702300e1c793627b0a016a6233811a640f47c18"
[[package]]
name = "annotated-types"
@ -47,7 +47,7 @@ name = "attrs"
version = "23.2.0"
requires_python = ">=3.7"
summary = "Classes Without Boilerplate"
groups = ["default", "dev"]
groups = ["default", "dev", "tests"]
files = [
{file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"},
{file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"},
@ -123,12 +123,28 @@ files = [
{file = "blosc2-2.7.0.tar.gz", hash = "sha256:9b982c1d40560eefb4a01d67c57e786d39a5ee9696f3deadd32ebf5f8885eb2a"},
]
[[package]]
name = "cattrs"
version = "23.2.3"
requires_python = ">=3.8"
summary = "Composable complex class support for attrs and dataclasses."
groups = ["dev", "tests"]
dependencies = [
"attrs>=23.1.0",
"exceptiongroup>=1.1.1; python_version < \"3.11\"",
"typing-extensions!=4.6.3,>=4.1.0; python_version < \"3.11\"",
]
files = [
{file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"},
{file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"},
]
[[package]]
name = "certifi"
version = "2024.6.2"
requires_python = ">=3.6"
summary = "Python package for providing Mozilla's CA Bundle."
groups = ["default", "dev"]
groups = ["default", "dev", "tests"]
files = [
{file = "certifi-2024.6.2-py3-none-any.whl", hash = "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56"},
{file = "certifi-2024.6.2.tar.gz", hash = "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516"},
@ -162,7 +178,7 @@ name = "charset-normalizer"
version = "3.3.2"
requires_python = ">=3.7.0"
summary = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
groups = ["default", "dev"]
groups = ["default", "dev", "tests"]
files = [
{file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"},
{file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"},
@ -243,7 +259,7 @@ name = "colorama"
version = "0.4.6"
requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
summary = "Cross-platform colored terminal text."
groups = ["default", "dev"]
groups = ["default", "dev", "tests"]
files = [
{file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"},
{file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"},
@ -254,7 +270,7 @@ name = "coverage"
version = "6.5.0"
requires_python = ">=3.7"
summary = "Code coverage measurement for Python"
groups = ["dev"]
groups = ["dev", "tests"]
files = [
{file = "coverage-6.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef8674b0ee8cc11e2d574e3e2998aea5df5ab242e012286824ea3c6970580e53"},
{file = "coverage-6.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:784f53ebc9f3fd0e2a3f6a78b2be1bd1f5575d7863e10c6e12504f240fd06660"},
@ -285,7 +301,7 @@ version = "6.5.0"
extras = ["toml"]
requires_python = ">=3.7"
summary = "Code coverage measurement for Python"
groups = ["dev"]
groups = ["dev", "tests"]
dependencies = [
"coverage==6.5.0",
"tomli; python_full_version <= \"3.11.0a6\"",
@ -319,7 +335,7 @@ name = "coveralls"
version = "3.3.1"
requires_python = ">= 3.5"
summary = "Show coverage stats online via coveralls.io"
groups = ["dev"]
groups = ["dev", "tests"]
dependencies = [
"coverage!=6.0.*,!=6.1,!=6.1.1,<7.0,>=4.1",
"docopt>=0.6.1",
@ -385,7 +401,7 @@ files = [
name = "docopt"
version = "0.6.2"
summary = "Pythonic argument parser, that will make you smile"
groups = ["dev"]
groups = ["dev", "tests"]
files = [
{file = "docopt-0.6.2.tar.gz", hash = "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"},
]
@ -406,7 +422,7 @@ name = "exceptiongroup"
version = "1.2.1"
requires_python = ">=3.7"
summary = "Backport of PEP 654 (exception groups)"
groups = ["default", "dev"]
groups = ["default", "dev", "tests"]
marker = "python_version < \"3.11\""
files = [
{file = "exceptiongroup-1.2.1-py3-none-any.whl", hash = "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad"},
@ -440,7 +456,7 @@ name = "future-fstrings"
version = "1.2.0"
requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
summary = "A backport of fstrings to python<3.6"
groups = ["dev"]
groups = ["dev", "tests"]
files = [
{file = "future_fstrings-1.2.0-py2.py3-none-any.whl", hash = "sha256:90e49598b553d8746c4dc7d9442e0359d038c3039d802c91c0a55505da318c63"},
{file = "future_fstrings-1.2.0.tar.gz", hash = "sha256:6cf41cbe97c398ab5a81168ce0dbb8ad95862d3caf23c21e4430627b90844089"},
@ -451,7 +467,7 @@ name = "gprof2dot"
version = "2024.6.6"
requires_python = ">=3.8"
summary = "Generate a dot graph from the output of several profilers."
groups = ["dev"]
groups = ["dev", "tests"]
files = [
{file = "gprof2dot-2024.6.6-py2.py3-none-any.whl", hash = "sha256:45b14ad7ce64e299c8f526881007b9eb2c6b75505d5613e96e66ee4d5ab33696"},
{file = "gprof2dot-2024.6.6.tar.gz", hash = "sha256:fa1420c60025a9eb7734f65225b4da02a10fc6dd741b37fa129bc6b41951e5ab"},
@ -547,7 +563,7 @@ name = "idna"
version = "3.7"
requires_python = ">=3.5"
summary = "Internationalized Domain Names in Applications (IDNA)"
groups = ["default", "dev"]
groups = ["default", "dev", "tests"]
files = [
{file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"},
{file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"},
@ -573,7 +589,7 @@ name = "iniconfig"
version = "2.0.0"
requires_python = ">=3.7"
summary = "brain-dead simple config-ini parsing"
groups = ["default", "dev"]
groups = ["default", "dev", "tests"]
files = [
{file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"},
{file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"},
@ -978,7 +994,7 @@ name = "networkx"
version = "3.3"
requires_python = ">=3.10"
summary = "Python package for creating and manipulating graphs and networks"
groups = ["dev"]
groups = ["dev", "tests"]
files = [
{file = "networkx-3.3-py3-none-any.whl", hash = "sha256:28575580c6ebdaf4505b22c6256a2b9de86b316dc63ba9e93abde3d78dfdbcf2"},
{file = "networkx-3.3.tar.gz", hash = "sha256:0c127d8b2f4865f59ae9cb8aafcd60b5c70f3241ebd66f7defad7c4ab90126c9"},
@ -1103,7 +1119,7 @@ name = "packaging"
version = "24.1"
requires_python = ">=3.8"
summary = "Core utilities for Python packages"
groups = ["default", "dev"]
groups = ["default", "dev", "tests"]
files = [
{file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"},
{file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"},
@ -1150,7 +1166,7 @@ name = "platformdirs"
version = "4.2.2"
requires_python = ">=3.8"
summary = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`."
groups = ["dev"]
groups = ["dev", "tests"]
files = [
{file = "platformdirs-4.2.2-py3-none-any.whl", hash = "sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee"},
{file = "platformdirs-4.2.2.tar.gz", hash = "sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3"},
@ -1161,7 +1177,7 @@ name = "pluggy"
version = "1.5.0"
requires_python = ">=3.8"
summary = "plugin and hook calling mechanisms for python"
groups = ["default", "dev"]
groups = ["default", "dev", "tests"]
files = [
{file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"},
{file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"},
@ -1397,7 +1413,7 @@ name = "pytest"
version = "7.4.4"
requires_python = ">=3.7"
summary = "pytest: simple powerful testing with Python"
groups = ["default", "dev"]
groups = ["default", "dev", "tests"]
dependencies = [
"colorama; sys_platform == \"win32\"",
"exceptiongroup>=1.0.0rc8; python_version < \"3.11\"",
@ -1416,7 +1432,7 @@ name = "pytest-cov"
version = "4.1.0"
requires_python = ">=3.7"
summary = "Pytest plugin for measuring coverage."
groups = ["dev"]
groups = ["dev", "tests"]
dependencies = [
"coverage[toml]>=5.2.1",
"pytest>=4.6",
@ -1430,7 +1446,7 @@ files = [
name = "pytest-depends"
version = "1.0.1"
summary = "Tests that depend on other tests"
groups = ["dev"]
groups = ["dev", "tests"]
dependencies = [
"colorama",
"future-fstrings",
@ -1459,7 +1475,7 @@ name = "pytest-md"
version = "0.2.0"
requires_python = ">=3.6"
summary = "Plugin for generating Markdown reports for pytest results"
groups = ["dev"]
groups = ["dev", "tests"]
dependencies = [
"pytest>=4.2.1",
]
@ -1472,7 +1488,7 @@ files = [
name = "pytest-profiling"
version = "1.7.0"
summary = "Profiling plugin for py.test"
groups = ["dev"]
groups = ["dev", "tests"]
dependencies = [
"gprof2dot",
"pytest",
@ -1617,7 +1633,7 @@ name = "requests"
version = "2.32.3"
requires_python = ">=3.8"
summary = "Python HTTP for Humans."
groups = ["default", "dev"]
groups = ["default", "dev", "tests"]
dependencies = [
"certifi>=2017.4.17",
"charset-normalizer<4,>=2",
@ -1629,6 +1645,25 @@ files = [
{file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"},
]
[[package]]
name = "requests-cache"
version = "1.2.1"
requires_python = ">=3.8"
summary = "A persistent cache for python requests"
groups = ["dev", "tests"]
dependencies = [
"attrs>=21.2",
"cattrs>=22.2",
"platformdirs>=2.5",
"requests>=2.22",
"url-normalize>=1.4",
"urllib3>=1.25.5",
]
files = [
{file = "requests_cache-1.2.1-py3-none-any.whl", hash = "sha256:1285151cddf5331067baa82598afe2d47c7495a1334bfe7a7d329b43e9fd3603"},
{file = "requests_cache-1.2.1.tar.gz", hash = "sha256:68abc986fdc5b8d0911318fbb5f7c80eebcd4d01bfacc6685ecf8876052511d1"},
]
[[package]]
name = "rfc3339-validator"
version = "0.1.4"
@ -1844,7 +1879,7 @@ name = "six"
version = "1.16.0"
requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
summary = "Python 2 and 3 compatibility utilities"
groups = ["default", "dev"]
groups = ["default", "dev", "tests"]
files = [
{file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"},
{file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
@ -1934,7 +1969,7 @@ name = "sybil"
version = "5.0.3"
requires_python = ">=3.7"
summary = "Automated testing for the examples in your code and documentation."
groups = ["dev"]
groups = ["dev", "tests"]
files = [
{file = "sybil-5.0.3-py3-none-any.whl", hash = "sha256:6f3c30822169895c4fb34c8366bdb132cf62bb68fb1d03d2ebb05282eab08c95"},
{file = "sybil-5.0.3.tar.gz", hash = "sha256:20dfe3a35a8d1ffcb4311434d1abf38c030c91064d75ff6b56ddd1060e08e758"},
@ -1945,7 +1980,7 @@ name = "tomli"
version = "2.0.1"
requires_python = ">=3.7"
summary = "A lil' TOML parser"
groups = ["default", "dev"]
groups = ["default", "dev", "tests"]
marker = "python_version < \"3.11\""
files = [
{file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"},
@ -1993,7 +2028,7 @@ name = "typing-extensions"
version = "4.12.2"
requires_python = ">=3.8"
summary = "Backported and Experimental Type Hints for Python 3.8+"
groups = ["default", "dev"]
groups = ["default", "dev", "tests"]
files = [
{file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"},
{file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"},
@ -2010,12 +2045,26 @@ files = [
{file = "uri_template-1.3.0-py3-none-any.whl", hash = "sha256:a44a133ea12d44a0c0f06d7d42a52d71282e77e2f937d8abd5655b8d56fc1363"},
]
[[package]]
name = "url-normalize"
version = "1.4.3"
requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"
summary = "URL normalization for Python"
groups = ["dev", "tests"]
dependencies = [
"six",
]
files = [
{file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"},
{file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"},
]
[[package]]
name = "urllib3"
version = "2.2.2"
requires_python = ">=3.8"
summary = "HTTP library with thread-safe connection pooling, file post, and more."
groups = ["default", "dev"]
groups = ["default", "dev", "tests"]
files = [
{file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"},
{file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"},

View file

@ -43,6 +43,7 @@ tests = [
"coveralls<4.0.0,>=3.3.1",
"pytest-profiling<2.0.0,>=1.7.0",
"sybil<6.0.0,>=5.0.3",
"requests-cache>=1.2.1",
]
dev = [
"nwb-linkml[tests]",

View file

@ -24,10 +24,11 @@ import json
import os
import shutil
import subprocess
import sys
import warnings
from pathlib import Path
from types import ModuleType
from typing import TYPE_CHECKING, Dict, List, Never, Optional, Union, overload
from typing import TYPE_CHECKING, Dict, List, Optional, Union, overload
import h5py
import numpy as np
@ -41,6 +42,11 @@ if TYPE_CHECKING:
from nwb_linkml.models import NWBFile
from nwb_linkml.providers.schema import SchemaProvider
if sys.version_info.minor >= 11:
from typing import Never
else:
from typing_extensions import Never
class HDF5IO:
"""

View file

@ -12,8 +12,8 @@ so we will make our own mapping class here and re-evaluate whether they should b
import contextlib
import datetime
import inspect
import sys
from abc import abstractmethod
from enum import StrEnum
from pathlib import Path
from typing import Dict, List, Literal, Optional, Tuple, Type, Union
@ -27,6 +27,14 @@ from nwb_linkml.providers.schema import SchemaProvider
from nwb_linkml.types.hdf5 import HDF5_Path
from nwb_linkml.types.ndarray import NDArrayProxy
if sys.version_info.minor >= 11:
from enum import StrEnum
else:
from enum import Enum
class StrEnum(str, Enum):
"""StrEnum-ish class for python 3.10"""
class ReadPhases(StrEnum):
plan = "plan"

View file

@ -1,7 +1,9 @@
import os
from doctest import ELLIPSIS, NORMALIZE_WHITESPACE
from pathlib import Path
import pytest
import requests_cache
from sybil import Sybil
from sybil.parsers.rest import DocTestParser, PythonCodeBlockParser
@ -16,6 +18,39 @@ pytest_collect_file = Sybil(
).pytest()
def pytest_addoption(parser):
parser.addoption(
"--with-output",
action="store_true",
help="dump output in compliance test for richer debugging information",
)
parser.addoption(
"--without-cache", action="store_true", help="Don't use a sqlite cache for network requests"
)
@pytest.fixture(autouse=True, scope="session")
def set_config_vars(tmp_output_dir):
os.environ["NWB_LINKML_CACHE_DIR"] = str(tmp_output_dir)
@pytest.fixture(scope="session", autouse=True)
def patch_requests_cache(pytestconfig):
"""
Cache network requests - for each unique network request, store it in
an sqlite cache. only do unique requests once per session.
"""
if pytestconfig.getoption("--without-cache"):
yield
return
cache_file = Path(__file__).parent / "output" / "requests-cache.sqlite"
requests_cache.install_cache(
str(cache_file),
backend="sqlite",
urls_expire_after={"localhost": requests_cache.DO_NOT_CACHE},
)
requests_cache.clear()
yield
# delete cache file unless we have requested it to persist for inspection
if not pytestconfig.getoption("--with-output"):
cache_file.unlink(missing_ok=True)

View file

@ -7,7 +7,7 @@ import pytest
from nwb_linkml.io.hdf5 import HDF5IO, truncate_file
@pytest.mark.xfail()
@pytest.mark.skip()
@pytest.mark.parametrize("dset", ["aibs.nwb", "aibs_ecephys.nwb"])
def test_hdf_read(data_dir, dset):
NWBFILE = data_dir / dset