From 2e405064439f93b969a2892230d01be53e95d957 Mon Sep 17 00:00:00 2001 From: sneakers-the-rat Date: Tue, 2 Jul 2024 00:34:01 -0700 Subject: [PATCH] requests caching, python 10 --- .gitignore | 3 +- nwb_linkml/pdm.lock | 109 ++++++++++++++++------- nwb_linkml/pyproject.toml | 1 + nwb_linkml/src/nwb_linkml/io/hdf5.py | 8 +- nwb_linkml/src/nwb_linkml/maps/hdf5.py | 10 ++- nwb_linkml/tests/conftest.py | 35 ++++++++ nwb_linkml/tests/test_io/test_io_hdf5.py | 2 +- 7 files changed, 134 insertions(+), 34 deletions(-) diff --git a/.gitignore b/.gitignore index b221948..8b7aa7b 100644 --- a/.gitignore +++ b/.gitignore @@ -164,4 +164,5 @@ __tmp__ prof jupyter_execute -.pdm-python \ No newline at end of file +.pdm-python +.venv* \ No newline at end of file diff --git a/nwb_linkml/pdm.lock b/nwb_linkml/pdm.lock index ac66fb6..1d543d2 100644 --- a/nwb_linkml/pdm.lock +++ b/nwb_linkml/pdm.lock @@ -2,10 +2,10 @@ # It is not intended for manual editing. [metadata] -groups = ["default", "dev"] +groups = ["default", "dev", "tests"] strategy = ["cross_platform", "inherit_metadata"] lock_version = "4.4.2" -content_hash = "sha256:94d398adc70609c1254f79f19c1b8ee5feb0adfc6c38b58415f7f506b94e8782" +content_hash = "sha256:8cb98f940354e71443df87fd1702300e1c793627b0a016a6233811a640f47c18" [[package]] name = "annotated-types" @@ -47,7 +47,7 @@ name = "attrs" version = "23.2.0" requires_python = ">=3.7" summary = "Classes Without Boilerplate" -groups = ["default", "dev"] +groups = ["default", "dev", "tests"] files = [ {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, @@ -123,12 +123,28 @@ files = [ {file = "blosc2-2.7.0.tar.gz", hash = "sha256:9b982c1d40560eefb4a01d67c57e786d39a5ee9696f3deadd32ebf5f8885eb2a"}, ] +[[package]] +name = "cattrs" +version = "23.2.3" +requires_python = ">=3.8" +summary = "Composable complex class support for attrs and dataclasses." +groups = ["dev", "tests"] +dependencies = [ + "attrs>=23.1.0", + "exceptiongroup>=1.1.1; python_version < \"3.11\"", + "typing-extensions!=4.6.3,>=4.1.0; python_version < \"3.11\"", +] +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + [[package]] name = "certifi" version = "2024.6.2" requires_python = ">=3.6" summary = "Python package for providing Mozilla's CA Bundle." -groups = ["default", "dev"] +groups = ["default", "dev", "tests"] files = [ {file = "certifi-2024.6.2-py3-none-any.whl", hash = "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56"}, {file = "certifi-2024.6.2.tar.gz", hash = "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516"}, @@ -162,7 +178,7 @@ name = "charset-normalizer" version = "3.3.2" requires_python = ">=3.7.0" summary = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -groups = ["default", "dev"] +groups = ["default", "dev", "tests"] files = [ {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, @@ -243,7 +259,7 @@ name = "colorama" version = "0.4.6" requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" summary = "Cross-platform colored terminal text." -groups = ["default", "dev"] +groups = ["default", "dev", "tests"] files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, @@ -254,7 +270,7 @@ name = "coverage" version = "6.5.0" requires_python = ">=3.7" summary = "Code coverage measurement for Python" -groups = ["dev"] +groups = ["dev", "tests"] files = [ {file = "coverage-6.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef8674b0ee8cc11e2d574e3e2998aea5df5ab242e012286824ea3c6970580e53"}, {file = "coverage-6.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:784f53ebc9f3fd0e2a3f6a78b2be1bd1f5575d7863e10c6e12504f240fd06660"}, @@ -285,7 +301,7 @@ version = "6.5.0" extras = ["toml"] requires_python = ">=3.7" summary = "Code coverage measurement for Python" -groups = ["dev"] +groups = ["dev", "tests"] dependencies = [ "coverage==6.5.0", "tomli; python_full_version <= \"3.11.0a6\"", @@ -319,7 +335,7 @@ name = "coveralls" version = "3.3.1" requires_python = ">= 3.5" summary = "Show coverage stats online via coveralls.io" -groups = ["dev"] +groups = ["dev", "tests"] dependencies = [ "coverage!=6.0.*,!=6.1,!=6.1.1,<7.0,>=4.1", "docopt>=0.6.1", @@ -385,7 +401,7 @@ files = [ name = "docopt" version = "0.6.2" summary = "Pythonic argument parser, that will make you smile" -groups = ["dev"] +groups = ["dev", "tests"] files = [ {file = "docopt-0.6.2.tar.gz", hash = "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"}, ] @@ -406,7 +422,7 @@ name = "exceptiongroup" version = "1.2.1" requires_python = ">=3.7" summary = "Backport of PEP 654 (exception groups)" -groups = ["default", "dev"] +groups = ["default", "dev", "tests"] marker = "python_version < \"3.11\"" files = [ {file = "exceptiongroup-1.2.1-py3-none-any.whl", hash = "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad"}, @@ -440,7 +456,7 @@ name = "future-fstrings" version = "1.2.0" requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" summary = "A backport of fstrings to python<3.6" -groups = ["dev"] +groups = ["dev", "tests"] files = [ {file = "future_fstrings-1.2.0-py2.py3-none-any.whl", hash = "sha256:90e49598b553d8746c4dc7d9442e0359d038c3039d802c91c0a55505da318c63"}, {file = "future_fstrings-1.2.0.tar.gz", hash = "sha256:6cf41cbe97c398ab5a81168ce0dbb8ad95862d3caf23c21e4430627b90844089"}, @@ -451,7 +467,7 @@ name = "gprof2dot" version = "2024.6.6" requires_python = ">=3.8" summary = "Generate a dot graph from the output of several profilers." -groups = ["dev"] +groups = ["dev", "tests"] files = [ {file = "gprof2dot-2024.6.6-py2.py3-none-any.whl", hash = "sha256:45b14ad7ce64e299c8f526881007b9eb2c6b75505d5613e96e66ee4d5ab33696"}, {file = "gprof2dot-2024.6.6.tar.gz", hash = "sha256:fa1420c60025a9eb7734f65225b4da02a10fc6dd741b37fa129bc6b41951e5ab"}, @@ -547,7 +563,7 @@ name = "idna" version = "3.7" requires_python = ">=3.5" summary = "Internationalized Domain Names in Applications (IDNA)" -groups = ["default", "dev"] +groups = ["default", "dev", "tests"] files = [ {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, @@ -573,7 +589,7 @@ name = "iniconfig" version = "2.0.0" requires_python = ">=3.7" summary = "brain-dead simple config-ini parsing" -groups = ["default", "dev"] +groups = ["default", "dev", "tests"] files = [ {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, @@ -978,7 +994,7 @@ name = "networkx" version = "3.3" requires_python = ">=3.10" summary = "Python package for creating and manipulating graphs and networks" -groups = ["dev"] +groups = ["dev", "tests"] files = [ {file = "networkx-3.3-py3-none-any.whl", hash = "sha256:28575580c6ebdaf4505b22c6256a2b9de86b316dc63ba9e93abde3d78dfdbcf2"}, {file = "networkx-3.3.tar.gz", hash = "sha256:0c127d8b2f4865f59ae9cb8aafcd60b5c70f3241ebd66f7defad7c4ab90126c9"}, @@ -1103,7 +1119,7 @@ name = "packaging" version = "24.1" requires_python = ">=3.8" summary = "Core utilities for Python packages" -groups = ["default", "dev"] +groups = ["default", "dev", "tests"] files = [ {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, @@ -1150,7 +1166,7 @@ name = "platformdirs" version = "4.2.2" requires_python = ">=3.8" summary = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." -groups = ["dev"] +groups = ["dev", "tests"] files = [ {file = "platformdirs-4.2.2-py3-none-any.whl", hash = "sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee"}, {file = "platformdirs-4.2.2.tar.gz", hash = "sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3"}, @@ -1161,7 +1177,7 @@ name = "pluggy" version = "1.5.0" requires_python = ">=3.8" summary = "plugin and hook calling mechanisms for python" -groups = ["default", "dev"] +groups = ["default", "dev", "tests"] files = [ {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, @@ -1397,7 +1413,7 @@ name = "pytest" version = "7.4.4" requires_python = ">=3.7" summary = "pytest: simple powerful testing with Python" -groups = ["default", "dev"] +groups = ["default", "dev", "tests"] dependencies = [ "colorama; sys_platform == \"win32\"", "exceptiongroup>=1.0.0rc8; python_version < \"3.11\"", @@ -1416,7 +1432,7 @@ name = "pytest-cov" version = "4.1.0" requires_python = ">=3.7" summary = "Pytest plugin for measuring coverage." -groups = ["dev"] +groups = ["dev", "tests"] dependencies = [ "coverage[toml]>=5.2.1", "pytest>=4.6", @@ -1430,7 +1446,7 @@ files = [ name = "pytest-depends" version = "1.0.1" summary = "Tests that depend on other tests" -groups = ["dev"] +groups = ["dev", "tests"] dependencies = [ "colorama", "future-fstrings", @@ -1459,7 +1475,7 @@ name = "pytest-md" version = "0.2.0" requires_python = ">=3.6" summary = "Plugin for generating Markdown reports for pytest results" -groups = ["dev"] +groups = ["dev", "tests"] dependencies = [ "pytest>=4.2.1", ] @@ -1472,7 +1488,7 @@ files = [ name = "pytest-profiling" version = "1.7.0" summary = "Profiling plugin for py.test" -groups = ["dev"] +groups = ["dev", "tests"] dependencies = [ "gprof2dot", "pytest", @@ -1617,7 +1633,7 @@ name = "requests" version = "2.32.3" requires_python = ">=3.8" summary = "Python HTTP for Humans." -groups = ["default", "dev"] +groups = ["default", "dev", "tests"] dependencies = [ "certifi>=2017.4.17", "charset-normalizer<4,>=2", @@ -1629,6 +1645,25 @@ files = [ {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, ] +[[package]] +name = "requests-cache" +version = "1.2.1" +requires_python = ">=3.8" +summary = "A persistent cache for python requests" +groups = ["dev", "tests"] +dependencies = [ + "attrs>=21.2", + "cattrs>=22.2", + "platformdirs>=2.5", + "requests>=2.22", + "url-normalize>=1.4", + "urllib3>=1.25.5", +] +files = [ + {file = "requests_cache-1.2.1-py3-none-any.whl", hash = "sha256:1285151cddf5331067baa82598afe2d47c7495a1334bfe7a7d329b43e9fd3603"}, + {file = "requests_cache-1.2.1.tar.gz", hash = "sha256:68abc986fdc5b8d0911318fbb5f7c80eebcd4d01bfacc6685ecf8876052511d1"}, +] + [[package]] name = "rfc3339-validator" version = "0.1.4" @@ -1844,7 +1879,7 @@ name = "six" version = "1.16.0" requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" summary = "Python 2 and 3 compatibility utilities" -groups = ["default", "dev"] +groups = ["default", "dev", "tests"] files = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, @@ -1934,7 +1969,7 @@ name = "sybil" version = "5.0.3" requires_python = ">=3.7" summary = "Automated testing for the examples in your code and documentation." -groups = ["dev"] +groups = ["dev", "tests"] files = [ {file = "sybil-5.0.3-py3-none-any.whl", hash = "sha256:6f3c30822169895c4fb34c8366bdb132cf62bb68fb1d03d2ebb05282eab08c95"}, {file = "sybil-5.0.3.tar.gz", hash = "sha256:20dfe3a35a8d1ffcb4311434d1abf38c030c91064d75ff6b56ddd1060e08e758"}, @@ -1945,7 +1980,7 @@ name = "tomli" version = "2.0.1" requires_python = ">=3.7" summary = "A lil' TOML parser" -groups = ["default", "dev"] +groups = ["default", "dev", "tests"] marker = "python_version < \"3.11\"" files = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, @@ -1993,7 +2028,7 @@ name = "typing-extensions" version = "4.12.2" requires_python = ">=3.8" summary = "Backported and Experimental Type Hints for Python 3.8+" -groups = ["default", "dev"] +groups = ["default", "dev", "tests"] files = [ {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, @@ -2010,12 +2045,26 @@ files = [ {file = "uri_template-1.3.0-py3-none-any.whl", hash = "sha256:a44a133ea12d44a0c0f06d7d42a52d71282e77e2f937d8abd5655b8d56fc1363"}, ] +[[package]] +name = "url-normalize" +version = "1.4.3" +requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +summary = "URL normalization for Python" +groups = ["dev", "tests"] +dependencies = [ + "six", +] +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + [[package]] name = "urllib3" version = "2.2.2" requires_python = ">=3.8" summary = "HTTP library with thread-safe connection pooling, file post, and more." -groups = ["default", "dev"] +groups = ["default", "dev", "tests"] files = [ {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"}, {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"}, diff --git a/nwb_linkml/pyproject.toml b/nwb_linkml/pyproject.toml index edb53af..5d64d38 100644 --- a/nwb_linkml/pyproject.toml +++ b/nwb_linkml/pyproject.toml @@ -43,6 +43,7 @@ tests = [ "coveralls<4.0.0,>=3.3.1", "pytest-profiling<2.0.0,>=1.7.0", "sybil<6.0.0,>=5.0.3", + "requests-cache>=1.2.1", ] dev = [ "nwb-linkml[tests]", diff --git a/nwb_linkml/src/nwb_linkml/io/hdf5.py b/nwb_linkml/src/nwb_linkml/io/hdf5.py index ee71654..c051211 100644 --- a/nwb_linkml/src/nwb_linkml/io/hdf5.py +++ b/nwb_linkml/src/nwb_linkml/io/hdf5.py @@ -24,10 +24,11 @@ import json import os import shutil import subprocess +import sys import warnings from pathlib import Path from types import ModuleType -from typing import TYPE_CHECKING, Dict, List, Never, Optional, Union, overload +from typing import TYPE_CHECKING, Dict, List, Optional, Union, overload import h5py import numpy as np @@ -41,6 +42,11 @@ if TYPE_CHECKING: from nwb_linkml.models import NWBFile from nwb_linkml.providers.schema import SchemaProvider +if sys.version_info.minor >= 11: + from typing import Never +else: + from typing_extensions import Never + class HDF5IO: """ diff --git a/nwb_linkml/src/nwb_linkml/maps/hdf5.py b/nwb_linkml/src/nwb_linkml/maps/hdf5.py index b97f8b6..90d8038 100644 --- a/nwb_linkml/src/nwb_linkml/maps/hdf5.py +++ b/nwb_linkml/src/nwb_linkml/maps/hdf5.py @@ -12,8 +12,8 @@ so we will make our own mapping class here and re-evaluate whether they should b import contextlib import datetime import inspect +import sys from abc import abstractmethod -from enum import StrEnum from pathlib import Path from typing import Dict, List, Literal, Optional, Tuple, Type, Union @@ -27,6 +27,14 @@ from nwb_linkml.providers.schema import SchemaProvider from nwb_linkml.types.hdf5 import HDF5_Path from nwb_linkml.types.ndarray import NDArrayProxy +if sys.version_info.minor >= 11: + from enum import StrEnum +else: + from enum import Enum + + class StrEnum(str, Enum): + """StrEnum-ish class for python 3.10""" + class ReadPhases(StrEnum): plan = "plan" diff --git a/nwb_linkml/tests/conftest.py b/nwb_linkml/tests/conftest.py index 183e742..ddc5e98 100644 --- a/nwb_linkml/tests/conftest.py +++ b/nwb_linkml/tests/conftest.py @@ -1,7 +1,9 @@ import os from doctest import ELLIPSIS, NORMALIZE_WHITESPACE +from pathlib import Path import pytest +import requests_cache from sybil import Sybil from sybil.parsers.rest import DocTestParser, PythonCodeBlockParser @@ -16,6 +18,39 @@ pytest_collect_file = Sybil( ).pytest() +def pytest_addoption(parser): + parser.addoption( + "--with-output", + action="store_true", + help="dump output in compliance test for richer debugging information", + ) + parser.addoption( + "--without-cache", action="store_true", help="Don't use a sqlite cache for network requests" + ) + + @pytest.fixture(autouse=True, scope="session") def set_config_vars(tmp_output_dir): os.environ["NWB_LINKML_CACHE_DIR"] = str(tmp_output_dir) + + +@pytest.fixture(scope="session", autouse=True) +def patch_requests_cache(pytestconfig): + """ + Cache network requests - for each unique network request, store it in + an sqlite cache. only do unique requests once per session. + """ + if pytestconfig.getoption("--without-cache"): + yield + return + cache_file = Path(__file__).parent / "output" / "requests-cache.sqlite" + requests_cache.install_cache( + str(cache_file), + backend="sqlite", + urls_expire_after={"localhost": requests_cache.DO_NOT_CACHE}, + ) + requests_cache.clear() + yield + # delete cache file unless we have requested it to persist for inspection + if not pytestconfig.getoption("--with-output"): + cache_file.unlink(missing_ok=True) diff --git a/nwb_linkml/tests/test_io/test_io_hdf5.py b/nwb_linkml/tests/test_io/test_io_hdf5.py index 6caabca..c64cf48 100644 --- a/nwb_linkml/tests/test_io/test_io_hdf5.py +++ b/nwb_linkml/tests/test_io/test_io_hdf5.py @@ -7,7 +7,7 @@ import pytest from nwb_linkml.io.hdf5 import HDF5IO, truncate_file -@pytest.mark.xfail() +@pytest.mark.skip() @pytest.mark.parametrize("dset", ["aibs.nwb", "aibs_ecephys.nwb"]) def test_hdf_read(data_dir, dset): NWBFILE = data_dir / dset