diff --git a/docs/_static/css/notebooks.css b/docs/_static/css/notebooks.css new file mode 100644 index 0000000..b7fcb03 --- /dev/null +++ b/docs/_static/css/notebooks.css @@ -0,0 +1,31 @@ +div.cell.tag_hide-cell details.above-input > summary, +div.cell.tag_hide-input details.above-input > summary, +div.cell.tag_hide-output details.below-input > summary{ + background-color: var(--color-admonition-title-background--admonition-todo); + color: var(--color-content-foreground); + border: unset; + border-left: 2px solid var(--mystnb-source-margin-color); + opacity: unset; + padding: 0.25em 0 0.25em 1em; +} + +div.cell.tag_hide-cell details.above-input > summary > span, +div.cell.tag_hide-input details.above-input > summary > span, +div.cell.tag_hide-output details.below-input > summary > span +{ + opacity: unset; +} + +div.cell details.above-input div.cell_input { + border: unset; + background-color: unset; + border-left: 2px solid var(--mystnb-source-margin-color); +} + +div.cell details.above-input div.cell_input div.highlight { + background: var(--color-admonition-background); +} + +.output.text_html pre { + font-size: 0.8em; +} \ No newline at end of file diff --git a/docs/api/testing/index.md b/docs/api/testing/index.md index 687835b..7b2a976 100644 --- a/docs/api/testing/index.md +++ b/docs/api/testing/index.md @@ -2,7 +2,18 @@ Utilities for testing and 3rd-party interface development. +See also the [narrative testing docs](../../contributing/testing.md) + ```{toctree} +:maxdepth: 2 + cases helpers +interfaces +``` + +```{eval-rst} +.. automodule:: numpydantic.testing + :members: + :undoc-members: ``` \ No newline at end of file diff --git a/docs/api/testing/interfaces.md b/docs/api/testing/interfaces.md new file mode 100644 index 0000000..c68c772 --- /dev/null +++ b/docs/api/testing/interfaces.md @@ -0,0 +1,7 @@ +# interfaces + +```{eval-rst} +.. automodule:: numpydantic.testing.interfaces + :members: + :undoc-members: +``` \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 963b0b3..32eb942 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -49,6 +49,7 @@ intersphinx_mapping = { html_theme = "furo" html_static_path = ["_static"] +html_css_files = ["css/notebooks.css"] # autodoc autodoc_pydantic_model_show_json_error_strategy = "coerce" diff --git a/docs/contributing/coc.md b/docs/contributing/coc.md new file mode 100644 index 0000000..8504e4c --- /dev/null +++ b/docs/contributing/coc.md @@ -0,0 +1,5 @@ +# Code of Conduct + +```{todo} +jonny write the code of conduct +``` \ No newline at end of file diff --git a/docs/contributing/index.md b/docs/contributing/index.md new file mode 100644 index 0000000..fd5a4c9 --- /dev/null +++ b/docs/contributing/index.md @@ -0,0 +1,8 @@ +# Contributing + +```{toctree} +coc +process +interface +testing +``` \ No newline at end of file diff --git a/docs/contributing/interface.md b/docs/contributing/interface.md new file mode 100644 index 0000000..d3e29a1 --- /dev/null +++ b/docs/contributing/interface.md @@ -0,0 +1,5 @@ +# Writing an Interface + +```{todo} +Jonny write the interface contrib docs +``` \ No newline at end of file diff --git a/docs/contributing/process.md b/docs/contributing/process.md new file mode 100644 index 0000000..51f29ba --- /dev/null +++ b/docs/contributing/process.md @@ -0,0 +1,15 @@ +# Contribution Process + +```{todo} +Jonny write the contribution docs +``` + +### Issues + +### Development Environment + +### Testing + +### Linting + +### Pull Requests \ No newline at end of file diff --git a/docs/contributing/testing.md b/docs/contributing/testing.md new file mode 100644 index 0000000..d7e9be0 --- /dev/null +++ b/docs/contributing/testing.md @@ -0,0 +1,213 @@ +--- +file_format: mystnb +mystnb: + output_stderr: remove + render_text_lexer: python + render_markdown_format: myst +myst: + enable_extensions: ["colon_fence"] +--- +# Testing + +```{code-cell} +--- +tags: [hide-cell] +--- + +from pathlib import Path +from rich.console import Console +from rich.theme import Theme +from rich.style import Style +from rich.color import Color + +theme = Theme({ + "repr.call": Style(color=Color.from_rgb(110,191,38), bold=True), + "repr.attrib_name": Style(color="slate_blue1"), + "repr.number": Style(color="deep_sky_blue1"), + "repr.none": Style(color="bright_magenta", italic=True), + "repr.attrib_name": Style(color="white"), + "repr.tag_contents": Style(color="light_steel_blue"), + "repr.str": Style(color="violet") +}) +console = Console(theme=theme) + +``` + +```{note} +Also see the [`numpydantic.testing` API docs](../api/testing/index.md) +and the [Writing an Interface](../interfaces.md) guide +``` + +Numpydantic exposes a system for combinatoric testing across dtypes, shapes, +and interfaces in the {mod}`numpydantic.testing` module. + +These helper classes and functions are included in the distributed package +so they can be used for downstream development of independent interfaces +(though we always welcome contributions!) + +## Validation Cases + +Each test case is parameterized by a {class}`.ValidationCase`. + +The case is intended to be able to be partially filled in so that multiple +validation cases can be merged together, but also used independently +by falling back on default values. + +There are three major parts to a validation case: + +- **Annotation specification:** {attr}`~.ValidationCase.annotation_dtype` and + {attr}`~.ValidationCase.annotation_shape` specifies how the + {class}`.NDArray` {attr}`.ValidationCase.annotation` that is used to test + against is generated +- **Array specification:** {attr}`~.ValidationCase.dtype` and {attr}`~.ValidationCase.shape` + specify that array that will be generated to test against the annotation +- **Interface specification:** An {class}`.InterfaceCase` that refers to + an {class}`.Interface`, and provides array generation and other auxilary logic. + +Typically, one specifies a dtype along with an annotation dtype or +a shape along with an annotation shape (or implicitly against the defaults for either), +along with a value for `passes` that indicates if that combination is valid. + +```{code-cell} +from numpydantic.testing import ValidationCase + +dtype_case = ValidationCase( + id="int_int", + dtype=int, + annotation_dtype=int, + passes=True +) +shape_case = ValidationCase( + id="cool_shape", + shape=(1,2,3), + annotation_shape=(1,"*","2-4"), + passes=True +) + +merged = dtype_case.merge(shape_case) +console.print(merged.model_dump(exclude={'annotation', 'model'}, exclude_unset=True)) +``` + +When merging validation cases, the merged case only `passes` if all the +original cases do. + +```{code-cell} +from numpydantic.testing import ValidationCase + +dtype_case = ValidationCase( + id="int_int", + dtype=int, + annotation_dtype=int, + passes=True +) +shape_case = ValidationCase( + id="uncool_shape", + shape=(1,2,3), + annotation_shape=(9,8,7), + passes=False +) + +merged = dtype_case.merge(shape_case) +console.print(merged.model_dump(exclude={'annotation', 'model'}, exclude_unset=True)) +``` + +We provide a convenience function {func}`.merged_product` for creating a merged product of +multiple sets of test cases. + +For example, you may want to create a set of dtype and shape cases and validate +against all combinations + +```{code-cell} +from numpydantic.testing.helpers import merged_product + +dtype_cases = [ + ValidationCase(dtype=int, annotation_dtype=int, passes=True), + ValidationCase(dtype=int, annotation_dtype=float, passes=False) +] +shape_cases = [ + ValidationCase(shape=(1,2,3), annotation_shape=(1,2,3), passes=True), + ValidationCase(shape=(4,5,6), annotation_shape=(1,2,3), passes=False) +] + +iterator = merged_product(dtype_cases, shape_cases) + +console.print([i.model_dump(exclude_unset=True, exclude={'model', 'annotation'}) for i in iterator]) + +``` + +You can pass constraints to the {func}`.merged_product` iterator to +filter cases that match some value, for example to get only the cases that pass: + +```{code-cell} +iterator = merged_product(dtype_cases, shape_cases, conditions={"passes": True}) +console.print([i.model_dump(exclude_unset=True, exclude={'model', 'annotation'}) for i in iterator]) +``` + +## Interface Cases + +Validation cases can be paired with interface cases that handle +generating arrays for the given interface from the specification in the +validation case. + +Since some array interfaces like Zarr have multiple possible forms +of an array (in memory, on disk, in a zip file, etc.) an interface +may have multiple cases that are important to test against. + +The {meth}`.InterfaceCase.make_array` method does what you'd expect it to, +creating an array, and returning the appropriate input type for the interface: + +```{code-cell} +from numpydantic.testing.interfaces import NumpyCase, ZarrNestedCase + +NumpyCase.make_array(shape=(1,2,3), dtype=float) +``` + +```{code-cell} +ZarrNestedCase.make_array(shape=(1,2,3), dtype=float, path=Path("__tmp__/zarr_dir")) +``` + +Interface cases also define when an interface should skip a given test +parameterization. For example, some array formats can't support arbitrary +object serialization, and the video class can only support 8-bit arrays +of a specific shape + +```{code-cell} +from numpydantic.testing.interfaces import VideoCase + +VideoCase.skip(shape=(1,1), dtype=float) +``` + +This, and the array generation methods are propagated up into +a ValidationCase that contains them + +```{code-cell} +case = ValidationCase(shape=(1,2,3), dtype=float, interface=VideoCase) +case.skip() +``` + +The {func}`.merged_product` iterator automatically excludes any +combinations of interfaces and test parameterizations that should be skipped. + +## Making Fixtures + +Pytest fixtures are a useful way to re-use validation case products. +To keep things tidy, you may want to use marks and ids when creating them +so that you can run tests against specific interfaces or conditions +with the `pytest -m mark` system. + +```python +import pytest + +@pytest.fixture( + params=( + pytest.param( + p, + id=p.id, + marks=getattr(pytest.mark, p.interface.interface.name) + ) + for p in iterator + ) +) +def my_cases(request): + return request.param +``` diff --git a/docs/index.md b/docs/index.md index 127719f..313601e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -514,6 +514,7 @@ api/meta api/schema api/serialization api/types +api/testing/index ``` @@ -523,6 +524,7 @@ api/types :hidden: true changelog +contributing/index development todo ``` diff --git a/src/numpydantic/testing/cases.py b/src/numpydantic/testing/cases.py index d3250cd..d0f44ee 100644 --- a/src/numpydantic/testing/cases.py +++ b/src/numpydantic/testing/cases.py @@ -83,6 +83,9 @@ SHAPE_CASES = ( id="Union incorrect both", ), ) +""" +Base Shape cases +""" DTYPE_CASES = [ @@ -163,6 +166,9 @@ DTYPE_CASES = [ annotation_dtype=UNION_TYPE, dtype=str, passes=False, id="union-type-str" ), ] +""" +Base Dtype cases +""" if YES_PIPE: @@ -214,19 +220,40 @@ INTERFACE_CASES = [ ValidationCase(interface=ZarrNestedCase, id="zarr_nested"), ValidationCase(interface=VideoCase, id="video"), ] +""" +All the interface cases +""" DTYPE_AND_SHAPE_CASES = merged_product(SHAPE_CASES, DTYPE_CASES) +""" +Merged product of dtype and shape cases +""" DTYPE_AND_SHAPE_CASES_PASSING = merged_product( SHAPE_CASES, DTYPE_CASES, conditions={"passes": True} ) +""" +Merged product of dtype and shape cases that are valid +""" DTYPE_AND_INTERFACE_CASES = merged_product(INTERFACE_CASES, DTYPE_CASES) +""" +Merged product of dtype and interface cases +""" DTYPE_AND_INTERFACE_CASES_PASSING = merged_product( INTERFACE_CASES, DTYPE_CASES, conditions={"passes": True} ) +""" +Merged product of dtype and interface cases that pass +""" ALL_CASES = merged_product(SHAPE_CASES, DTYPE_CASES, INTERFACE_CASES) +""" +Merged product of all cases - dtype, shape, and interface +""" ALL_CASES_PASSING = merged_product( SHAPE_CASES, DTYPE_CASES, INTERFACE_CASES, conditions={"passes": True} ) +""" +Merged product of all cases, but only those that pass +"""