diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index eba8485..b1283e1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,7 +36,10 @@ jobs: nwb_models/pyproject.toml - name: Install dependencies - run: pip install -e .[tests] + run: | + pip install -e .[tests] + pip install -e ../nwb_schema_language + pip install -e ../nwb_models working-directory: nwb_linkml - name: Run Tests diff --git a/docs/meta/todo.md b/docs/meta/todo.md index d2bf9ac..dd9f750 100644 --- a/docs/meta/todo.md +++ b/docs/meta/todo.md @@ -49,6 +49,10 @@ Remove monkeypatches/overrides once PRs are closed Tests - [ ] Ensure schemas and pydantic modules in repos are up to date +Loading +- [ ] Top-level containers are still a little janky, eg. how `ProcessingModule` just accepts + extra args rather than properly abstracting `value` as a `__getitem__(self, key) -> T:` + ## Docs TODOs ```{todolist} diff --git a/nwb_linkml/conftest.py b/nwb_linkml/conftest.py index 450875f..88c09a6 100644 --- a/nwb_linkml/conftest.py +++ b/nwb_linkml/conftest.py @@ -71,7 +71,7 @@ adapter_parser = Sybil( doctest_parser = Sybil( parsers=[DocTestParser(optionflags=ELLIPSIS + NORMALIZE_WHITESPACE), PythonCodeBlockParser()], - patterns=["*.py"], + patterns=["providers/git.py"], ) pytest_collect_file = (adapter_parser + doctest_parser).pytest() diff --git a/nwb_linkml/pdm.lock b/nwb_linkml/pdm.lock index 7a1aca7..f6f2c7c 100644 --- a/nwb_linkml/pdm.lock +++ b/nwb_linkml/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev", "plot", "tests"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:f219083028bd024c53bc55626c8b6088d6eb5c2ade56bd694a7a112098aa9bfc" +content_hash = "sha256:1c297e11f6dc9e4f6b8d29df872177d2ce65bbd334c0b65aa5175dfb125c4d9f" [[metadata.targets]] requires_python = ">=3.10,<3.13" @@ -549,7 +549,7 @@ name = "h5py" version = "3.11.0" requires_python = ">=3.8" summary = "Read and write HDF5 files from Python" -groups = ["default"] +groups = ["default", "dev", "tests"] dependencies = [ "numpy>=1.17.3", ] @@ -580,6 +580,26 @@ files = [ {file = "hbreader-0.9.1.tar.gz", hash = "sha256:d2c132f8ba6276d794c66224c3297cec25c8079d0a4cf019c061611e0a3b94fa"}, ] +[[package]] +name = "hdmf" +version = "3.14.3" +requires_python = ">=3.8" +summary = "A hierarchical data modeling framework for modern science data standards" +groups = ["dev", "tests"] +dependencies = [ + "h5py>=2.10", + "importlib-resources; python_version < \"3.9\"", + "jsonschema>=2.6.0", + "numpy>=1.18", + "pandas>=1.0.5", + "ruamel-yaml>=0.16", + "scipy>=1.4", +] +files = [ + {file = "hdmf-3.14.3-py3-none-any.whl", hash = "sha256:1417ccc0d336d535192b7a3db4c7354cbc15123f1ccb3cdd82e363308e78f9bc"}, + {file = "hdmf-3.14.3.tar.gz", hash = "sha256:e9548fc7bdbb534a2750092b6b9819df2ce50e27430866c3c32061a2306271cc"}, +] + [[package]] name = "idna" version = "3.8" @@ -751,7 +771,7 @@ name = "jsonschema" version = "4.23.0" requires_python = ">=3.8" summary = "An implementation of JSON Schema validation for Python" -groups = ["default"] +groups = ["default", "dev", "tests"] dependencies = [ "attrs>=22.2.0", "importlib-resources>=1.4.0; python_version < \"3.9\"", @@ -770,7 +790,7 @@ name = "jsonschema-specifications" version = "2023.12.1" requires_python = ">=3.8" summary = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" -groups = ["default"] +groups = ["default", "dev", "tests"] dependencies = [ "importlib-resources>=1.4.0; python_version < \"3.9\"", "referencing>=0.31.0", @@ -976,7 +996,7 @@ name = "networkx" version = "3.3" requires_python = ">=3.10" summary = "Python package for creating and manipulating graphs and networks" -groups = ["dev", "tests"] +groups = ["default", "dev", "tests"] files = [ {file = "networkx-3.3-py3-none-any.whl", hash = "sha256:28575580c6ebdaf4505b22c6256a2b9de86b316dc63ba9e93abde3d78dfdbcf2"}, {file = "networkx-3.3.tar.gz", hash = "sha256:0c127d8b2f4865f59ae9cb8aafcd60b5c70f3241ebd66f7defad7c4ab90126c9"}, @@ -984,45 +1004,36 @@ files = [ [[package]] name = "numpy" -version = "2.1.0" -requires_python = ">=3.10" +version = "1.26.4" +requires_python = ">=3.9" summary = "Fundamental package for array computing in Python" -groups = ["default"] +groups = ["default", "dev", "tests"] files = [ - {file = "numpy-2.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6326ab99b52fafdcdeccf602d6286191a79fe2fda0ae90573c5814cd2b0bc1b8"}, - {file = "numpy-2.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0937e54c09f7a9a68da6889362ddd2ff584c02d015ec92672c099b61555f8911"}, - {file = "numpy-2.1.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:30014b234f07b5fec20f4146f69e13cfb1e33ee9a18a1879a0142fbb00d47673"}, - {file = "numpy-2.1.0-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:899da829b362ade41e1e7eccad2cf274035e1cb36ba73034946fccd4afd8606b"}, - {file = "numpy-2.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08801848a40aea24ce16c2ecde3b756f9ad756586fb2d13210939eb69b023f5b"}, - {file = "numpy-2.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:398049e237d1aae53d82a416dade04defed1a47f87d18d5bd615b6e7d7e41d1f"}, - {file = "numpy-2.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0abb3916a35d9090088a748636b2c06dc9a6542f99cd476979fb156a18192b84"}, - {file = "numpy-2.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:10e2350aea18d04832319aac0f887d5fcec1b36abd485d14f173e3e900b83e33"}, - {file = "numpy-2.1.0-cp310-cp310-win32.whl", hash = "sha256:f6b26e6c3b98adb648243670fddc8cab6ae17473f9dc58c51574af3e64d61211"}, - {file = "numpy-2.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:f505264735ee074250a9c78247ee8618292091d9d1fcc023290e9ac67e8f1afa"}, - {file = "numpy-2.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:76368c788ccb4f4782cf9c842b316140142b4cbf22ff8db82724e82fe1205dce"}, - {file = "numpy-2.1.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:f8e93a01a35be08d31ae33021e5268f157a2d60ebd643cfc15de6ab8e4722eb1"}, - {file = "numpy-2.1.0-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:9523f8b46485db6939bd069b28b642fec86c30909cea90ef550373787f79530e"}, - {file = "numpy-2.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54139e0eb219f52f60656d163cbe67c31ede51d13236c950145473504fa208cb"}, - {file = "numpy-2.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5ebbf9fbdabed208d4ecd2e1dfd2c0741af2f876e7ae522c2537d404ca895c3"}, - {file = "numpy-2.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:378cb4f24c7d93066ee4103204f73ed046eb88f9ad5bb2275bb9fa0f6a02bd36"}, - {file = "numpy-2.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8f699a709120b220dfe173f79c73cb2a2cab2c0b88dd59d7b49407d032b8ebd"}, - {file = "numpy-2.1.0-cp311-cp311-win32.whl", hash = "sha256:ffbd6faeb190aaf2b5e9024bac9622d2ee549b7ec89ef3a9373fa35313d44e0e"}, - {file = "numpy-2.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:0af3a5987f59d9c529c022c8c2a64805b339b7ef506509fba7d0556649b9714b"}, - {file = "numpy-2.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fe76d75b345dc045acdbc006adcb197cc680754afd6c259de60d358d60c93736"}, - {file = "numpy-2.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f358ea9e47eb3c2d6eba121ab512dfff38a88db719c38d1e67349af210bc7529"}, - {file = "numpy-2.1.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:dd94ce596bda40a9618324547cfaaf6650b1a24f5390350142499aa4e34e53d1"}, - {file = "numpy-2.1.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:b47c551c6724960479cefd7353656498b86e7232429e3a41ab83be4da1b109e8"}, - {file = "numpy-2.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0756a179afa766ad7cb6f036de622e8a8f16ffdd55aa31f296c870b5679d745"}, - {file = "numpy-2.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24003ba8ff22ea29a8c306e61d316ac74111cebf942afbf692df65509a05f111"}, - {file = "numpy-2.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b34fa5e3b5d6dc7e0a4243fa0f81367027cb6f4a7215a17852979634b5544ee0"}, - {file = "numpy-2.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c4f982715e65036c34897eb598d64aef15150c447be2cfc6643ec7a11af06574"}, - {file = "numpy-2.1.0-cp312-cp312-win32.whl", hash = "sha256:c4cd94dfefbefec3f8b544f61286584292d740e6e9d4677769bc76b8f41deb02"}, - {file = "numpy-2.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:a0cdef204199278f5c461a0bed6ed2e052998276e6d8ab2963d5b5c39a0500bc"}, - {file = "numpy-2.1.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:15ef8b2177eeb7e37dd5ef4016f30b7659c57c2c0b57a779f1d537ff33a72c7b"}, - {file = "numpy-2.1.0-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:e5f0642cdf4636198a4990de7a71b693d824c56a757862230454629cf62e323d"}, - {file = "numpy-2.1.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f15976718c004466406342789f31b6673776360f3b1e3c575f25302d7e789575"}, - {file = "numpy-2.1.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:6c1de77ded79fef664d5098a66810d4d27ca0224e9051906e634b3f7ead134c2"}, - {file = "numpy-2.1.0.tar.gz", hash = "sha256:7dc90da0081f7e1da49ec4e398ede6a8e9cc4f5ebe5f9e06b443ed889ee9aaa2"}, + {file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"}, + {file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2"}, + {file = "numpy-1.26.4-cp310-cp310-win32.whl", hash = "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07"}, + {file = "numpy-1.26.4-cp310-cp310-win_amd64.whl", hash = "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a"}, + {file = "numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20"}, + {file = "numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0"}, + {file = "numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110"}, + {file = "numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818"}, + {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"}, ] [[package]] @@ -1102,7 +1113,7 @@ name = "pandas" version = "2.2.2" requires_python = ">=3.9" summary = "Powerful data structures for data analysis, time series, and statistics" -groups = ["default"] +groups = ["default", "dev", "tests"] dependencies = [ "numpy>=1.22.4; python_version < \"3.11\"", "numpy>=1.23.2; python_version == \"3.11\"", @@ -1350,6 +1361,24 @@ files = [ {file = "PyJSG-0.11.10.tar.gz", hash = "sha256:4bd6e3ff2833fa2b395bbe803a2d72a5f0bab5b7285bccd0da1a1bc0aee88bfa"}, ] +[[package]] +name = "pynwb" +version = "2.8.1" +requires_python = ">=3.8" +summary = "Package for working with Neurodata stored in the NWB format." +groups = ["dev", "tests"] +dependencies = [ + "h5py>=2.10", + "hdmf>=3.14.0", + "numpy<2.0,>=1.18", + "pandas>=1.1.5", + "python-dateutil>=2.7.3", +] +files = [ + {file = "pynwb-2.8.1-py3-none-any.whl", hash = "sha256:f3c392652b26396e135cf6f1abd570d413c9eb7bf5bdb1a89d899852338fdf6c"}, + {file = "pynwb-2.8.1.tar.gz", hash = "sha256:498e4bc46a7b0a1331a0f754bac72ea7f9d10d1bba35af3c7be78a61bb1d104b"}, +] + [[package]] name = "pyparsing" version = "3.1.4" @@ -1469,7 +1498,7 @@ name = "python-dateutil" version = "2.9.0.post0" requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" summary = "Extensions to the standard Python datetime module" -groups = ["default"] +groups = ["default", "dev", "tests"] dependencies = [ "six>=1.5", ] @@ -1506,7 +1535,7 @@ files = [ name = "pytz" version = "2024.1" summary = "World timezone definitions, modern and historical" -groups = ["default"] +groups = ["default", "dev", "tests"] files = [ {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, @@ -1597,7 +1626,7 @@ name = "referencing" version = "0.35.1" requires_python = ">=3.8" summary = "JSON Referencing + Python" -groups = ["default"] +groups = ["default", "dev", "tests"] dependencies = [ "attrs>=22.2.0", "rpds-py>=0.7.0", @@ -1701,7 +1730,7 @@ name = "rpds-py" version = "0.20.0" requires_python = ">=3.8" summary = "Python bindings to Rust's persistent data structures (rpds)" -groups = ["default"] +groups = ["default", "dev", "tests"] files = [ {file = "rpds_py-0.20.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:3ad0fda1635f8439cde85c700f964b23ed5fc2d28016b32b9ee5fe30da5c84e2"}, {file = "rpds_py-0.20.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9bb4a0d90fdb03437c109a17eade42dfbf6190408f29b2744114d11586611d6f"}, @@ -1762,7 +1791,7 @@ name = "ruamel-yaml" version = "0.18.6" requires_python = ">=3.7" summary = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" -groups = ["default"] +groups = ["default", "dev", "tests"] dependencies = [ "ruamel-yaml-clib>=0.2.7; platform_python_implementation == \"CPython\" and python_version < \"3.13\"", ] @@ -1776,7 +1805,7 @@ name = "ruamel-yaml-clib" version = "0.2.8" requires_python = ">=3.6" summary = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" -groups = ["default"] +groups = ["default", "dev", "tests"] marker = "platform_python_implementation == \"CPython\" and python_version < \"3.13\"" files = [ {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b42169467c42b692c19cf539c38d4602069d8c1505e97b86387fcf7afb766e1d"}, @@ -1833,6 +1862,43 @@ files = [ {file = "ruff-0.6.2.tar.gz", hash = "sha256:239ee6beb9e91feb8e0ec384204a763f36cb53fb895a1a364618c6abb076b3be"}, ] +[[package]] +name = "scipy" +version = "1.14.1" +requires_python = ">=3.10" +summary = "Fundamental algorithms for scientific computing in Python" +groups = ["dev", "tests"] +dependencies = [ + "numpy<2.3,>=1.23.5", +] +files = [ + {file = "scipy-1.14.1-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:b28d2ca4add7ac16ae8bb6632a3c86e4b9e4d52d3e34267f6e1b0c1f8d87e389"}, + {file = "scipy-1.14.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:d0d2821003174de06b69e58cef2316a6622b60ee613121199cb2852a873f8cf3"}, + {file = "scipy-1.14.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:8bddf15838ba768bb5f5083c1ea012d64c9a444e16192762bd858f1e126196d0"}, + {file = "scipy-1.14.1-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:97c5dddd5932bd2a1a31c927ba5e1463a53b87ca96b5c9bdf5dfd6096e27efc3"}, + {file = "scipy-1.14.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2ff0a7e01e422c15739ecd64432743cf7aae2b03f3084288f399affcefe5222d"}, + {file = "scipy-1.14.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e32dced201274bf96899e6491d9ba3e9a5f6b336708656466ad0522d8528f69"}, + {file = "scipy-1.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8426251ad1e4ad903a4514712d2fa8fdd5382c978010d1c6f5f37ef286a713ad"}, + {file = "scipy-1.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:a49f6ed96f83966f576b33a44257d869756df6cf1ef4934f59dd58b25e0327e5"}, + {file = "scipy-1.14.1-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:2da0469a4ef0ecd3693761acbdc20f2fdeafb69e6819cc081308cc978153c675"}, + {file = "scipy-1.14.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:c0ee987efa6737242745f347835da2cc5bb9f1b42996a4d97d5c7ff7928cb6f2"}, + {file = "scipy-1.14.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3a1b111fac6baec1c1d92f27e76511c9e7218f1695d61b59e05e0fe04dc59617"}, + {file = "scipy-1.14.1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8475230e55549ab3f207bff11ebfc91c805dc3463ef62eda3ccf593254524ce8"}, + {file = "scipy-1.14.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:278266012eb69f4a720827bdd2dc54b2271c97d84255b2faaa8f161a158c3b37"}, + {file = "scipy-1.14.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fef8c87f8abfb884dac04e97824b61299880c43f4ce675dd2cbeadd3c9b466d2"}, + {file = "scipy-1.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b05d43735bb2f07d689f56f7b474788a13ed8adc484a85aa65c0fd931cf9ccd2"}, + {file = "scipy-1.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:716e389b694c4bb564b4fc0c51bc84d381735e0d39d3f26ec1af2556ec6aad94"}, + {file = "scipy-1.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:631f07b3734d34aced009aaf6fedfd0eb3498a97e581c3b1e5f14a04164a456d"}, + {file = "scipy-1.14.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:af29a935803cc707ab2ed7791c44288a682f9c8107bc00f0eccc4f92c08d6e07"}, + {file = "scipy-1.14.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:2843f2d527d9eebec9a43e6b406fb7266f3af25a751aa91d62ff416f54170bc5"}, + {file = "scipy-1.14.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:eb58ca0abd96911932f688528977858681a59d61a7ce908ffd355957f7025cfc"}, + {file = "scipy-1.14.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30ac8812c1d2aab7131a79ba62933a2a76f582d5dbbc695192453dae67ad6310"}, + {file = "scipy-1.14.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f9ea80f2e65bdaa0b7627fb00cbeb2daf163caa015e59b7516395fe3bd1e066"}, + {file = "scipy-1.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:edaf02b82cd7639db00dbff629995ef185c8df4c3ffa71a5562a595765a06ce1"}, + {file = "scipy-1.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2ff38e22128e6c03ff73b6bb0f85f897d2362f8c052e3b8ad00532198fbdae3f"}, + {file = "scipy-1.14.1.tar.gz", hash = "sha256:5a275584e726026a5699459aa72f828a610821006228e841b94275c4a7c08417"}, +] + [[package]] name = "setuptools" version = "74.0.0" @@ -2023,7 +2089,7 @@ name = "tzdata" version = "2024.1" requires_python = ">=2" summary = "Provider of IANA time zone data" -groups = ["default"] +groups = ["default", "dev", "tests"] files = [ {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, diff --git a/nwb_linkml/pyproject.toml b/nwb_linkml/pyproject.toml index 41cf80a..c8ccd36 100644 --- a/nwb_linkml/pyproject.toml +++ b/nwb_linkml/pyproject.toml @@ -9,7 +9,7 @@ license = {text = "AGPL-3.0"} readme = "README.md" requires-python = "<3.13,>=3.10" dependencies = [ - "nwb-models>=0.1.0", + "nwb-models>=0.2.0", "pyyaml>=6.0", "linkml-runtime>=1.7.7", "nwb-schema-language>=0.1.3", @@ -22,9 +22,10 @@ dependencies = [ "pydantic-settings>=2.0.3", "tqdm>=4.66.1", 'typing-extensions>=4.12.2;python_version<"3.11"', - "numpydantic>=1.3.3", + "numpydantic>=1.5.0", "black>=24.4.2", "pandas>=2.2.2", + "networkx>=3.3", ] [project.urls] @@ -44,6 +45,7 @@ tests = [ "pytest-cov<5.0.0,>=4.1.0", "sybil>=6.0.3", "requests-cache>=1.2.1", + "pynwb>=2.8.1", ] dev = [ "nwb-linkml[tests]", diff --git a/nwb_linkml/src/nwb_linkml/adapters/adapter.py b/nwb_linkml/src/nwb_linkml/adapters/adapter.py index 13e86fd..acbc896 100644 --- a/nwb_linkml/src/nwb_linkml/adapters/adapter.py +++ b/nwb_linkml/src/nwb_linkml/adapters/adapter.py @@ -2,6 +2,7 @@ Base class for adapters """ +import os import sys from abc import abstractmethod from dataclasses import dataclass, field @@ -101,6 +102,19 @@ class Adapter(BaseModel): """Abstract base class for adapters""" _logger: Optional[Logger] = None + _debug: Optional[bool] = None + + @property + def debug(self) -> bool: + """ + Whether we are in debug mode, which adds extra metadata in generated elements. + + Set explicitly via ``_debug`` , or else checks for the truthiness of the + environment variable ``NWB_LINKML_DEBUG`` + """ + if self._debug is None: + self._debug = bool(os.environ.get("NWB_LINKML_DEBUG", False)) + return self._debug @property def logger(self) -> Logger: diff --git a/nwb_linkml/src/nwb_linkml/adapters/attribute.py b/nwb_linkml/src/nwb_linkml/adapters/attribute.py index ddf6edb..7ae2ea1 100644 --- a/nwb_linkml/src/nwb_linkml/adapters/attribute.py +++ b/nwb_linkml/src/nwb_linkml/adapters/attribute.py @@ -10,7 +10,7 @@ from linkml_runtime.linkml_model.meta import SlotDefinition from nwb_linkml.adapters.adapter import Adapter, BuildResult, is_1d from nwb_linkml.adapters.array import ArrayAdapter from nwb_linkml.maps import Map -from nwb_linkml.maps.dtype import handle_dtype +from nwb_linkml.maps.dtype import handle_dtype, inlined from nwb_schema_language import Attribute @@ -104,6 +104,7 @@ class MapScalar(AttributeMap): range=handle_dtype(attr.dtype), description=attr.doc, required=attr.required, + inlined=inlined(attr.dtype), **cls.handle_defaults(attr), ) return BuildResult(slots=[slot]) @@ -151,6 +152,7 @@ class MapArray(AttributeMap): multivalued=multivalued, description=attr.doc, required=attr.required, + inlined=inlined(attr.dtype), **expressions, **cls.handle_defaults(attr), ) @@ -171,7 +173,10 @@ class AttributeAdapter(Adapter): Build the slot definitions, every attribute should have a map. """ map = self.match() - return map.apply(self.cls) + res = map.apply(self.cls) + if self.debug: # pragma: no cover - only used in development + res = self._amend_debug(res, map) + return res def match(self) -> Optional[Type[AttributeMap]]: """ @@ -195,3 +200,13 @@ class AttributeAdapter(Adapter): return None else: return matches[0] + + def _amend_debug( + self, res: BuildResult, map: Optional[Type[AttributeMap]] = None + ) -> BuildResult: # pragma: no cover - only used in development + map_name = "None" if map is None else map.__name__ + for cls in res.classes: + cls.annotations["attribute_map"] = {"tag": "attribute_map", "value": map_name} + for slot in res.slots: + slot.annotations["attribute_map"] = {"tag": "attribute_map", "value": map_name} + return res diff --git a/nwb_linkml/src/nwb_linkml/adapters/classes.py b/nwb_linkml/src/nwb_linkml/adapters/classes.py index 0097e47..c008f71 100644 --- a/nwb_linkml/src/nwb_linkml/adapters/classes.py +++ b/nwb_linkml/src/nwb_linkml/adapters/classes.py @@ -92,6 +92,13 @@ class ClassAdapter(Adapter): # Get vanilla top-level attributes kwargs["attributes"].extend(self.build_attrs(self.cls)) + if self.debug: # pragma: no cover - only used in development + kwargs["annotations"] = {} + kwargs["annotations"]["group_adapter"] = { + "tag": "group_adapter", + "value": "container_slot", + } + if extra_attrs is not None: if isinstance(extra_attrs, SlotDefinition): extra_attrs = [extra_attrs] @@ -230,18 +237,23 @@ class ClassAdapter(Adapter): ifabsent=f"string({name})", equals_string=equals_string, range="string", + identifier=True, ) else: - name_slot = SlotDefinition(name="name", required=True, range="string") + name_slot = SlotDefinition(name="name", required=True, range="string", identifier=True) return name_slot def build_self_slot(self) -> SlotDefinition: """ If we are a child class, we make a slot so our parent can refer to us """ - return SlotDefinition( + slot = SlotDefinition( name=self._get_slot_name(), description=self.cls.doc, range=self._get_full_name(), + inlined=True, **QUANTITY_MAP[self.cls.quantity], ) + if self.debug: # pragma: no cover - only used in development + slot.annotations["group_adapter"] = {"tag": "group_adapter", "value": "self_slot"} + return slot diff --git a/nwb_linkml/src/nwb_linkml/adapters/dataset.py b/nwb_linkml/src/nwb_linkml/adapters/dataset.py index ef5eb61..f0b0053 100644 --- a/nwb_linkml/src/nwb_linkml/adapters/dataset.py +++ b/nwb_linkml/src/nwb_linkml/adapters/dataset.py @@ -11,7 +11,7 @@ from nwb_linkml.adapters.adapter import BuildResult, has_attrs, is_1d, is_compou from nwb_linkml.adapters.array import ArrayAdapter from nwb_linkml.adapters.classes import ClassAdapter from nwb_linkml.maps import QUANTITY_MAP, Map -from nwb_linkml.maps.dtype import flat_to_linkml, handle_dtype +from nwb_linkml.maps.dtype import flat_to_linkml, handle_dtype, inlined from nwb_linkml.maps.naming import camel_to_snake from nwb_schema_language import Dataset @@ -147,6 +147,7 @@ class MapScalarAttributes(DatasetMap): name: name: name ifabsent: string(starting_time) + identifier: true range: string required: true equals_string: starting_time @@ -245,6 +246,7 @@ class MapListlike(DatasetMap): attributes: name: name: name + identifier: true range: string required: true value: @@ -257,6 +259,8 @@ class MapListlike(DatasetMap): range: Image required: true multivalued: true + inlined: true + inlined_as_list: true tree_root: true """ @@ -299,6 +303,8 @@ class MapListlike(DatasetMap): description=cls.doc, required=cls.quantity not in ("*", "?"), annotations=[{"source_type": "reference"}], + inlined=True, + inlined_as_list=True, ) res.classes[0].attributes["value"] = slot return res @@ -384,13 +390,11 @@ class MapArraylike(DatasetMap): - ``False`` """ - dtype = handle_dtype(cls.dtype) return ( cls.name and (all([cls.dims, cls.shape]) or cls.neurodata_type_inc == "VectorData") and not has_attrs(cls) and not is_compound(cls) - and dtype in flat_to_linkml ) @classmethod @@ -418,6 +422,7 @@ class MapArraylike(DatasetMap): range=handle_dtype(cls.dtype), description=cls.doc, required=cls.quantity not in ("*", "?"), + inlined=inlined(cls.dtype), **expressions, ) ] @@ -430,6 +435,10 @@ class MapArrayLikeAttributes(DatasetMap): The most general case - treat everything that isn't handled by one of the special cases as an array! + We specifically include classes that have no attributes but also don't have a name, + as they still require their own class (unlike :class:`.MapArrayLike` above, where we + just generate an anonymous slot.) + Examples: .. adapter:: DatasetAdapter @@ -478,6 +487,7 @@ class MapArrayLikeAttributes(DatasetMap): attributes: name: name: name + identifier: true range: string required: true resolution: @@ -525,7 +535,7 @@ class MapArrayLikeAttributes(DatasetMap): return ( all([cls.dims, cls.shape]) and cls.neurodata_type_inc != "VectorData" - and has_attrs(cls) + and (has_attrs(cls) or not cls.name) and not is_compound(cls) and (dtype == "AnyType" or dtype in flat_to_linkml) ) @@ -540,7 +550,9 @@ class MapArrayLikeAttributes(DatasetMap): array_adapter = ArrayAdapter(cls.dims, cls.shape) expressions = array_adapter.make_slot() # make a slot for the arraylike class - array_slot = SlotDefinition(name="value", range=handle_dtype(cls.dtype), **expressions) + array_slot = SlotDefinition( + name="value", range=handle_dtype(cls.dtype), inlined=inlined(cls.dtype), **expressions + ) res.classes[0].attributes.update({"value": array_slot}) return res @@ -579,6 +591,7 @@ class MapClassRange(DatasetMap): description=cls.doc, range=f"{cls.neurodata_type_inc}", annotations=[{"named": True}, {"source_type": "neurodata_type_inc"}], + inlined=True, **QUANTITY_MAP[cls.quantity], ) res = BuildResult(slots=[this_slot]) @@ -590,102 +603,6 @@ class MapClassRange(DatasetMap): # -------------------------------------------------- -class MapVectorClassRange(DatasetMap): - """ - Map a ``VectorData`` class that is a reference to another class as simply - a multivalued slot range, rather than an independent class - """ - - @classmethod - def check(c, cls: Dataset) -> bool: - """ - Check that we are a VectorData object without any additional attributes - with a dtype that refers to another class - """ - dtype = handle_dtype(cls.dtype) - return ( - cls.neurodata_type_inc == "VectorData" - and cls.name - and not has_attrs(cls) - and not (cls.shape or cls.dims) - and not is_compound(cls) - and dtype not in flat_to_linkml - ) - - @classmethod - def apply( - c, cls: Dataset, res: Optional[BuildResult] = None, name: Optional[str] = None - ) -> BuildResult: - """ - Create a slot that replaces the base class just as a list[ClassRef] - """ - this_slot = SlotDefinition( - name=cls.name, - description=cls.doc, - multivalued=True, - range=handle_dtype(cls.dtype), - required=cls.quantity not in ("*", "?"), - ) - res = BuildResult(slots=[this_slot]) - return res - - -# -# class Map1DVector(DatasetMap): -# """ -# ``VectorData`` is subclassed with a name but without dims or attributes, -# treat this as a normal 1D array slot that replaces any class that would be built for this -# -# eg. all the datasets in epoch.TimeIntervals: -# -# .. code-block:: yaml -# -# groups: -# - neurodata_type_def: TimeIntervals -# neurodata_type_inc: DynamicTable -# doc: A container for aggregating epoch data and the TimeSeries that each epoch applies -# to. -# datasets: -# - name: start_time -# neurodata_type_inc: VectorData -# dtype: float32 -# doc: Start time of epoch, in seconds. -# -# """ -# -# @classmethod -# def check(c, cls: Dataset) -> bool: -# """ -# Check that we're a 1d VectorData class -# """ -# return ( -# cls.neurodata_type_inc == "VectorData" -# and not cls.dims -# and not cls.shape -# and not cls.attributes -# and not cls.neurodata_type_def -# and not is_compound(cls) -# and cls.name -# ) -# -# @classmethod -# def apply( -# c, cls: Dataset, res: Optional[BuildResult] = None, name: Optional[str] = None -# ) -> BuildResult: -# """ -# Return a simple multivalued slot -# """ -# this_slot = SlotDefinition( -# name=cls.name, -# description=cls.doc, -# range=handle_dtype(cls.dtype), -# multivalued=True, -# ) -# # No need to make a class for us, so we replace the existing build results -# res = BuildResult(slots=[this_slot]) -# return res - - class MapNVectors(DatasetMap): """ An unnamed container that indicates an arbitrary quantity of some other neurodata type. @@ -795,6 +712,7 @@ class MapCompoundDtype(DatasetMap): description=a_dtype.doc, range=handle_dtype(a_dtype.dtype), array=ArrayExpression(exact_number_dimensions=1), + inlined=inlined(a_dtype.dtype), **QUANTITY_MAP[cls.quantity], ) res.classes[0].attributes.update(slots) @@ -826,6 +744,8 @@ class DatasetAdapter(ClassAdapter): if map is not None: res = map.apply(self.cls, res, self._get_full_name()) + if self.debug: # pragma: no cover - only used in development + res = self._amend_debug(res, map) return res def match(self) -> Optional[Type[DatasetMap]]: @@ -850,3 +770,13 @@ class DatasetAdapter(ClassAdapter): return None else: return matches[0] + + def _amend_debug( + self, res: BuildResult, map: Optional[Type[DatasetMap]] = None + ) -> BuildResult: # pragma: no cover - only used in development + map_name = "None" if map is None else map.__name__ + for cls in res.classes: + cls.annotations["dataset_map"] = {"tag": "dataset_map", "value": map_name} + for slot in res.slots: + slot.annotations["dataset_map"] = {"tag": "dataset_map", "value": map_name} + return res diff --git a/nwb_linkml/src/nwb_linkml/adapters/group.py b/nwb_linkml/src/nwb_linkml/adapters/group.py index 13a03b7..0703aa0 100644 --- a/nwb_linkml/src/nwb_linkml/adapters/group.py +++ b/nwb_linkml/src/nwb_linkml/adapters/group.py @@ -68,11 +68,17 @@ class GroupAdapter(ClassAdapter): if not self.cls.links: return [] + annotations = [{"tag": "source_type", "value": "link"}] + + if self.debug: # pragma: no cover - only used in development + annotations.append({"tag": "group_adapter", "value": "link"}) + slots = [ SlotDefinition( name=link.name, any_of=[{"range": link.target_type}, {"range": "string"}], - annotations=[{"tag": "source_type", "value": "link"}], + annotations=annotations, + inlined=True, **QUANTITY_MAP[link.quantity], ) for link in self.cls.links @@ -111,6 +117,9 @@ class GroupAdapter(ClassAdapter): inlined_as_list=False, ) + if self.debug: # pragma: no cover - only used in development + slot.annotations["group_adapter"] = {"tag": "group_adapter", "value": "container_group"} + if self.parent is not None: # if we have a parent, # just return the slot itself without the class @@ -144,17 +153,20 @@ class GroupAdapter(ClassAdapter): """ name = camel_to_snake(self.cls.neurodata_type_inc) if not self.cls.name else cls.name - return BuildResult( - slots=[ - SlotDefinition( - name=name, - range=self.cls.neurodata_type_inc, - description=self.cls.doc, - **QUANTITY_MAP[cls.quantity], - ) - ] + slot = SlotDefinition( + name=name, + range=self.cls.neurodata_type_inc, + description=self.cls.doc, + inlined=True, + inlined_as_list=False, + **QUANTITY_MAP[cls.quantity], ) + if self.debug: # pragma: no cover - only used in development + slot.annotations["group_adapter"] = {"tag": "group_adapter", "value": "container_slot"} + + return BuildResult(slots=[slot]) + def build_subclasses(self) -> BuildResult: """ Build nested groups and datasets @@ -166,20 +178,9 @@ class GroupAdapter(ClassAdapter): # for creating slots vs. classes is handled by the adapter class dataset_res = BuildResult() for dset in self.cls.datasets: - # if dset.name == 'timestamps': - # pdb.set_trace() dset_adapter = DatasetAdapter(cls=dset, parent=self) dataset_res += dset_adapter.build() - # Actually i'm not sure we have to special case this, we could handle it in - # i/o instead - - # Groups are a bit more complicated because they can also behave like - # range declarations: - # eg. a group can have multiple groups with `neurodata_type_inc`, no name, - # and quantity of *, - # the group can then contain any number of groups of those included types as direct children - group_res = BuildResult() for group in self.cls.groups: @@ -190,6 +191,33 @@ class GroupAdapter(ClassAdapter): return res + def build_self_slot(self) -> SlotDefinition: + """ + If we are a child class, we make a slot so our parent can refer to us + + Groups are a bit more complicated because they can also behave like + range declarations: + eg. a group can have multiple groups with `neurodata_type_inc`, no name, + and quantity of *, + the group can then contain any number of groups of those included types as direct children + + We make sure that we're inlined as a dict so our parent class can refer to us like:: + + parent.{slot_name}[{name}] = self + + """ + slot = SlotDefinition( + name=self._get_slot_name(), + description=self.cls.doc, + range=self._get_full_name(), + inlined=True, + inlined_as_list=True, + **QUANTITY_MAP[self.cls.quantity], + ) + if self.debug: # pragma: no cover - only used in development + slot.annotations["group_adapter"] = {"tag": "group_adapter", "value": "container_slot"} + return slot + def _check_if_container(self, group: Group) -> bool: """ Check if a given subgroup is a container subgroup, diff --git a/nwb_linkml/src/nwb_linkml/adapters/namespaces.py b/nwb_linkml/src/nwb_linkml/adapters/namespaces.py index 266906e..c6abd70 100644 --- a/nwb_linkml/src/nwb_linkml/adapters/namespaces.py +++ b/nwb_linkml/src/nwb_linkml/adapters/namespaces.py @@ -48,7 +48,16 @@ class NamespacesAdapter(Adapter): need_imports = [] for needed in ns_adapter.needed_imports.values(): - need_imports.extend([n for n in needed if n not in ns_adapter.needed_imports]) + # try to locate imports implied by the namespace schema, + # but are either not provided by the current namespace + # or are otherwise already provided in `imported` by the loader function + need_imports.extend( + [ + n + for n in needed + if n not in ns_adapter.needed_imports and n not in ns_adapter.versions + ] + ) for needed in need_imports: if needed in DEFAULT_REPOS: @@ -56,6 +65,8 @@ class NamespacesAdapter(Adapter): needed_adapter = NamespacesAdapter.from_yaml(needed_source_ns) ns_adapter.imported.append(needed_adapter) + ns_adapter.populate_imports() + return ns_adapter def build( @@ -176,7 +187,6 @@ class NamespacesAdapter(Adapter): else: raise KeyError(f"No schema found that define {name}") - @model_validator(mode="after") def populate_imports(self) -> "NamespacesAdapter": """ Populate the imports that are needed for each schema file diff --git a/nwb_linkml/src/nwb_linkml/generators/pydantic.py b/nwb_linkml/src/nwb_linkml/generators/pydantic.py index 0f824af..1928cf5 100644 --- a/nwb_linkml/src/nwb_linkml/generators/pydantic.py +++ b/nwb_linkml/src/nwb_linkml/generators/pydantic.py @@ -6,11 +6,10 @@ See class and module docstrings for details :) """ import re -import sys from dataclasses import dataclass, field from pathlib import Path from types import ModuleType -from typing import ClassVar, Dict, List, Optional, Tuple +from typing import Callable, ClassVar, Dict, List, Literal, Optional, Tuple from linkml.generators import PydanticGenerator from linkml.generators.pydanticgen.array import ArrayRepresentation, NumpydanticArray @@ -23,11 +22,14 @@ from linkml_runtime.linkml_model.meta import ( SlotDefinition, SlotDefinitionName, ) -from linkml_runtime.utils.compile_python import file_text from linkml_runtime.utils.formatutils import remove_empty_items from linkml_runtime.utils.schemaview import SchemaView -from nwb_linkml.includes.base import BASEMODEL_GETITEM +from nwb_linkml.includes.base import ( + BASEMODEL_COERCE_CHILD, + BASEMODEL_COERCE_VALUE, + BASEMODEL_GETITEM, +) from nwb_linkml.includes.hdmf import ( DYNAMIC_TABLE_IMPORTS, DYNAMIC_TABLE_INJECTS, @@ -36,7 +38,7 @@ from nwb_linkml.includes.hdmf import ( ) from nwb_linkml.includes.types import ModelTypeString, NamedImports, NamedString, _get_name -OPTIONAL_PATTERN = re.compile(r"Optional\[([\w\.]*)\]") +OPTIONAL_PATTERN = re.compile(r"Optional\[(.*)\]") @dataclass @@ -52,6 +54,8 @@ class NWBPydanticGenerator(PydanticGenerator): ), 'object_id: Optional[str] = Field(None, description="Unique UUID for each object")', BASEMODEL_GETITEM, + BASEMODEL_COERCE_VALUE, + BASEMODEL_COERCE_CHILD, ) split: bool = True imports: list[Import] = field(default_factory=lambda: [Import(module="numpy", alias="np")]) @@ -66,6 +70,7 @@ class NWBPydanticGenerator(PydanticGenerator): emit_metadata: bool = True gen_classvars: bool = True gen_slots: bool = True + extra_fields: Literal["allow", "forbid", "ignore"] = "allow" skip_meta: ClassVar[Tuple[str]] = ("domain_of", "alias") @@ -131,6 +136,8 @@ class NWBPydanticGenerator(PydanticGenerator): """Customize dynamictable behavior""" cls = AfterGenerateClass.inject_dynamictable(cls) cls = AfterGenerateClass.wrap_dynamictable_columns(cls, sv) + cls = AfterGenerateClass.inject_elementidentifiers(cls, sv, self._get_element_import) + cls = AfterGenerateClass.strip_vector_data_slots(cls, sv) return cls def before_render_template(self, template: PydanticModule, sv: SchemaView) -> PydanticModule: @@ -204,15 +211,17 @@ class AfterGenerateSlot: # merge injects/imports from the numpydantic array without using the merge method if slot.injected_classes is None: slot.injected_classes = NumpydanticArray.INJECTS.copy() - else: + else: # pragma: no cover - for completeness, shouldn't happen slot.injected_classes.extend(NumpydanticArray.INJECTS.copy()) - if isinstance(slot.imports, list): + if isinstance( + slot.imports, list + ): # pragma: no cover - for completeness, shouldn't happen slot.imports = ( Imports(imports=slot.imports) + NumpydanticArray.IMPORTS.model_copy() ) elif isinstance(slot.imports, Imports): slot.imports += NumpydanticArray.IMPORTS.model_copy() - else: + else: # pragma: no cover - for completeness, shouldn't happen slot.imports = NumpydanticArray.IMPORTS.model_copy() return slot @@ -224,17 +233,20 @@ class AfterGenerateSlot: """ if "named" in slot.source.annotations and slot.source.annotations["named"].value: - slot.attribute.range = f"Named[{slot.attribute.range}]" + + slot.attribute.range = wrap_preserving_optional(slot.attribute.range, "Named") named_injects = [ModelTypeString, _get_name, NamedString] if slot.injected_classes is None: slot.injected_classes = named_injects - else: + else: # pragma: no cover - for completeness, shouldn't happen slot.injected_classes.extend([ModelTypeString, _get_name, NamedString]) - if isinstance(slot.imports, list): + if isinstance( + slot.imports, list + ): # pragma: no cover - for completeness, shouldn't happen slot.imports = Imports(imports=slot.imports) + NamedImports elif isinstance(slot.imports, Imports): slot.imports += NamedImports - else: + else: # pragma: no cover - for completeness, shouldn't happen slot.imports = NamedImports return slot @@ -254,41 +266,57 @@ class AfterGenerateClass: Returns: """ - if cls.cls.name in "DynamicTable": - cls.cls.bases = ["DynamicTableMixin"] + if cls.cls.name == "DynamicTable": + cls.cls.bases = ["DynamicTableMixin", "ConfiguredBaseModel"] - if cls.injected_classes is None: + if ( + cls.injected_classes is None + ): # pragma: no cover - for completeness, shouldn't happen cls.injected_classes = DYNAMIC_TABLE_INJECTS.copy() else: cls.injected_classes.extend(DYNAMIC_TABLE_INJECTS.copy()) if isinstance(cls.imports, Imports): cls.imports += DYNAMIC_TABLE_IMPORTS - elif isinstance(cls.imports, list): + elif isinstance( + cls.imports, list + ): # pragma: no cover - for completeness, shouldn't happen cls.imports = Imports(imports=cls.imports) + DYNAMIC_TABLE_IMPORTS - else: + else: # pragma: no cover - for completeness, shouldn't happen cls.imports = DYNAMIC_TABLE_IMPORTS.model_copy() elif cls.cls.name == "VectorData": - cls.cls.bases = ["VectorDataMixin"] + cls.cls.bases = ["VectorDataMixin", "ConfiguredBaseModel"] + # make ``value`` generic on T + if "value" in cls.cls.attributes: + cls.cls.attributes["value"].range = "Optional[T]" elif cls.cls.name == "VectorIndex": - cls.cls.bases = ["VectorIndexMixin"] + cls.cls.bases = ["VectorIndexMixin", "ConfiguredBaseModel"] elif cls.cls.name == "DynamicTableRegion": - cls.cls.bases = ["DynamicTableRegionMixin", "VectorData"] + cls.cls.bases = ["DynamicTableRegionMixin", "VectorData", "ConfiguredBaseModel"] elif cls.cls.name == "AlignedDynamicTable": cls.cls.bases = ["AlignedDynamicTableMixin", "DynamicTable"] + elif cls.cls.name == "ElementIdentifiers": + cls.cls.bases = ["ElementIdentifiersMixin", "Data", "ConfiguredBaseModel"] + # make ``value`` generic on T + if "value" in cls.cls.attributes: + cls.cls.attributes["value"].range = "Optional[T]" elif cls.cls.name == "TimeSeriesReferenceVectorData": # in core.nwb.base, so need to inject and import again cls.cls.bases = ["TimeSeriesReferenceVectorDataMixin", "VectorData"] - if cls.injected_classes is None: + if ( + cls.injected_classes is None + ): # pragma: no cover - for completeness, shouldn't happen cls.injected_classes = TSRVD_INJECTS.copy() else: cls.injected_classes.extend(TSRVD_INJECTS.copy()) if isinstance(cls.imports, Imports): cls.imports += TSRVD_IMPORTS - elif isinstance(cls.imports, list): + elif isinstance( + cls.imports, list + ): # pragma: no cover - for completeness, shouldn't happen cls.imports = Imports(imports=cls.imports) + TSRVD_IMPORTS - else: + else: # pragma: no cover - for completeness, shouldn't happen cls.imports = TSRVD_IMPORTS.model_copy() return cls @@ -305,34 +333,60 @@ class AfterGenerateClass: ): for an_attr in cls.cls.attributes: if "NDArray" in (slot_range := cls.cls.attributes[an_attr].range): - if an_attr.endswith("_index"): - cls.cls.attributes[an_attr].range = "".join( - ["VectorIndex[", slot_range, "]"] - ) - else: - cls.cls.attributes[an_attr].range = "".join( - ["VectorData[", slot_range, "]"] - ) + if an_attr == "id": + cls.cls.attributes[an_attr].range = "ElementIdentifiers" + return cls + + wrap_cls = "VectorIndex" if an_attr.endswith("_index") else "VectorData" + + cls.cls.attributes[an_attr].range = wrap_preserving_optional( + slot_range, wrap_cls + ) + + return cls + + @staticmethod + def inject_elementidentifiers( + cls: ClassResult, sv: SchemaView, import_method: Callable[[str], Import] + ) -> ClassResult: + """ + Inject ElementIdentifiers into module that define dynamictables - + needed to handle ID columns + """ + if ( + cls.source.is_a == "DynamicTable" + or "DynamicTable" in sv.class_ancestors(cls.source.name) + ) and sv.schema.name != "hdmf-common.table": + imp = import_method("ElementIdentifiers") + cls.imports += [imp] + return cls + + @staticmethod + def strip_vector_data_slots(cls: ClassResult, sv: SchemaView) -> ClassResult: + """ + Remove spurious ``vector_data`` slots from DynamicTables + """ + if "vector_data" in cls.cls.attributes: + del cls.cls.attributes["vector_data"] return cls -def compile_python( - text_or_fn: str, package_path: Path = None, module_name: str = "test" -) -> ModuleType: +def wrap_preserving_optional(annotation: str, wrap: str) -> str: """ - Compile the text or file and return the resulting module - @param text_or_fn: Python text or file name that references python file - @param package_path: Root package path. If omitted and we've got a python file, - the package is the containing - directory - @return: Compiled module - """ - python_txt = file_text(text_or_fn) - if package_path is None and python_txt != text_or_fn: - package_path = Path(text_or_fn) - spec = compile(python_txt, "", "exec") - module = ModuleType(module_name) + Add a wrapping type to a type annotation string, + preserving any `Optional[]` annotation, bumping it to the outside - exec(spec, module.__dict__) - sys.modules[module_name] = module - return module + Examples: + + >>> wrap_preserving_optional('Optional[list[str]]', 'NewType') + 'Optional[NewType[list[str]]]' + + """ + + is_optional = OPTIONAL_PATTERN.match(annotation) + if is_optional: + annotation = is_optional.groups()[0] + annotation = f"Optional[{wrap}[{annotation}]]" + else: + annotation = f"{wrap}[{annotation}]" + return annotation diff --git a/nwb_linkml/src/nwb_linkml/includes/base.py b/nwb_linkml/src/nwb_linkml/includes/base.py index ed69bf3..3ecae8c 100644 --- a/nwb_linkml/src/nwb_linkml/includes/base.py +++ b/nwb_linkml/src/nwb_linkml/includes/base.py @@ -12,3 +12,38 @@ BASEMODEL_GETITEM = """ else: raise KeyError("No value or data field to index from") """ + +BASEMODEL_COERCE_VALUE = """ + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + \"\"\"Try to rescue instantiation by using the value field\"\"\" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 +""" + +BASEMODEL_COERCE_CHILD = """ + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + \"\"\"Recast parent classes into child classes\"\"\" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v +""" diff --git a/nwb_linkml/src/nwb_linkml/includes/hdmf.py b/nwb_linkml/src/nwb_linkml/includes/hdmf.py index 8bc9107..7a7d294 100644 --- a/nwb_linkml/src/nwb_linkml/includes/hdmf.py +++ b/nwb_linkml/src/nwb_linkml/includes/hdmf.py @@ -53,8 +53,11 @@ class DynamicTableMixin(BaseModel): NON_COLUMN_FIELDS: ClassVar[tuple[str]] = ( "id", "name", + "categories", "colnames", "description", + "hdf5_path", + "object_id", ) # overridden by subclass but implemented here for testing and typechecking purposes :) @@ -138,7 +141,7 @@ class DynamicTableMixin(BaseModel): # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -246,11 +249,14 @@ class DynamicTableMixin(BaseModel): if k not in cls.NON_COLUMN_FIELDS and not k.endswith("_index") and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] model["colnames"] = colnames else: # add any columns not explicitly given an order at the end colnames = model["colnames"].copy() + if isinstance(colnames, np.ndarray): + colnames = colnames.tolist() colnames.extend( [ k @@ -259,6 +265,7 @@ class DynamicTableMixin(BaseModel): and not k.endswith("_index") and k not in model["colnames"] and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] ) model["colnames"] = colnames @@ -277,17 +284,25 @@ class DynamicTableMixin(BaseModel): if isinstance(model, dict): for key, val in model.items(): - if key in cls.model_fields: + if key in cls.model_fields or key in cls.NON_COLUMN_FIELDS: continue if not isinstance(val, (VectorData, VectorIndex)): try: - if key.endswith("_index"): - model[key] = VectorIndex(name=key, description="", value=val) + to_cast = VectorIndex if key.endswith("_index") else VectorData + if isinstance(val, dict): + model[key] = to_cast(**val) else: - model[key] = VectorData(name=key, description="", value=val) + model[key] = to_cast(name=key, description="", value=val) except ValidationError as e: # pragma: no cover - raise ValidationError( - f"field {key} cannot be cast to VectorData from {val}" + raise ValidationError.from_exception_data( + title=f"field {key} cannot be cast to VectorData from {val}", + line_errors=[ + { + "type": "ValueError", + "loc": ("DynamicTableMixin", "cast_extra_columns"), + "input": val, + } + ], ) from e return model @@ -320,9 +335,9 @@ class DynamicTableMixin(BaseModel): """ Ensure that all columns are equal length """ - lengths = [len(v) for v in self._columns.values()] + [len(self.id)] + lengths = [len(v) for v in self._columns.values() if v is not None] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "DynamicTable columns are not of equal length! " f"Got colnames:\n{self.colnames}\nand lengths: {lengths}" ) return self @@ -370,7 +385,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -571,10 +586,13 @@ class AlignedDynamicTableMixin(BaseModel): __pydantic_extra__: Dict[str, Union["DynamicTableMixin", "VectorDataMixin", "VectorIndexMixin"]] NON_CATEGORY_FIELDS: ClassVar[tuple[str]] = ( + "id", "name", "categories", "colnames", "description", + "hdf5_path", + "object_id", ) name: str = "aligned_table" @@ -604,28 +622,29 @@ class AlignedDynamicTableMixin(BaseModel): elif isinstance(item, tuple) and len(item) == 2 and isinstance(item[1], str): # get a slice of a single table return self._categories[item[1]][item[0]] - elif isinstance(item, (int, slice, Iterable)): + elif isinstance(item, (int, slice, Iterable, np.int_)): # get a slice of all the tables ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) - tables = [ids] + ids = pd.Index(data=ids, name="id") + tables = [] for category_name, category in self._categories.items(): table = category[item] if isinstance(table, pd.DataFrame): table = table.reset_index() + table.index = ids elif isinstance(table, np.ndarray): - table = pd.DataFrame({category_name: [table]}) + table = pd.DataFrame({category_name: [table]}, index=ids) elif isinstance(table, Iterable): - table = pd.DataFrame({category_name: table}) + table = pd.DataFrame({category_name: table}, index=ids) else: raise ValueError( f"Don't know how to construct category table for {category_name}" ) tables.append(table) - names = [self.name] + self.categories + # names = [self.name] + self.categories # construct below in case we need to support array indexing in the future else: raise ValueError( @@ -633,8 +652,7 @@ class AlignedDynamicTableMixin(BaseModel): "need an int, string, slice, ndarray, or tuple[int | slice, str]" ) - df = pd.concat(tables, axis=1, keys=names) - df.set_index((self.name, "id"), drop=True, inplace=True) + df = pd.concat(tables, axis=1, keys=self.categories) return df def __getattr__(self, item: str) -> Any: @@ -692,14 +710,19 @@ class AlignedDynamicTableMixin(BaseModel): model["categories"] = categories else: # add any columns not explicitly given an order at the end - categories = [ - k - for k in model - if k not in cls.NON_COLUMN_FIELDS - and not k.endswith("_index") - and k not in model["categories"] - ] - model["categories"].extend(categories) + categories = model["categories"].copy() + if isinstance(categories, np.ndarray): + categories = categories.tolist() + categories.extend( + [ + k + for k in model + if k not in cls.NON_CATEGORY_FIELDS + and not k.endswith("_index") + and k not in model["categories"] + ] + ) + model["categories"] = categories return model @model_validator(mode="after") @@ -733,7 +756,7 @@ class AlignedDynamicTableMixin(BaseModel): """ lengths = [len(v) for v in self._categories.values()] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "AlignedDynamicTableColumns are not of equal length! " f"Got colnames:\n{self.categories}\nand lengths: {lengths}" ) return self @@ -828,6 +851,13 @@ class TimeSeriesReferenceVectorDataMixin(VectorDataMixin): ) +class ElementIdentifiersMixin(VectorDataMixin): + """ + Mixin class for ElementIdentifiers - allow treating + as generic, and give general indexing methods from VectorData + """ + + DYNAMIC_TABLE_IMPORTS = Imports( imports=[ Import(module="pandas", alias="pd"), @@ -871,6 +901,7 @@ DYNAMIC_TABLE_INJECTS = [ DynamicTableRegionMixin, DynamicTableMixin, AlignedDynamicTableMixin, + ElementIdentifiersMixin, ] TSRVD_IMPORTS = Imports( @@ -912,3 +943,8 @@ if "pytest" in sys.modules: """TimeSeriesReferenceVectorData subclass for testing""" pass + + class ElementIdentifiers(ElementIdentifiersMixin): + """ElementIdentifiers subclass for testing""" + + pass diff --git a/nwb_linkml/src/nwb_linkml/io/hdf5.py b/nwb_linkml/src/nwb_linkml/io/hdf5.py index ade89d9..bf4fbe6 100644 --- a/nwb_linkml/src/nwb_linkml/io/hdf5.py +++ b/nwb_linkml/src/nwb_linkml/io/hdf5.py @@ -22,6 +22,7 @@ Other TODO: import json import os +import re import shutil import subprocess import sys @@ -31,11 +32,18 @@ from types import ModuleType from typing import TYPE_CHECKING, Dict, List, Optional, Union, overload import h5py +import networkx as nx import numpy as np +from numpydantic.interface.hdf5 import H5ArrayPath from pydantic import BaseModel from tqdm import tqdm -from nwb_linkml.maps.hdf5 import ReadPhases, ReadQueue, flatten_hdf +from nwb_linkml.maps.hdf5 import ( + get_attr_references, + get_dataset_references, + get_references, + resolve_hardlink, +) if TYPE_CHECKING: from nwb_linkml.providers.schema import SchemaProvider @@ -47,6 +55,221 @@ else: from typing_extensions import Never +SKIP_PATTERN = re.compile("(^/specifications.*)|(\.specloc)") +"""Nodes to always skip in reading e.g. because they are handled elsewhere""" + + +def hdf_dependency_graph(h5f: Path | h5py.File | h5py.Group) -> nx.DiGraph: + """ + Directed dependency graph of dataset and group nodes in an NWBFile such that + each node ``n_i`` is connected to node ``n_j`` if + + * ``n_j`` is ``n_i``'s child + * ``n_i`` contains a reference to ``n_j`` + + Resolve references in + + * Attributes + * Dataset columns + * Compound dtypes + + Edges are labeled with ``reference`` or ``child`` depending on the type of edge it is, + and attributes from the hdf5 file are added as node attributes. + + Args: + h5f (:class:`pathlib.Path` | :class:`h5py.File`): NWB file to graph + + Returns: + :class:`networkx.DiGraph` + """ + + if isinstance(h5f, (Path, str)): + h5f = h5py.File(h5f, "r") + + g = nx.DiGraph() + + def _visit_item(name: str, node: h5py.Dataset | h5py.Group) -> None: + if SKIP_PATTERN.match(node.name): + return + # find references in attributes + refs = get_references(node) + # add edges from references + edges = [(node.name, ref) for ref in refs if not SKIP_PATTERN.match(ref)] + g.add_edges_from(edges, label="reference") + + # add children, if group + if isinstance(node, h5py.Group): + children = [ + resolve_hardlink(child) + for child in node.values() + if not SKIP_PATTERN.match(child.name) + ] + edges = [(node.name, ref) for ref in children if not SKIP_PATTERN.match(ref)] + g.add_edges_from(edges, label="child") + + # ensure node added to graph + if len(edges) == 0: + g.add_node(node.name) + + # store attrs in node + g.nodes[node.name].update(node.attrs) + + # apply to root + _visit_item(h5f.name, h5f) + + h5f.visititems(_visit_item) + return g + + +def filter_dependency_graph(g: nx.DiGraph) -> nx.DiGraph: + """ + Remove nodes from a dependency graph if they + + * have no neurodata type AND + * have no outbound edges + + OR + + * are a VectorIndex (which are handled by the dynamictable mixins) + """ + remove_nodes = [] + node: str + for node in g.nodes: + ndtype = g.nodes[node].get("neurodata_type", None) + if (ndtype is None and g.out_degree(node) == 0) or SKIP_PATTERN.match(node): + remove_nodes.append(node) + + g.remove_nodes_from(remove_nodes) + return g + + +def _load_node( + path: str, h5f: h5py.File, provider: "SchemaProvider", context: dict +) -> dict | BaseModel: + """ + Load an individual node in the graph, then removes it from the graph + Args: + path: + g: + context: + + Returns: + + """ + obj = h5f.get(path) + + if isinstance(obj, h5py.Dataset): + args = _load_dataset(obj, h5f, context) + elif isinstance(obj, h5py.Group): + args = _load_group(obj, h5f, context) + else: + raise TypeError(f"Nodes can only be h5py Datasets and Groups, got {obj}") + + if "neurodata_type" in obj.attrs: + model = provider.get_class(obj.attrs["namespace"], obj.attrs["neurodata_type"]) + return model(**args) + else: + if "name" in args: + del args["name"] + if "hdf5_path" in args: + del args["hdf5_path"] + return args + + +def _load_dataset( + dataset: h5py.Dataset, h5f: h5py.File, context: dict +) -> Union[dict, str, int, float]: + """ + Resolves datasets that do not have a ``neurodata_type`` as a dictionary or a scalar. + + If the dataset is a single value without attrs, load it and return as a scalar value. + Otherwise return a :class:`.H5ArrayPath` as a reference to the dataset in the `value` key. + """ + res = {} + if dataset.shape == (): + val = dataset[()] + if isinstance(val, h5py.h5r.Reference): + val = context.get(h5f[val].name) + # if this is just a scalar value, return it + if not dataset.attrs: + return val + + res["value"] = val + elif len(dataset) > 0 and isinstance(dataset[0], h5py.h5r.Reference): + # vector of references + res["value"] = [context.get(h5f[ref].name) for ref in dataset[:]] + elif len(dataset.dtype) > 1: + # compound dataset - check if any of the fields are references + for name in dataset.dtype.names: + if isinstance(dataset[name][0], h5py.h5r.Reference): + res[name] = [context.get(h5f[ref].name) for ref in dataset[name]] + else: + res[name] = H5ArrayPath(h5f.filename, dataset.name, name) + else: + res["value"] = H5ArrayPath(h5f.filename, dataset.name) + + res.update(dataset.attrs) + if "namespace" in res: + del res["namespace"] + if "neurodata_type" in res: + del res["neurodata_type"] + res["name"] = dataset.name.split("/")[-1] + res["hdf5_path"] = dataset.name + + # resolve attr references + for k, v in res.items(): + if isinstance(v, h5py.h5r.Reference): + ref_path = h5f[v].name + if SKIP_PATTERN.match(ref_path): + res[k] = ref_path + else: + res[k] = context[ref_path] + + if len(res) == 1: + return res["value"] + else: + return res + + +def _load_group(group: h5py.Group, h5f: h5py.File, context: dict) -> dict: + """ + Load a group! + """ + res = {} + res.update(group.attrs) + for child_name, child in group.items(): + if child.name in context: + res[child_name] = context[child.name] + elif isinstance(child, h5py.Dataset): + res[child_name] = _load_dataset(child, h5f, context) + elif isinstance(child, h5py.Group): + res[child_name] = _load_group(child, h5f, context) + else: + raise TypeError( + "Can only handle preinstantiated child objects in context, datasets, and group," + f" got {child} for {child_name}" + ) + if "namespace" in res: + del res["namespace"] + if "neurodata_type" in res: + del res["neurodata_type"] + name = group.name.split("/")[-1] + if name: + res["name"] = name + res["hdf5_path"] = group.name + + # resolve attr references + for k, v in res.items(): + if isinstance(v, h5py.h5r.Reference): + ref_path = h5f[v].name + if SKIP_PATTERN.match(ref_path): + res[k] = ref_path + else: + res[k] = context[ref_path] + + return res + + class HDF5IO: """ Read (and eventually write) from an NWB HDF5 file. @@ -106,28 +329,22 @@ class HDF5IO: h5f = h5py.File(str(self.path)) src = h5f.get(path) if path else h5f + graph = hdf_dependency_graph(src) + graph = filter_dependency_graph(graph) - # get all children of selected item - if isinstance(src, (h5py.File, h5py.Group)): - children = flatten_hdf(src) - else: - raise NotImplementedError("directly read individual datasets") - - queue = ReadQueue(h5f=self.path, queue=children, provider=provider) - - # Apply initial planning phase of reading - queue.apply_phase(ReadPhases.plan) - # Read operations gather the data before casting into models - queue.apply_phase(ReadPhases.read) - # Construction operations actually cast the models - # this often needs to run several times as models with dependencies wait for their - # dependents to be cast - queue.apply_phase(ReadPhases.construct) + # topo sort to get read order + # TODO: This could be parallelized using `topological_generations`, + # but it's not clear what the perf bonus would be because there are many generations + # with few items + topo_order = list(reversed(list(nx.topological_sort(graph)))) + context = {} + for node in topo_order: + res = _load_node(node, h5f, provider, context) + context[node] = res if path is None: - return queue.completed["/"].result - else: - return queue.completed[path].result + path = "/" + return context[path] def write(self, path: Path) -> Never: """ @@ -167,7 +384,7 @@ class HDF5IO: """ from nwb_linkml.providers.schema import SchemaProvider - h5f = h5py.File(str(self.path)) + h5f = h5py.File(str(self.path), "r") schema = read_specs_as_dicts(h5f.get("specifications")) # get versions for each namespace @@ -269,7 +486,7 @@ def find_references(h5f: h5py.File, path: str) -> List[str]: return references -def truncate_file(source: Path, target: Optional[Path] = None, n: int = 10) -> Path: +def truncate_file(source: Path, target: Optional[Path] = None, n: int = 10) -> Path | None: """ Create a truncated HDF5 file where only the first few samples are kept. @@ -285,6 +502,14 @@ def truncate_file(source: Path, target: Optional[Path] = None, n: int = 10) -> P Returns: :class:`pathlib.Path` path of the truncated file """ + if shutil.which("h5repack") is None: + warnings.warn( + "Truncation requires h5repack to be available, " + "or else the truncated files will be no smaller than the originals", + stacklevel=2, + ) + return + target = source.parent / (source.stem + "_truncated.hdf5") if target is None else Path(target) source = Path(source) @@ -300,17 +525,34 @@ def truncate_file(source: Path, target: Optional[Path] = None, n: int = 10) -> P os.chmod(target, 0o774) to_resize = [] + attr_refs = {} + dataset_refs = {} def _need_resizing(name: str, obj: h5py.Dataset | h5py.Group) -> None: if isinstance(obj, h5py.Dataset) and obj.size > n: to_resize.append(name) - print("Resizing datasets...") + def _find_attr_refs(name: str, obj: h5py.Dataset | h5py.Group) -> None: + """Find all references in object attrs""" + refs = get_attr_references(obj) + if refs: + attr_refs[name] = refs + + def _find_dataset_refs(name: str, obj: h5py.Dataset | h5py.Group) -> None: + """Find all references in datasets themselves""" + refs = get_dataset_references(obj) + if refs: + dataset_refs[name] = refs + # first we get the items that need to be resized and then resize them below # problems with writing to the file from within the visititems call + print("Planning resize...") h5f_target = h5py.File(str(target), "r+") h5f_target.visititems(_need_resizing) + h5f_target.visititems(_find_attr_refs) + h5f_target.visititems(_find_dataset_refs) + print("Resizing datasets...") for resize in to_resize: obj = h5f_target.get(resize) try: @@ -320,10 +562,14 @@ def truncate_file(source: Path, target: Optional[Path] = None, n: int = 10) -> P # so we have to copy and create a new dataset tmp_name = obj.name + "__tmp" original_name = obj.name + obj.parent.move(obj.name, tmp_name) old_obj = obj.parent.get(tmp_name) - new_obj = obj.parent.create_dataset(original_name, data=old_obj[0:n]) + new_obj = obj.parent.create_dataset( + original_name, data=old_obj[0:n], dtype=old_obj.dtype + ) for k, v in old_obj.attrs.items(): + new_obj.attrs[k] = v del new_obj.parent[tmp_name] @@ -331,16 +577,18 @@ def truncate_file(source: Path, target: Optional[Path] = None, n: int = 10) -> P h5f_target.close() # use h5repack to actually remove the items from the dataset - if shutil.which("h5repack") is None: - warnings.warn( - "Truncated file made, but since h5repack not found in path, file won't be any smaller", - stacklevel=2, - ) - return target - print("Repacking hdf5...") res = subprocess.run( - ["h5repack", "-f", "GZIP=9", str(target), str(target_tmp)], capture_output=True + [ + "h5repack", + "--verbose=2", + "--enable-error-stack", + "-f", + "GZIP=9", + str(target), + str(target_tmp), + ], + capture_output=True, ) if res.returncode != 0: warnings.warn(f"h5repack did not return 0: {res.stderr} {res.stdout}", stacklevel=2) @@ -348,6 +596,36 @@ def truncate_file(source: Path, target: Optional[Path] = None, n: int = 10) -> P target_tmp.unlink() return target + h5f_target = h5py.File(str(target_tmp), "r+") + + # recreate references after repacking, because repacking ruins them if they + # are in a compound dtype + for obj_name, obj_refs in attr_refs.items(): + obj = h5f_target.get(obj_name) + for attr_name, ref_target in obj_refs.items(): + ref_target = h5f_target.get(ref_target) + obj.attrs[attr_name] = ref_target.ref + + for obj_name, obj_refs in dataset_refs.items(): + obj = h5f_target.get(obj_name) + if isinstance(obj_refs, list): + if len(obj_refs) == 1: + ref_target = h5f_target.get(obj_refs[0]) + obj[()] = ref_target.ref + else: + targets = [h5f_target.get(ref).ref for ref in obj_refs[:n]] + obj[:] = targets + else: + # dict for a compound dataset + for col_name, column_refs in obj_refs.items(): + targets = [h5f_target.get(ref).ref for ref in column_refs[:n]] + data = obj[:] + data[col_name] = targets + obj[:] = data + + h5f_target.flush() + h5f_target.close() + target.unlink() target_tmp.rename(target) diff --git a/nwb_linkml/src/nwb_linkml/io/schema.py b/nwb_linkml/src/nwb_linkml/io/schema.py index 42718f5..8f960c7 100644 --- a/nwb_linkml/src/nwb_linkml/io/schema.py +++ b/nwb_linkml/src/nwb_linkml/io/schema.py @@ -131,6 +131,8 @@ def load_namespace_adapter( else: adapter = NamespacesAdapter(namespaces=namespaces, schemas=sch) + adapter.populate_imports() + return adapter diff --git a/nwb_linkml/src/nwb_linkml/maps/dtype.py b/nwb_linkml/src/nwb_linkml/maps/dtype.py index d618dbe..2497a65 100644 --- a/nwb_linkml/src/nwb_linkml/maps/dtype.py +++ b/nwb_linkml/src/nwb_linkml/maps/dtype.py @@ -3,7 +3,7 @@ Dtype mappings """ from datetime import datetime -from typing import Any +from typing import Any, Optional import numpy as np @@ -160,14 +160,28 @@ def handle_dtype(dtype: DTypeType | None) -> str: elif isinstance(dtype, FlatDtype): return dtype.value elif isinstance(dtype, list) and isinstance(dtype[0], CompoundDtype): - # there is precisely one class that uses compound dtypes: - # TimeSeriesReferenceVectorData - # compoundDtypes are able to define a ragged table according to the schema - # but are used in this single case equivalently to attributes. - # so we'll... uh... treat them as slots. - # TODO + # Compound Dtypes are handled by the MapCompoundDtype dataset map, + # but this function is also used within ``check`` methods, so we should always + # return something from it rather than raise return "AnyType" else: # flat dtype return dtype + + +def inlined(dtype: DTypeType | None) -> Optional[bool]: + """ + Check if a slot should be inlined based on its dtype + + for now that is equivalent to checking whether that dtype is another a reference dtype, + but the function remains semantically reserved for answering this question w.r.t. dtype. + + Returns ``None`` if not inlined to not clutter generated models with unnecessary props + """ + return ( + True + if isinstance(dtype, ReferenceDtype) + or (isinstance(dtype, CompoundDtype) and isinstance(dtype.dtype, ReferenceDtype)) + else None + ) diff --git a/nwb_linkml/src/nwb_linkml/maps/hdf5.py b/nwb_linkml/src/nwb_linkml/maps/hdf5.py index a7b052f..a507678 100644 --- a/nwb_linkml/src/nwb_linkml/maps/hdf5.py +++ b/nwb_linkml/src/nwb_linkml/maps/hdf5.py @@ -5,832 +5,47 @@ We have sort of diverged from the initial idea of a generalized map as in :class so we will make our own mapping class here and re-evaluate whether they should be unified later """ -# FIXME: return and document whatever is left of this godforsaken module after refactoring # ruff: noqa: D102 # ruff: noqa: D101 -import contextlib -import datetime -import inspect -import sys -from abc import abstractmethod -from pathlib import Path -from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Tuple, Type, Union +from typing import List, Union import h5py -from numpydantic.interface.hdf5 import H5ArrayPath -from pydantic import BaseModel, ConfigDict, Field - -from nwb_linkml.annotations import unwrap_optional -from nwb_linkml.maps import Map -from nwb_linkml.types.hdf5 import HDF5_Path - -if sys.version_info.minor >= 11: - from enum import StrEnum -else: - from enum import Enum - - class StrEnum(str, Enum): - """StrEnum-ish class for python 3.10""" -if TYPE_CHECKING: - from nwb_linkml.providers.schema import SchemaProvider +def get_attr_references(obj: h5py.Dataset | h5py.Group) -> dict[str, str]: + """ + Get any references in object attributes + """ + refs = { + k: obj.file.get(ref).name + for k, ref in obj.attrs.items() + if isinstance(ref, h5py.h5r.Reference) + } + return refs -class ReadPhases(StrEnum): - plan = "plan" - """Before reading starts, building an index of objects to read""" - read = "read" - """Main reading operation""" - construct = "construct" - """After reading, casting the results of the read into their models""" - - -class H5SourceItem(BaseModel): +def get_dataset_references(obj: h5py.Dataset | h5py.Group) -> list[str] | dict[str, str]: """ - Descriptor of items for each element when :func:`.flatten_hdf` flattens an hdf5 file. - - Consumed by :class:`.HDF5Map` classes, orchestrated by :class:`.ReadQueue` - """ - - path: str - """Absolute hdf5 path of element""" - h5f_path: str - """Path to the source hdf5 file""" - leaf: bool - """ - If ``True``, this item has no children - (and thus we should start instantiating it before ascending to parent classes) - """ - h5_type: Literal["group", "dataset"] - """What kind of hdf5 element this is""" - depends: List[str] = Field(default_factory=list) - """ - Paths of other source items that this item depends on before it can be instantiated. - eg. from softlinks - """ - attrs: dict = Field(default_factory=dict) - """Any static attrs that can be had from the element""" - namespace: Optional[str] = None - """Optional: The namespace that the neurodata type belongs to""" - neurodata_type: Optional[str] = None - """Optional: the neurodata type for this dataset or group""" - - model_config = ConfigDict(arbitrary_types_allowed=True) - - @property - def parts(self) -> List[str]: - """path split by /""" - return self.path.split("/") - - -class H5ReadResult(BaseModel): - """ - Result returned by each of our mapping operations. - - Also used as the source for operations in the ``construct`` :class:`.ReadPhases` - """ - - path: str - """absolute hdf5 path of element""" - source: Union[H5SourceItem, "H5ReadResult"] - """ - Source that this result is based on. - The map can modify this item, so the container should update the source - queue on each pass - """ - completed: bool = False - """ - Was this item completed by this map step? False for cases where eg. - we still have dependencies that need to be completed before this one - """ - result: Optional[dict | str | int | float | BaseModel] = None - """ - If completed, built result. A dict that can be instantiated into the model. - If completed is True and result is None, then remove this object - """ - model: Optional[Type[BaseModel]] = None - """ - The model that this item should be cast into - """ - completes: List[HDF5_Path] = Field(default_factory=list) - """ - If this result completes any other fields, we remove them from the build queue. - """ - namespace: Optional[str] = None - """ - Optional: the namespace of the neurodata type for this object - """ - neurodata_type: Optional[str] = None - """ - Optional: The neurodata type to use for this object - """ - applied: List[str] = Field(default_factory=list) - """ - Which map operations were applied to this item - """ - errors: List[str] = Field(default_factory=list) - """ - Problems that occurred during resolution - """ - depends: List[HDF5_Path] = Field(default_factory=list) - """ - Other items that the final resolution of this item depends on - """ - - -FlatH5 = Dict[str, H5SourceItem] - - -class HDF5Map(Map): - phase: ReadPhases - priority: int = 0 - """ - Within a phase, sort mapping operations from low to high priority - (maybe this should be renamed because highest priority last doesn't make a lot of sense) - """ - - @classmethod - @abstractmethod - def check( - cls, - src: H5SourceItem | H5ReadResult, - provider: "SchemaProvider", - completed: Dict[str, H5ReadResult], - ) -> bool: - """Check if this map applies to the given item to read""" - - @classmethod - @abstractmethod - def apply( - cls, - src: H5SourceItem | H5ReadResult, - provider: "SchemaProvider", - completed: Dict[str, H5ReadResult], - ) -> H5ReadResult: - """Actually apply the map!""" - - -# -------------------------------------------------- -# Planning maps -# -------------------------------------------------- - - -def check_empty(obj: h5py.Group) -> bool: - """ - Check if a group has no attrs or children OR has no attrs and all its children - also have no attrs and no children - - Returns: - bool + Get references in datasets """ + refs = [] + # For datasets, apply checks depending on shape of data. if isinstance(obj, h5py.Dataset): - return False - - # check if we are empty - no_attrs = False - if len(obj.attrs) == 0: - no_attrs = True - - no_children = False - if len(obj.keys()) == 0: - no_children = True - - # check if immediate children are empty - # handles empty groups of empty groups - children_empty = False - if all( - [ - isinstance(item, h5py.Group) and len(item.keys()) == 0 and len(item.attrs) == 0 - for item in obj.values() - ] - ): - children_empty = True - - # if we have no attrs and we are a leaf OR our children are empty, remove us - return bool(no_attrs and (no_children or children_empty)) - - -class PruneEmpty(HDF5Map): - """Remove groups with no attrs""" - - phase = ReadPhases.plan - - @classmethod - def check( - cls, src: H5SourceItem, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> bool: - if src.h5_type == "group": - with h5py.File(src.h5f_path, "r") as h5f: - obj = h5f.get(src.path) - return check_empty(obj) - - @classmethod - def apply( - cls, src: H5SourceItem, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> H5ReadResult: - return H5ReadResult.model_construct(path=src.path, source=src, completed=True) - - -# -# class ResolveDynamicTable(HDF5Map): -# """ -# Handle loading a dynamic table! -# -# Dynamic tables are sort of odd in that their models don't include their fields -# (except as a list of strings in ``colnames`` ), -# so we need to create a new model that includes fields for each column, -# and then we include the datasets as :class:`~numpydantic.interface.hdf5.H5ArrayPath` -# objects which lazy load the arrays in a thread/process safe way. -# -# This map also resolves the child elements, -# indicating so by the ``completes`` field in the :class:`.ReadResult` -# """ -# -# phase = ReadPhases.read -# priority = 1 -# -# @classmethod -# def check( -# cls, src: H5SourceItem, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] -# ) -> bool: -# if src.h5_type == "dataset": -# return False -# if "neurodata_type" in src.attrs: -# if src.attrs["neurodata_type"] == "DynamicTable": -# return True -# # otherwise, see if it's a subclass -# model = provider.get_class(src.attrs["namespace"], src.attrs["neurodata_type"]) -# # just inspect the MRO as strings rather than trying to check subclasses because -# # we might replace DynamicTable in the future, and there isn't a stable DynamicTable -# # class to inherit from anyway because of the whole multiple versions thing -# parents = [parent.__name__ for parent in model.__mro__] -# return "DynamicTable" in parents -# else: -# return False -# -# @classmethod -# def apply( -# cls, src: H5SourceItem, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] -# ) -> H5ReadResult: -# with h5py.File(src.h5f_path, "r") as h5f: -# obj = h5f.get(src.path) -# -# # make a populated model :) -# base_model = provider.get_class(src.namespace, src.neurodata_type) -# model = dynamictable_to_model(obj, base=base_model) -# -# completes = [HDF5_Path(child.name) for child in obj.values()] -# -# return H5ReadResult( -# path=src.path, -# source=src, -# result=model, -# completes=completes, -# completed=True, -# applied=["ResolveDynamicTable"], -# ) - - -class ResolveModelGroup(HDF5Map): - """ - HDF5 Groups that have a model, as indicated by ``neurodata_type`` in their attrs. - We use the model to determine what fields we should get, and then stash references - to the children to process later as :class:`.HDF5_Path` - - **Special Case:** Some groups like ``ProcessingGroup`` and others that have an arbitrary - number of named children have a special ``children`` field that is a dictionary mapping - names to the objects themselves. - - So for example, this: - - /processing/ - eye_tracking/ - cr_ellipse_fits/ - center_x - center_y - ... - eye_ellipse_fits/ - ... - pupil_ellipse_fits/ - ... - eye_tracking_rig_metadata/ - ... - - would pack the ``eye_tracking`` group (a ``ProcessingModule`` ) as: - - { - "name": "eye_tracking", - "children": { - "cr_ellipse_fits": HDF5_Path('/processing/eye_tracking/cr_ellipse_fits'), - "eye_ellipse_fits" : HDF5_Path('/processing/eye_tracking/eye_ellipse_fits'), - ... - } - } - - We will do some nice things in the model metaclass to make it possible to access the children - like ``nwbfile.processing.cr_ellipse_fits.center_x`` - rather than having to switch between indexing and attribute access :) - """ - - phase = ReadPhases.read - priority = 10 # do this generally last - - @classmethod - def check( - cls, src: H5SourceItem, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> bool: - return bool("neurodata_type" in src.attrs and src.h5_type == "group") - - @classmethod - def apply( - cls, src: H5SourceItem, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> H5ReadResult: - model = provider.get_class(src.namespace, src.neurodata_type) - res = {} - depends = [] - with h5py.File(src.h5f_path, "r") as h5f: - obj = h5f.get(src.path) - for key in model.model_fields: - if key == "children": - res[key] = {name: resolve_hardlink(child) for name, child in obj.items()} - depends.extend([resolve_hardlink(child) for child in obj.values()]) - elif key in obj.attrs: - res[key] = obj.attrs[key] - continue - elif key in obj: - # make sure it's not empty - if check_empty(obj[key]): - continue - # stash a reference to this, we'll compile it at the end - depends.append(resolve_hardlink(obj[key])) - res[key] = resolve_hardlink(obj[key]) - - res["hdf5_path"] = src.path - res["name"] = src.parts[-1] - return H5ReadResult( - path=src.path, - source=src, - completed=True, - result=res, - model=model, - namespace=src.namespace, - neurodata_type=src.neurodata_type, - applied=["ResolveModelGroup"], - depends=depends, - ) - - -class ResolveDatasetAsDict(HDF5Map): - """ - Resolve datasets that do not have a ``neurodata_type`` of their own as a dictionary - that will be packaged into a model in the next step. Grabs the array in an - :class:`~numpydantic.interface.hdf5.H5ArrayPath` - under an ``array`` key, and then grabs any additional ``attrs`` as well. - - Mutually exclusive with :class:`.ResolveScalars` - this only applies to datasets that are larger - than a single entry. - """ - - phase = ReadPhases.read - priority = 11 - - @classmethod - def check( - cls, src: H5SourceItem, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> bool: - if src.h5_type == "dataset" and "neurodata_type" not in src.attrs: - with h5py.File(src.h5f_path, "r") as h5f: - obj = h5f.get(src.path) - return obj.shape != () - else: - return False - - @classmethod - def apply( - cls, src: H5SourceItem, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> H5ReadResult: - - res = { - "array": H5ArrayPath(file=src.h5f_path, path=src.path), - "hdf5_path": src.path, - "name": src.parts[-1], - **src.attrs, - } - return H5ReadResult( - path=src.path, source=src, completed=True, result=res, applied=["ResolveDatasetAsDict"] - ) - - -class ResolveScalars(HDF5Map): - phase = ReadPhases.read - priority = 11 # catchall - - @classmethod - def check( - cls, src: H5SourceItem, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> bool: - if src.h5_type == "dataset" and "neurodata_type" not in src.attrs: - with h5py.File(src.h5f_path, "r") as h5f: - obj = h5f.get(src.path) - return obj.shape == () - else: - return False - - @classmethod - def apply( - cls, src: H5SourceItem, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> H5ReadResult: - with h5py.File(src.h5f_path, "r") as h5f: - obj = h5f.get(src.path) - res = obj[()] - return H5ReadResult( - path=src.path, source=src, completed=True, result=res, applied=["ResolveScalars"] - ) - - -class ResolveContainerGroups(HDF5Map): - """ - Groups like ``/acquisition``` and others that have no ``neurodata_type`` - (and thus no model) are returned as a dictionary with :class:`.HDF5_Path` references to - the children they contain - """ - - phase = ReadPhases.read - priority = 9 - - @classmethod - def check( - cls, src: H5SourceItem, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> bool: - if src.h5_type == "group" and "neurodata_type" not in src.attrs and len(src.attrs) == 0: - with h5py.File(src.h5f_path, "r") as h5f: - obj = h5f.get(src.path) - return len(obj.keys()) > 0 - else: - return False - - @classmethod - def apply( - cls, src: H5SourceItem, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> H5ReadResult: - """Simple, just return a dict with references to its children""" - depends = [] - with h5py.File(src.h5f_path, "r") as h5f: - obj = h5f.get(src.path) - children = {} - for k, v in obj.items(): - children[k] = HDF5_Path(v.name) - depends.append(HDF5_Path(v.name)) - - # res = { - # 'name': src.parts[-1], - # 'hdf5_path': src.path, - # **children - # } - - return H5ReadResult( - path=src.path, - source=src, - completed=True, - result=children, - depends=depends, - applied=["ResolveContainerGroups"], - ) - - -# -------------------------------------------------- -# Completion Steps -# -------------------------------------------------- - - -class CompletePassThrough(HDF5Map): - """ - Passthrough map for the construction phase for models that don't need any more work done - - - :class:`.ResolveDynamicTable` - - :class:`.ResolveDatasetAsDict` - - :class:`.ResolveScalars` - """ - - phase = ReadPhases.construct - priority = 1 - - @classmethod - def check( - cls, src: H5ReadResult, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> bool: - passthrough_ops = ("ResolveDynamicTable", "ResolveDatasetAsDict", "ResolveScalars") - - return any(hasattr(src, "applied") and op in src.applied for op in passthrough_ops) - - @classmethod - def apply( - cls, src: H5ReadResult, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> H5ReadResult: - return src - - -class CompleteContainerGroups(HDF5Map): - """ - Complete container groups (usually top-level groups like /acquisition) - that do not have a ndueodata type of their own by resolving them as dictionaries - of values (that will then be given to their parent model) - - """ - - phase = ReadPhases.construct - priority = 3 - - @classmethod - def check( - cls, src: H5ReadResult, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> bool: - return ( - src.model is None - and src.neurodata_type is None - and src.source.h5_type == "group" - and all([depend in completed for depend in src.depends]) - ) - - @classmethod - def apply( - cls, src: H5ReadResult, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> H5ReadResult: - res, errors, completes = resolve_references(src.result, completed) - - return H5ReadResult( - result=res, - errors=errors, - completes=completes, - **src.model_dump(exclude={"result", "errors", "completes"}), - ) - - -class CompleteModelGroups(HDF5Map): - phase = ReadPhases.construct - priority = 4 - - @classmethod - def check( - cls, src: H5ReadResult, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> bool: - return ( - src.model is not None - and src.source.h5_type == "group" - and src.neurodata_type != "NWBFile" - and all([depend in completed for depend in src.depends]) - ) - - @classmethod - def apply( - cls, src: H5ReadResult, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> H5ReadResult: - # gather any results that were left for completion elsewhere - # first get all already-completed items - res = {k: v for k, v in src.result.items() if not isinstance(v, HDF5_Path)} - unpacked_results, errors, completes = resolve_references(src.result, completed) - res.update(unpacked_results) - - # now that we have the model in hand, we can solve any datasets that had an array - # but whose attributes are fixed (and thus should just be an array, rather than a subclass) - for k, v in src.model.model_fields.items(): - annotation = unwrap_optional(v.annotation) - if ( - inspect.isclass(annotation) - and not issubclass(annotation, BaseModel) - and isinstance(res, dict) - and k in res - and isinstance(res[k], dict) - and "array" in res[k] - ): - res[k] = res[k]["array"] - - instance = src.model(**res) - return H5ReadResult( - path=src.path, - source=src, - result=instance, - model=src.model, - completed=True, - completes=completes, - neurodata_type=src.neurodata_type, - namespace=src.namespace, - applied=src.applied + ["CompleteModelGroups"], - errors=errors, - ) - - -class CompleteNWBFile(HDF5Map): - """ - The Top-Level NWBFile class is so special cased we just make its own completion special case! - - .. todo:: - - This is truly hideous, just meant as a way to get to the finish line on a late night, - will be cleaned up later - - """ - - phase = ReadPhases.construct - priority = 11 - - @classmethod - def check( - cls, src: H5ReadResult, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> bool: - return src.neurodata_type == "NWBFile" and all( - [depend in completed for depend in src.depends] - ) - - @classmethod - def apply( - cls, src: H5ReadResult, provider: "SchemaProvider", completed: Dict[str, H5ReadResult] - ) -> H5ReadResult: - res = {k: v for k, v in src.result.items() if not isinstance(v, HDF5_Path)} - unpacked_results, errors, completes = resolve_references(src.result, completed) - res.update(unpacked_results) - - res["name"] = "root" - res["file_create_date"] = [ - datetime.datetime.fromisoformat(ts.decode("utf-8")) - for ts in res["file_create_date"]["array"][:] - ] - if "stimulus" not in res: - res["stimulus"] = provider.get_class("core", "NWBFileStimulus")() - electrode_groups = [] - egroup_keys = list(res["general"].get("extracellular_ephys", {}).keys()) - egroup_dict = {} - for k in egroup_keys: - if k != "electrodes": - egroup = res["general"]["extracellular_ephys"][k] - electrode_groups.append(egroup) - egroup_dict[egroup.hdf5_path] = egroup - del res["general"]["extracellular_ephys"][k] - if len(electrode_groups) > 0: - res["general"]["extracellular_ephys"]["electrode_group"] = electrode_groups - trode_type = provider.get_class("core", "NWBFileGeneralExtracellularEphysElectrodes") - # anmro = list(type(res['general']['extracellular_ephys']['electrodes']).__mro__) - # anmro.insert(1, trode_type) - trodes_original = res["general"]["extracellular_ephys"]["electrodes"] - trodes = trode_type.model_construct(trodes_original.model_dump()) - res["general"]["extracellular_ephys"]["electrodes"] = trodes - - instance = src.model(**res) - return H5ReadResult( - path=src.path, - source=src, - result=instance, - model=src.model, - completed=True, - completes=completes, - neurodata_type=src.neurodata_type, - namespace=src.namespace, - applied=src.applied + ["CompleteModelGroups"], - errors=errors, - ) - - -class ReadQueue(BaseModel): - """Container model to store items as they are built""" - - h5f: Path = Field( - description=( - "Path to the source hdf5 file used when resolving the queue! " - "Each translation step should handle opening and closing the file, " - "rather than passing a handle around" - ) - ) - provider: "SchemaProvider" = Field( - description="SchemaProvider used by each of the items in the read queue" - ) - queue: Dict[str, H5SourceItem | H5ReadResult] = Field( - default_factory=dict, - description="Items left to be instantiated, keyed by hdf5 path", - ) - completed: Dict[str, H5ReadResult] = Field( - default_factory=dict, - description="Items that have already been instantiated, keyed by hdf5 path", - ) - model_config = ConfigDict(arbitrary_types_allowed=True) - phases_completed: List[ReadPhases] = Field( - default_factory=list, description="Phases that have already been completed" - ) - - def apply_phase(self, phase: ReadPhases, max_passes: int = 5) -> None: - phase_maps = [m for m in HDF5Map.__subclasses__() if m.phase == phase] - phase_maps = sorted(phase_maps, key=lambda x: x.priority) - - results = [] - - # TODO: Thread/multiprocess this - for item in self.queue.values(): - for op in phase_maps: - if op.check(item, self.provider, self.completed): - # Formerly there was an "exclusive" property in the maps which let - # potentially multiple operations be applied per stage, - # except if an operation was `exclusive` which would break - # iteration over the operations. - # This was removed because it was badly implemented, - # but if there is ever a need to do that, - # then we would need to decide what to do with the multiple results. - results.append(op.apply(item, self.provider, self.completed)) - break # out of inner iteration - - # remake the source queue and save results - completes = [] - for res in results: - # remove the original item - del self.queue[res.path] - if res.completed: - # if the item has been finished and there is some result, add it to the results - if res.result is not None: - self.completed[res.path] = res - # otherwise if the item has been completed and there was no result, - # just drop it. - - # if we have completed other things, delete them from the queue - completes.extend(res.completes) - - else: - # if we didn't complete the item (eg. we found we needed more dependencies), - # add the updated source to the queue again - if phase != ReadPhases.construct: - self.queue[res.path] = res.source - else: - self.queue[res.path] = res - - # delete the ones that were already completed but might have been - # incorrectly added back in the pile - for c in completes: - with contextlib.suppress(KeyError): - del self.queue[c] - - # if we have nothing left in our queue, we have completed this phase - # and prepare only ever has one pass - if phase == ReadPhases.plan: - self.phases_completed.append(phase) - return - - if len(self.queue) == 0: - self.phases_completed.append(phase) - if phase != ReadPhases.construct: - # if we're not in the last phase, move our completed to our queue - self.queue = self.completed - self.completed = {} - elif max_passes > 0: - self.apply_phase(phase, max_passes=max_passes - 1) - - -def flatten_hdf( - h5f: h5py.File | h5py.Group, skip: str = "specifications" -) -> Dict[str, H5SourceItem]: - """ - Flatten all child elements of hdf element into a dict of :class:`.H5SourceItem` s - keyed by their path - - Args: - h5f (:class:`h5py.File` | :class:`h5py.Group`): HDF file or group to flatten! - """ - items = {} - - def _itemize(name: str, obj: h5py.Dataset | h5py.Group) -> None: - if skip in name: - return - - leaf = isinstance(obj, h5py.Dataset) or len(obj.keys()) == 0 - - if isinstance(obj, h5py.Dataset): - h5_type = "dataset" - elif isinstance(obj, h5py.Group): - h5_type = "group" - else: - raise ValueError(f"Object must be a dataset or group! {obj}") - - # get references in attrs and datasets to populate dependencies - # depends = get_references(obj) - - if not name.startswith("/"): - name = "/" + name - - attrs = dict(obj.attrs.items()) - - items[name] = H5SourceItem.model_construct( - path=name, - h5f_path=h5f.file.filename, - leaf=leaf, - # depends = depends, - h5_type=h5_type, - attrs=attrs, - namespace=attrs.get("namespace"), - neurodata_type=attrs.get("neurodata_type"), - ) - - h5f.visititems(_itemize) - # then add the root item - _itemize(h5f.name, h5f) - return items + if obj.shape == (): + # scalar + if isinstance(obj[()], h5py.h5r.Reference): + refs = [obj.file.get(obj[()]).name] + elif len(obj) > 0 and isinstance(obj[0], h5py.h5r.Reference): + # single-column + refs = [obj.file.get(ref).name for ref in obj[:]] + elif len(obj.dtype) > 1: + # "compound" datasets + refs = {} + for name in obj.dtype.names: + if isinstance(obj[name][0], h5py.h5r.Reference): + refs[name] = [obj.file.get(ref).name for ref in obj[name]] + return refs def get_references(obj: h5py.Dataset | h5py.Group) -> List[str]: @@ -851,60 +66,21 @@ def get_references(obj: h5py.Dataset | h5py.Group) -> List[str]: List[str]: List of paths that are referenced within this object """ # Find references in attrs - refs = [ref for ref in obj.attrs.values() if isinstance(ref, h5py.h5r.Reference)] + attr_refs = get_attr_references(obj) + dataset_refs = get_dataset_references(obj) - # For datasets, apply checks depending on shape of data. - if isinstance(obj, h5py.Dataset): - if obj.shape == (): - # scalar - if isinstance(obj[()], h5py.h5r.Reference): - refs.append(obj[()]) - elif isinstance(obj[0], h5py.h5r.Reference): - # single-column - refs.extend(obj[:].tolist()) - elif len(obj.dtype) > 1: - # "compound" datasets - for name in obj.dtype.names: - if isinstance(obj[name][0], h5py.h5r.Reference): - refs.extend(obj[name].tolist()) - - # dereference and get name of reference - if isinstance(obj, h5py.Dataset): - depends = list(set([obj.parent.get(i).name for i in refs])) + # flatten to list + refs = [ref for ref in attr_refs.values()] + if isinstance(dataset_refs, list): + refs.extend(dataset_refs) else: - depends = list(set([obj.get(i).name for i in refs])) - return depends + for v in dataset_refs.values(): + refs.extend(v) + + return refs -def resolve_references( - src: dict, completed: Dict[str, H5ReadResult] -) -> Tuple[dict, List[str], List[HDF5_Path]]: - """ - Recursively replace references to other completed items with their results - - """ - completes = [] - errors = [] - res = {} - for path, item in src.items(): - if isinstance(item, HDF5_Path): - other_item = completed.get(item) - if other_item is None: - errors.append(f"Couldn't find: {item}") - res[path] = other_item.result - completes.append(item) - - elif isinstance(item, dict): - inner_res, inner_error, inner_completes = resolve_references(item, completed) - res[path] = inner_res - errors.extend(inner_error) - completes.extend(inner_completes) - else: - res[path] = item - return res, errors, completes - - -def resolve_hardlink(obj: Union[h5py.Group, h5py.Dataset]) -> HDF5_Path: +def resolve_hardlink(obj: Union[h5py.Group, h5py.Dataset]) -> str: """ Unhelpfully, hardlinks are pretty challenging to detect with h5py, so we have to do extra work to check if an item is "real" or a hardlink to another item. @@ -916,4 +92,4 @@ def resolve_hardlink(obj: Union[h5py.Group, h5py.Dataset]) -> HDF5_Path: We basically dereference the object and return that path instead of the path given by the object's ``name`` """ - return HDF5_Path(obj.file[obj.ref].name) + return obj.file[obj.ref].name diff --git a/nwb_linkml/src/nwb_linkml/providers/linkml.py b/nwb_linkml/src/nwb_linkml/providers/linkml.py index 4af2bec..fe8dec5 100644 --- a/nwb_linkml/src/nwb_linkml/providers/linkml.py +++ b/nwb_linkml/src/nwb_linkml/providers/linkml.py @@ -127,6 +127,7 @@ class LinkMLProvider(Provider): for schema_needs in adapter.needed_imports.values(): for needed in schema_needs: adapter.imported.append(ns_adapters[needed]) + adapter.populate_imports() # then do the build res = {} diff --git a/nwb_linkml/src/nwb_linkml/providers/provider.py b/nwb_linkml/src/nwb_linkml/providers/provider.py index 87f6567..ff349af 100644 --- a/nwb_linkml/src/nwb_linkml/providers/provider.py +++ b/nwb_linkml/src/nwb_linkml/providers/provider.py @@ -97,9 +97,9 @@ class Provider(ABC): module_path = Path(importlib.util.find_spec("nwb_models").origin).parent if self.PROVIDES == "linkml": - namespace_path = module_path / "schema" / "linkml" / namespace + namespace_path = module_path / "schema" / "linkml" / namespace_module elif self.PROVIDES == "pydantic": - namespace_path = module_path / "models" / "pydantic" / namespace + namespace_path = module_path / "models" / "pydantic" / namespace_module if version is not None: version_path = namespace_path / version_module_case(version) diff --git a/nwb_linkml/src/nwb_linkml/providers/pydantic.py b/nwb_linkml/src/nwb_linkml/providers/pydantic.py index 5d5975c..c44c85c 100644 --- a/nwb_linkml/src/nwb_linkml/providers/pydantic.py +++ b/nwb_linkml/src/nwb_linkml/providers/pydantic.py @@ -278,7 +278,7 @@ class PydanticProvider(Provider): nwb_models.models.pydantic.{namespace}.{version} """ name_pieces = [ - "nwb_linkml", + "nwb_models", "models", "pydantic", module_case(namespace), diff --git a/nwb_linkml/src/nwb_linkml/providers/schema.py b/nwb_linkml/src/nwb_linkml/providers/schema.py index adadb00..7555f93 100644 --- a/nwb_linkml/src/nwb_linkml/providers/schema.py +++ b/nwb_linkml/src/nwb_linkml/providers/schema.py @@ -131,7 +131,7 @@ class SchemaProvider(Provider): results = {} for ns, ns_result in linkml_res.items(): results[ns] = pydantic_provider.build( - ns_result["namespace"], versions=self.versions, **pydantic_kwargs + ns_result.namespace, versions=self.versions, **pydantic_kwargs ) return results diff --git a/nwb_linkml/src/nwb_linkml/types/hdf5.py b/nwb_linkml/src/nwb_linkml/types/hdf5.py deleted file mode 100644 index 9f74576..0000000 --- a/nwb_linkml/src/nwb_linkml/types/hdf5.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -Types used with hdf5 io -""" - -from typing import Any - -from pydantic import GetCoreSchemaHandler -from pydantic_core import CoreSchema, core_schema - - -class HDF5_Path(str): - """ - Trivial subclass of string to indicate that it is a reference to a location within an HDF5 file - """ - - @classmethod - def __get_pydantic_core_schema__( - cls, source_type: Any, handler: GetCoreSchemaHandler - ) -> CoreSchema: - return core_schema.no_info_after_validator_function(cls, handler(str)) diff --git a/nwb_linkml/tests/conftest.py b/nwb_linkml/tests/conftest.py index 6133148..7bb3632 100644 --- a/nwb_linkml/tests/conftest.py +++ b/nwb_linkml/tests/conftest.py @@ -9,10 +9,16 @@ from .fixtures import * # noqa: F403 def pytest_addoption(parser): + parser.addoption( + "--clean", + action="store_true", + default=False, + help="Don't reuse cached resources like cloned git repos or generated files", + ) parser.addoption( "--with-output", action="store_true", - help="dump output in compliance test for richer debugging information", + help="keep test outputs for richer debugging information", ) parser.addoption( "--without-cache", action="store_true", help="Don't use a sqlite cache for network requests" diff --git a/nwb_linkml/tests/data/aibs.nwb b/nwb_linkml/tests/data/aibs.nwb index 6000e09..1380c55 100644 Binary files a/nwb_linkml/tests/data/aibs.nwb and b/nwb_linkml/tests/data/aibs.nwb differ diff --git a/nwb_linkml/tests/data/aibs_ecephys.nwb b/nwb_linkml/tests/data/aibs_ecephys.nwb index 4a5ad9c..cc15038 100644 Binary files a/nwb_linkml/tests/data/aibs_ecephys.nwb and b/nwb_linkml/tests/data/aibs_ecephys.nwb differ diff --git a/nwb_linkml/tests/data/test_nwb.yaml b/nwb_linkml/tests/data/test_nwb.yaml new file mode 100644 index 0000000..defa118 --- /dev/null +++ b/nwb_linkml/tests/data/test_nwb.yaml @@ -0,0 +1,61 @@ +# manually transcribed target version of nwb-linkml dataset +# matching the one created by fixtures.py:nwb_file +meta: + id: my_dataset + + prefixes: + nwbfile: + - path: "test_nwb.nwb" + - hash: "blake2b:blahblahblahblah" + + imports: + core: + as: nwb + version: "2.7.0" + from: + - pypi: + package: nwb-models + hdmf-common: + as: hdmf + version: "1.8.0" + from: + - pypi: + package: nwb-models + +extracellular_ephys: &ecephys + electrodes: + group: + - @shank0 + - @shank0 + - @shank0 + - @shank1 + - # etc. + shank0: + device: @general.devices.array + shank1: + device: @general.devices.array + # etc. + +data: !nwb.NWBFile + file_create_date: [ 2024-01-01 ] + identifier: "1111-1111-1111-1111" + session_description: All that you touch, you change. + session_start_time: 2024-01-01T01:01:01 + general: + devices: + - Heka ITC-1600: + - Microscope: + description: My two-photon microscope + manufacturer: The best microscope manufacturer + - array: + description: old reliable + manufacturer: diy + extracellular_ephys: nwbfile:/general/extracellular_ephys + experiment_description: All that you change, changes you. + experimenter: [ "Lauren Oya Olamina" ] + institution: Earthseed Research Institute + keywords: + - behavior + - belief + related_publications: doi:10.1016/j.neuron.2016.12.011 + diff --git a/nwb_linkml/tests/data/test_nwb_condensed_sketch.yaml b/nwb_linkml/tests/data/test_nwb_condensed_sketch.yaml new file mode 100644 index 0000000..d9ca7c7 --- /dev/null +++ b/nwb_linkml/tests/data/test_nwb_condensed_sketch.yaml @@ -0,0 +1,76 @@ +# Sketch of a condensed expression syntax for creation with nwb-linkml +# just a sketch! keeping here for continued work but currently unused. +--- +id: my_dataset + +prefixes: + nwbfile: + - path: "test_nwb.nwb" + - hash: "blake2b:blahblahblahblah" + +imports: + core: + as: nwb + version: "2.7.0" + from: + - pypi: + package: nwb-models + hdmf-common: + as: hdmf + version: "1.8.0" + from: + - pypi: + package: nwb-models +--- + +extracellular_ephys: &ecephys + electrodes: + group: + - @shank{{i}} + - @shank{{i}} + - @shank{{i}} + # could have expression here like { range(3) } => i + # - ... { range(3) } => i + # or blank ... implies use expression from outer scope + - ... + shank{{i}}: + device: @general.devices.array + ...: { range(3) } => i + +# expands to +extracellular_ephys: + electrodes: + group: + - @shank0 + - @shank0 + - @shank0 + - @shank1 + - # etc. + shank0: + device: @general.devices.array + shank1: + device: @general.devices.array + # etc. + +data: !{{ nwb.NWBFile }} <== :nwbfile + file_create_date: [ 2024-01-01 ] + identifier: "1111-1111-1111-1111" + session_description: All that you touch, you change. + session_start_time: 2024-01-01T01:01:01 + general: + devices: + - Heka ITC-1600: + - Microscope: + - array: + description: old reliable + manufacturer: diy + extracellular_ephys: *ecephys + + experiment_description: All that you change, changes you. + experimenter: [ "Lauren Oya Olamina" ] + institution: Earthseed Research Institute + keywords: + - behavior + - belief + related_publications: doi:10.1016/j.neuron.2016.12.011 + diff --git a/nwb_linkml/tests/fixtures/__init__.py b/nwb_linkml/tests/fixtures/__init__.py new file mode 100644 index 0000000..f135929 --- /dev/null +++ b/nwb_linkml/tests/fixtures/__init__.py @@ -0,0 +1,29 @@ +from .nwb import nwb_file, nwb_file_base +from .paths import data_dir, tmp_output_dir, tmp_output_dir_func, tmp_output_dir_mod +from .schema import ( + NWBSchemaTest, + TestSchemas, + linkml_schema, + linkml_schema_bare, + nwb_core_fixture, + nwb_core_linkml, + nwb_core_module, + nwb_schema, +) + +__all__ = [ + "NWBSchemaTest", + "TestSchemas", + "data_dir", + "linkml_schema", + "linkml_schema_bare", + "nwb_core_fixture", + "nwb_core_linkml", + "nwb_core_module", + "nwb_file", + "nwb_file_base", + "nwb_schema", + "tmp_output_dir", + "tmp_output_dir_func", + "tmp_output_dir_mod", +] diff --git a/nwb_linkml/tests/fixtures/nwb.py b/nwb_linkml/tests/fixtures/nwb.py new file mode 100644 index 0000000..c878c74 --- /dev/null +++ b/nwb_linkml/tests/fixtures/nwb.py @@ -0,0 +1,477 @@ +from datetime import datetime +from itertools import product +from pathlib import Path + +import numpy as np +import pytest +from hdmf.common import DynamicTable, VectorData +from pynwb import NWBHDF5IO, NWBFile, TimeSeries +from pynwb.base import TimeSeriesReference, TimeSeriesReferenceVectorData +from pynwb.behavior import Position, SpatialSeries +from pynwb.ecephys import LFP, ElectricalSeries +from pynwb.file import Subject +from pynwb.icephys import VoltageClampSeries, VoltageClampStimulusSeries +from pynwb.image import ImageSeries +from pynwb.ophys import ( + CorrectedImageStack, + Fluorescence, + ImageSegmentation, + MotionCorrection, + OnePhotonSeries, + OpticalChannel, + RoiResponseSeries, + TwoPhotonSeries, +) + + +@pytest.fixture(scope="session") +def nwb_file_base() -> NWBFile: + nwbfile = NWBFile( + session_description="All that you touch, you change.", # required + identifier="1111-1111-1111-1111", # required + session_start_time=datetime(year=2024, month=1, day=1), # required + session_id="session_1234", # optional + experimenter=[ + "Lauren Oya Olamina", + ], # optional + institution="Earthseed Research Institute", # optional + experiment_description="All that you change, changes you.", # optional + keywords=["behavior", "belief"], # optional + related_publications="doi:10.1016/j.neuron.2016.12.011", # optional + ) + subject = Subject( + subject_id="001", + age="P90D", + description="mouse 5", + species="Mus musculus", + sex="M", + ) + nwbfile.subject = subject + return nwbfile + + +def _nwb_timeseries(nwbfile: NWBFile) -> NWBFile: + data = np.arange(100, 200, 10) + timestamps = np.arange(10.0) + time_series_with_timestamps = TimeSeries( + name="test_timeseries", + description="an example time series", + data=data, + unit="m", + timestamps=timestamps, + ) + nwbfile.add_acquisition(time_series_with_timestamps) + return nwbfile + + +def _nwb_position(nwbfile: NWBFile) -> NWBFile: + position_data = np.array([np.linspace(0, 10, 50), np.linspace(0, 8, 50)]).T + position_timestamps = np.linspace(0, 50).astype(float) / 200 + + spatial_series_obj = SpatialSeries( + name="SpatialSeries", + description="(x,y) position in open field", + data=position_data, + timestamps=position_timestamps, + reference_frame="(0,0) is bottom left corner", + ) + # name is set to "Position" by default + position_obj = Position(spatial_series=spatial_series_obj) + behavior_module = nwbfile.create_processing_module( + name="behavior", description="processed behavioral data" + ) + behavior_module.add(position_obj) + + nwbfile.add_trial_column( + name="correct", + description="whether the trial was correct", + ) + nwbfile.add_trial(start_time=1.0, stop_time=5.0, correct=True) + nwbfile.add_trial(start_time=6.0, stop_time=10.0, correct=False) + return nwbfile + + +def _nwb_ecephys(nwbfile: NWBFile) -> NWBFile: + """ + Extracellular Ephys + https://pynwb.readthedocs.io/en/latest/tutorials/domain/ecephys.html + """ + generator = np.random.default_rng() + device = nwbfile.create_device(name="array", description="old reliable", manufacturer="diy") + nwbfile.add_electrode_column(name="label", description="label of electrode") + + nshanks = 4 + nchannels_per_shank = 3 + electrode_counter = 0 + + for ishank in range(nshanks): + # create an electrode group for this shank + electrode_group = nwbfile.create_electrode_group( + name=f"shank{ishank}", + description=f"electrode group for shank {ishank}", + device=device, + location="brain area", + ) + # add electrodes to the electrode table + for ielec in range(nchannels_per_shank): + nwbfile.add_electrode( + group=electrode_group, + label=f"shank{ishank}elec{ielec}", + location="brain area", + ) + electrode_counter += 1 + all_table_region = nwbfile.create_electrode_table_region( + region=list(range(electrode_counter)), # reference row indices 0 to N-1 + description="all electrodes", + ) + raw_data = generator.standard_normal((50, 12)) + raw_electrical_series = ElectricalSeries( + name="ElectricalSeries", + description="Raw acquisition traces", + data=raw_data, + electrodes=all_table_region, + starting_time=0.0, + # timestamp of the first sample in seconds relative to the session start time + rate=20000.0, # in Hz + ) + nwbfile.add_acquisition(raw_electrical_series) + + # -------------------------------------------------- + # LFP + # -------------------------------------------------- + generator = np.random.default_rng() + lfp_data = generator.standard_normal((50, 12)) + lfp_electrical_series = ElectricalSeries( + name="ElectricalSeries", + description="LFP data", + data=lfp_data, + electrodes=all_table_region, + starting_time=0.0, + rate=200.0, + ) + lfp = LFP(electrical_series=lfp_electrical_series) + ecephys_module = nwbfile.create_processing_module( + name="ecephys", description="processed extracellular electrophysiology data" + ) + ecephys_module.add(lfp) + + return nwbfile + + +def _nwb_units(nwbfile: NWBFile) -> NWBFile: + generator = np.random.default_rng() + # Spike Times + nwbfile.add_unit_column(name="quality", description="sorting quality") + firing_rate = 20 + n_units = 10 + res = 1000 + duration = 20 + for _ in range(n_units): + spike_times = np.where(generator.random(res * duration) < (firing_rate / res))[0] / res + nwbfile.add_unit(spike_times=spike_times, quality="good") + return nwbfile + + +def _nwb_icephys(nwbfile: NWBFile) -> NWBFile: + device = nwbfile.create_device(name="Heka ITC-1600") + electrode = nwbfile.create_icephys_electrode( + name="elec0", description="a mock intracellular electrode", device=device + ) + stimulus = VoltageClampStimulusSeries( + name="ccss", + data=[1, 2, 3, 4, 5], + starting_time=123.6, + rate=10e3, + electrode=electrode, + gain=0.02, + sweep_number=np.uint64(15), + ) + + # Create and icephys response + response = VoltageClampSeries( + name="vcs", + data=[0.1, 0.2, 0.3, 0.4, 0.5], + conversion=1e-12, + resolution=np.nan, + starting_time=123.6, + rate=20e3, + electrode=electrode, + gain=0.02, + capacitance_slow=100e-12, + resistance_comp_correction=70.0, + sweep_number=np.uint64(15), + ) + # we can also add stimulus template data as follows + rowindex = nwbfile.add_intracellular_recording( + electrode=electrode, stimulus=stimulus, response=response, id=10 + ) + + rowindex2 = nwbfile.add_intracellular_recording( + electrode=electrode, + stimulus=stimulus, + stimulus_start_index=1, + stimulus_index_count=3, + response=response, + response_start_index=2, + response_index_count=3, + id=11, + ) + rowindex3 = nwbfile.add_intracellular_recording(electrode=electrode, response=response, id=12) + + nwbfile.intracellular_recordings.add_column( + name="recording_tag", + data=["A1", "A2", "A3"], + description="String with a recording tag", + ) + location_column = VectorData( + name="location", + data=["Mordor", "Gondor", "Rohan"], + description="Recording location in Middle Earth", + ) + + lab_category = DynamicTable( + name="recording_lab_data", + description="category table for lab-specific recording metadata", + colnames=[ + "location", + ], + columns=[ + location_column, + ], + ) + # Add the table as a new category to our intracellular_recordings + nwbfile.intracellular_recordings.add_category(category=lab_category) + nwbfile.intracellular_recordings.add_column( + name="voltage_threshold", + data=[0.1, 0.12, 0.13], + description="Just an example column on the electrodes category table", + category="electrodes", + ) + stimulus_template = VoltageClampStimulusSeries( + name="ccst", + data=[0, 1, 2, 3, 4], + starting_time=0.0, + rate=10e3, + electrode=electrode, + gain=0.02, + ) + nwbfile.add_stimulus_template(stimulus_template) + + nwbfile.intracellular_recordings.add_column( + name="stimulus_template", + data=[ + TimeSeriesReference(0, 5, stimulus_template), + # (start_index, index_count, stimulus_template) + TimeSeriesReference(1, 3, stimulus_template), + TimeSeriesReference.empty(stimulus_template), + ], + # if there was no data for that recording, use empty reference + description=( + "Column storing the reference to the stimulus template for the recording (rows)." + ), + category="stimuli", + col_cls=TimeSeriesReferenceVectorData, + ) + + icephys_simultaneous_recordings = nwbfile.get_icephys_simultaneous_recordings() + icephys_simultaneous_recordings.add_column( + name="simultaneous_recording_tag", + description="A custom tag for simultaneous_recordings", + ) + simultaneous_index = nwbfile.add_icephys_simultaneous_recording( + recordings=[rowindex, rowindex2, rowindex3], + id=12, + simultaneous_recording_tag="LabTag1", + ) + repetition_index = nwbfile.add_icephys_repetition( + sequential_recordings=[simultaneous_index], id=17 + ) + nwbfile.add_icephys_experimental_condition(repetitions=[repetition_index], id=19) + nwbfile.icephys_experimental_conditions.add_column( + name="tag", + data=np.arange(1), + description="integer tag for a experimental condition", + ) + return nwbfile + + +def _nwb_ca_imaging(nwbfile: NWBFile) -> NWBFile: + """ + Calcium Imaging + https://pynwb.readthedocs.io/en/latest/tutorials/domain/ophys.html + """ + generator = np.random.default_rng() + device = nwbfile.create_device( + name="Microscope", + description="My two-photon microscope", + manufacturer="The best microscope manufacturer", + ) + optical_channel = OpticalChannel( + name="OpticalChannel", + description="an optical channel", + emission_lambda=500.0, + ) + imaging_plane = nwbfile.create_imaging_plane( + name="ImagingPlane", + optical_channel=optical_channel, + imaging_rate=30.0, + description="a very interesting part of the brain", + device=device, + excitation_lambda=600.0, + indicator="GFP", + location="V1", + grid_spacing=[0.01, 0.01], + grid_spacing_unit="meters", + origin_coords=[1.0, 2.0, 3.0], + origin_coords_unit="meters", + ) + one_p_series = OnePhotonSeries( + name="OnePhotonSeries", + description="Raw 1p data", + data=np.ones((1000, 100, 100)), + imaging_plane=imaging_plane, + rate=1.0, + unit="normalized amplitude", + ) + nwbfile.add_acquisition(one_p_series) + two_p_series = TwoPhotonSeries( + name="TwoPhotonSeries", + description="Raw 2p data", + data=np.ones((1000, 100, 100)), + imaging_plane=imaging_plane, + rate=1.0, + unit="normalized amplitude", + ) + + nwbfile.add_acquisition(two_p_series) + + corrected = ImageSeries( + name="corrected", # this must be named "corrected" + description="A motion corrected image stack", + data=np.ones((1000, 100, 100)), + unit="na", + format="raw", + starting_time=0.0, + rate=1.0, + ) + + xy_translation = TimeSeries( + name="xy_translation", + description="x,y translation in pixels", + data=np.ones((1000, 2)), + unit="pixels", + starting_time=0.0, + rate=1.0, + ) + + corrected_image_stack = CorrectedImageStack( + corrected=corrected, + original=one_p_series, + xy_translation=xy_translation, + ) + + motion_correction = MotionCorrection(corrected_image_stacks=[corrected_image_stack]) + + ophys_module = nwbfile.create_processing_module( + name="ophys", description="optical physiology processed data" + ) + + ophys_module.add(motion_correction) + + img_seg = ImageSegmentation() + + ps = img_seg.create_plane_segmentation( + name="PlaneSegmentation", + description="output from segmenting my favorite imaging plane", + imaging_plane=imaging_plane, + reference_images=one_p_series, # optional + ) + + ophys_module.add(img_seg) + + for _ in range(30): + image_mask = np.zeros((100, 100)) + + # randomly generate example image masks + x = generator.integers(0, 95) + y = generator.integers(0, 95) + image_mask[x : x + 5, y : y + 5] = 1 + + # add image mask to plane segmentation + ps.add_roi(image_mask=image_mask) + + ps2 = img_seg.create_plane_segmentation( + name="PlaneSegmentation2", + description="output from segmenting my favorite imaging plane", + imaging_plane=imaging_plane, + reference_images=one_p_series, # optional + ) + + for _ in range(30): + # randomly generate example starting points for region + x = generator.integers(0, 95) + y = generator.integers(0, 95) + + # define an example 4 x 3 region of pixels of weight '1' + pixel_mask = [(ix, iy, 1) for ix in range(x, x + 4) for iy in range(y, y + 3)] + + # add pixel mask to plane segmentation + ps2.add_roi(pixel_mask=pixel_mask) + + ps3 = img_seg.create_plane_segmentation( + name="PlaneSegmentation3", + description="output from segmenting my favorite imaging plane", + imaging_plane=imaging_plane, + reference_images=one_p_series, # optional + ) + + for _ in range(30): + # randomly generate example starting points for region + x = generator.integers(0, 95) + y = generator.integers(0, 95) + z = generator.integers(0, 15) + + # define an example 4 x 3 x 2 voxel region of weight '0.5' + voxel_mask = [] + for ix, iy, iz in product(range(x, x + 4), range(y, y + 3), range(z, z + 2)): + voxel_mask.append((ix, iy, iz, 0.5)) + + # add voxel mask to plane segmentation + ps3.add_roi(voxel_mask=voxel_mask) + rt_region = ps.create_roi_table_region(region=[0, 1], description="the first of two ROIs") + roi_resp_series = RoiResponseSeries( + name="RoiResponseSeries", + description="Fluorescence responses for two ROIs", + data=np.ones((50, 2)), # 50 samples, 2 ROIs + rois=rt_region, + unit="lumens", + rate=30.0, + ) + fl = Fluorescence(roi_response_series=roi_resp_series) + ophys_module.add(fl) + return nwbfile + + +@pytest.fixture(scope="session") +def nwb_file(tmp_output_dir, nwb_file_base, request: pytest.FixtureRequest) -> Path: + """ + NWB File created with pynwb that uses all the weird language features + + Borrowing code from pynwb docs in one humonogous fixture function + since there's not really a reason to + """ + nwb_path = tmp_output_dir / "test_nwb.nwb" + if nwb_path.exists() and not request.config.getoption("--clean"): + return nwb_path + + nwbfile = nwb_file_base + nwbfile = _nwb_timeseries(nwbfile) + nwbfile = _nwb_position(nwbfile) + nwbfile = _nwb_ecephys(nwbfile) + nwbfile = _nwb_units(nwbfile) + nwbfile = _nwb_icephys(nwbfile) + + with NWBHDF5IO(nwb_path, "w") as io: + io.write(nwbfile) + + return nwb_path diff --git a/nwb_linkml/tests/fixtures/paths.py b/nwb_linkml/tests/fixtures/paths.py new file mode 100644 index 0000000..f2d0e1e --- /dev/null +++ b/nwb_linkml/tests/fixtures/paths.py @@ -0,0 +1,63 @@ +import shutil +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="session") +def tmp_output_dir(request: pytest.FixtureRequest) -> Path: + path = Path(__file__).parents[1].resolve() / "__tmp__" + if path.exists(): + if request.config.getoption("--clean"): + shutil.rmtree(path) + else: + for subdir in path.iterdir(): + if subdir.name == "git": + # don't wipe out git repos every time, they don't rly change + continue + elif ( + subdir.is_file() + and subdir.parent != path + or subdir.is_file() + and subdir.suffix == ".nwb" + ): + continue + elif subdir.is_file(): + subdir.unlink(missing_ok=True) + else: + shutil.rmtree(str(subdir)) + path.mkdir(exist_ok=True) + + return path + + +@pytest.fixture(scope="function") +def tmp_output_dir_func(tmp_output_dir) -> Path: + """ + tmp output dir that gets cleared between every function + cleans at the start rather than at cleanup in case the output is to be inspected + """ + subpath = tmp_output_dir / "__tmpfunc__" + if subpath.exists(): + shutil.rmtree(str(subpath)) + subpath.mkdir() + return subpath + + +@pytest.fixture(scope="module") +def tmp_output_dir_mod(tmp_output_dir) -> Path: + """ + tmp output dir that gets cleared between every function + cleans at the start rather than at cleanup in case the output is to be inspected + """ + subpath = tmp_output_dir / "__tmpmod__" + if subpath.exists(): + shutil.rmtree(str(subpath)) + subpath.mkdir() + return subpath + + +@pytest.fixture(scope="session") +def data_dir() -> Path: + path = Path(__file__).parents[1].resolve() / "data" + return path diff --git a/nwb_linkml/tests/fixtures.py b/nwb_linkml/tests/fixtures/schema.py similarity index 83% rename from nwb_linkml/tests/fixtures.py rename to nwb_linkml/tests/fixtures/schema.py index a38e3e0..788d12b 100644 --- a/nwb_linkml/tests/fixtures.py +++ b/nwb_linkml/tests/fixtures/schema.py @@ -1,4 +1,3 @@ -import shutil from dataclasses import dataclass, field from pathlib import Path from types import ModuleType @@ -14,70 +13,12 @@ from linkml_runtime.linkml_model import ( TypeDefinition, ) -from nwb_linkml.adapters.namespaces import NamespacesAdapter +from nwb_linkml.adapters import NamespacesAdapter from nwb_linkml.io import schema as io from nwb_linkml.providers import LinkMLProvider, PydanticProvider from nwb_linkml.providers.linkml import LinkMLSchemaBuild from nwb_schema_language import Attribute, Dataset, Group -__all__ = [ - "NWBSchemaTest", - "TestSchemas", - "data_dir", - "linkml_schema", - "linkml_schema_bare", - "nwb_core_fixture", - "nwb_schema", - "tmp_output_dir", - "tmp_output_dir_func", - "tmp_output_dir_mod", -] - - -@pytest.fixture(scope="session") -def tmp_output_dir() -> Path: - path = Path(__file__).parent.resolve() / "__tmp__" - if path.exists(): - for subdir in path.iterdir(): - if subdir.name == "git": - # don't wipe out git repos every time, they don't rly change - continue - elif subdir.is_file() and subdir.parent != path: - continue - elif subdir.is_file(): - subdir.unlink(missing_ok=True) - else: - shutil.rmtree(str(subdir)) - path.mkdir(exist_ok=True) - - return path - - -@pytest.fixture(scope="function") -def tmp_output_dir_func(tmp_output_dir) -> Path: - """ - tmp output dir that gets cleared between every function - cleans at the start rather than at cleanup in case the output is to be inspected - """ - subpath = tmp_output_dir / "__tmpfunc__" - if subpath.exists(): - shutil.rmtree(str(subpath)) - subpath.mkdir() - return subpath - - -@pytest.fixture(scope="module") -def tmp_output_dir_mod(tmp_output_dir) -> Path: - """ - tmp output dir that gets cleared between every function - cleans at the start rather than at cleanup in case the output is to be inspected - """ - subpath = tmp_output_dir / "__tmpmod__" - if subpath.exists(): - shutil.rmtree(str(subpath)) - subpath.mkdir() - return subpath - @pytest.fixture(scope="session", params=[{"core_version": "2.7.0", "hdmf_version": "1.8.0"}]) def nwb_core_fixture(request) -> NamespacesAdapter: @@ -108,12 +49,6 @@ def nwb_core_module(nwb_core_linkml: LinkMLSchemaBuild, tmp_output_dir) -> Modul return mod -@pytest.fixture(scope="session") -def data_dir() -> Path: - path = Path(__file__).parent.resolve() / "data" - return path - - @dataclass class TestSchemas: __test__ = False diff --git a/nwb_linkml/tests/test_adapters/test_adapter_classes.py b/nwb_linkml/tests/test_adapters/test_adapter_classes.py index ee6e7f6..48fe603 100644 --- a/nwb_linkml/tests/test_adapters/test_adapter_classes.py +++ b/nwb_linkml/tests/test_adapters/test_adapter_classes.py @@ -151,7 +151,7 @@ def test_name_slot(): assert slot.name == "name" assert slot.required assert slot.range == "string" - assert slot.identifier is None + assert slot.identifier assert slot.ifabsent is None assert slot.equals_string is None @@ -160,7 +160,7 @@ def test_name_slot(): assert slot.name == "name" assert slot.required assert slot.range == "string" - assert slot.identifier is None + assert slot.identifier assert slot.ifabsent == "string(FixedName)" assert slot.equals_string == "FixedName" diff --git a/nwb_linkml/tests/test_generators/test_generator_pydantic.py b/nwb_linkml/tests/test_generators/test_generator_pydantic.py index fdab147..12021f4 100644 --- a/nwb_linkml/tests/test_generators/test_generator_pydantic.py +++ b/nwb_linkml/tests/test_generators/test_generator_pydantic.py @@ -5,6 +5,8 @@ Note that since this is largely a subclass, we don't test all of the functionali because it's tested in the base linkml package. """ +# ruff: noqa: F821 - until the tests here settle down + import re import sys import typing @@ -16,7 +18,7 @@ import pytest from numpydantic.ndarray import NDArrayMeta from pydantic import BaseModel -from nwb_linkml.generators.pydantic import NWBPydanticGenerator, compile_python +from nwb_linkml.generators.pydantic import NWBPydanticGenerator from ..fixtures import ( TestSchemas, diff --git a/nwb_linkml/tests/test_includes/test_hdmf.py b/nwb_linkml/tests/test_includes/test_hdmf.py index 920e8b4..a8b14b7 100644 --- a/nwb_linkml/tests/test_includes/test_hdmf.py +++ b/nwb_linkml/tests/test_includes/test_hdmf.py @@ -284,14 +284,14 @@ def test_dynamictable_assert_equal_length(): "existing_col": np.arange(10), "new_col_1": hdmf.VectorData(value=np.arange(11)), } - with pytest.raises(ValidationError, match="Columns are not of equal length"): + with pytest.raises(ValidationError, match="columns are not of equal length"): _ = MyDT(**cols) cols = { "existing_col": np.arange(11), "new_col_1": hdmf.VectorData(value=np.arange(10)), } - with pytest.raises(ValidationError, match="Columns are not of equal length"): + with pytest.raises(ValidationError, match="columns are not of equal length"): _ = MyDT(**cols) # wrong lengths are fine as long as the index is good @@ -308,7 +308,7 @@ def test_dynamictable_assert_equal_length(): "new_col_1": hdmf.VectorData(value=np.arange(100)), "new_col_1_index": hdmf.VectorIndex(value=np.arange(0, 100, 5) + 5), } - with pytest.raises(ValidationError, match="Columns are not of equal length"): + with pytest.raises(ValidationError, match="columns are not of equal length"): _ = MyDT(**cols) @@ -344,7 +344,7 @@ def test_vectordata_indexing(): """ n_rows = 50 value_array, index_array = _ragged_array(n_rows) - value_array = np.concat(value_array) + value_array = np.concatenate(value_array) data = hdmf.VectorData(value=value_array) @@ -551,13 +551,13 @@ def test_aligned_dynamictable_indexing(aligned_table): row.columns == pd.MultiIndex.from_tuples( [ - ("table1", "index"), + ("table1", "id"), ("table1", "col1"), ("table1", "col2"), - ("table2", "index"), + ("table2", "id"), ("table2", "col3"), ("table2", "col4"), - ("table3", "index"), + ("table3", "id"), ("table3", "col5"), ("table3", "col6"), ] @@ -592,7 +592,7 @@ def test_mixed_aligned_dynamictable(aligned_table): AlignedTable, cols = aligned_table value_array, index_array = _ragged_array(10) - value_array = np.concat(value_array) + value_array = np.concatenate(value_array) data = hdmf.VectorData(value=value_array) index = hdmf.VectorIndex(value=index_array) @@ -754,11 +754,11 @@ def test_aligned_dynamictable_ictable(intracellular_recordings_table): rows.columns == pd.MultiIndex.from_tuples( [ - ("electrodes", "index"), + ("electrodes", "id"), ("electrodes", "electrode"), - ("stimuli", "index"), + ("stimuli", "id"), ("stimuli", "stimulus"), - ("responses", "index"), + ("responses", "id"), ("responses", "response"), ] ) diff --git a/nwb_linkml/tests/test_io/test_io_hdf5.py b/nwb_linkml/tests/test_io/test_io_hdf5.py index c64cf48..4222a2c 100644 --- a/nwb_linkml/tests/test_io/test_io_hdf5.py +++ b/nwb_linkml/tests/test_io/test_io_hdf5.py @@ -1,10 +1,10 @@ -import pdb - import h5py +import networkx as nx import numpy as np import pytest -from nwb_linkml.io.hdf5 import HDF5IO, truncate_file +from nwb_linkml.io.hdf5 import HDF5IO, filter_dependency_graph, hdf_dependency_graph, truncate_file +from nwb_linkml.maps.hdf5 import resolve_hardlink @pytest.mark.skip() @@ -13,7 +13,7 @@ def test_hdf_read(data_dir, dset): NWBFILE = data_dir / dset io = HDF5IO(path=NWBFILE) # the test for now is just whether we can read it lol - model = io.read() + _ = io.read() def test_truncate_file(tmp_output_dir): @@ -86,15 +86,60 @@ def test_truncate_file(tmp_output_dir): assert target_h5f["data"]["dataset_contig"].attrs["anattr"] == 1 -@pytest.mark.skip() -def test_flatten_hdf(): - from nwb_linkml.maps.hdf5 import flatten_hdf +def test_dependencies_hardlink(nwb_file): + """ + Test that hardlinks are resolved (eg. from /processing/ecephys/LFP/ElectricalSeries/electrodes + to /acquisition/ElectricalSeries/electrodes + Args: + nwb_file: - path = "/Users/jonny/Dropbox/lab/p2p_ld/data/nwb/sub-738651046_ses-760693773.nwb" - import h5py + Returns: - h5f = h5py.File(path) - flat = flatten_hdf(h5f) - assert not any(["specifications" in v.path for v in flat.values()]) - pdb.set_trace() - raise NotImplementedError("Just a stub for local testing for now, finish me!") + """ + parent = "/processing/ecephys/LFP/ElectricalSeries" + source = "/processing/ecephys/LFP/ElectricalSeries/electrodes" + target = "/acquisition/ElectricalSeries/electrodes" + + # assert that the hardlink exists in the test file + with h5py.File(str(nwb_file), "r") as h5f: + node = h5f.get(source) + linked_node = resolve_hardlink(node) + assert linked_node == target + + graph = hdf_dependency_graph(nwb_file) + # the parent should link to the target as a child + assert (parent, target) in graph.edges([parent]) + assert graph.edges[parent, target]["label"] == "child" + + +@pytest.mark.dev +def test_dependency_graph_images(nwb_file, tmp_output_dir): + """ + Generate images of the dependency graph + """ + graph = hdf_dependency_graph(nwb_file) + A_unfiltered = nx.nx_agraph.to_agraph(graph) + A_unfiltered.draw(tmp_output_dir / "test_nwb_unfiltered.png", prog="dot") + graph = filter_dependency_graph(graph) + A_filtered = nx.nx_agraph.to_agraph(graph) + A_filtered.draw(tmp_output_dir / "test_nwb_filtered.png", prog="dot") + + +@pytest.mark.parametrize( + "dset", + [ + {"name": "aibs.nwb", "source": "sub-738651046_ses-760693773.nwb"}, + { + "name": "aibs_ecephys.nwb", + "source": "sub-738651046_ses-760693773_probe-769322820_ecephys.nwb", + }, + ], +) +@pytest.mark.dev +def test_make_truncated_datasets(tmp_output_dir, data_dir, dset): + input_file = tmp_output_dir / dset["source"] + output_file = data_dir / dset["name"] + if not input_file.exists(): + return + + truncate_file(input_file, output_file, 10) diff --git a/nwb_linkml/tests/test_io/test_io_nwb.py b/nwb_linkml/tests/test_io/test_io_nwb.py new file mode 100644 index 0000000..1ad51ed --- /dev/null +++ b/nwb_linkml/tests/test_io/test_io_nwb.py @@ -0,0 +1,110 @@ +""" +Placeholder test module to test reading from pynwb-generated NWB file +""" + +from datetime import datetime + +import numpy as np +import pandas as pd +import pytest +from numpydantic.interface.hdf5 import H5Proxy +from pydantic import BaseModel +from pynwb import NWBHDF5IO +from pynwb import NWBFile as PyNWBFile + +from nwb_linkml.io.hdf5 import HDF5IO +from nwb_models.models import NWBFile + + +def test_read_from_nwbfile(nwb_file): + """ + Read data from a pynwb HDF5 NWB file + + Placeholder that just ensures that reads work and all pydantic models validate, + testing of correctness of read will happen elsewhere. + """ + res = HDF5IO(nwb_file).read() + + +@pytest.fixture(scope="module") +def read_nwbfile(nwb_file) -> NWBFile: + res = HDF5IO(nwb_file).read() + return res + + +@pytest.fixture(scope="module") +def read_pynwb(nwb_file) -> PyNWBFile: + nwbf = NWBHDF5IO(nwb_file, "r") + res = nwbf.read() + yield res + nwbf.close() + + +def _compare_attrs(model: BaseModel, pymodel: object): + for field, value in model.model_dump().items(): + if isinstance(value, (dict, H5Proxy)): + continue + if hasattr(pymodel, field): + pynwb_val = getattr(pymodel, field) + if isinstance(pynwb_val, list): + if isinstance(pynwb_val[0], datetime): + # need to normalize UTC numpy.datetime64 with datetime with tz + continue + assert all([val == pval for val, pval in zip(value, pynwb_val)]) + else: + if not pynwb_val: + # pynwb instantiates some stuff as empty dicts where we use ``None`` + assert bool(pynwb_val) == bool(value) + else: + assert value == pynwb_val + + +def test_nwbfile_base(read_nwbfile, read_pynwb): + """ + Base attributes on top-level nwbfile are correct + """ + _compare_attrs(read_nwbfile, read_pynwb) + + +def test_timeseries(read_nwbfile, read_pynwb): + py_acq = read_pynwb.get_acquisition("test_timeseries") + acq = read_nwbfile.acquisition["test_timeseries"] + _compare_attrs(acq, py_acq) + # data and timeseries should be equal + assert np.array_equal(acq.data[:], py_acq.data[:]) + assert np.array_equal(acq.timestamps[:], py_acq.timestamps[:]) + + +def test_position(read_nwbfile, read_pynwb): + trials = read_nwbfile.intervals.trials[:] + py_trials = read_pynwb.trials.to_dataframe() + pd.testing.assert_frame_equal(py_trials, trials) + + spatial = read_nwbfile.processing["behavior"].Position.SpatialSeries + py_spatial = read_pynwb.processing["behavior"]["Position"]["SpatialSeries"] + _compare_attrs(spatial, py_spatial) + assert np.array_equal(spatial[:], py_spatial.data[:]) + assert np.array_equal(spatial.timestamps[:], py_spatial.timestamps[:]) + + +def test_ecephys(read_nwbfile, read_pynwb): + pass + + +def test_units(read_nwbfile, read_pynwb): + pass + + +def test_icephys(read_nwbfile, read_pynwb): + pass + + +def test_ca_imaging(read_nwbfile, read_pynwb): + pass + + +def test_read_from_yaml(nwb_file): + """ + Read data from a yaml-fied NWB file + """ + pass diff --git a/nwb_models/README.md b/nwb_models/README.md index cc36d0a..4758202 100644 --- a/nwb_models/README.md +++ b/nwb_models/README.md @@ -1 +1,3 @@ # nwb-models + +(README forthcoming, for now see [`nwb-linkml`](https://pypi.org/project/nwb-linkml)) \ No newline at end of file diff --git a/nwb_models/pyproject.toml b/nwb_models/pyproject.toml index 21078a2..59b0b6d 100644 --- a/nwb_models/pyproject.toml +++ b/nwb_models/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "nwb-models" -version = "0.1.0" +version = "0.2.0" description = "Pydantic/LinkML models for Neurodata Without Borders" authors = [ {name = "sneakers-the-rat", email = "sneakers-the-rat@protonmail.com"}, diff --git a/nwb_models/src/nwb_models/models/pydantic/__init__.py b/nwb_models/src/nwb_models/models/pydantic/__init__.py index fa3cf1e..e69de29 100644 --- a/nwb_models/src/nwb_models/models/pydantic/__init__.py +++ b/nwb_models/src/nwb_models/models/pydantic/__init__.py @@ -1 +0,0 @@ -from .pydantic.core.v2_7_0.namespace import * diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_base.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_base.py index db6e75c..263d389 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_base.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_behavior.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_behavior.py index 31bf322..5691dab 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_behavior.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_behavior.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_device.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_device.py index bf15387..ab24817 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_device.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_device.py @@ -21,7 +21,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -40,6 +40,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_ecephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_ecephys.py index 7a99a15..136ec40 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_ecephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_ecephys.py @@ -38,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -57,6 +57,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_epoch.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_epoch.py index 7475c41..4ab3c01 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_epoch.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_epoch.py @@ -20,7 +20,12 @@ from pydantic import ( ) from ...core.v2_2_0.core_nwb_base import TimeSeries -from ...hdmf_common.v1_1_0.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_1_0.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -31,7 +36,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -50,6 +55,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -127,7 +163,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags: VectorData[Optional[NDArray[Any, str]]] = Field( + tags: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""User-defined tags that identify or categorize events.""", json_schema_extra={ @@ -136,7 +172,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags_index: Named[Optional[VectorIndex]] = Field( + tags_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for tags.""", json_schema_extra={ @@ -151,7 +187,7 @@ class TimeIntervals(DynamicTable): timeseries: Optional[TimeIntervalsTimeseries] = Field( None, description="""An index into a TimeSeries object.""" ) - timeseries_index: Named[Optional[VectorIndex]] = Field( + timeseries_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for timeseries.""", json_schema_extra={ @@ -168,14 +204,11 @@ class TimeIntervals(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_file.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_file.py index e03f10b..ae16391 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_file.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_file.py @@ -24,7 +24,12 @@ from ...core.v2_2_0.core_nwb_icephys import IntracellularElectrode, SweepTable from ...core.v2_2_0.core_nwb_misc import Units from ...core.v2_2_0.core_nwb_ogen import OptogeneticStimulusSite from ...core.v2_2_0.core_nwb_ophys import ImagingPlane -from ...hdmf_common.v1_1_0.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_1_0.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -35,7 +40,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -54,6 +59,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -464,7 +500,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_x: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_x: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""x coordinate in electrode group""", json_schema_extra={ @@ -473,7 +509,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_y: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_y: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""y coordinate in electrode group""", json_schema_extra={ @@ -482,7 +518,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_z: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_z: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""z coordinate in electrode group""", json_schema_extra={ @@ -491,7 +527,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - reference: VectorData[Optional[NDArray[Any, str]]] = Field( + reference: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""Description of the reference used for this electrode.""", json_schema_extra={ @@ -505,14 +541,11 @@ class ExtracellularEphysElectrodes(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_icephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_icephys.py index be3a0ab..439d5af 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_icephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_icephys.py @@ -26,7 +26,12 @@ from ...core.v2_2_0.core_nwb_base import ( TimeSeriesSync, ) from ...core.v2_2_0.core_nwb_device import Device -from ...hdmf_common.v1_1_0.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_1_0.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -37,7 +42,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +61,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -897,14 +933,11 @@ class SweepTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_image.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_image.py index 73b924f..33784d6 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_image.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_image.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -84,17 +115,16 @@ class GrayscaleImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "x"}, {"alias": "y"}]}} + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBImage(Image): @@ -107,17 +137,24 @@ class RGBImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 3 r_g_b"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b", "exact_cardinality": 3}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBAImage(Image): @@ -130,17 +167,24 @@ class RGBAImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 4 r_g_b_a"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b_a", "exact_cardinality": 4}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class ImageSeries(TimeSeries): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_misc.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_misc.py index f83925f..e8a4896 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_misc.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_misc.py @@ -24,6 +24,7 @@ from ...core.v2_2_0.core_nwb_ecephys import ElectrodeGroup from ...hdmf_common.v1_1_0.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -37,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +57,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -443,14 +475,11 @@ class DecompositionSeriesBands(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) @@ -466,7 +495,7 @@ class Units(DynamicTable): ) name: str = Field("Units", json_schema_extra={"linkml_meta": {"ifabsent": "string(Units)"}}) - spike_times_index: Named[Optional[VectorIndex]] = Field( + spike_times_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the spike_times dataset.""", json_schema_extra={ @@ -481,7 +510,7 @@ class Units(DynamicTable): spike_times: Optional[UnitsSpikeTimes] = Field( None, description="""Spike times for each unit.""" ) - obs_intervals_index: Named[Optional[VectorIndex]] = Field( + obs_intervals_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the obs_intervals dataset.""", json_schema_extra={ @@ -493,7 +522,7 @@ class Units(DynamicTable): } }, ) - obs_intervals: VectorData[Optional[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( + obs_intervals: Optional[VectorData[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( Field( None, description="""Observation intervals for each unit.""", @@ -509,7 +538,7 @@ class Units(DynamicTable): }, ) ) - electrodes_index: Named[Optional[VectorIndex]] = Field( + electrodes_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into electrodes.""", json_schema_extra={ @@ -521,7 +550,7 @@ class Units(DynamicTable): } }, ) - electrodes: Named[Optional[DynamicTableRegion]] = Field( + electrodes: Optional[Named[DynamicTableRegion]] = Field( None, description="""Electrode that each spike unit came from, specified using a DynamicTableRegion.""", json_schema_extra={ @@ -536,16 +565,16 @@ class Units(DynamicTable): electrode_group: Optional[List[ElectrodeGroup]] = Field( None, description="""Electrode group that each spike unit came from.""" ) - waveform_mean: VectorData[ - Optional[ + waveform_mean: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], ] ] ] = Field(None, description="""Spike waveform mean for each spike unit.""") - waveform_sd: VectorData[ - Optional[ + waveform_sd: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], @@ -557,14 +586,11 @@ class Units(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_ogen.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_ogen.py index 853250a..998dda0 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_ogen.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_ogen.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_ophys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_ophys.py index 6ce44c7..70db9d7 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_ophys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_ophys.py @@ -39,7 +39,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -58,6 +58,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_retinotopy.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_retinotopy.py index ab2f91e..17edeec 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_retinotopy.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/core_nwb_retinotopy.py @@ -31,7 +31,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -50,6 +50,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -166,17 +197,16 @@ class RetinotopyImage(GrayscaleImage): ) field_of_view: List[float] = Field(..., description="""Size of viewing area, in meters.""") format: str = Field(..., description="""Format of image. Right now only 'raw' is supported.""") + value: Optional[NDArray[Shape["* x, * y"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "x"}, {"alias": "y"}]}} + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class ImagingRetinotopy(NWBDataInterface): @@ -204,7 +234,7 @@ class ImagingRetinotopy(NWBDataInterface): } }, ) - axis_1_power_map: Named[Optional[AxisMap]] = Field( + axis_1_power_map: Optional[Named[AxisMap]] = Field( None, description="""Power response on the first measured axis. Response is scaled so 0.0 is no power in the response and 1.0 is maximum relative power.""", json_schema_extra={ @@ -228,7 +258,7 @@ class ImagingRetinotopy(NWBDataInterface): } }, ) - axis_2_power_map: Named[Optional[AxisMap]] = Field( + axis_2_power_map: Optional[Named[AxisMap]] = Field( None, description="""Power response to stimulus on the second measured axis.""", json_schema_extra={ @@ -306,17 +336,16 @@ class ImagingRetinotopyFocalDepthImage(RetinotopyImage): ) field_of_view: List[float] = Field(..., description="""Size of viewing area, in meters.""") format: str = Field(..., description="""Format of image. Right now only 'raw' is supported.""") + value: Optional[NDArray[Shape["* x, * y"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "x"}, {"alias": "y"}]}} + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/namespace.py index 710c80c..d4b265d 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_0/namespace.py @@ -149,7 +149,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -168,6 +168,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_base.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_base.py index 635f77a..f0f43be 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_base.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_behavior.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_behavior.py index c0a675b..e96918c 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_behavior.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_behavior.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_device.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_device.py index 1da62bb..80de9c0 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_device.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_device.py @@ -21,7 +21,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -40,6 +40,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_ecephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_ecephys.py index 25525e8..169dd5e 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_ecephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_ecephys.py @@ -38,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -57,6 +57,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_epoch.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_epoch.py index 247b667..ed1353e 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_epoch.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_epoch.py @@ -20,7 +20,12 @@ from pydantic import ( ) from ...core.v2_2_1.core_nwb_base import TimeSeries -from ...hdmf_common.v1_1_2.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_1_2.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -31,7 +36,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -50,6 +55,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -127,7 +163,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags: VectorData[Optional[NDArray[Any, str]]] = Field( + tags: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""User-defined tags that identify or categorize events.""", json_schema_extra={ @@ -136,7 +172,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags_index: Named[Optional[VectorIndex]] = Field( + tags_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for tags.""", json_schema_extra={ @@ -151,7 +187,7 @@ class TimeIntervals(DynamicTable): timeseries: Optional[TimeIntervalsTimeseries] = Field( None, description="""An index into a TimeSeries object.""" ) - timeseries_index: Named[Optional[VectorIndex]] = Field( + timeseries_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for timeseries.""", json_schema_extra={ @@ -168,14 +204,11 @@ class TimeIntervals(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_file.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_file.py index 49ebbf0..b5a0b9b 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_file.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_file.py @@ -24,7 +24,12 @@ from ...core.v2_2_1.core_nwb_icephys import IntracellularElectrode, SweepTable from ...core.v2_2_1.core_nwb_misc import Units from ...core.v2_2_1.core_nwb_ogen import OptogeneticStimulusSite from ...core.v2_2_1.core_nwb_ophys import ImagingPlane -from ...hdmf_common.v1_1_2.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_1_2.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -35,7 +40,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -54,6 +59,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -464,7 +500,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_x: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_x: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""x coordinate in electrode group""", json_schema_extra={ @@ -473,7 +509,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_y: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_y: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""y coordinate in electrode group""", json_schema_extra={ @@ -482,7 +518,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_z: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_z: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""z coordinate in electrode group""", json_schema_extra={ @@ -491,7 +527,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - reference: VectorData[Optional[NDArray[Any, str]]] = Field( + reference: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""Description of the reference used for this electrode.""", json_schema_extra={ @@ -505,14 +541,11 @@ class ExtracellularEphysElectrodes(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_icephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_icephys.py index 34a9d42..991c1e8 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_icephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_icephys.py @@ -26,7 +26,12 @@ from ...core.v2_2_1.core_nwb_base import ( TimeSeriesSync, ) from ...core.v2_2_1.core_nwb_device import Device -from ...hdmf_common.v1_1_2.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_1_2.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -37,7 +42,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +61,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -897,14 +933,11 @@ class SweepTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_image.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_image.py index 0824778..52c10a5 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_image.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_image.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -84,17 +115,16 @@ class GrayscaleImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "x"}, {"alias": "y"}]}} + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBImage(Image): @@ -107,17 +137,24 @@ class RGBImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 3 r_g_b"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b", "exact_cardinality": 3}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBAImage(Image): @@ -130,17 +167,24 @@ class RGBAImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 4 r_g_b_a"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b_a", "exact_cardinality": 4}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class ImageSeries(TimeSeries): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_misc.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_misc.py index ecd0946..19a036f 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_misc.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_misc.py @@ -24,6 +24,7 @@ from ...core.v2_2_1.core_nwb_ecephys import ElectrodeGroup from ...hdmf_common.v1_1_2.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -37,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +57,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -443,14 +475,11 @@ class DecompositionSeriesBands(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) @@ -466,7 +495,7 @@ class Units(DynamicTable): ) name: str = Field("Units", json_schema_extra={"linkml_meta": {"ifabsent": "string(Units)"}}) - spike_times_index: Named[Optional[VectorIndex]] = Field( + spike_times_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the spike_times dataset.""", json_schema_extra={ @@ -481,7 +510,7 @@ class Units(DynamicTable): spike_times: Optional[UnitsSpikeTimes] = Field( None, description="""Spike times for each unit.""" ) - obs_intervals_index: Named[Optional[VectorIndex]] = Field( + obs_intervals_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the obs_intervals dataset.""", json_schema_extra={ @@ -493,7 +522,7 @@ class Units(DynamicTable): } }, ) - obs_intervals: VectorData[Optional[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( + obs_intervals: Optional[VectorData[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( Field( None, description="""Observation intervals for each unit.""", @@ -509,7 +538,7 @@ class Units(DynamicTable): }, ) ) - electrodes_index: Named[Optional[VectorIndex]] = Field( + electrodes_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into electrodes.""", json_schema_extra={ @@ -521,7 +550,7 @@ class Units(DynamicTable): } }, ) - electrodes: Named[Optional[DynamicTableRegion]] = Field( + electrodes: Optional[Named[DynamicTableRegion]] = Field( None, description="""Electrode that each spike unit came from, specified using a DynamicTableRegion.""", json_schema_extra={ @@ -536,16 +565,16 @@ class Units(DynamicTable): electrode_group: Optional[List[ElectrodeGroup]] = Field( None, description="""Electrode group that each spike unit came from.""" ) - waveform_mean: VectorData[ - Optional[ + waveform_mean: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], ] ] ] = Field(None, description="""Spike waveform mean for each spike unit.""") - waveform_sd: VectorData[ - Optional[ + waveform_sd: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], @@ -557,14 +586,11 @@ class Units(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_ogen.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_ogen.py index 1577358..609baf0 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_ogen.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_ogen.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_ophys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_ophys.py index ca843fc..a951c51 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_ophys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_ophys.py @@ -39,7 +39,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -58,6 +58,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_retinotopy.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_retinotopy.py index 69e47a3..1c6f4ad 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_retinotopy.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/core_nwb_retinotopy.py @@ -31,7 +31,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -50,6 +50,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -166,17 +197,16 @@ class RetinotopyImage(GrayscaleImage): ) field_of_view: List[float] = Field(..., description="""Size of viewing area, in meters.""") format: str = Field(..., description="""Format of image. Right now only 'raw' is supported.""") + value: Optional[NDArray[Shape["* x, * y"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "x"}, {"alias": "y"}]}} + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class ImagingRetinotopy(NWBDataInterface): @@ -204,7 +234,7 @@ class ImagingRetinotopy(NWBDataInterface): } }, ) - axis_1_power_map: Named[Optional[AxisMap]] = Field( + axis_1_power_map: Optional[Named[AxisMap]] = Field( None, description="""Power response on the first measured axis. Response is scaled so 0.0 is no power in the response and 1.0 is maximum relative power.""", json_schema_extra={ @@ -228,7 +258,7 @@ class ImagingRetinotopy(NWBDataInterface): } }, ) - axis_2_power_map: Named[Optional[AxisMap]] = Field( + axis_2_power_map: Optional[Named[AxisMap]] = Field( None, description="""Power response to stimulus on the second measured axis.""", json_schema_extra={ @@ -306,17 +336,16 @@ class ImagingRetinotopyFocalDepthImage(RetinotopyImage): ) field_of_view: List[float] = Field(..., description="""Size of viewing area, in meters.""") format: str = Field(..., description="""Format of image. Right now only 'raw' is supported.""") + value: Optional[NDArray[Shape["* x, * y"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "x"}, {"alias": "y"}]}} + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/namespace.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/namespace.py index 5dd8a41..7f2ade1 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_1/namespace.py @@ -149,7 +149,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -168,6 +168,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_base.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_base.py index dd580be..956e37d 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_base.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_behavior.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_behavior.py index bb900e7..271fceb 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_behavior.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_behavior.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_device.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_device.py index dbb96bf..28aa954 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_device.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_device.py @@ -21,7 +21,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -40,6 +40,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_ecephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_ecephys.py index 5749b00..9664726 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_ecephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_ecephys.py @@ -38,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -57,6 +57,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_epoch.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_epoch.py index 6e766ad..c12a965 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_epoch.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_epoch.py @@ -20,7 +20,12 @@ from pydantic import ( ) from ...core.v2_2_2.core_nwb_base import TimeSeries -from ...hdmf_common.v1_1_3.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_1_3.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -31,7 +36,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -50,6 +55,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -127,7 +163,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags: VectorData[Optional[NDArray[Any, str]]] = Field( + tags: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""User-defined tags that identify or categorize events.""", json_schema_extra={ @@ -136,7 +172,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags_index: Named[Optional[VectorIndex]] = Field( + tags_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for tags.""", json_schema_extra={ @@ -151,7 +187,7 @@ class TimeIntervals(DynamicTable): timeseries: Optional[TimeIntervalsTimeseries] = Field( None, description="""An index into a TimeSeries object.""" ) - timeseries_index: Named[Optional[VectorIndex]] = Field( + timeseries_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for timeseries.""", json_schema_extra={ @@ -168,14 +204,11 @@ class TimeIntervals(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_file.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_file.py index 3824ab2..ec66471 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_file.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_file.py @@ -24,7 +24,12 @@ from ...core.v2_2_2.core_nwb_icephys import IntracellularElectrode, SweepTable from ...core.v2_2_2.core_nwb_misc import Units from ...core.v2_2_2.core_nwb_ogen import OptogeneticStimulusSite from ...core.v2_2_2.core_nwb_ophys import ImagingPlane -from ...hdmf_common.v1_1_3.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_1_3.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -35,7 +40,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -54,6 +59,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -464,7 +500,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_x: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_x: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""x coordinate in electrode group""", json_schema_extra={ @@ -473,7 +509,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_y: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_y: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""y coordinate in electrode group""", json_schema_extra={ @@ -482,7 +518,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_z: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_z: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""z coordinate in electrode group""", json_schema_extra={ @@ -491,7 +527,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - reference: VectorData[Optional[NDArray[Any, str]]] = Field( + reference: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""Description of the reference used for this electrode.""", json_schema_extra={ @@ -505,14 +541,11 @@ class ExtracellularEphysElectrodes(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_icephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_icephys.py index 9d4a696..9b7729d 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_icephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_icephys.py @@ -26,7 +26,12 @@ from ...core.v2_2_2.core_nwb_base import ( TimeSeriesSync, ) from ...core.v2_2_2.core_nwb_device import Device -from ...hdmf_common.v1_1_3.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_1_3.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -37,7 +42,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +61,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -897,14 +933,11 @@ class SweepTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_image.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_image.py index fec53fa..6e805b1 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_image.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_image.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -84,17 +115,16 @@ class GrayscaleImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "x"}, {"alias": "y"}]}} + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBImage(Image): @@ -107,17 +137,24 @@ class RGBImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 3 r_g_b"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b", "exact_cardinality": 3}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBAImage(Image): @@ -130,17 +167,24 @@ class RGBAImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 4 r_g_b_a"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b_a", "exact_cardinality": 4}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class ImageSeries(TimeSeries): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_misc.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_misc.py index 5c32d21..d80af52 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_misc.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_misc.py @@ -24,6 +24,7 @@ from ...core.v2_2_2.core_nwb_ecephys import ElectrodeGroup from ...hdmf_common.v1_1_3.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -37,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +57,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -443,14 +475,11 @@ class DecompositionSeriesBands(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) @@ -466,7 +495,7 @@ class Units(DynamicTable): ) name: str = Field("Units", json_schema_extra={"linkml_meta": {"ifabsent": "string(Units)"}}) - spike_times_index: Named[Optional[VectorIndex]] = Field( + spike_times_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the spike_times dataset.""", json_schema_extra={ @@ -481,7 +510,7 @@ class Units(DynamicTable): spike_times: Optional[UnitsSpikeTimes] = Field( None, description="""Spike times for each unit.""" ) - obs_intervals_index: Named[Optional[VectorIndex]] = Field( + obs_intervals_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the obs_intervals dataset.""", json_schema_extra={ @@ -493,7 +522,7 @@ class Units(DynamicTable): } }, ) - obs_intervals: VectorData[Optional[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( + obs_intervals: Optional[VectorData[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( Field( None, description="""Observation intervals for each unit.""", @@ -509,7 +538,7 @@ class Units(DynamicTable): }, ) ) - electrodes_index: Named[Optional[VectorIndex]] = Field( + electrodes_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into electrodes.""", json_schema_extra={ @@ -521,7 +550,7 @@ class Units(DynamicTable): } }, ) - electrodes: Named[Optional[DynamicTableRegion]] = Field( + electrodes: Optional[Named[DynamicTableRegion]] = Field( None, description="""Electrode that each spike unit came from, specified using a DynamicTableRegion.""", json_schema_extra={ @@ -536,16 +565,16 @@ class Units(DynamicTable): electrode_group: Optional[List[ElectrodeGroup]] = Field( None, description="""Electrode group that each spike unit came from.""" ) - waveform_mean: VectorData[ - Optional[ + waveform_mean: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], ] ] ] = Field(None, description="""Spike waveform mean for each spike unit.""") - waveform_sd: VectorData[ - Optional[ + waveform_sd: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], @@ -557,14 +586,11 @@ class Units(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_ogen.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_ogen.py index 8c1e21d..debdaf9 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_ogen.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_ogen.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_ophys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_ophys.py index b7ed446..e7b56da 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_ophys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_ophys.py @@ -39,7 +39,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -58,6 +58,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_retinotopy.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_retinotopy.py index f92004d..bfa2ad5 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_retinotopy.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/core_nwb_retinotopy.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/namespace.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/namespace.py index 536af15..9ba793b 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_2/namespace.py @@ -152,7 +152,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -171,6 +171,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_base.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_base.py index 82fa6ff..0e81486 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_base.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_behavior.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_behavior.py index e14c8d5..42613b4 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_behavior.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_behavior.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_device.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_device.py index 09cf60b..1aeeb6c 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_device.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_device.py @@ -21,7 +21,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -40,6 +40,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_ecephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_ecephys.py index 25a0a42..d4f5172 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_ecephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_ecephys.py @@ -38,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -57,6 +57,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_epoch.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_epoch.py index 3cd5078..61f894b 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_epoch.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_epoch.py @@ -20,7 +20,12 @@ from pydantic import ( ) from ...core.v2_2_4.core_nwb_base import TimeSeries -from ...hdmf_common.v1_1_3.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_1_3.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -31,7 +36,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -50,6 +55,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -127,7 +163,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags: VectorData[Optional[NDArray[Any, str]]] = Field( + tags: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""User-defined tags that identify or categorize events.""", json_schema_extra={ @@ -136,7 +172,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags_index: Named[Optional[VectorIndex]] = Field( + tags_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for tags.""", json_schema_extra={ @@ -151,7 +187,7 @@ class TimeIntervals(DynamicTable): timeseries: Optional[TimeIntervalsTimeseries] = Field( None, description="""An index into a TimeSeries object.""" ) - timeseries_index: Named[Optional[VectorIndex]] = Field( + timeseries_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for timeseries.""", json_schema_extra={ @@ -168,14 +204,11 @@ class TimeIntervals(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_file.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_file.py index 0542071..9167a4d 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_file.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_file.py @@ -25,7 +25,12 @@ from ...core.v2_2_4.core_nwb_icephys import IntracellularElectrode, SweepTable from ...core.v2_2_4.core_nwb_misc import Units from ...core.v2_2_4.core_nwb_ogen import OptogeneticStimulusSite from ...core.v2_2_4.core_nwb_ophys import ImagingPlane -from ...hdmf_common.v1_1_3.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_1_3.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -36,7 +41,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -55,6 +60,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -440,7 +476,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_x: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_x: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""x coordinate in electrode group""", json_schema_extra={ @@ -449,7 +485,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_y: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_y: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""y coordinate in electrode group""", json_schema_extra={ @@ -458,7 +494,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_z: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_z: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""z coordinate in electrode group""", json_schema_extra={ @@ -467,7 +503,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - reference: VectorData[Optional[NDArray[Any, str]]] = Field( + reference: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""Description of the reference used for this electrode.""", json_schema_extra={ @@ -481,14 +517,11 @@ class ExtracellularEphysElectrodes(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_icephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_icephys.py index 8f20762..8067eb7 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_icephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_icephys.py @@ -26,7 +26,12 @@ from ...core.v2_2_4.core_nwb_base import ( TimeSeriesSync, ) from ...core.v2_2_4.core_nwb_device import Device -from ...hdmf_common.v1_1_3.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_1_3.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -37,7 +42,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +61,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -897,14 +933,11 @@ class SweepTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_image.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_image.py index 850e89f..05c1d6e 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_image.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_image.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -84,17 +115,16 @@ class GrayscaleImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "x"}, {"alias": "y"}]}} + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBImage(Image): @@ -107,17 +137,24 @@ class RGBImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 3 r_g_b"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b", "exact_cardinality": 3}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBAImage(Image): @@ -130,17 +167,24 @@ class RGBAImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 4 r_g_b_a"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b_a", "exact_cardinality": 4}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class ImageSeries(TimeSeries): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_misc.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_misc.py index 22fc753..5ff807c 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_misc.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_misc.py @@ -24,6 +24,7 @@ from ...core.v2_2_4.core_nwb_ecephys import ElectrodeGroup from ...hdmf_common.v1_1_3.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -37,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +57,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -443,14 +475,11 @@ class DecompositionSeriesBands(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) @@ -466,7 +495,7 @@ class Units(DynamicTable): ) name: str = Field("Units", json_schema_extra={"linkml_meta": {"ifabsent": "string(Units)"}}) - spike_times_index: Named[Optional[VectorIndex]] = Field( + spike_times_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the spike_times dataset.""", json_schema_extra={ @@ -481,7 +510,7 @@ class Units(DynamicTable): spike_times: Optional[UnitsSpikeTimes] = Field( None, description="""Spike times for each unit.""" ) - obs_intervals_index: Named[Optional[VectorIndex]] = Field( + obs_intervals_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the obs_intervals dataset.""", json_schema_extra={ @@ -493,7 +522,7 @@ class Units(DynamicTable): } }, ) - obs_intervals: VectorData[Optional[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( + obs_intervals: Optional[VectorData[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( Field( None, description="""Observation intervals for each unit.""", @@ -509,7 +538,7 @@ class Units(DynamicTable): }, ) ) - electrodes_index: Named[Optional[VectorIndex]] = Field( + electrodes_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into electrodes.""", json_schema_extra={ @@ -521,7 +550,7 @@ class Units(DynamicTable): } }, ) - electrodes: Named[Optional[DynamicTableRegion]] = Field( + electrodes: Optional[Named[DynamicTableRegion]] = Field( None, description="""Electrode that each spike unit came from, specified using a DynamicTableRegion.""", json_schema_extra={ @@ -536,16 +565,16 @@ class Units(DynamicTable): electrode_group: Optional[List[ElectrodeGroup]] = Field( None, description="""Electrode group that each spike unit came from.""" ) - waveform_mean: VectorData[ - Optional[ + waveform_mean: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], ] ] ] = Field(None, description="""Spike waveform mean for each spike unit.""") - waveform_sd: VectorData[ - Optional[ + waveform_sd: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], @@ -557,14 +586,11 @@ class Units(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_ogen.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_ogen.py index 50d25ad..20f6353 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_ogen.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_ogen.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_ophys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_ophys.py index 1a0bf16..b91e448 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_ophys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_ophys.py @@ -31,6 +31,7 @@ from ...core.v2_2_4.core_nwb_image import ImageSeries, ImageSeriesExternalFile from ...hdmf_common.v1_1_3.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -44,7 +45,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -63,6 +64,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -322,7 +354,7 @@ class PlaneSegmentation(DynamicTable): None, description="""ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero.""", ) - pixel_mask_index: Named[Optional[VectorIndex]] = Field( + pixel_mask_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into pixel_mask.""", json_schema_extra={ @@ -338,7 +370,7 @@ class PlaneSegmentation(DynamicTable): None, description="""Pixel masks for each ROI: a list of indices and weights for the ROI. Pixel masks are concatenated and parsing of this dataset is maintained by the PlaneSegmentation""", ) - voxel_mask_index: Named[Optional[VectorIndex]] = Field( + voxel_mask_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into voxel_mask.""", json_schema_extra={ @@ -373,14 +405,11 @@ class PlaneSegmentation(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_retinotopy.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_retinotopy.py index 111a502..362bc59 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_retinotopy.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/core_nwb_retinotopy.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/namespace.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/namespace.py index 10795d3..23ec3dd 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_4/namespace.py @@ -159,7 +159,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -178,6 +178,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_base.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_base.py index 5012a43..86fe03f 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_base.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_behavior.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_behavior.py index e7c936a..f4f5e96 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_behavior.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_behavior.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_device.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_device.py index 5d308f9..5abfc5d 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_device.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_device.py @@ -21,7 +21,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -40,6 +40,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_ecephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_ecephys.py index 4e993f0..48d2503 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_ecephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_ecephys.py @@ -38,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -57,6 +57,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_epoch.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_epoch.py index bd7e37e..6a8ba5a 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_epoch.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_epoch.py @@ -20,7 +20,12 @@ from pydantic import ( ) from ...core.v2_2_5.core_nwb_base import TimeSeries -from ...hdmf_common.v1_1_3.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_1_3.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -31,7 +36,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -50,6 +55,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -127,7 +163,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags: VectorData[Optional[NDArray[Any, str]]] = Field( + tags: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""User-defined tags that identify or categorize events.""", json_schema_extra={ @@ -136,7 +172,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags_index: Named[Optional[VectorIndex]] = Field( + tags_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for tags.""", json_schema_extra={ @@ -151,7 +187,7 @@ class TimeIntervals(DynamicTable): timeseries: Optional[TimeIntervalsTimeseries] = Field( None, description="""An index into a TimeSeries object.""" ) - timeseries_index: Named[Optional[VectorIndex]] = Field( + timeseries_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for timeseries.""", json_schema_extra={ @@ -168,14 +204,11 @@ class TimeIntervals(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_file.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_file.py index 1998de2..59aa79e 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_file.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_file.py @@ -25,7 +25,12 @@ from ...core.v2_2_5.core_nwb_icephys import IntracellularElectrode, SweepTable from ...core.v2_2_5.core_nwb_misc import Units from ...core.v2_2_5.core_nwb_ogen import OptogeneticStimulusSite from ...core.v2_2_5.core_nwb_ophys import ImagingPlane -from ...hdmf_common.v1_1_3.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_1_3.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -36,7 +41,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -55,6 +60,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -440,7 +476,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_x: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_x: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""x coordinate in electrode group""", json_schema_extra={ @@ -449,7 +485,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_y: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_y: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""y coordinate in electrode group""", json_schema_extra={ @@ -458,7 +494,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_z: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_z: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""z coordinate in electrode group""", json_schema_extra={ @@ -467,7 +503,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - reference: VectorData[Optional[NDArray[Any, str]]] = Field( + reference: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""Description of the reference used for this electrode.""", json_schema_extra={ @@ -481,14 +517,11 @@ class ExtracellularEphysElectrodes(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_icephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_icephys.py index 5a57663..ee68bff 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_icephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_icephys.py @@ -26,7 +26,12 @@ from ...core.v2_2_5.core_nwb_base import ( TimeSeriesSync, ) from ...core.v2_2_5.core_nwb_device import Device -from ...hdmf_common.v1_1_3.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_1_3.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -37,7 +42,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +61,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -897,14 +933,11 @@ class SweepTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_image.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_image.py index 7f20c3a..f3d0d5f 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_image.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_image.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -84,17 +115,16 @@ class GrayscaleImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "x"}, {"alias": "y"}]}} + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBImage(Image): @@ -107,17 +137,24 @@ class RGBImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 3 r_g_b"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b", "exact_cardinality": 3}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBAImage(Image): @@ -130,17 +167,24 @@ class RGBAImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 4 r_g_b_a"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b_a", "exact_cardinality": 4}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class ImageSeries(TimeSeries): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_misc.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_misc.py index 36c0f1e..5faeb05 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_misc.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_misc.py @@ -24,6 +24,7 @@ from ...core.v2_2_5.core_nwb_ecephys import ElectrodeGroup from ...hdmf_common.v1_1_3.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -37,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +57,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -443,14 +475,11 @@ class DecompositionSeriesBands(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) @@ -466,7 +495,7 @@ class Units(DynamicTable): ) name: str = Field("Units", json_schema_extra={"linkml_meta": {"ifabsent": "string(Units)"}}) - spike_times_index: Named[Optional[VectorIndex]] = Field( + spike_times_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the spike_times dataset.""", json_schema_extra={ @@ -481,7 +510,7 @@ class Units(DynamicTable): spike_times: Optional[UnitsSpikeTimes] = Field( None, description="""Spike times for each unit.""" ) - obs_intervals_index: Named[Optional[VectorIndex]] = Field( + obs_intervals_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the obs_intervals dataset.""", json_schema_extra={ @@ -493,7 +522,7 @@ class Units(DynamicTable): } }, ) - obs_intervals: VectorData[Optional[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( + obs_intervals: Optional[VectorData[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( Field( None, description="""Observation intervals for each unit.""", @@ -509,7 +538,7 @@ class Units(DynamicTable): }, ) ) - electrodes_index: Named[Optional[VectorIndex]] = Field( + electrodes_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into electrodes.""", json_schema_extra={ @@ -521,7 +550,7 @@ class Units(DynamicTable): } }, ) - electrodes: Named[Optional[DynamicTableRegion]] = Field( + electrodes: Optional[Named[DynamicTableRegion]] = Field( None, description="""Electrode that each spike unit came from, specified using a DynamicTableRegion.""", json_schema_extra={ @@ -536,16 +565,16 @@ class Units(DynamicTable): electrode_group: Optional[List[ElectrodeGroup]] = Field( None, description="""Electrode group that each spike unit came from.""" ) - waveform_mean: VectorData[ - Optional[ + waveform_mean: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], ] ] ] = Field(None, description="""Spike waveform mean for each spike unit.""") - waveform_sd: VectorData[ - Optional[ + waveform_sd: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], @@ -557,14 +586,11 @@ class Units(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_ogen.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_ogen.py index 294866b..6c81182 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_ogen.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_ogen.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_ophys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_ophys.py index 4860989..98c3a53 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_ophys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_ophys.py @@ -31,6 +31,7 @@ from ...core.v2_2_5.core_nwb_image import ImageSeries, ImageSeriesExternalFile from ...hdmf_common.v1_1_3.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -44,7 +45,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -63,6 +64,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -324,7 +356,7 @@ class PlaneSegmentation(DynamicTable): None, description="""ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero.""", ) - pixel_mask_index: Named[Optional[VectorIndex]] = Field( + pixel_mask_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into pixel_mask.""", json_schema_extra={ @@ -340,7 +372,7 @@ class PlaneSegmentation(DynamicTable): None, description="""Pixel masks for each ROI: a list of indices and weights for the ROI. Pixel masks are concatenated and parsing of this dataset is maintained by the PlaneSegmentation""", ) - voxel_mask_index: Named[Optional[VectorIndex]] = Field( + voxel_mask_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into voxel_mask.""", json_schema_extra={ @@ -375,14 +407,11 @@ class PlaneSegmentation(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_retinotopy.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_retinotopy.py index 6f05814..5466646 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_retinotopy.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/core_nwb_retinotopy.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/namespace.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/namespace.py index 5f82379..5d12f36 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_2_5/namespace.py @@ -159,7 +159,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -178,6 +178,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_base.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_base.py index 34bb4c6..ad3c5f4 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_base.py @@ -23,7 +23,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -42,6 +42,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -274,7 +305,7 @@ class ProcessingModule(NWBContainer): {"from_schema": "core.nwb.base", "tree_root": True} ) - value: Optional[List[Union[DynamicTable, NWBDataInterface]]] = Field( + value: Optional[Dict[str, Union[DynamicTable, NWBDataInterface]]] = Field( None, json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBDataInterface"}, {"range": "DynamicTable"}]} @@ -294,7 +325,7 @@ class Images(NWBDataInterface): name: str = Field("Images", json_schema_extra={"linkml_meta": {"ifabsent": "string(Images)"}}) description: str = Field(..., description="""Description of this collection of images.""") - image: List[Image] = Field(..., description="""Images stored in this collection.""") + image: List[str] = Field(..., description="""Images stored in this collection.""") # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_behavior.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_behavior.py index ef00827..8358db6 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_behavior.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_behavior.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -167,7 +198,7 @@ class BehavioralEpochs(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[IntervalSeries]] = Field( + value: Optional[Dict[str, IntervalSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "IntervalSeries"}]}} ) name: str = Field(...) @@ -182,7 +213,7 @@ class BehavioralEvents(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[TimeSeries]] = Field( + value: Optional[Dict[str, TimeSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}} ) name: str = Field(...) @@ -197,7 +228,7 @@ class BehavioralTimeSeries(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[TimeSeries]] = Field( + value: Optional[Dict[str, TimeSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}} ) name: str = Field(...) @@ -212,7 +243,7 @@ class PupilTracking(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[TimeSeries]] = Field( + value: Optional[Dict[str, TimeSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}} ) name: str = Field(...) @@ -227,7 +258,7 @@ class EyeTracking(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[SpatialSeries]] = Field( + value: Optional[Dict[str, SpatialSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpatialSeries"}]}} ) name: str = Field(...) @@ -242,7 +273,7 @@ class CompassDirection(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[SpatialSeries]] = Field( + value: Optional[Dict[str, SpatialSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpatialSeries"}]}} ) name: str = Field(...) @@ -257,7 +288,7 @@ class Position(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[SpatialSeries]] = Field( + value: Optional[Dict[str, SpatialSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpatialSeries"}]}} ) name: str = Field(...) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_device.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_device.py index 26e327b..5c0f451 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_device.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_device.py @@ -21,7 +21,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -40,6 +40,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_ecephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_ecephys.py index 7092c13..2676bd5 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_ecephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_ecephys.py @@ -38,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -57,6 +57,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -354,7 +385,7 @@ class EventWaveform(NWBDataInterface): {"from_schema": "core.nwb.ecephys", "tree_root": True} ) - value: Optional[List[SpikeEventSeries]] = Field( + value: Optional[Dict[str, SpikeEventSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpikeEventSeries"}]}} ) name: str = Field(...) @@ -369,7 +400,7 @@ class FilteredEphys(NWBDataInterface): {"from_schema": "core.nwb.ecephys", "tree_root": True} ) - value: Optional[List[ElectricalSeries]] = Field( + value: Optional[Dict[str, ElectricalSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "ElectricalSeries"}]}} ) name: str = Field(...) @@ -384,7 +415,7 @@ class LFP(NWBDataInterface): {"from_schema": "core.nwb.ecephys", "tree_root": True} ) - value: Optional[List[ElectricalSeries]] = Field( + value: Optional[Dict[str, ElectricalSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "ElectricalSeries"}]}} ) name: str = Field(...) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_epoch.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_epoch.py index d5015fe..93ea1ba 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_epoch.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_epoch.py @@ -20,7 +20,12 @@ from pydantic import ( ) from ...core.v2_3_0.core_nwb_base import TimeSeries -from ...hdmf_common.v1_5_0.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_5_0.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -31,7 +36,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -50,6 +55,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -127,7 +163,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags: VectorData[Optional[NDArray[Any, str]]] = Field( + tags: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""User-defined tags that identify or categorize events.""", json_schema_extra={ @@ -136,7 +172,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags_index: Named[Optional[VectorIndex]] = Field( + tags_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for tags.""", json_schema_extra={ @@ -151,7 +187,7 @@ class TimeIntervals(DynamicTable): timeseries: Optional[TimeIntervalsTimeseries] = Field( None, description="""An index into a TimeSeries object.""" ) - timeseries_index: Named[Optional[VectorIndex]] = Field( + timeseries_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for timeseries.""", json_schema_extra={ @@ -168,14 +204,11 @@ class TimeIntervals(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class TimeIntervalsTimeseries(VectorData): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_file.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_file.py index 2000dfd..d692065 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_file.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_file.py @@ -25,7 +25,7 @@ from ...core.v2_3_0.core_nwb_icephys import IntracellularElectrode, SweepTable from ...core.v2_3_0.core_nwb_misc import Units from ...core.v2_3_0.core_nwb_ogen import OptogeneticStimulusSite from ...core.v2_3_0.core_nwb_ophys import ImagingPlane -from ...hdmf_common.v1_5_0.hdmf_common_table import DynamicTable, VectorData +from ...hdmf_common.v1_5_0.hdmf_common_table import DynamicTable, ElementIdentifiers, VectorData metamodel_version = "None" @@ -36,7 +36,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -55,6 +55,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -152,28 +183,28 @@ class NWBFile(NWBContainer): ..., description="""Date and time corresponding to time zero of all timestamps. The date is stored in UTC with local timezone offset as ISO 8601 extended formatted string: 2018-09-28T14:43:54.123+02:00. Dates stored in UTC end in \"Z\" with no timezone offset. Date accuracy is up to milliseconds. All times stored in the file use this time as reference (i.e., time zero).""", ) - acquisition: Optional[List[Union[DynamicTable, NWBDataInterface]]] = Field( + acquisition: Optional[Dict[str, Union[DynamicTable, NWBDataInterface]]] = Field( None, description="""Data streams recorded from the system, including ephys, ophys, tracking, etc. This group should be read-only after the experiment is completed and timestamps are corrected to a common timebase. The data stored here may be links to raw data stored in external NWB files. This will allow keeping bulky raw data out of the file while preserving the option of keeping some/all in the file. Acquired data includes tracking and experimental data streams (i.e., everything measured from the system). If bulky data is stored in the /acquisition group, the data can exist in a separate NWB file that is linked to by the file being used for processing and analysis.""", json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBDataInterface"}, {"range": "DynamicTable"}]} }, ) - analysis: Optional[List[Union[DynamicTable, NWBContainer]]] = Field( + analysis: Optional[Dict[str, Union[DynamicTable, NWBContainer]]] = Field( None, description="""Lab-specific and custom scientific analysis of data. There is no defined format for the content of this group - the format is up to the individual user/lab. To facilitate sharing analysis data between labs, the contents here should be stored in standard types (e.g., neurodata_types) and appropriately documented. The file can store lab-specific and custom data analysis without restriction on its form or schema, reducing data formatting restrictions on end users. Such data should be placed in the analysis group. The analysis data should be documented so that it could be shared with other labs.""", json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBContainer"}, {"range": "DynamicTable"}]} }, ) - scratch: Optional[List[Union[DynamicTable, NWBContainer]]] = Field( + scratch: Optional[Dict[str, Union[DynamicTable, NWBContainer]]] = Field( None, description="""A place to store one-off analysis results. Data placed here is not intended for sharing. By placing data here, users acknowledge that there is no guarantee that their data meets any standard.""", json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBContainer"}, {"range": "DynamicTable"}]} }, ) - processing: Optional[List[ProcessingModule]] = Field( + processing: Optional[Dict[str, ProcessingModule]] = Field( None, description="""The home for ProcessingModules. These modules perform intermediate analysis of data that is necessary to perform before scientific analysis. Examples include spike clustering, extracting position from tracking data, stitching together image slices. ProcessingModules can be large and express many data sets from relatively complex analysis (e.g., spike detection and clustering) or small, representing extraction of position information from tracking video, or even binary lick/no-lick decisions. Common software tools (e.g., klustakwik, MClust) are expected to read/write data here. 'Processing' refers to intermediate analysis of the acquired data to make it more amenable to scientific analysis.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "ProcessingModule"}]}}, @@ -206,12 +237,12 @@ class NWBFileStimulus(ConfiguredBaseModel): "linkml_meta": {"equals_string": "stimulus", "ifabsent": "string(stimulus)"} }, ) - presentation: Optional[List[TimeSeries]] = Field( + presentation: Optional[Dict[str, TimeSeries]] = Field( None, description="""Stimuli presented during the experiment.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}}, ) - templates: Optional[List[TimeSeries]] = Field( + templates: Optional[Dict[str, TimeSeries]] = Field( None, description="""Template stimuli. Timestamps in templates are based on stimulus design and are relative to the beginning of the stimulus. When templates are used, the stimulus instances must convert presentation times to the experiment`s time reference frame.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}}, @@ -289,11 +320,11 @@ class NWBFileGeneral(ConfiguredBaseModel): None, description="""Information about virus(es) used in experiments, including virus ID, source, date made, injection location, volume, etc.""", ) - lab_meta_data: Optional[List[LabMetaData]] = Field( + lab_meta_data: Optional[Dict[str, LabMetaData]] = Field( None, description="""Place-holder than can be extended so that lab-specific meta-data can be placed in /general.""", ) - devices: Optional[List[Device]] = Field( + devices: Optional[Dict[str, Device]] = Field( None, description="""Description of hardware devices used during experiment, e.g., monitors, ADC boards, microscopes, etc.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "Device"}]}}, @@ -308,12 +339,12 @@ class NWBFileGeneral(ConfiguredBaseModel): intracellular_ephys: Optional[GeneralIntracellularEphys] = Field( None, description="""Metadata related to intracellular electrophysiology.""" ) - optogenetics: Optional[List[OptogeneticStimulusSite]] = Field( + optogenetics: Optional[Dict[str, OptogeneticStimulusSite]] = Field( None, description="""Metadata describing optogenetic stimuluation.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "OptogeneticStimulusSite"}]}}, ) - optophysiology: Optional[List[ImagingPlane]] = Field( + optophysiology: Optional[Dict[str, ImagingPlane]] = Field( None, description="""Metadata related to optophysiology.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "ImagingPlane"}]}}, @@ -353,7 +384,7 @@ class GeneralExtracellularEphys(ConfiguredBaseModel): } }, ) - electrode_group: Optional[List[ElectrodeGroup]] = Field( + electrode_group: Optional[Dict[str, ElectrodeGroup]] = Field( None, description="""Physical group of electrodes.""" ) electrodes: Optional[ExtracellularEphysElectrodes] = Field( @@ -428,8 +459,14 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - group: List[ElectrodeGroup] = Field( - ..., description="""Reference to the ElectrodeGroup this electrode is a part of.""" + group: VectorData[NDArray[Any, ElectrodeGroup]] = Field( + ..., + description="""Reference to the ElectrodeGroup this electrode is a part of.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) group_name: VectorData[NDArray[Any, str]] = Field( ..., @@ -440,7 +477,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_x: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_x: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""x coordinate in electrode group""", json_schema_extra={ @@ -449,7 +486,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_y: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_y: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""y coordinate in electrode group""", json_schema_extra={ @@ -458,7 +495,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_z: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_z: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""z coordinate in electrode group""", json_schema_extra={ @@ -467,7 +504,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - reference: VectorData[Optional[NDArray[Any, str]]] = Field( + reference: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""Description of the reference used for this electrode.""", json_schema_extra={ @@ -481,14 +518,11 @@ class ExtracellularEphysElectrodes(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class GeneralIntracellularEphys(ConfiguredBaseModel): @@ -511,7 +545,7 @@ class GeneralIntracellularEphys(ConfiguredBaseModel): None, description="""Description of filtering used. Includes filtering type and parameters, frequency fall-off, etc. If this changes between TimeSeries, filter description should be stored as a text attribute for each TimeSeries.""", ) - intracellular_electrode: Optional[List[IntracellularElectrode]] = Field( + intracellular_electrode: Optional[Dict[str, IntracellularElectrode]] = Field( None, description="""An intracellular electrode.""" ) sweep_table: Optional[SweepTable] = Field( @@ -542,7 +576,7 @@ class NWBFileIntervals(ConfiguredBaseModel): invalid_times: Optional[TimeIntervals] = Field( None, description="""Time intervals that should be removed from analysis.""" ) - time_intervals: Optional[List[TimeIntervals]] = Field( + time_intervals: Optional[Dict[str, TimeIntervals]] = Field( None, description="""Optional additional table(s) for describing other experimental time intervals.""", ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_icephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_icephys.py index aad3631..1fb2a04 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_icephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_icephys.py @@ -26,7 +26,12 @@ from ...core.v2_3_0.core_nwb_base import ( TimeSeriesSync, ) from ...core.v2_3_0.core_nwb_device import Device -from ...hdmf_common.v1_5_0.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_5_0.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -37,7 +42,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +61,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -879,8 +915,14 @@ class SweepTable(DynamicTable): } }, ) - series: List[PatchClampSeries] = Field( - ..., description="""The PatchClampSeries with the sweep number in that row.""" + series: VectorData[NDArray[Any, PatchClampSeries]] = Field( + ..., + description="""The PatchClampSeries with the sweep number in that row.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) series_index: Named[VectorIndex] = Field( ..., @@ -899,14 +941,11 @@ class SweepTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_image.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_image.py index fb2bc82..8758ca8 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_image.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_image.py @@ -23,7 +23,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -42,6 +42,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -85,17 +116,16 @@ class GrayscaleImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "x"}, {"alias": "y"}]}} + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBImage(Image): @@ -108,17 +138,24 @@ class RGBImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 3 r_g_b"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b", "exact_cardinality": 3}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBAImage(Image): @@ -131,17 +168,24 @@ class RGBAImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 4 r_g_b_a"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b_a", "exact_cardinality": 4}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class ImageSeries(TimeSeries): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_misc.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_misc.py index 8d984db..ac3b366 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_misc.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_misc.py @@ -24,6 +24,7 @@ from ...core.v2_3_0.core_nwb_ecephys import ElectrodeGroup from ...hdmf_common.v1_5_0.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -37,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +57,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -312,7 +344,7 @@ class DecompositionSeries(TimeSeries): ..., description="""Data decomposed into frequency bands.""" ) metric: str = Field(..., description="""The metric used, e.g. phase, amplitude, power.""") - source_channels: Named[Optional[DynamicTableRegion]] = Field( + source_channels: Optional[Named[DynamicTableRegion]] = Field( None, description="""DynamicTableRegion pointer to the channels that this decomposition series was generated from.""", json_schema_extra={ @@ -455,14 +487,11 @@ class DecompositionSeriesBands(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class Units(DynamicTable): @@ -475,7 +504,7 @@ class Units(DynamicTable): ) name: str = Field("Units", json_schema_extra={"linkml_meta": {"ifabsent": "string(Units)"}}) - spike_times_index: Named[Optional[VectorIndex]] = Field( + spike_times_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the spike_times dataset.""", json_schema_extra={ @@ -490,7 +519,7 @@ class Units(DynamicTable): spike_times: Optional[UnitsSpikeTimes] = Field( None, description="""Spike times for each unit.""" ) - obs_intervals_index: Named[Optional[VectorIndex]] = Field( + obs_intervals_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the obs_intervals dataset.""", json_schema_extra={ @@ -502,7 +531,7 @@ class Units(DynamicTable): } }, ) - obs_intervals: VectorData[Optional[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( + obs_intervals: Optional[VectorData[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( Field( None, description="""Observation intervals for each unit.""", @@ -518,7 +547,7 @@ class Units(DynamicTable): }, ) ) - electrodes_index: Named[Optional[VectorIndex]] = Field( + electrodes_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into electrodes.""", json_schema_extra={ @@ -530,7 +559,7 @@ class Units(DynamicTable): } }, ) - electrodes: Named[Optional[DynamicTableRegion]] = Field( + electrodes: Optional[Named[DynamicTableRegion]] = Field( None, description="""Electrode that each spike unit came from, specified using a DynamicTableRegion.""", json_schema_extra={ @@ -542,26 +571,32 @@ class Units(DynamicTable): } }, ) - electrode_group: Optional[List[ElectrodeGroup]] = Field( - None, description="""Electrode group that each spike unit came from.""" + electrode_group: Optional[VectorData[NDArray[Any, ElectrodeGroup]]] = Field( + None, + description="""Electrode group that each spike unit came from.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) - waveform_mean: VectorData[ - Optional[ + waveform_mean: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], ] ] ] = Field(None, description="""Spike waveform mean for each spike unit.""") - waveform_sd: VectorData[ - Optional[ + waveform_sd: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], ] ] ] = Field(None, description="""Spike waveform standard deviation for each spike unit.""") - waveforms: VectorData[Optional[NDArray[Shape["* num_waveforms, * num_samples"], float]]] = ( + waveforms: Optional[VectorData[NDArray[Shape["* num_waveforms, * num_samples"], float]]] = ( Field( None, description="""Individual waveforms for each spike on each electrode. This is a doubly indexed column. The 'waveforms_index' column indexes which waveforms in this column belong to the same spike event for a given unit, where each waveform was recorded from a different electrode. The 'waveforms_index_index' column indexes the 'waveforms_index' column to indicate which spike events belong to a given unit. For example, if the 'waveforms_index_index' column has values [2, 5, 6], then the first 2 elements of the 'waveforms_index' column correspond to the 2 spike events of the first unit, the next 3 elements of the 'waveforms_index' column correspond to the 3 spike events of the second unit, and the next 1 element of the 'waveforms_index' column corresponds to the 1 spike event of the third unit. If the 'waveforms_index' column has values [3, 6, 8, 10, 12, 13], then the first 3 elements of the 'waveforms' column contain the 3 spike waveforms that were recorded from 3 different electrodes for the first spike time of the first unit. See https://nwb-schema.readthedocs.io/en/stable/format_description.html#doubly-ragged-arrays for a graphical representation of this example. When there is only one electrode for each unit (i.e., each spike time is associated with a single waveform), then the 'waveforms_index' column will have values 1, 2, ..., N, where N is the number of spike events. The number of electrodes for each spike event should be the same within a given unit. The 'electrodes' column should be used to indicate which electrodes are associated with each unit, and the order of the waveforms within a given unit x spike event should be in the same order as the electrodes referenced in the 'electrodes' column of this table. The number of samples for each waveform must be the same.""", @@ -572,7 +607,7 @@ class Units(DynamicTable): }, ) ) - waveforms_index: Named[Optional[VectorIndex]] = Field( + waveforms_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the waveforms dataset. One value for every spike event. See 'waveforms' for more detail.""", json_schema_extra={ @@ -584,7 +619,7 @@ class Units(DynamicTable): } }, ) - waveforms_index_index: Named[Optional[VectorIndex]] = Field( + waveforms_index_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the waveforms_index dataset. One value for every unit (row in the table). See 'waveforms' for more detail.""", json_schema_extra={ @@ -601,14 +636,11 @@ class Units(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class UnitsSpikeTimes(VectorData): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_ogen.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_ogen.py index 2e49cf9..bf95c5c 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_ogen.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_ogen.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_ophys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_ophys.py index 29cac39..670269a 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_ophys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_ophys.py @@ -31,6 +31,7 @@ from ...core.v2_3_0.core_nwb_image import ImageSeries, ImageSeriesExternalFile from ...hdmf_common.v1_5_0.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -44,7 +45,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -63,6 +64,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -283,7 +315,7 @@ class DfOverF(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[RoiResponseSeries]] = Field( + value: Optional[Dict[str, RoiResponseSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "RoiResponseSeries"}]}} ) name: str = Field(...) @@ -298,7 +330,7 @@ class Fluorescence(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[RoiResponseSeries]] = Field( + value: Optional[Dict[str, RoiResponseSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "RoiResponseSeries"}]}} ) name: str = Field(...) @@ -313,7 +345,7 @@ class ImageSegmentation(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[PlaneSegmentation]] = Field( + value: Optional[Dict[str, PlaneSegmentation]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "PlaneSegmentation"}]}} ) name: str = Field(...) @@ -329,11 +361,18 @@ class PlaneSegmentation(DynamicTable): ) name: str = Field(...) - image_mask: Optional[PlaneSegmentationImageMask] = Field( + image_mask: Optional[ + VectorData[ + Union[ + NDArray[Shape["* num_roi, * num_x, * num_y"], Any], + NDArray[Shape["* num_roi, * num_x, * num_y, * num_z"], Any], + ] + ] + ] = Field( None, description="""ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero.""", ) - pixel_mask_index: Named[Optional[VectorIndex]] = Field( + pixel_mask_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into pixel_mask.""", json_schema_extra={ @@ -349,7 +388,7 @@ class PlaneSegmentation(DynamicTable): None, description="""Pixel masks for each ROI: a list of indices and weights for the ROI. Pixel masks are concatenated and parsing of this dataset is maintained by the PlaneSegmentation""", ) - voxel_mask_index: Named[Optional[VectorIndex]] = Field( + voxel_mask_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into voxel_mask.""", json_schema_extra={ @@ -365,7 +404,7 @@ class PlaneSegmentation(DynamicTable): None, description="""Voxel masks for each ROI: a list of indices and weights for the ROI. Voxel masks are concatenated and parsing of this dataset is maintained by the PlaneSegmentation""", ) - reference_images: Optional[List[ImageSeries]] = Field( + reference_images: Optional[Dict[str, ImageSeries]] = Field( None, description="""Image stacks that the segmentation masks apply to.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "ImageSeries"}]}}, @@ -384,38 +423,11 @@ class PlaneSegmentation(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) - - -class PlaneSegmentationImageMask(VectorData): - """ - ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero. - """ - - linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta({"from_schema": "core.nwb.ophys"}) - - name: Literal["image_mask"] = Field( - "image_mask", - json_schema_extra={ - "linkml_meta": {"equals_string": "image_mask", "ifabsent": "string(image_mask)"} - }, - ) - description: str = Field(..., description="""Description of what these vectors represent.""") - value: Optional[ - Union[ - NDArray[Shape["* dim0"], Any], - NDArray[Shape["* dim0, * dim1"], Any], - NDArray[Shape["* dim0, * dim1, * dim2"], Any], - NDArray[Shape["* dim0, * dim1, * dim2, * dim3"], Any], - ] - ] = Field(None) class PlaneSegmentationPixelMask(VectorData): @@ -538,7 +550,7 @@ class ImagingPlane(NWBContainer): None, description="""Describes reference frame of origin_coords and grid_spacing. For example, this can be a text description of the anatomical location and orientation of the grid defined by origin_coords and grid_spacing or the vectors needed to transform or rotate the grid to a common anatomical axis (e.g., AP/DV/ML). This field is necessary to interpret origin_coords and grid_spacing. If origin_coords and grid_spacing are not present, then this field is not required. For example, if the microscope takes 10 x 10 x 2 images, where the first value of the data matrix (index (0, 0, 0)) corresponds to (-1.2, -0.6, -2) mm relative to bregma, the spacing between pixels is 0.2 mm in x, 0.2 mm in y and 0.5 mm in z, and larger numbers in x means more anterior, larger numbers in y means more rightward, and larger numbers in z means more ventral, then enter the following -- origin_coords = (-1.2, -0.6, -2) grid_spacing = (0.2, 0.2, 0.5) reference_frame = \"Origin coordinates are relative to bregma. First dimension corresponds to anterior-posterior axis (larger index = more anterior). Second dimension corresponds to medial-lateral axis (larger index = more rightward). Third dimension corresponds to dorsal-ventral axis (larger index = more ventral).\"""", ) - optical_channel: List[OpticalChannel] = Field( + optical_channel: Dict[str, OpticalChannel] = Field( ..., description="""An optical channel used to record from an imaging plane.""" ) device: Union[Device, str] = Field( @@ -652,7 +664,7 @@ class MotionCorrection(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[CorrectedImageStack]] = Field( + value: Optional[Dict[str, CorrectedImageStack]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "CorrectedImageStack"}]}} ) name: str = Field(...) @@ -694,7 +706,6 @@ DfOverF.model_rebuild() Fluorescence.model_rebuild() ImageSegmentation.model_rebuild() PlaneSegmentation.model_rebuild() -PlaneSegmentationImageMask.model_rebuild() PlaneSegmentationPixelMask.model_rebuild() PlaneSegmentationVoxelMask.model_rebuild() ImagingPlane.model_rebuild() diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_retinotopy.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_retinotopy.py index d58860e..5c78658 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_retinotopy.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/core_nwb_retinotopy.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/namespace.py index b5ffa4b..2125d57 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_3_0/namespace.py @@ -117,7 +117,6 @@ from ...core.v2_3_0.core_nwb_ophys import ( MotionCorrection, OpticalChannel, PlaneSegmentation, - PlaneSegmentationImageMask, PlaneSegmentationPixelMask, PlaneSegmentationVoxelMask, RoiResponseSeries, @@ -134,7 +133,7 @@ from ...core.v2_3_0.core_nwb_retinotopy import ( ImagingRetinotopyVasculatureImage, ) from ...hdmf_common.v1_5_0.hdmf_common_base import Container, Data, SimpleMultiContainer -from ...hdmf_common.v1_5_0.hdmf_common_sparse import CSRMatrix, CSRMatrixData +from ...hdmf_common.v1_5_0.hdmf_common_sparse import CSRMatrix from ...hdmf_common.v1_5_0.hdmf_common_table import ( AlignedDynamicTable, DynamicTable, @@ -162,7 +161,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -181,6 +180,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_base.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_base.py index b557e8c..f8b6d99 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_base.py @@ -36,7 +36,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -55,6 +55,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -88,7 +119,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -469,7 +500,7 @@ class ProcessingModule(NWBContainer): {"from_schema": "core.nwb.base", "tree_root": True} ) - value: Optional[List[Union[DynamicTable, NWBDataInterface]]] = Field( + value: Optional[Dict[str, Union[DynamicTable, NWBDataInterface]]] = Field( None, json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBDataInterface"}, {"range": "DynamicTable"}]} @@ -489,7 +520,7 @@ class Images(NWBDataInterface): name: str = Field("Images", json_schema_extra={"linkml_meta": {"ifabsent": "string(Images)"}}) description: str = Field(..., description="""Description of this collection of images.""") - image: List[Image] = Field(..., description="""Images stored in this collection.""") + image: List[str] = Field(..., description="""Images stored in this collection.""") # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_behavior.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_behavior.py index c96aee7..7c0abb8 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_behavior.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_behavior.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -167,7 +198,7 @@ class BehavioralEpochs(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[IntervalSeries]] = Field( + value: Optional[Dict[str, IntervalSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "IntervalSeries"}]}} ) name: str = Field(...) @@ -182,7 +213,7 @@ class BehavioralEvents(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[TimeSeries]] = Field( + value: Optional[Dict[str, TimeSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}} ) name: str = Field(...) @@ -197,7 +228,7 @@ class BehavioralTimeSeries(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[TimeSeries]] = Field( + value: Optional[Dict[str, TimeSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}} ) name: str = Field(...) @@ -212,7 +243,7 @@ class PupilTracking(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[TimeSeries]] = Field( + value: Optional[Dict[str, TimeSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}} ) name: str = Field(...) @@ -227,7 +258,7 @@ class EyeTracking(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[SpatialSeries]] = Field( + value: Optional[Dict[str, SpatialSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpatialSeries"}]}} ) name: str = Field(...) @@ -242,7 +273,7 @@ class CompassDirection(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[SpatialSeries]] = Field( + value: Optional[Dict[str, SpatialSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpatialSeries"}]}} ) name: str = Field(...) @@ -257,7 +288,7 @@ class Position(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[SpatialSeries]] = Field( + value: Optional[Dict[str, SpatialSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpatialSeries"}]}} ) name: str = Field(...) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_device.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_device.py index ebf61b7..436d2d4 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_device.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_device.py @@ -21,7 +21,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -40,6 +40,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_ecephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_ecephys.py index bfe65b3..ac26b29 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_ecephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_ecephys.py @@ -38,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -57,6 +57,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -354,7 +385,7 @@ class EventWaveform(NWBDataInterface): {"from_schema": "core.nwb.ecephys", "tree_root": True} ) - value: Optional[List[SpikeEventSeries]] = Field( + value: Optional[Dict[str, SpikeEventSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpikeEventSeries"}]}} ) name: str = Field(...) @@ -369,7 +400,7 @@ class FilteredEphys(NWBDataInterface): {"from_schema": "core.nwb.ecephys", "tree_root": True} ) - value: Optional[List[ElectricalSeries]] = Field( + value: Optional[Dict[str, ElectricalSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "ElectricalSeries"}]}} ) name: str = Field(...) @@ -384,7 +415,7 @@ class LFP(NWBDataInterface): {"from_schema": "core.nwb.ecephys", "tree_root": True} ) - value: Optional[List[ElectricalSeries]] = Field( + value: Optional[Dict[str, ElectricalSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "ElectricalSeries"}]}} ) name: str = Field(...) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_epoch.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_epoch.py index 603c5d3..25894a3 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_epoch.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_epoch.py @@ -20,7 +20,12 @@ from pydantic import ( ) from ...core.v2_4_0.core_nwb_base import TimeSeries -from ...hdmf_common.v1_5_0.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_5_0.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -31,7 +36,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -50,6 +55,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -127,7 +163,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags: VectorData[Optional[NDArray[Any, str]]] = Field( + tags: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""User-defined tags that identify or categorize events.""", json_schema_extra={ @@ -136,7 +172,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags_index: Named[Optional[VectorIndex]] = Field( + tags_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for tags.""", json_schema_extra={ @@ -151,7 +187,7 @@ class TimeIntervals(DynamicTable): timeseries: Optional[TimeIntervalsTimeseries] = Field( None, description="""An index into a TimeSeries object.""" ) - timeseries_index: Named[Optional[VectorIndex]] = Field( + timeseries_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for timeseries.""", json_schema_extra={ @@ -168,14 +204,11 @@ class TimeIntervals(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class TimeIntervalsTimeseries(VectorData): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_file.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_file.py index 186458d..84d5b9a 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_file.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_file.py @@ -33,7 +33,7 @@ from ...core.v2_4_0.core_nwb_icephys import ( from ...core.v2_4_0.core_nwb_misc import Units from ...core.v2_4_0.core_nwb_ogen import OptogeneticStimulusSite from ...core.v2_4_0.core_nwb_ophys import ImagingPlane -from ...hdmf_common.v1_5_0.hdmf_common_table import DynamicTable, VectorData +from ...hdmf_common.v1_5_0.hdmf_common_table import DynamicTable, ElementIdentifiers, VectorData metamodel_version = "None" @@ -44,7 +44,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -63,6 +63,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -160,28 +191,28 @@ class NWBFile(NWBContainer): ..., description="""Date and time corresponding to time zero of all timestamps. The date is stored in UTC with local timezone offset as ISO 8601 extended formatted string: 2018-09-28T14:43:54.123+02:00. Dates stored in UTC end in \"Z\" with no timezone offset. Date accuracy is up to milliseconds. All times stored in the file use this time as reference (i.e., time zero).""", ) - acquisition: Optional[List[Union[DynamicTable, NWBDataInterface]]] = Field( + acquisition: Optional[Dict[str, Union[DynamicTable, NWBDataInterface]]] = Field( None, description="""Data streams recorded from the system, including ephys, ophys, tracking, etc. This group should be read-only after the experiment is completed and timestamps are corrected to a common timebase. The data stored here may be links to raw data stored in external NWB files. This will allow keeping bulky raw data out of the file while preserving the option of keeping some/all in the file. Acquired data includes tracking and experimental data streams (i.e., everything measured from the system). If bulky data is stored in the /acquisition group, the data can exist in a separate NWB file that is linked to by the file being used for processing and analysis.""", json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBDataInterface"}, {"range": "DynamicTable"}]} }, ) - analysis: Optional[List[Union[DynamicTable, NWBContainer]]] = Field( + analysis: Optional[Dict[str, Union[DynamicTable, NWBContainer]]] = Field( None, description="""Lab-specific and custom scientific analysis of data. There is no defined format for the content of this group - the format is up to the individual user/lab. To facilitate sharing analysis data between labs, the contents here should be stored in standard types (e.g., neurodata_types) and appropriately documented. The file can store lab-specific and custom data analysis without restriction on its form or schema, reducing data formatting restrictions on end users. Such data should be placed in the analysis group. The analysis data should be documented so that it could be shared with other labs.""", json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBContainer"}, {"range": "DynamicTable"}]} }, ) - scratch: Optional[List[Union[DynamicTable, NWBContainer]]] = Field( + scratch: Optional[Dict[str, Union[DynamicTable, NWBContainer]]] = Field( None, description="""A place to store one-off analysis results. Data placed here is not intended for sharing. By placing data here, users acknowledge that there is no guarantee that their data meets any standard.""", json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBContainer"}, {"range": "DynamicTable"}]} }, ) - processing: Optional[List[ProcessingModule]] = Field( + processing: Optional[Dict[str, ProcessingModule]] = Field( None, description="""The home for ProcessingModules. These modules perform intermediate analysis of data that is necessary to perform before scientific analysis. Examples include spike clustering, extracting position from tracking data, stitching together image slices. ProcessingModules can be large and express many data sets from relatively complex analysis (e.g., spike detection and clustering) or small, representing extraction of position information from tracking video, or even binary lick/no-lick decisions. Common software tools (e.g., klustakwik, MClust) are expected to read/write data here. 'Processing' refers to intermediate analysis of the acquired data to make it more amenable to scientific analysis.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "ProcessingModule"}]}}, @@ -214,12 +245,12 @@ class NWBFileStimulus(ConfiguredBaseModel): "linkml_meta": {"equals_string": "stimulus", "ifabsent": "string(stimulus)"} }, ) - presentation: Optional[List[TimeSeries]] = Field( + presentation: Optional[Dict[str, TimeSeries]] = Field( None, description="""Stimuli presented during the experiment.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}}, ) - templates: Optional[List[TimeSeries]] = Field( + templates: Optional[Dict[str, TimeSeries]] = Field( None, description="""Template stimuli. Timestamps in templates are based on stimulus design and are relative to the beginning of the stimulus. When templates are used, the stimulus instances must convert presentation times to the experiment`s time reference frame.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}}, @@ -297,11 +328,11 @@ class NWBFileGeneral(ConfiguredBaseModel): None, description="""Information about virus(es) used in experiments, including virus ID, source, date made, injection location, volume, etc.""", ) - lab_meta_data: Optional[List[LabMetaData]] = Field( + lab_meta_data: Optional[Dict[str, LabMetaData]] = Field( None, description="""Place-holder than can be extended so that lab-specific meta-data can be placed in /general.""", ) - devices: Optional[List[Device]] = Field( + devices: Optional[Dict[str, Device]] = Field( None, description="""Description of hardware devices used during experiment, e.g., monitors, ADC boards, microscopes, etc.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "Device"}]}}, @@ -316,12 +347,12 @@ class NWBFileGeneral(ConfiguredBaseModel): intracellular_ephys: Optional[GeneralIntracellularEphys] = Field( None, description="""Metadata related to intracellular electrophysiology.""" ) - optogenetics: Optional[List[OptogeneticStimulusSite]] = Field( + optogenetics: Optional[Dict[str, OptogeneticStimulusSite]] = Field( None, description="""Metadata describing optogenetic stimuluation.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "OptogeneticStimulusSite"}]}}, ) - optophysiology: Optional[List[ImagingPlane]] = Field( + optophysiology: Optional[Dict[str, ImagingPlane]] = Field( None, description="""Metadata related to optophysiology.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "ImagingPlane"}]}}, @@ -361,7 +392,7 @@ class GeneralExtracellularEphys(ConfiguredBaseModel): } }, ) - electrode_group: Optional[List[ElectrodeGroup]] = Field( + electrode_group: Optional[Dict[str, ElectrodeGroup]] = Field( None, description="""Physical group of electrodes.""" ) electrodes: Optional[ExtracellularEphysElectrodes] = Field( @@ -436,8 +467,14 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - group: List[ElectrodeGroup] = Field( - ..., description="""Reference to the ElectrodeGroup this electrode is a part of.""" + group: VectorData[NDArray[Any, ElectrodeGroup]] = Field( + ..., + description="""Reference to the ElectrodeGroup this electrode is a part of.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) group_name: VectorData[NDArray[Any, str]] = Field( ..., @@ -448,7 +485,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_x: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_x: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""x coordinate in electrode group""", json_schema_extra={ @@ -457,7 +494,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_y: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_y: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""y coordinate in electrode group""", json_schema_extra={ @@ -466,7 +503,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_z: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_z: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""z coordinate in electrode group""", json_schema_extra={ @@ -475,7 +512,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - reference: VectorData[Optional[NDArray[Any, str]]] = Field( + reference: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""Description of the reference used for this electrode.""", json_schema_extra={ @@ -489,14 +526,11 @@ class ExtracellularEphysElectrodes(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class GeneralIntracellularEphys(ConfiguredBaseModel): @@ -519,7 +553,7 @@ class GeneralIntracellularEphys(ConfiguredBaseModel): None, description="""[DEPRECATED] Use IntracellularElectrode.filtering instead. Description of filtering used. Includes filtering type and parameters, frequency fall-off, etc. If this changes between TimeSeries, filter description should be stored as a text attribute for each TimeSeries.""", ) - intracellular_electrode: Optional[List[IntracellularElectrode]] = Field( + intracellular_electrode: Optional[Dict[str, IntracellularElectrode]] = Field( None, description="""An intracellular electrode.""" ) sweep_table: Optional[SweepTable] = Field( @@ -571,7 +605,7 @@ class NWBFileIntervals(ConfiguredBaseModel): invalid_times: Optional[TimeIntervals] = Field( None, description="""Time intervals that should be removed from analysis.""" ) - time_intervals: Optional[List[TimeIntervals]] = Field( + time_intervals: Optional[Dict[str, TimeIntervals]] = Field( None, description="""Optional additional table(s) for describing other experimental time intervals.""", ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_icephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_icephys.py index aa2accc..d4ebcb3 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_icephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_icephys.py @@ -31,6 +31,7 @@ from ...hdmf_common.v1_5_0.hdmf_common_table import ( AlignedDynamicTable, DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -44,7 +45,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -63,6 +64,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -886,8 +918,14 @@ class SweepTable(DynamicTable): } }, ) - series: List[PatchClampSeries] = Field( - ..., description="""The PatchClampSeries with the sweep number in that row.""" + series: VectorData[NDArray[Any, PatchClampSeries]] = Field( + ..., + description="""The PatchClampSeries with the sweep number in that row.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) series_index: Named[VectorIndex] = Field( ..., @@ -906,14 +944,11 @@ class SweepTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class IntracellularElectrodesTable(DynamicTable): @@ -936,21 +971,24 @@ class IntracellularElectrodesTable(DynamicTable): } }, ) - electrode: List[IntracellularElectrode] = Field( - ..., description="""Column for storing the reference to the intracellular electrode.""" + electrode: VectorData[NDArray[Any, IntracellularElectrode]] = Field( + ..., + description="""Column for storing the reference to the intracellular electrode.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) colnames: List[str] = Field( ..., description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class IntracellularStimuliTable(DynamicTable): @@ -989,14 +1027,11 @@ class IntracellularStimuliTable(DynamicTable): ..., description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class IntracellularResponsesTable(DynamicTable): @@ -1035,14 +1070,11 @@ class IntracellularResponsesTable(DynamicTable): ..., description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class IntracellularRecordingsTable(AlignedDynamicTable): @@ -1094,21 +1126,18 @@ class IntracellularRecordingsTable(AlignedDynamicTable): responses: IntracellularResponsesTable = Field( ..., description="""Table for storing intracellular response related metadata.""" ) - value: Optional[List[DynamicTable]] = Field( + value: Optional[Dict[str, DynamicTable]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "DynamicTable"}]}} ) colnames: List[str] = Field( ..., description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class SimultaneousRecordingsTable(DynamicTable): @@ -1150,14 +1179,11 @@ class SimultaneousRecordingsTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class SimultaneousRecordingsTableRecordings(DynamicTableRegion): @@ -1238,14 +1264,11 @@ class SequentialRecordingsTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class SequentialRecordingsTableSimultaneousRecordings(DynamicTableRegion): @@ -1317,14 +1340,11 @@ class RepetitionsTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class RepetitionsTableSequentialRecordings(DynamicTableRegion): @@ -1398,14 +1418,11 @@ class ExperimentalConditionsTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class ExperimentalConditionsTableRepetitions(DynamicTableRegion): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_image.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_image.py index 5d1e5d8..8fd3288 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_image.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_image.py @@ -23,7 +23,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -42,6 +42,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -85,17 +116,16 @@ class GrayscaleImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "x"}, {"alias": "y"}]}} + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBImage(Image): @@ -108,17 +138,24 @@ class RGBImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 3 r_g_b"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b", "exact_cardinality": 3}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBAImage(Image): @@ -131,17 +168,24 @@ class RGBAImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 4 r_g_b_a"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b_a", "exact_cardinality": 4}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class ImageSeries(TimeSeries): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_misc.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_misc.py index 6c06a17..3ab6b75 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_misc.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_misc.py @@ -24,6 +24,7 @@ from ...core.v2_4_0.core_nwb_ecephys import ElectrodeGroup from ...hdmf_common.v1_5_0.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -37,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +57,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -312,7 +344,7 @@ class DecompositionSeries(TimeSeries): ..., description="""Data decomposed into frequency bands.""" ) metric: str = Field(..., description="""The metric used, e.g. phase, amplitude, power.""") - source_channels: Named[Optional[DynamicTableRegion]] = Field( + source_channels: Optional[Named[DynamicTableRegion]] = Field( None, description="""DynamicTableRegion pointer to the channels that this decomposition series was generated from.""", json_schema_extra={ @@ -455,14 +487,11 @@ class DecompositionSeriesBands(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class Units(DynamicTable): @@ -475,7 +504,7 @@ class Units(DynamicTable): ) name: str = Field("Units", json_schema_extra={"linkml_meta": {"ifabsent": "string(Units)"}}) - spike_times_index: Named[Optional[VectorIndex]] = Field( + spike_times_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the spike_times dataset.""", json_schema_extra={ @@ -490,7 +519,7 @@ class Units(DynamicTable): spike_times: Optional[UnitsSpikeTimes] = Field( None, description="""Spike times for each unit.""" ) - obs_intervals_index: Named[Optional[VectorIndex]] = Field( + obs_intervals_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the obs_intervals dataset.""", json_schema_extra={ @@ -502,7 +531,7 @@ class Units(DynamicTable): } }, ) - obs_intervals: VectorData[Optional[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( + obs_intervals: Optional[VectorData[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( Field( None, description="""Observation intervals for each unit.""", @@ -518,7 +547,7 @@ class Units(DynamicTable): }, ) ) - electrodes_index: Named[Optional[VectorIndex]] = Field( + electrodes_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into electrodes.""", json_schema_extra={ @@ -530,7 +559,7 @@ class Units(DynamicTable): } }, ) - electrodes: Named[Optional[DynamicTableRegion]] = Field( + electrodes: Optional[Named[DynamicTableRegion]] = Field( None, description="""Electrode that each spike unit came from, specified using a DynamicTableRegion.""", json_schema_extra={ @@ -542,26 +571,32 @@ class Units(DynamicTable): } }, ) - electrode_group: Optional[List[ElectrodeGroup]] = Field( - None, description="""Electrode group that each spike unit came from.""" + electrode_group: Optional[VectorData[NDArray[Any, ElectrodeGroup]]] = Field( + None, + description="""Electrode group that each spike unit came from.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) - waveform_mean: VectorData[ - Optional[ + waveform_mean: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], ] ] ] = Field(None, description="""Spike waveform mean for each spike unit.""") - waveform_sd: VectorData[ - Optional[ + waveform_sd: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], ] ] ] = Field(None, description="""Spike waveform standard deviation for each spike unit.""") - waveforms: VectorData[Optional[NDArray[Shape["* num_waveforms, * num_samples"], float]]] = ( + waveforms: Optional[VectorData[NDArray[Shape["* num_waveforms, * num_samples"], float]]] = ( Field( None, description="""Individual waveforms for each spike on each electrode. This is a doubly indexed column. The 'waveforms_index' column indexes which waveforms in this column belong to the same spike event for a given unit, where each waveform was recorded from a different electrode. The 'waveforms_index_index' column indexes the 'waveforms_index' column to indicate which spike events belong to a given unit. For example, if the 'waveforms_index_index' column has values [2, 5, 6], then the first 2 elements of the 'waveforms_index' column correspond to the 2 spike events of the first unit, the next 3 elements of the 'waveforms_index' column correspond to the 3 spike events of the second unit, and the next 1 element of the 'waveforms_index' column corresponds to the 1 spike event of the third unit. If the 'waveforms_index' column has values [3, 6, 8, 10, 12, 13], then the first 3 elements of the 'waveforms' column contain the 3 spike waveforms that were recorded from 3 different electrodes for the first spike time of the first unit. See https://nwb-schema.readthedocs.io/en/stable/format_description.html#doubly-ragged-arrays for a graphical representation of this example. When there is only one electrode for each unit (i.e., each spike time is associated with a single waveform), then the 'waveforms_index' column will have values 1, 2, ..., N, where N is the number of spike events. The number of electrodes for each spike event should be the same within a given unit. The 'electrodes' column should be used to indicate which electrodes are associated with each unit, and the order of the waveforms within a given unit x spike event should be in the same order as the electrodes referenced in the 'electrodes' column of this table. The number of samples for each waveform must be the same.""", @@ -572,7 +607,7 @@ class Units(DynamicTable): }, ) ) - waveforms_index: Named[Optional[VectorIndex]] = Field( + waveforms_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the waveforms dataset. One value for every spike event. See 'waveforms' for more detail.""", json_schema_extra={ @@ -584,7 +619,7 @@ class Units(DynamicTable): } }, ) - waveforms_index_index: Named[Optional[VectorIndex]] = Field( + waveforms_index_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the waveforms_index dataset. One value for every unit (row in the table). See 'waveforms' for more detail.""", json_schema_extra={ @@ -601,14 +636,11 @@ class Units(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class UnitsSpikeTimes(VectorData): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_ogen.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_ogen.py index 07100b3..350a398 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_ogen.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_ogen.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_ophys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_ophys.py index 0d335ca..9f6e191 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_ophys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_ophys.py @@ -31,6 +31,7 @@ from ...core.v2_4_0.core_nwb_image import ImageSeries, ImageSeriesExternalFile from ...hdmf_common.v1_5_0.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -44,7 +45,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -63,6 +64,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -283,7 +315,7 @@ class DfOverF(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[RoiResponseSeries]] = Field( + value: Optional[Dict[str, RoiResponseSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "RoiResponseSeries"}]}} ) name: str = Field(...) @@ -298,7 +330,7 @@ class Fluorescence(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[RoiResponseSeries]] = Field( + value: Optional[Dict[str, RoiResponseSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "RoiResponseSeries"}]}} ) name: str = Field(...) @@ -313,7 +345,7 @@ class ImageSegmentation(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[PlaneSegmentation]] = Field( + value: Optional[Dict[str, PlaneSegmentation]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "PlaneSegmentation"}]}} ) name: str = Field(...) @@ -329,11 +361,18 @@ class PlaneSegmentation(DynamicTable): ) name: str = Field(...) - image_mask: Optional[PlaneSegmentationImageMask] = Field( + image_mask: Optional[ + VectorData[ + Union[ + NDArray[Shape["* num_roi, * num_x, * num_y"], Any], + NDArray[Shape["* num_roi, * num_x, * num_y, * num_z"], Any], + ] + ] + ] = Field( None, description="""ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero.""", ) - pixel_mask_index: Named[Optional[VectorIndex]] = Field( + pixel_mask_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into pixel_mask.""", json_schema_extra={ @@ -349,7 +388,7 @@ class PlaneSegmentation(DynamicTable): None, description="""Pixel masks for each ROI: a list of indices and weights for the ROI. Pixel masks are concatenated and parsing of this dataset is maintained by the PlaneSegmentation""", ) - voxel_mask_index: Named[Optional[VectorIndex]] = Field( + voxel_mask_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into voxel_mask.""", json_schema_extra={ @@ -365,7 +404,7 @@ class PlaneSegmentation(DynamicTable): None, description="""Voxel masks for each ROI: a list of indices and weights for the ROI. Voxel masks are concatenated and parsing of this dataset is maintained by the PlaneSegmentation""", ) - reference_images: Optional[List[ImageSeries]] = Field( + reference_images: Optional[Dict[str, ImageSeries]] = Field( None, description="""Image stacks that the segmentation masks apply to.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "ImageSeries"}]}}, @@ -384,38 +423,11 @@ class PlaneSegmentation(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) - - -class PlaneSegmentationImageMask(VectorData): - """ - ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero. - """ - - linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta({"from_schema": "core.nwb.ophys"}) - - name: Literal["image_mask"] = Field( - "image_mask", - json_schema_extra={ - "linkml_meta": {"equals_string": "image_mask", "ifabsent": "string(image_mask)"} - }, - ) - description: str = Field(..., description="""Description of what these vectors represent.""") - value: Optional[ - Union[ - NDArray[Shape["* dim0"], Any], - NDArray[Shape["* dim0, * dim1"], Any], - NDArray[Shape["* dim0, * dim1, * dim2"], Any], - NDArray[Shape["* dim0, * dim1, * dim2, * dim3"], Any], - ] - ] = Field(None) class PlaneSegmentationPixelMask(VectorData): @@ -538,7 +550,7 @@ class ImagingPlane(NWBContainer): None, description="""Describes reference frame of origin_coords and grid_spacing. For example, this can be a text description of the anatomical location and orientation of the grid defined by origin_coords and grid_spacing or the vectors needed to transform or rotate the grid to a common anatomical axis (e.g., AP/DV/ML). This field is necessary to interpret origin_coords and grid_spacing. If origin_coords and grid_spacing are not present, then this field is not required. For example, if the microscope takes 10 x 10 x 2 images, where the first value of the data matrix (index (0, 0, 0)) corresponds to (-1.2, -0.6, -2) mm relative to bregma, the spacing between pixels is 0.2 mm in x, 0.2 mm in y and 0.5 mm in z, and larger numbers in x means more anterior, larger numbers in y means more rightward, and larger numbers in z means more ventral, then enter the following -- origin_coords = (-1.2, -0.6, -2) grid_spacing = (0.2, 0.2, 0.5) reference_frame = \"Origin coordinates are relative to bregma. First dimension corresponds to anterior-posterior axis (larger index = more anterior). Second dimension corresponds to medial-lateral axis (larger index = more rightward). Third dimension corresponds to dorsal-ventral axis (larger index = more ventral).\"""", ) - optical_channel: List[OpticalChannel] = Field( + optical_channel: Dict[str, OpticalChannel] = Field( ..., description="""An optical channel used to record from an imaging plane.""" ) device: Union[Device, str] = Field( @@ -652,7 +664,7 @@ class MotionCorrection(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[CorrectedImageStack]] = Field( + value: Optional[Dict[str, CorrectedImageStack]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "CorrectedImageStack"}]}} ) name: str = Field(...) @@ -694,7 +706,6 @@ DfOverF.model_rebuild() Fluorescence.model_rebuild() ImageSegmentation.model_rebuild() PlaneSegmentation.model_rebuild() -PlaneSegmentationImageMask.model_rebuild() PlaneSegmentationPixelMask.model_rebuild() PlaneSegmentationVoxelMask.model_rebuild() ImagingPlane.model_rebuild() diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_retinotopy.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_retinotopy.py index a42a469..ffc194e 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_retinotopy.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/core_nwb_retinotopy.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/namespace.py index 4bc04cd..620dcf2 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_4_0/namespace.py @@ -130,7 +130,6 @@ from ...core.v2_4_0.core_nwb_ophys import ( MotionCorrection, OpticalChannel, PlaneSegmentation, - PlaneSegmentationImageMask, PlaneSegmentationPixelMask, PlaneSegmentationVoxelMask, RoiResponseSeries, @@ -147,7 +146,7 @@ from ...core.v2_4_0.core_nwb_retinotopy import ( ImagingRetinotopyVasculatureImage, ) from ...hdmf_common.v1_5_0.hdmf_common_base import Container, Data, SimpleMultiContainer -from ...hdmf_common.v1_5_0.hdmf_common_sparse import CSRMatrix, CSRMatrixData +from ...hdmf_common.v1_5_0.hdmf_common_sparse import CSRMatrix from ...hdmf_common.v1_5_0.hdmf_common_table import ( AlignedDynamicTable, DynamicTable, @@ -175,7 +174,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -194,6 +193,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_base.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_base.py index 3a2170a..2db9763 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_base.py @@ -47,7 +47,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -66,6 +66,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -99,7 +130,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -520,7 +551,7 @@ class ProcessingModule(NWBContainer): {"from_schema": "core.nwb.base", "tree_root": True} ) - value: Optional[List[Union[DynamicTable, NWBDataInterface]]] = Field( + value: Optional[Dict[str, Union[DynamicTable, NWBDataInterface]]] = Field( None, json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBDataInterface"}, {"range": "DynamicTable"}]} @@ -540,8 +571,8 @@ class Images(NWBDataInterface): name: str = Field("Images", json_schema_extra={"linkml_meta": {"ifabsent": "string(Images)"}}) description: str = Field(..., description="""Description of this collection of images.""") - image: List[Image] = Field(..., description="""Images stored in this collection.""") - order_of_images: Named[Optional[ImageReferences]] = Field( + image: List[str] = Field(..., description="""Images stored in this collection.""") + order_of_images: Optional[Named[ImageReferences]] = Field( None, description="""Ordered dataset of references to Image objects stored in the parent group. Each Image object in the Images group should be stored once and only once, so the dataset should have the same length as the number of images.""", json_schema_extra={ diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_behavior.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_behavior.py index d4e6a03..89c1038 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_behavior.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_behavior.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -169,7 +200,7 @@ class BehavioralEpochs(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[IntervalSeries]] = Field( + value: Optional[Dict[str, IntervalSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "IntervalSeries"}]}} ) name: str = Field(...) @@ -184,7 +215,7 @@ class BehavioralEvents(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[TimeSeries]] = Field( + value: Optional[Dict[str, TimeSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}} ) name: str = Field(...) @@ -199,7 +230,7 @@ class BehavioralTimeSeries(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[TimeSeries]] = Field( + value: Optional[Dict[str, TimeSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}} ) name: str = Field(...) @@ -214,7 +245,7 @@ class PupilTracking(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[TimeSeries]] = Field( + value: Optional[Dict[str, TimeSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}} ) name: str = Field(...) @@ -229,7 +260,7 @@ class EyeTracking(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[SpatialSeries]] = Field( + value: Optional[Dict[str, SpatialSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpatialSeries"}]}} ) name: str = Field(...) @@ -244,7 +275,7 @@ class CompassDirection(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[SpatialSeries]] = Field( + value: Optional[Dict[str, SpatialSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpatialSeries"}]}} ) name: str = Field(...) @@ -259,7 +290,7 @@ class Position(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[SpatialSeries]] = Field( + value: Optional[Dict[str, SpatialSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpatialSeries"}]}} ) name: str = Field(...) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_device.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_device.py index 16f07fc..e4cf279 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_device.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_device.py @@ -21,7 +21,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -40,6 +40,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_ecephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_ecephys.py index 343564a..91c2222 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_ecephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_ecephys.py @@ -38,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -57,6 +57,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -354,7 +385,7 @@ class EventWaveform(NWBDataInterface): {"from_schema": "core.nwb.ecephys", "tree_root": True} ) - value: Optional[List[SpikeEventSeries]] = Field( + value: Optional[Dict[str, SpikeEventSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpikeEventSeries"}]}} ) name: str = Field(...) @@ -369,7 +400,7 @@ class FilteredEphys(NWBDataInterface): {"from_schema": "core.nwb.ecephys", "tree_root": True} ) - value: Optional[List[ElectricalSeries]] = Field( + value: Optional[Dict[str, ElectricalSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "ElectricalSeries"}]}} ) name: str = Field(...) @@ -384,7 +415,7 @@ class LFP(NWBDataInterface): {"from_schema": "core.nwb.ecephys", "tree_root": True} ) - value: Optional[List[ElectricalSeries]] = Field( + value: Optional[Dict[str, ElectricalSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "ElectricalSeries"}]}} ) name: str = Field(...) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_epoch.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_epoch.py index 2fafb90..ab92eb7 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_epoch.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_epoch.py @@ -20,7 +20,12 @@ from pydantic import ( ) from ...core.v2_5_0.core_nwb_base import TimeSeriesReferenceVectorData -from ...hdmf_common.v1_5_0.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_5_0.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -31,7 +36,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -50,6 +55,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -127,7 +163,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags: VectorData[Optional[NDArray[Any, str]]] = Field( + tags: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""User-defined tags that identify or categorize events.""", json_schema_extra={ @@ -136,7 +172,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags_index: Named[Optional[VectorIndex]] = Field( + tags_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for tags.""", json_schema_extra={ @@ -148,7 +184,7 @@ class TimeIntervals(DynamicTable): } }, ) - timeseries: Named[Optional[TimeSeriesReferenceVectorData]] = Field( + timeseries: Optional[Named[TimeSeriesReferenceVectorData]] = Field( None, description="""An index into a TimeSeries object.""", json_schema_extra={ @@ -160,7 +196,7 @@ class TimeIntervals(DynamicTable): } }, ) - timeseries_index: Named[Optional[VectorIndex]] = Field( + timeseries_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for timeseries.""", json_schema_extra={ @@ -177,14 +213,11 @@ class TimeIntervals(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_file.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_file.py index 039eb68..6c056a6 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_file.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_file.py @@ -34,7 +34,7 @@ from ...core.v2_5_0.core_nwb_icephys import ( from ...core.v2_5_0.core_nwb_misc import Units from ...core.v2_5_0.core_nwb_ogen import OptogeneticStimulusSite from ...core.v2_5_0.core_nwb_ophys import ImagingPlane -from ...hdmf_common.v1_5_0.hdmf_common_table import DynamicTable, VectorData +from ...hdmf_common.v1_5_0.hdmf_common_table import DynamicTable, ElementIdentifiers, VectorData metamodel_version = "None" @@ -45,7 +45,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -64,6 +64,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -161,28 +192,28 @@ class NWBFile(NWBContainer): ..., description="""Date and time corresponding to time zero of all timestamps. The date is stored in UTC with local timezone offset as ISO 8601 extended formatted string: 2018-09-28T14:43:54.123+02:00. Dates stored in UTC end in \"Z\" with no timezone offset. Date accuracy is up to milliseconds. All times stored in the file use this time as reference (i.e., time zero).""", ) - acquisition: Optional[List[Union[DynamicTable, NWBDataInterface]]] = Field( + acquisition: Optional[Dict[str, Union[DynamicTable, NWBDataInterface]]] = Field( None, description="""Data streams recorded from the system, including ephys, ophys, tracking, etc. This group should be read-only after the experiment is completed and timestamps are corrected to a common timebase. The data stored here may be links to raw data stored in external NWB files. This will allow keeping bulky raw data out of the file while preserving the option of keeping some/all in the file. Acquired data includes tracking and experimental data streams (i.e., everything measured from the system). If bulky data is stored in the /acquisition group, the data can exist in a separate NWB file that is linked to by the file being used for processing and analysis.""", json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBDataInterface"}, {"range": "DynamicTable"}]} }, ) - analysis: Optional[List[Union[DynamicTable, NWBContainer]]] = Field( + analysis: Optional[Dict[str, Union[DynamicTable, NWBContainer]]] = Field( None, description="""Lab-specific and custom scientific analysis of data. There is no defined format for the content of this group - the format is up to the individual user/lab. To facilitate sharing analysis data between labs, the contents here should be stored in standard types (e.g., neurodata_types) and appropriately documented. The file can store lab-specific and custom data analysis without restriction on its form or schema, reducing data formatting restrictions on end users. Such data should be placed in the analysis group. The analysis data should be documented so that it could be shared with other labs.""", json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBContainer"}, {"range": "DynamicTable"}]} }, ) - scratch: Optional[List[Union[DynamicTable, NWBContainer]]] = Field( + scratch: Optional[Dict[str, Union[DynamicTable, NWBContainer]]] = Field( None, description="""A place to store one-off analysis results. Data placed here is not intended for sharing. By placing data here, users acknowledge that there is no guarantee that their data meets any standard.""", json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBContainer"}, {"range": "DynamicTable"}]} }, ) - processing: Optional[List[ProcessingModule]] = Field( + processing: Optional[Dict[str, ProcessingModule]] = Field( None, description="""The home for ProcessingModules. These modules perform intermediate analysis of data that is necessary to perform before scientific analysis. Examples include spike clustering, extracting position from tracking data, stitching together image slices. ProcessingModules can be large and express many data sets from relatively complex analysis (e.g., spike detection and clustering) or small, representing extraction of position information from tracking video, or even binary lick/no-lick decisions. Common software tools (e.g., klustakwik, MClust) are expected to read/write data here. 'Processing' refers to intermediate analysis of the acquired data to make it more amenable to scientific analysis.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "ProcessingModule"}]}}, @@ -215,12 +246,12 @@ class NWBFileStimulus(ConfiguredBaseModel): "linkml_meta": {"equals_string": "stimulus", "ifabsent": "string(stimulus)"} }, ) - presentation: Optional[List[TimeSeries]] = Field( + presentation: Optional[Dict[str, TimeSeries]] = Field( None, description="""Stimuli presented during the experiment.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}}, ) - templates: Optional[List[Union[Images, TimeSeries]]] = Field( + templates: Optional[Dict[str, Union[Images, TimeSeries]]] = Field( None, description="""Template stimuli. Timestamps in templates are based on stimulus design and are relative to the beginning of the stimulus. When templates are used, the stimulus instances must convert presentation times to the experiment`s time reference frame.""", json_schema_extra={ @@ -300,11 +331,11 @@ class NWBFileGeneral(ConfiguredBaseModel): None, description="""Information about virus(es) used in experiments, including virus ID, source, date made, injection location, volume, etc.""", ) - lab_meta_data: Optional[List[LabMetaData]] = Field( + lab_meta_data: Optional[Dict[str, LabMetaData]] = Field( None, description="""Place-holder than can be extended so that lab-specific meta-data can be placed in /general.""", ) - devices: Optional[List[Device]] = Field( + devices: Optional[Dict[str, Device]] = Field( None, description="""Description of hardware devices used during experiment, e.g., monitors, ADC boards, microscopes, etc.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "Device"}]}}, @@ -319,12 +350,12 @@ class NWBFileGeneral(ConfiguredBaseModel): intracellular_ephys: Optional[GeneralIntracellularEphys] = Field( None, description="""Metadata related to intracellular electrophysiology.""" ) - optogenetics: Optional[List[OptogeneticStimulusSite]] = Field( + optogenetics: Optional[Dict[str, OptogeneticStimulusSite]] = Field( None, description="""Metadata describing optogenetic stimuluation.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "OptogeneticStimulusSite"}]}}, ) - optophysiology: Optional[List[ImagingPlane]] = Field( + optophysiology: Optional[Dict[str, ImagingPlane]] = Field( None, description="""Metadata related to optophysiology.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "ImagingPlane"}]}}, @@ -364,7 +395,7 @@ class GeneralExtracellularEphys(ConfiguredBaseModel): } }, ) - electrode_group: Optional[List[ElectrodeGroup]] = Field( + electrode_group: Optional[Dict[str, ElectrodeGroup]] = Field( None, description="""Physical group of electrodes.""" ) electrodes: Optional[ExtracellularEphysElectrodes] = Field( @@ -385,7 +416,7 @@ class ExtracellularEphysElectrodes(DynamicTable): "linkml_meta": {"equals_string": "electrodes", "ifabsent": "string(electrodes)"} }, ) - x: VectorData[Optional[NDArray[Any, float]]] = Field( + x: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""x coordinate of the channel location in the brain (+x is posterior).""", json_schema_extra={ @@ -394,7 +425,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - y: VectorData[Optional[NDArray[Any, float]]] = Field( + y: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""y coordinate of the channel location in the brain (+y is inferior).""", json_schema_extra={ @@ -403,7 +434,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - z: VectorData[Optional[NDArray[Any, float]]] = Field( + z: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""z coordinate of the channel location in the brain (+z is right).""", json_schema_extra={ @@ -412,7 +443,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - imp: VectorData[Optional[NDArray[Any, float]]] = Field( + imp: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""Impedance of the channel, in ohms.""", json_schema_extra={ @@ -430,7 +461,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - filtering: VectorData[Optional[NDArray[Any, str]]] = Field( + filtering: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""Description of hardware filtering, including the filter name and frequency cutoffs.""", json_schema_extra={ @@ -439,8 +470,14 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - group: List[ElectrodeGroup] = Field( - ..., description="""Reference to the ElectrodeGroup this electrode is a part of.""" + group: VectorData[NDArray[Any, ElectrodeGroup]] = Field( + ..., + description="""Reference to the ElectrodeGroup this electrode is a part of.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) group_name: VectorData[NDArray[Any, str]] = Field( ..., @@ -451,7 +488,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_x: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_x: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""x coordinate in electrode group""", json_schema_extra={ @@ -460,7 +497,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_y: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_y: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""y coordinate in electrode group""", json_schema_extra={ @@ -469,7 +506,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_z: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_z: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""z coordinate in electrode group""", json_schema_extra={ @@ -478,7 +515,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - reference: VectorData[Optional[NDArray[Any, str]]] = Field( + reference: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""Description of the reference electrode and/or reference scheme used for this electrode, e.g., \"stainless steel skull screw\" or \"online common average referencing\".""", json_schema_extra={ @@ -492,14 +529,11 @@ class ExtracellularEphysElectrodes(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class GeneralIntracellularEphys(ConfiguredBaseModel): @@ -522,7 +556,7 @@ class GeneralIntracellularEphys(ConfiguredBaseModel): None, description="""[DEPRECATED] Use IntracellularElectrode.filtering instead. Description of filtering used. Includes filtering type and parameters, frequency fall-off, etc. If this changes between TimeSeries, filter description should be stored as a text attribute for each TimeSeries.""", ) - intracellular_electrode: Optional[List[IntracellularElectrode]] = Field( + intracellular_electrode: Optional[Dict[str, IntracellularElectrode]] = Field( None, description="""An intracellular electrode.""" ) sweep_table: Optional[SweepTable] = Field( @@ -574,7 +608,7 @@ class NWBFileIntervals(ConfiguredBaseModel): invalid_times: Optional[TimeIntervals] = Field( None, description="""Time intervals that should be removed from analysis.""" ) - time_intervals: Optional[List[TimeIntervals]] = Field( + time_intervals: Optional[Dict[str, TimeIntervals]] = Field( None, description="""Optional additional table(s) for describing other experimental time intervals.""", ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_icephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_icephys.py index bef122c..b500a82 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_icephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_icephys.py @@ -31,6 +31,7 @@ from ...hdmf_common.v1_5_0.hdmf_common_table import ( AlignedDynamicTable, DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -44,7 +45,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -63,6 +64,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -887,8 +919,14 @@ class SweepTable(DynamicTable): } }, ) - series: List[PatchClampSeries] = Field( - ..., description="""The PatchClampSeries with the sweep number in that row.""" + series: VectorData[NDArray[Any, PatchClampSeries]] = Field( + ..., + description="""The PatchClampSeries with the sweep number in that row.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) series_index: Named[VectorIndex] = Field( ..., @@ -907,14 +945,11 @@ class SweepTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class IntracellularElectrodesTable(DynamicTable): @@ -937,21 +972,24 @@ class IntracellularElectrodesTable(DynamicTable): } }, ) - electrode: List[IntracellularElectrode] = Field( - ..., description="""Column for storing the reference to the intracellular electrode.""" + electrode: VectorData[NDArray[Any, IntracellularElectrode]] = Field( + ..., + description="""Column for storing the reference to the intracellular electrode.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) colnames: List[str] = Field( ..., description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class IntracellularStimuliTable(DynamicTable): @@ -990,14 +1028,11 @@ class IntracellularStimuliTable(DynamicTable): ..., description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class IntracellularResponsesTable(DynamicTable): @@ -1036,14 +1071,11 @@ class IntracellularResponsesTable(DynamicTable): ..., description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class IntracellularRecordingsTable(AlignedDynamicTable): @@ -1095,21 +1127,18 @@ class IntracellularRecordingsTable(AlignedDynamicTable): responses: IntracellularResponsesTable = Field( ..., description="""Table for storing intracellular response related metadata.""" ) - value: Optional[List[DynamicTable]] = Field( + value: Optional[Dict[str, DynamicTable]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "DynamicTable"}]}} ) colnames: List[str] = Field( ..., description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class SimultaneousRecordingsTable(DynamicTable): @@ -1151,14 +1180,11 @@ class SimultaneousRecordingsTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class SimultaneousRecordingsTableRecordings(DynamicTableRegion): @@ -1239,14 +1265,11 @@ class SequentialRecordingsTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class SequentialRecordingsTableSimultaneousRecordings(DynamicTableRegion): @@ -1318,14 +1341,11 @@ class RepetitionsTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class RepetitionsTableSequentialRecordings(DynamicTableRegion): @@ -1399,14 +1419,11 @@ class ExperimentalConditionsTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class ExperimentalConditionsTableRepetitions(DynamicTableRegion): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_image.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_image.py index 21e5d0a..520d249 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_image.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_image.py @@ -29,7 +29,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -48,6 +48,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -91,17 +122,16 @@ class GrayscaleImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "x"}, {"alias": "y"}]}} + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBImage(Image): @@ -114,17 +144,24 @@ class RGBImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 3 r_g_b"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b", "exact_cardinality": 3}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBAImage(Image): @@ -137,17 +174,24 @@ class RGBAImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 4 r_g_b_a"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b_a", "exact_cardinality": 4}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class ImageSeries(TimeSeries): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_misc.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_misc.py index 6be8f5c..7901288 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_misc.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_misc.py @@ -24,6 +24,7 @@ from ...core.v2_5_0.core_nwb_ecephys import ElectrodeGroup from ...hdmf_common.v1_5_0.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -37,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +57,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -312,7 +344,7 @@ class DecompositionSeries(TimeSeries): ..., description="""Data decomposed into frequency bands.""" ) metric: str = Field(..., description="""The metric used, e.g. phase, amplitude, power.""") - source_channels: Named[Optional[DynamicTableRegion]] = Field( + source_channels: Optional[Named[DynamicTableRegion]] = Field( None, description="""DynamicTableRegion pointer to the channels that this decomposition series was generated from.""", json_schema_extra={ @@ -455,14 +487,11 @@ class DecompositionSeriesBands(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class Units(DynamicTable): @@ -475,7 +504,7 @@ class Units(DynamicTable): ) name: str = Field("Units", json_schema_extra={"linkml_meta": {"ifabsent": "string(Units)"}}) - spike_times_index: Named[Optional[VectorIndex]] = Field( + spike_times_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the spike_times dataset.""", json_schema_extra={ @@ -490,7 +519,7 @@ class Units(DynamicTable): spike_times: Optional[UnitsSpikeTimes] = Field( None, description="""Spike times for each unit.""" ) - obs_intervals_index: Named[Optional[VectorIndex]] = Field( + obs_intervals_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the obs_intervals dataset.""", json_schema_extra={ @@ -502,7 +531,7 @@ class Units(DynamicTable): } }, ) - obs_intervals: VectorData[Optional[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( + obs_intervals: Optional[VectorData[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( Field( None, description="""Observation intervals for each unit.""", @@ -518,7 +547,7 @@ class Units(DynamicTable): }, ) ) - electrodes_index: Named[Optional[VectorIndex]] = Field( + electrodes_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into electrodes.""", json_schema_extra={ @@ -530,7 +559,7 @@ class Units(DynamicTable): } }, ) - electrodes: Named[Optional[DynamicTableRegion]] = Field( + electrodes: Optional[Named[DynamicTableRegion]] = Field( None, description="""Electrode that each spike unit came from, specified using a DynamicTableRegion.""", json_schema_extra={ @@ -542,26 +571,32 @@ class Units(DynamicTable): } }, ) - electrode_group: Optional[List[ElectrodeGroup]] = Field( - None, description="""Electrode group that each spike unit came from.""" + electrode_group: Optional[VectorData[NDArray[Any, ElectrodeGroup]]] = Field( + None, + description="""Electrode group that each spike unit came from.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) - waveform_mean: VectorData[ - Optional[ + waveform_mean: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], ] ] ] = Field(None, description="""Spike waveform mean for each spike unit.""") - waveform_sd: VectorData[ - Optional[ + waveform_sd: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], ] ] ] = Field(None, description="""Spike waveform standard deviation for each spike unit.""") - waveforms: VectorData[Optional[NDArray[Shape["* num_waveforms, * num_samples"], float]]] = ( + waveforms: Optional[VectorData[NDArray[Shape["* num_waveforms, * num_samples"], float]]] = ( Field( None, description="""Individual waveforms for each spike on each electrode. This is a doubly indexed column. The 'waveforms_index' column indexes which waveforms in this column belong to the same spike event for a given unit, where each waveform was recorded from a different electrode. The 'waveforms_index_index' column indexes the 'waveforms_index' column to indicate which spike events belong to a given unit. For example, if the 'waveforms_index_index' column has values [2, 5, 6], then the first 2 elements of the 'waveforms_index' column correspond to the 2 spike events of the first unit, the next 3 elements of the 'waveforms_index' column correspond to the 3 spike events of the second unit, and the next 1 element of the 'waveforms_index' column corresponds to the 1 spike event of the third unit. If the 'waveforms_index' column has values [3, 6, 8, 10, 12, 13], then the first 3 elements of the 'waveforms' column contain the 3 spike waveforms that were recorded from 3 different electrodes for the first spike time of the first unit. See https://nwb-schema.readthedocs.io/en/stable/format_description.html#doubly-ragged-arrays for a graphical representation of this example. When there is only one electrode for each unit (i.e., each spike time is associated with a single waveform), then the 'waveforms_index' column will have values 1, 2, ..., N, where N is the number of spike events. The number of electrodes for each spike event should be the same within a given unit. The 'electrodes' column should be used to indicate which electrodes are associated with each unit, and the order of the waveforms within a given unit x spike event should be in the same order as the electrodes referenced in the 'electrodes' column of this table. The number of samples for each waveform must be the same.""", @@ -572,7 +607,7 @@ class Units(DynamicTable): }, ) ) - waveforms_index: Named[Optional[VectorIndex]] = Field( + waveforms_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the waveforms dataset. One value for every spike event. See 'waveforms' for more detail.""", json_schema_extra={ @@ -584,7 +619,7 @@ class Units(DynamicTable): } }, ) - waveforms_index_index: Named[Optional[VectorIndex]] = Field( + waveforms_index_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the waveforms_index dataset. One value for every unit (row in the table). See 'waveforms' for more detail.""", json_schema_extra={ @@ -601,14 +636,11 @@ class Units(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class UnitsSpikeTimes(VectorData): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_ogen.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_ogen.py index 5d3e9ff..e39e6b2 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_ogen.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_ogen.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_ophys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_ophys.py index fef5a92..6107f4c 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_ophys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_ophys.py @@ -31,6 +31,7 @@ from ...core.v2_5_0.core_nwb_image import ImageSeries, ImageSeriesExternalFile from ...hdmf_common.v1_5_0.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -44,7 +45,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -63,6 +64,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -283,7 +315,7 @@ class DfOverF(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[RoiResponseSeries]] = Field( + value: Optional[Dict[str, RoiResponseSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "RoiResponseSeries"}]}} ) name: str = Field(...) @@ -298,7 +330,7 @@ class Fluorescence(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[RoiResponseSeries]] = Field( + value: Optional[Dict[str, RoiResponseSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "RoiResponseSeries"}]}} ) name: str = Field(...) @@ -313,7 +345,7 @@ class ImageSegmentation(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[PlaneSegmentation]] = Field( + value: Optional[Dict[str, PlaneSegmentation]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "PlaneSegmentation"}]}} ) name: str = Field(...) @@ -329,11 +361,18 @@ class PlaneSegmentation(DynamicTable): ) name: str = Field(...) - image_mask: Optional[PlaneSegmentationImageMask] = Field( + image_mask: Optional[ + VectorData[ + Union[ + NDArray[Shape["* num_roi, * num_x, * num_y"], Any], + NDArray[Shape["* num_roi, * num_x, * num_y, * num_z"], Any], + ] + ] + ] = Field( None, description="""ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero.""", ) - pixel_mask_index: Named[Optional[VectorIndex]] = Field( + pixel_mask_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into pixel_mask.""", json_schema_extra={ @@ -349,7 +388,7 @@ class PlaneSegmentation(DynamicTable): None, description="""Pixel masks for each ROI: a list of indices and weights for the ROI. Pixel masks are concatenated and parsing of this dataset is maintained by the PlaneSegmentation""", ) - voxel_mask_index: Named[Optional[VectorIndex]] = Field( + voxel_mask_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into voxel_mask.""", json_schema_extra={ @@ -365,7 +404,7 @@ class PlaneSegmentation(DynamicTable): None, description="""Voxel masks for each ROI: a list of indices and weights for the ROI. Voxel masks are concatenated and parsing of this dataset is maintained by the PlaneSegmentation""", ) - reference_images: Optional[List[ImageSeries]] = Field( + reference_images: Optional[Dict[str, ImageSeries]] = Field( None, description="""Image stacks that the segmentation masks apply to.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "ImageSeries"}]}}, @@ -384,38 +423,11 @@ class PlaneSegmentation(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) - - -class PlaneSegmentationImageMask(VectorData): - """ - ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero. - """ - - linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta({"from_schema": "core.nwb.ophys"}) - - name: Literal["image_mask"] = Field( - "image_mask", - json_schema_extra={ - "linkml_meta": {"equals_string": "image_mask", "ifabsent": "string(image_mask)"} - }, - ) - description: str = Field(..., description="""Description of what these vectors represent.""") - value: Optional[ - Union[ - NDArray[Shape["* dim0"], Any], - NDArray[Shape["* dim0, * dim1"], Any], - NDArray[Shape["* dim0, * dim1, * dim2"], Any], - NDArray[Shape["* dim0, * dim1, * dim2, * dim3"], Any], - ] - ] = Field(None) class PlaneSegmentationPixelMask(VectorData): @@ -538,7 +550,7 @@ class ImagingPlane(NWBContainer): None, description="""Describes reference frame of origin_coords and grid_spacing. For example, this can be a text description of the anatomical location and orientation of the grid defined by origin_coords and grid_spacing or the vectors needed to transform or rotate the grid to a common anatomical axis (e.g., AP/DV/ML). This field is necessary to interpret origin_coords and grid_spacing. If origin_coords and grid_spacing are not present, then this field is not required. For example, if the microscope takes 10 x 10 x 2 images, where the first value of the data matrix (index (0, 0, 0)) corresponds to (-1.2, -0.6, -2) mm relative to bregma, the spacing between pixels is 0.2 mm in x, 0.2 mm in y and 0.5 mm in z, and larger numbers in x means more anterior, larger numbers in y means more rightward, and larger numbers in z means more ventral, then enter the following -- origin_coords = (-1.2, -0.6, -2) grid_spacing = (0.2, 0.2, 0.5) reference_frame = \"Origin coordinates are relative to bregma. First dimension corresponds to anterior-posterior axis (larger index = more anterior). Second dimension corresponds to medial-lateral axis (larger index = more rightward). Third dimension corresponds to dorsal-ventral axis (larger index = more ventral).\"""", ) - optical_channel: List[OpticalChannel] = Field( + optical_channel: Dict[str, OpticalChannel] = Field( ..., description="""An optical channel used to record from an imaging plane.""" ) device: Union[Device, str] = Field( @@ -652,7 +664,7 @@ class MotionCorrection(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[CorrectedImageStack]] = Field( + value: Optional[Dict[str, CorrectedImageStack]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "CorrectedImageStack"}]}} ) name: str = Field(...) @@ -694,7 +706,6 @@ DfOverF.model_rebuild() Fluorescence.model_rebuild() ImageSegmentation.model_rebuild() PlaneSegmentation.model_rebuild() -PlaneSegmentationImageMask.model_rebuild() PlaneSegmentationPixelMask.model_rebuild() PlaneSegmentationVoxelMask.model_rebuild() ImagingPlane.model_rebuild() diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_retinotopy.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_retinotopy.py index e399448..b72f7b4 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_retinotopy.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/core_nwb_retinotopy.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/namespace.py index 5692d11..7aaa8a2 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_5_0/namespace.py @@ -131,7 +131,6 @@ from ...core.v2_5_0.core_nwb_ophys import ( MotionCorrection, OpticalChannel, PlaneSegmentation, - PlaneSegmentationImageMask, PlaneSegmentationPixelMask, PlaneSegmentationVoxelMask, RoiResponseSeries, @@ -148,7 +147,7 @@ from ...core.v2_5_0.core_nwb_retinotopy import ( ImagingRetinotopyVasculatureImage, ) from ...hdmf_common.v1_5_0.hdmf_common_base import Container, Data, SimpleMultiContainer -from ...hdmf_common.v1_5_0.hdmf_common_sparse import CSRMatrix, CSRMatrixData +from ...hdmf_common.v1_5_0.hdmf_common_sparse import CSRMatrix from ...hdmf_common.v1_5_0.hdmf_common_table import ( AlignedDynamicTable, DynamicTable, @@ -176,7 +175,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -195,6 +194,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_base.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_base.py index c4d356f..4837ae7 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_base.py @@ -47,7 +47,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -66,6 +66,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -99,7 +130,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -520,7 +551,7 @@ class ProcessingModule(NWBContainer): {"from_schema": "core.nwb.base", "tree_root": True} ) - value: Optional[List[Union[DynamicTable, NWBDataInterface]]] = Field( + value: Optional[Dict[str, Union[DynamicTable, NWBDataInterface]]] = Field( None, json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBDataInterface"}, {"range": "DynamicTable"}]} @@ -540,8 +571,8 @@ class Images(NWBDataInterface): name: str = Field("Images", json_schema_extra={"linkml_meta": {"ifabsent": "string(Images)"}}) description: str = Field(..., description="""Description of this collection of images.""") - image: List[Image] = Field(..., description="""Images stored in this collection.""") - order_of_images: Named[Optional[ImageReferences]] = Field( + image: List[str] = Field(..., description="""Images stored in this collection.""") + order_of_images: Optional[Named[ImageReferences]] = Field( None, description="""Ordered dataset of references to Image objects stored in the parent group. Each Image object in the Images group should be stored once and only once, so the dataset should have the same length as the number of images.""", json_schema_extra={ diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_behavior.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_behavior.py index 07f5165..7e4ad59 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_behavior.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_behavior.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -169,7 +200,7 @@ class BehavioralEpochs(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[IntervalSeries]] = Field( + value: Optional[Dict[str, IntervalSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "IntervalSeries"}]}} ) name: str = Field(...) @@ -184,7 +215,7 @@ class BehavioralEvents(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[TimeSeries]] = Field( + value: Optional[Dict[str, TimeSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}} ) name: str = Field(...) @@ -199,7 +230,7 @@ class BehavioralTimeSeries(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[TimeSeries]] = Field( + value: Optional[Dict[str, TimeSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}} ) name: str = Field(...) @@ -214,7 +245,7 @@ class PupilTracking(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[TimeSeries]] = Field( + value: Optional[Dict[str, TimeSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}} ) name: str = Field(...) @@ -229,7 +260,7 @@ class EyeTracking(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[SpatialSeries]] = Field( + value: Optional[Dict[str, SpatialSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpatialSeries"}]}} ) name: str = Field(...) @@ -244,7 +275,7 @@ class CompassDirection(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[SpatialSeries]] = Field( + value: Optional[Dict[str, SpatialSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpatialSeries"}]}} ) name: str = Field(...) @@ -259,7 +290,7 @@ class Position(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[SpatialSeries]] = Field( + value: Optional[Dict[str, SpatialSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpatialSeries"}]}} ) name: str = Field(...) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_device.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_device.py index c1a89c4..c57186b 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_device.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_device.py @@ -21,7 +21,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -40,6 +40,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_ecephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_ecephys.py index d83d650..0529035 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_ecephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_ecephys.py @@ -38,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -57,6 +57,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -354,7 +385,7 @@ class EventWaveform(NWBDataInterface): {"from_schema": "core.nwb.ecephys", "tree_root": True} ) - value: Optional[List[SpikeEventSeries]] = Field( + value: Optional[Dict[str, SpikeEventSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpikeEventSeries"}]}} ) name: str = Field(...) @@ -369,7 +400,7 @@ class FilteredEphys(NWBDataInterface): {"from_schema": "core.nwb.ecephys", "tree_root": True} ) - value: Optional[List[ElectricalSeries]] = Field( + value: Optional[Dict[str, ElectricalSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "ElectricalSeries"}]}} ) name: str = Field(...) @@ -384,7 +415,7 @@ class LFP(NWBDataInterface): {"from_schema": "core.nwb.ecephys", "tree_root": True} ) - value: Optional[List[ElectricalSeries]] = Field( + value: Optional[Dict[str, ElectricalSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "ElectricalSeries"}]}} ) name: str = Field(...) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_epoch.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_epoch.py index 46da361..d3fa53b 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_epoch.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_epoch.py @@ -20,7 +20,12 @@ from pydantic import ( ) from ...core.v2_6_0_alpha.core_nwb_base import TimeSeriesReferenceVectorData -from ...hdmf_common.v1_5_0.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_5_0.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -31,7 +36,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -50,6 +55,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -127,7 +163,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags: VectorData[Optional[NDArray[Any, str]]] = Field( + tags: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""User-defined tags that identify or categorize events.""", json_schema_extra={ @@ -136,7 +172,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags_index: Named[Optional[VectorIndex]] = Field( + tags_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for tags.""", json_schema_extra={ @@ -148,7 +184,7 @@ class TimeIntervals(DynamicTable): } }, ) - timeseries: Named[Optional[TimeSeriesReferenceVectorData]] = Field( + timeseries: Optional[Named[TimeSeriesReferenceVectorData]] = Field( None, description="""An index into a TimeSeries object.""", json_schema_extra={ @@ -160,7 +196,7 @@ class TimeIntervals(DynamicTable): } }, ) - timeseries_index: Named[Optional[VectorIndex]] = Field( + timeseries_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for timeseries.""", json_schema_extra={ @@ -177,14 +213,11 @@ class TimeIntervals(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_file.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_file.py index 6c39f8e..975e51c 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_file.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_file.py @@ -34,7 +34,7 @@ from ...core.v2_6_0_alpha.core_nwb_icephys import ( from ...core.v2_6_0_alpha.core_nwb_misc import Units from ...core.v2_6_0_alpha.core_nwb_ogen import OptogeneticStimulusSite from ...core.v2_6_0_alpha.core_nwb_ophys import ImagingPlane -from ...hdmf_common.v1_5_0.hdmf_common_table import DynamicTable, VectorData +from ...hdmf_common.v1_5_0.hdmf_common_table import DynamicTable, ElementIdentifiers, VectorData metamodel_version = "None" @@ -45,7 +45,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -64,6 +64,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -161,28 +192,28 @@ class NWBFile(NWBContainer): ..., description="""Date and time corresponding to time zero of all timestamps. The date is stored in UTC with local timezone offset as ISO 8601 extended formatted string: 2018-09-28T14:43:54.123+02:00. Dates stored in UTC end in \"Z\" with no timezone offset. Date accuracy is up to milliseconds. All times stored in the file use this time as reference (i.e., time zero).""", ) - acquisition: Optional[List[Union[DynamicTable, NWBDataInterface]]] = Field( + acquisition: Optional[Dict[str, Union[DynamicTable, NWBDataInterface]]] = Field( None, description="""Data streams recorded from the system, including ephys, ophys, tracking, etc. This group should be read-only after the experiment is completed and timestamps are corrected to a common timebase. The data stored here may be links to raw data stored in external NWB files. This will allow keeping bulky raw data out of the file while preserving the option of keeping some/all in the file. Acquired data includes tracking and experimental data streams (i.e., everything measured from the system). If bulky data is stored in the /acquisition group, the data can exist in a separate NWB file that is linked to by the file being used for processing and analysis.""", json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBDataInterface"}, {"range": "DynamicTable"}]} }, ) - analysis: Optional[List[Union[DynamicTable, NWBContainer]]] = Field( + analysis: Optional[Dict[str, Union[DynamicTable, NWBContainer]]] = Field( None, description="""Lab-specific and custom scientific analysis of data. There is no defined format for the content of this group - the format is up to the individual user/lab. To facilitate sharing analysis data between labs, the contents here should be stored in standard types (e.g., neurodata_types) and appropriately documented. The file can store lab-specific and custom data analysis without restriction on its form or schema, reducing data formatting restrictions on end users. Such data should be placed in the analysis group. The analysis data should be documented so that it could be shared with other labs.""", json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBContainer"}, {"range": "DynamicTable"}]} }, ) - scratch: Optional[List[Union[DynamicTable, NWBContainer]]] = Field( + scratch: Optional[Dict[str, Union[DynamicTable, NWBContainer]]] = Field( None, description="""A place to store one-off analysis results. Data placed here is not intended for sharing. By placing data here, users acknowledge that there is no guarantee that their data meets any standard.""", json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBContainer"}, {"range": "DynamicTable"}]} }, ) - processing: Optional[List[ProcessingModule]] = Field( + processing: Optional[Dict[str, ProcessingModule]] = Field( None, description="""The home for ProcessingModules. These modules perform intermediate analysis of data that is necessary to perform before scientific analysis. Examples include spike clustering, extracting position from tracking data, stitching together image slices. ProcessingModules can be large and express many data sets from relatively complex analysis (e.g., spike detection and clustering) or small, representing extraction of position information from tracking video, or even binary lick/no-lick decisions. Common software tools (e.g., klustakwik, MClust) are expected to read/write data here. 'Processing' refers to intermediate analysis of the acquired data to make it more amenable to scientific analysis.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "ProcessingModule"}]}}, @@ -215,12 +246,12 @@ class NWBFileStimulus(ConfiguredBaseModel): "linkml_meta": {"equals_string": "stimulus", "ifabsent": "string(stimulus)"} }, ) - presentation: Optional[List[TimeSeries]] = Field( + presentation: Optional[Dict[str, TimeSeries]] = Field( None, description="""Stimuli presented during the experiment.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}}, ) - templates: Optional[List[Union[Images, TimeSeries]]] = Field( + templates: Optional[Dict[str, Union[Images, TimeSeries]]] = Field( None, description="""Template stimuli. Timestamps in templates are based on stimulus design and are relative to the beginning of the stimulus. When templates are used, the stimulus instances must convert presentation times to the experiment`s time reference frame.""", json_schema_extra={ @@ -300,11 +331,11 @@ class NWBFileGeneral(ConfiguredBaseModel): None, description="""Information about virus(es) used in experiments, including virus ID, source, date made, injection location, volume, etc.""", ) - lab_meta_data: Optional[List[LabMetaData]] = Field( + lab_meta_data: Optional[Dict[str, LabMetaData]] = Field( None, description="""Place-holder than can be extended so that lab-specific meta-data can be placed in /general.""", ) - devices: Optional[List[Device]] = Field( + devices: Optional[Dict[str, Device]] = Field( None, description="""Description of hardware devices used during experiment, e.g., monitors, ADC boards, microscopes, etc.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "Device"}]}}, @@ -319,12 +350,12 @@ class NWBFileGeneral(ConfiguredBaseModel): intracellular_ephys: Optional[GeneralIntracellularEphys] = Field( None, description="""Metadata related to intracellular electrophysiology.""" ) - optogenetics: Optional[List[OptogeneticStimulusSite]] = Field( + optogenetics: Optional[Dict[str, OptogeneticStimulusSite]] = Field( None, description="""Metadata describing optogenetic stimuluation.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "OptogeneticStimulusSite"}]}}, ) - optophysiology: Optional[List[ImagingPlane]] = Field( + optophysiology: Optional[Dict[str, ImagingPlane]] = Field( None, description="""Metadata related to optophysiology.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "ImagingPlane"}]}}, @@ -364,7 +395,7 @@ class GeneralExtracellularEphys(ConfiguredBaseModel): } }, ) - electrode_group: Optional[List[ElectrodeGroup]] = Field( + electrode_group: Optional[Dict[str, ElectrodeGroup]] = Field( None, description="""Physical group of electrodes.""" ) electrodes: Optional[ExtracellularEphysElectrodes] = Field( @@ -385,7 +416,7 @@ class ExtracellularEphysElectrodes(DynamicTable): "linkml_meta": {"equals_string": "electrodes", "ifabsent": "string(electrodes)"} }, ) - x: VectorData[Optional[NDArray[Any, float]]] = Field( + x: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""x coordinate of the channel location in the brain (+x is posterior).""", json_schema_extra={ @@ -394,7 +425,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - y: VectorData[Optional[NDArray[Any, float]]] = Field( + y: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""y coordinate of the channel location in the brain (+y is inferior).""", json_schema_extra={ @@ -403,7 +434,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - z: VectorData[Optional[NDArray[Any, float]]] = Field( + z: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""z coordinate of the channel location in the brain (+z is right).""", json_schema_extra={ @@ -412,7 +443,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - imp: VectorData[Optional[NDArray[Any, float]]] = Field( + imp: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""Impedance of the channel, in ohms.""", json_schema_extra={ @@ -430,7 +461,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - filtering: VectorData[Optional[NDArray[Any, str]]] = Field( + filtering: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""Description of hardware filtering, including the filter name and frequency cutoffs.""", json_schema_extra={ @@ -439,8 +470,14 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - group: List[ElectrodeGroup] = Field( - ..., description="""Reference to the ElectrodeGroup this electrode is a part of.""" + group: VectorData[NDArray[Any, ElectrodeGroup]] = Field( + ..., + description="""Reference to the ElectrodeGroup this electrode is a part of.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) group_name: VectorData[NDArray[Any, str]] = Field( ..., @@ -451,7 +488,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_x: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_x: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""x coordinate in electrode group""", json_schema_extra={ @@ -460,7 +497,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_y: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_y: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""y coordinate in electrode group""", json_schema_extra={ @@ -469,7 +506,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_z: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_z: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""z coordinate in electrode group""", json_schema_extra={ @@ -478,7 +515,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - reference: VectorData[Optional[NDArray[Any, str]]] = Field( + reference: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""Description of the reference electrode and/or reference scheme used for this electrode, e.g., \"stainless steel skull screw\" or \"online common average referencing\".""", json_schema_extra={ @@ -492,14 +529,11 @@ class ExtracellularEphysElectrodes(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class GeneralIntracellularEphys(ConfiguredBaseModel): @@ -522,7 +556,7 @@ class GeneralIntracellularEphys(ConfiguredBaseModel): None, description="""[DEPRECATED] Use IntracellularElectrode.filtering instead. Description of filtering used. Includes filtering type and parameters, frequency fall-off, etc. If this changes between TimeSeries, filter description should be stored as a text attribute for each TimeSeries.""", ) - intracellular_electrode: Optional[List[IntracellularElectrode]] = Field( + intracellular_electrode: Optional[Dict[str, IntracellularElectrode]] = Field( None, description="""An intracellular electrode.""" ) sweep_table: Optional[SweepTable] = Field( @@ -574,7 +608,7 @@ class NWBFileIntervals(ConfiguredBaseModel): invalid_times: Optional[TimeIntervals] = Field( None, description="""Time intervals that should be removed from analysis.""" ) - time_intervals: Optional[List[TimeIntervals]] = Field( + time_intervals: Optional[Dict[str, TimeIntervals]] = Field( None, description="""Optional additional table(s) for describing other experimental time intervals.""", ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_icephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_icephys.py index 8142b53..1f9c04b 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_icephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_icephys.py @@ -31,6 +31,7 @@ from ...hdmf_common.v1_5_0.hdmf_common_table import ( AlignedDynamicTable, DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -44,7 +45,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -63,6 +64,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -887,8 +919,14 @@ class SweepTable(DynamicTable): } }, ) - series: List[PatchClampSeries] = Field( - ..., description="""The PatchClampSeries with the sweep number in that row.""" + series: VectorData[NDArray[Any, PatchClampSeries]] = Field( + ..., + description="""The PatchClampSeries with the sweep number in that row.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) series_index: Named[VectorIndex] = Field( ..., @@ -907,14 +945,11 @@ class SweepTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class IntracellularElectrodesTable(DynamicTable): @@ -937,21 +972,24 @@ class IntracellularElectrodesTable(DynamicTable): } }, ) - electrode: List[IntracellularElectrode] = Field( - ..., description="""Column for storing the reference to the intracellular electrode.""" + electrode: VectorData[NDArray[Any, IntracellularElectrode]] = Field( + ..., + description="""Column for storing the reference to the intracellular electrode.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) colnames: List[str] = Field( ..., description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class IntracellularStimuliTable(DynamicTable): @@ -990,14 +1028,11 @@ class IntracellularStimuliTable(DynamicTable): ..., description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class IntracellularResponsesTable(DynamicTable): @@ -1036,14 +1071,11 @@ class IntracellularResponsesTable(DynamicTable): ..., description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class IntracellularRecordingsTable(AlignedDynamicTable): @@ -1095,21 +1127,18 @@ class IntracellularRecordingsTable(AlignedDynamicTable): responses: IntracellularResponsesTable = Field( ..., description="""Table for storing intracellular response related metadata.""" ) - value: Optional[List[DynamicTable]] = Field( + value: Optional[Dict[str, DynamicTable]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "DynamicTable"}]}} ) colnames: List[str] = Field( ..., description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class SimultaneousRecordingsTable(DynamicTable): @@ -1151,14 +1180,11 @@ class SimultaneousRecordingsTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class SimultaneousRecordingsTableRecordings(DynamicTableRegion): @@ -1239,14 +1265,11 @@ class SequentialRecordingsTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class SequentialRecordingsTableSimultaneousRecordings(DynamicTableRegion): @@ -1318,14 +1341,11 @@ class RepetitionsTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class RepetitionsTableSequentialRecordings(DynamicTableRegion): @@ -1399,14 +1419,11 @@ class ExperimentalConditionsTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class ExperimentalConditionsTableRepetitions(DynamicTableRegion): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_image.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_image.py index e0506e9..af69abe 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_image.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_image.py @@ -29,7 +29,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -48,6 +48,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -91,17 +122,16 @@ class GrayscaleImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "x"}, {"alias": "y"}]}} + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBImage(Image): @@ -114,17 +144,24 @@ class RGBImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 3 r_g_b"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b", "exact_cardinality": 3}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBAImage(Image): @@ -137,17 +174,24 @@ class RGBAImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 4 r_g_b_a"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b_a", "exact_cardinality": 4}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class ImageSeries(TimeSeries): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_misc.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_misc.py index ee349a1..5c28736 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_misc.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_misc.py @@ -24,6 +24,7 @@ from ...core.v2_6_0_alpha.core_nwb_ecephys import ElectrodeGroup from ...hdmf_common.v1_5_0.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -37,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +57,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -312,7 +344,7 @@ class DecompositionSeries(TimeSeries): ..., description="""Data decomposed into frequency bands.""" ) metric: str = Field(..., description="""The metric used, e.g. phase, amplitude, power.""") - source_channels: Named[Optional[DynamicTableRegion]] = Field( + source_channels: Optional[Named[DynamicTableRegion]] = Field( None, description="""DynamicTableRegion pointer to the channels that this decomposition series was generated from.""", json_schema_extra={ @@ -455,14 +487,11 @@ class DecompositionSeriesBands(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class Units(DynamicTable): @@ -475,7 +504,7 @@ class Units(DynamicTable): ) name: str = Field("Units", json_schema_extra={"linkml_meta": {"ifabsent": "string(Units)"}}) - spike_times_index: Named[Optional[VectorIndex]] = Field( + spike_times_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the spike_times dataset.""", json_schema_extra={ @@ -490,7 +519,7 @@ class Units(DynamicTable): spike_times: Optional[UnitsSpikeTimes] = Field( None, description="""Spike times for each unit in seconds.""" ) - obs_intervals_index: Named[Optional[VectorIndex]] = Field( + obs_intervals_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the obs_intervals dataset.""", json_schema_extra={ @@ -502,7 +531,7 @@ class Units(DynamicTable): } }, ) - obs_intervals: VectorData[Optional[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( + obs_intervals: Optional[VectorData[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( Field( None, description="""Observation intervals for each unit.""", @@ -518,7 +547,7 @@ class Units(DynamicTable): }, ) ) - electrodes_index: Named[Optional[VectorIndex]] = Field( + electrodes_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into electrodes.""", json_schema_extra={ @@ -530,7 +559,7 @@ class Units(DynamicTable): } }, ) - electrodes: Named[Optional[DynamicTableRegion]] = Field( + electrodes: Optional[Named[DynamicTableRegion]] = Field( None, description="""Electrode that each spike unit came from, specified using a DynamicTableRegion.""", json_schema_extra={ @@ -542,26 +571,32 @@ class Units(DynamicTable): } }, ) - electrode_group: Optional[List[ElectrodeGroup]] = Field( - None, description="""Electrode group that each spike unit came from.""" + electrode_group: Optional[VectorData[NDArray[Any, ElectrodeGroup]]] = Field( + None, + description="""Electrode group that each spike unit came from.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) - waveform_mean: VectorData[ - Optional[ + waveform_mean: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], ] ] ] = Field(None, description="""Spike waveform mean for each spike unit.""") - waveform_sd: VectorData[ - Optional[ + waveform_sd: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], ] ] ] = Field(None, description="""Spike waveform standard deviation for each spike unit.""") - waveforms: VectorData[Optional[NDArray[Shape["* num_waveforms, * num_samples"], float]]] = ( + waveforms: Optional[VectorData[NDArray[Shape["* num_waveforms, * num_samples"], float]]] = ( Field( None, description="""Individual waveforms for each spike on each electrode. This is a doubly indexed column. The 'waveforms_index' column indexes which waveforms in this column belong to the same spike event for a given unit, where each waveform was recorded from a different electrode. The 'waveforms_index_index' column indexes the 'waveforms_index' column to indicate which spike events belong to a given unit. For example, if the 'waveforms_index_index' column has values [2, 5, 6], then the first 2 elements of the 'waveforms_index' column correspond to the 2 spike events of the first unit, the next 3 elements of the 'waveforms_index' column correspond to the 3 spike events of the second unit, and the next 1 element of the 'waveforms_index' column corresponds to the 1 spike event of the third unit. If the 'waveforms_index' column has values [3, 6, 8, 10, 12, 13], then the first 3 elements of the 'waveforms' column contain the 3 spike waveforms that were recorded from 3 different electrodes for the first spike time of the first unit. See https://nwb-schema.readthedocs.io/en/stable/format_description.html#doubly-ragged-arrays for a graphical representation of this example. When there is only one electrode for each unit (i.e., each spike time is associated with a single waveform), then the 'waveforms_index' column will have values 1, 2, ..., N, where N is the number of spike events. The number of electrodes for each spike event should be the same within a given unit. The 'electrodes' column should be used to indicate which electrodes are associated with each unit, and the order of the waveforms within a given unit x spike event should be in the same order as the electrodes referenced in the 'electrodes' column of this table. The number of samples for each waveform must be the same.""", @@ -572,7 +607,7 @@ class Units(DynamicTable): }, ) ) - waveforms_index: Named[Optional[VectorIndex]] = Field( + waveforms_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the waveforms dataset. One value for every spike event. See 'waveforms' for more detail.""", json_schema_extra={ @@ -584,7 +619,7 @@ class Units(DynamicTable): } }, ) - waveforms_index_index: Named[Optional[VectorIndex]] = Field( + waveforms_index_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the waveforms_index dataset. One value for every unit (row in the table). See 'waveforms' for more detail.""", json_schema_extra={ @@ -601,14 +636,11 @@ class Units(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class UnitsSpikeTimes(VectorData): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_ogen.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_ogen.py index 5565ce8..42fe82f 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_ogen.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_ogen.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_ophys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_ophys.py index 76d0e67..f6acd6c 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_ophys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_ophys.py @@ -31,6 +31,7 @@ from ...core.v2_6_0_alpha.core_nwb_image import ImageSeries, ImageSeriesExternal from ...hdmf_common.v1_5_0.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -44,7 +45,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -63,6 +64,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -382,7 +414,7 @@ class DfOverF(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[RoiResponseSeries]] = Field( + value: Optional[Dict[str, RoiResponseSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "RoiResponseSeries"}]}} ) name: str = Field(...) @@ -397,7 +429,7 @@ class Fluorescence(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[RoiResponseSeries]] = Field( + value: Optional[Dict[str, RoiResponseSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "RoiResponseSeries"}]}} ) name: str = Field(...) @@ -412,7 +444,7 @@ class ImageSegmentation(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[PlaneSegmentation]] = Field( + value: Optional[Dict[str, PlaneSegmentation]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "PlaneSegmentation"}]}} ) name: str = Field(...) @@ -428,11 +460,18 @@ class PlaneSegmentation(DynamicTable): ) name: str = Field(...) - image_mask: Optional[PlaneSegmentationImageMask] = Field( + image_mask: Optional[ + VectorData[ + Union[ + NDArray[Shape["* num_roi, * num_x, * num_y"], Any], + NDArray[Shape["* num_roi, * num_x, * num_y, * num_z"], Any], + ] + ] + ] = Field( None, description="""ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero.""", ) - pixel_mask_index: Named[Optional[VectorIndex]] = Field( + pixel_mask_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into pixel_mask.""", json_schema_extra={ @@ -448,7 +487,7 @@ class PlaneSegmentation(DynamicTable): None, description="""Pixel masks for each ROI: a list of indices and weights for the ROI. Pixel masks are concatenated and parsing of this dataset is maintained by the PlaneSegmentation""", ) - voxel_mask_index: Named[Optional[VectorIndex]] = Field( + voxel_mask_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into voxel_mask.""", json_schema_extra={ @@ -464,7 +503,7 @@ class PlaneSegmentation(DynamicTable): None, description="""Voxel masks for each ROI: a list of indices and weights for the ROI. Voxel masks are concatenated and parsing of this dataset is maintained by the PlaneSegmentation""", ) - reference_images: Optional[List[ImageSeries]] = Field( + reference_images: Optional[Dict[str, ImageSeries]] = Field( None, description="""Image stacks that the segmentation masks apply to.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "ImageSeries"}]}}, @@ -483,38 +522,11 @@ class PlaneSegmentation(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) - - -class PlaneSegmentationImageMask(VectorData): - """ - ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero. - """ - - linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta({"from_schema": "core.nwb.ophys"}) - - name: Literal["image_mask"] = Field( - "image_mask", - json_schema_extra={ - "linkml_meta": {"equals_string": "image_mask", "ifabsent": "string(image_mask)"} - }, - ) - description: str = Field(..., description="""Description of what these vectors represent.""") - value: Optional[ - Union[ - NDArray[Shape["* dim0"], Any], - NDArray[Shape["* dim0, * dim1"], Any], - NDArray[Shape["* dim0, * dim1, * dim2"], Any], - NDArray[Shape["* dim0, * dim1, * dim2, * dim3"], Any], - ] - ] = Field(None) class PlaneSegmentationPixelMask(VectorData): @@ -637,7 +649,7 @@ class ImagingPlane(NWBContainer): None, description="""Describes reference frame of origin_coords and grid_spacing. For example, this can be a text description of the anatomical location and orientation of the grid defined by origin_coords and grid_spacing or the vectors needed to transform or rotate the grid to a common anatomical axis (e.g., AP/DV/ML). This field is necessary to interpret origin_coords and grid_spacing. If origin_coords and grid_spacing are not present, then this field is not required. For example, if the microscope takes 10 x 10 x 2 images, where the first value of the data matrix (index (0, 0, 0)) corresponds to (-1.2, -0.6, -2) mm relative to bregma, the spacing between pixels is 0.2 mm in x, 0.2 mm in y and 0.5 mm in z, and larger numbers in x means more anterior, larger numbers in y means more rightward, and larger numbers in z means more ventral, then enter the following -- origin_coords = (-1.2, -0.6, -2) grid_spacing = (0.2, 0.2, 0.5) reference_frame = \"Origin coordinates are relative to bregma. First dimension corresponds to anterior-posterior axis (larger index = more anterior). Second dimension corresponds to medial-lateral axis (larger index = more rightward). Third dimension corresponds to dorsal-ventral axis (larger index = more ventral).\"""", ) - optical_channel: List[OpticalChannel] = Field( + optical_channel: Dict[str, OpticalChannel] = Field( ..., description="""An optical channel used to record from an imaging plane.""" ) device: Union[Device, str] = Field( @@ -751,7 +763,7 @@ class MotionCorrection(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[CorrectedImageStack]] = Field( + value: Optional[Dict[str, CorrectedImageStack]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "CorrectedImageStack"}]}} ) name: str = Field(...) @@ -794,7 +806,6 @@ DfOverF.model_rebuild() Fluorescence.model_rebuild() ImageSegmentation.model_rebuild() PlaneSegmentation.model_rebuild() -PlaneSegmentationImageMask.model_rebuild() PlaneSegmentationPixelMask.model_rebuild() PlaneSegmentationVoxelMask.model_rebuild() ImagingPlane.model_rebuild() diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_retinotopy.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_retinotopy.py index b3017f1..3a085f7 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_retinotopy.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/core_nwb_retinotopy.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/namespace.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/namespace.py index a6c0e87..21b7046 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_6_0_alpha/namespace.py @@ -133,7 +133,6 @@ from ...core.v2_6_0_alpha.core_nwb_ophys import ( OnePhotonSeries, OpticalChannel, PlaneSegmentation, - PlaneSegmentationImageMask, PlaneSegmentationPixelMask, PlaneSegmentationVoxelMask, RoiResponseSeries, @@ -150,7 +149,7 @@ from ...core.v2_6_0_alpha.core_nwb_retinotopy import ( ImagingRetinotopyVasculatureImage, ) from ...hdmf_common.v1_5_0.hdmf_common_base import Container, Data, SimpleMultiContainer -from ...hdmf_common.v1_5_0.hdmf_common_sparse import CSRMatrix, CSRMatrixData +from ...hdmf_common.v1_5_0.hdmf_common_sparse import CSRMatrix from ...hdmf_common.v1_5_0.hdmf_common_table import ( AlignedDynamicTable, DynamicTable, @@ -178,7 +177,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -197,6 +196,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_base.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_base.py index 961bfd4..a645a2f 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_base.py @@ -47,7 +47,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -66,6 +66,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -99,7 +130,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -520,7 +551,7 @@ class ProcessingModule(NWBContainer): {"from_schema": "core.nwb.base", "tree_root": True} ) - value: Optional[List[Union[DynamicTable, NWBDataInterface]]] = Field( + value: Optional[Dict[str, Union[DynamicTable, NWBDataInterface]]] = Field( None, json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBDataInterface"}, {"range": "DynamicTable"}]} @@ -540,8 +571,8 @@ class Images(NWBDataInterface): name: str = Field("Images", json_schema_extra={"linkml_meta": {"ifabsent": "string(Images)"}}) description: str = Field(..., description="""Description of this collection of images.""") - image: List[Image] = Field(..., description="""Images stored in this collection.""") - order_of_images: Named[Optional[ImageReferences]] = Field( + image: List[str] = Field(..., description="""Images stored in this collection.""") + order_of_images: Optional[Named[ImageReferences]] = Field( None, description="""Ordered dataset of references to Image objects stored in the parent group. Each Image object in the Images group should be stored once and only once, so the dataset should have the same length as the number of images.""", json_schema_extra={ diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_behavior.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_behavior.py index 43c1936..836c2e2 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_behavior.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_behavior.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -169,7 +200,7 @@ class BehavioralEpochs(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[IntervalSeries]] = Field( + value: Optional[Dict[str, IntervalSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "IntervalSeries"}]}} ) name: str = Field(...) @@ -184,7 +215,7 @@ class BehavioralEvents(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[TimeSeries]] = Field( + value: Optional[Dict[str, TimeSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}} ) name: str = Field(...) @@ -199,7 +230,7 @@ class BehavioralTimeSeries(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[TimeSeries]] = Field( + value: Optional[Dict[str, TimeSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}} ) name: str = Field(...) @@ -214,7 +245,7 @@ class PupilTracking(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[TimeSeries]] = Field( + value: Optional[Dict[str, TimeSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "TimeSeries"}]}} ) name: str = Field(...) @@ -229,7 +260,7 @@ class EyeTracking(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[SpatialSeries]] = Field( + value: Optional[Dict[str, SpatialSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpatialSeries"}]}} ) name: str = Field(...) @@ -244,7 +275,7 @@ class CompassDirection(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[SpatialSeries]] = Field( + value: Optional[Dict[str, SpatialSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpatialSeries"}]}} ) name: str = Field(...) @@ -259,7 +290,7 @@ class Position(NWBDataInterface): {"from_schema": "core.nwb.behavior", "tree_root": True} ) - value: Optional[List[SpatialSeries]] = Field( + value: Optional[Dict[str, SpatialSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpatialSeries"}]}} ) name: str = Field(...) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_device.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_device.py index 3867744..59b53c8 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_device.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_device.py @@ -21,7 +21,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -40,6 +40,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_ecephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_ecephys.py index 89eeeb0..dc96a98 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_ecephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_ecephys.py @@ -38,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -57,6 +57,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -354,7 +385,7 @@ class EventWaveform(NWBDataInterface): {"from_schema": "core.nwb.ecephys", "tree_root": True} ) - value: Optional[List[SpikeEventSeries]] = Field( + value: Optional[Dict[str, SpikeEventSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "SpikeEventSeries"}]}} ) name: str = Field(...) @@ -369,7 +400,7 @@ class FilteredEphys(NWBDataInterface): {"from_schema": "core.nwb.ecephys", "tree_root": True} ) - value: Optional[List[ElectricalSeries]] = Field( + value: Optional[Dict[str, ElectricalSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "ElectricalSeries"}]}} ) name: str = Field(...) @@ -384,7 +415,7 @@ class LFP(NWBDataInterface): {"from_schema": "core.nwb.ecephys", "tree_root": True} ) - value: Optional[List[ElectricalSeries]] = Field( + value: Optional[Dict[str, ElectricalSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "ElectricalSeries"}]}} ) name: str = Field(...) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_epoch.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_epoch.py index 95178b0..e8b5539 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_epoch.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_epoch.py @@ -20,7 +20,12 @@ from pydantic import ( ) from ...core.v2_7_0.core_nwb_base import TimeSeriesReferenceVectorData -from ...hdmf_common.v1_8_0.hdmf_common_table import DynamicTable, VectorData, VectorIndex +from ...hdmf_common.v1_8_0.hdmf_common_table import ( + DynamicTable, + ElementIdentifiers, + VectorData, + VectorIndex, +) metamodel_version = "None" @@ -31,7 +36,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -50,6 +55,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -127,7 +163,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags: VectorData[Optional[NDArray[Any, str]]] = Field( + tags: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""User-defined tags that identify or categorize events.""", json_schema_extra={ @@ -136,7 +172,7 @@ class TimeIntervals(DynamicTable): } }, ) - tags_index: Named[Optional[VectorIndex]] = Field( + tags_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for tags.""", json_schema_extra={ @@ -148,7 +184,7 @@ class TimeIntervals(DynamicTable): } }, ) - timeseries: Named[Optional[TimeSeriesReferenceVectorData]] = Field( + timeseries: Optional[Named[TimeSeriesReferenceVectorData]] = Field( None, description="""An index into a TimeSeries object.""", json_schema_extra={ @@ -160,7 +196,7 @@ class TimeIntervals(DynamicTable): } }, ) - timeseries_index: Named[Optional[VectorIndex]] = Field( + timeseries_index: Optional[Named[VectorIndex]] = Field( None, description="""Index for timeseries.""", json_schema_extra={ @@ -177,14 +213,11 @@ class TimeIntervals(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_file.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_file.py index fe7fabd..038a4ae 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_file.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_file.py @@ -34,7 +34,7 @@ from ...core.v2_7_0.core_nwb_icephys import ( from ...core.v2_7_0.core_nwb_misc import Units from ...core.v2_7_0.core_nwb_ogen import OptogeneticStimulusSite from ...core.v2_7_0.core_nwb_ophys import ImagingPlane -from ...hdmf_common.v1_8_0.hdmf_common_table import DynamicTable, VectorData +from ...hdmf_common.v1_8_0.hdmf_common_table import DynamicTable, ElementIdentifiers, VectorData metamodel_version = "None" @@ -45,7 +45,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -64,6 +64,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -161,28 +192,28 @@ class NWBFile(NWBContainer): ..., description="""Date and time corresponding to time zero of all timestamps. The date is stored in UTC with local timezone offset as ISO 8601 extended formatted string: 2018-09-28T14:43:54.123+02:00. Dates stored in UTC end in \"Z\" with no timezone offset. Date accuracy is up to milliseconds. All times stored in the file use this time as reference (i.e., time zero).""", ) - acquisition: Optional[List[Union[DynamicTable, NWBDataInterface]]] = Field( + acquisition: Optional[Dict[str, Union[DynamicTable, NWBDataInterface]]] = Field( None, description="""Data streams recorded from the system, including ephys, ophys, tracking, etc. This group should be read-only after the experiment is completed and timestamps are corrected to a common timebase. The data stored here may be links to raw data stored in external NWB files. This will allow keeping bulky raw data out of the file while preserving the option of keeping some/all in the file. Acquired data includes tracking and experimental data streams (i.e., everything measured from the system). If bulky data is stored in the /acquisition group, the data can exist in a separate NWB file that is linked to by the file being used for processing and analysis.""", json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBDataInterface"}, {"range": "DynamicTable"}]} }, ) - analysis: Optional[List[Union[DynamicTable, NWBContainer]]] = Field( + analysis: Optional[Dict[str, Union[DynamicTable, NWBContainer]]] = Field( None, description="""Lab-specific and custom scientific analysis of data. There is no defined format for the content of this group - the format is up to the individual user/lab. To facilitate sharing analysis data between labs, the contents here should be stored in standard types (e.g., neurodata_types) and appropriately documented. The file can store lab-specific and custom data analysis without restriction on its form or schema, reducing data formatting restrictions on end users. Such data should be placed in the analysis group. The analysis data should be documented so that it could be shared with other labs.""", json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBContainer"}, {"range": "DynamicTable"}]} }, ) - scratch: Optional[List[Union[DynamicTable, NWBContainer]]] = Field( + scratch: Optional[Dict[str, Union[DynamicTable, NWBContainer]]] = Field( None, description="""A place to store one-off analysis results. Data placed here is not intended for sharing. By placing data here, users acknowledge that there is no guarantee that their data meets any standard.""", json_schema_extra={ "linkml_meta": {"any_of": [{"range": "NWBContainer"}, {"range": "DynamicTable"}]} }, ) - processing: Optional[List[ProcessingModule]] = Field( + processing: Optional[Dict[str, ProcessingModule]] = Field( None, description="""The home for ProcessingModules. These modules perform intermediate analysis of data that is necessary to perform before scientific analysis. Examples include spike clustering, extracting position from tracking data, stitching together image slices. ProcessingModules can be large and express many data sets from relatively complex analysis (e.g., spike detection and clustering) or small, representing extraction of position information from tracking video, or even binary lick/no-lick decisions. Common software tools (e.g., klustakwik, MClust) are expected to read/write data here. 'Processing' refers to intermediate analysis of the acquired data to make it more amenable to scientific analysis.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "ProcessingModule"}]}}, @@ -215,7 +246,7 @@ class NWBFileStimulus(ConfiguredBaseModel): "linkml_meta": {"equals_string": "stimulus", "ifabsent": "string(stimulus)"} }, ) - presentation: Optional[List[Union[DynamicTable, NWBDataInterface, TimeSeries]]] = Field( + presentation: Optional[Dict[str, Union[DynamicTable, NWBDataInterface, TimeSeries]]] = Field( None, description="""Stimuli presented during the experiment.""", json_schema_extra={ @@ -228,7 +259,7 @@ class NWBFileStimulus(ConfiguredBaseModel): } }, ) - templates: Optional[List[Union[Images, TimeSeries]]] = Field( + templates: Optional[Dict[str, Union[Images, TimeSeries]]] = Field( None, description="""Template stimuli. Timestamps in templates are based on stimulus design and are relative to the beginning of the stimulus. When templates are used, the stimulus instances must convert presentation times to the experiment`s time reference frame.""", json_schema_extra={ @@ -308,11 +339,11 @@ class NWBFileGeneral(ConfiguredBaseModel): None, description="""Information about virus(es) used in experiments, including virus ID, source, date made, injection location, volume, etc.""", ) - lab_meta_data: Optional[List[LabMetaData]] = Field( + lab_meta_data: Optional[Dict[str, LabMetaData]] = Field( None, description="""Place-holder than can be extended so that lab-specific meta-data can be placed in /general.""", ) - devices: Optional[List[Device]] = Field( + devices: Optional[Dict[str, Device]] = Field( None, description="""Description of hardware devices used during experiment, e.g., monitors, ADC boards, microscopes, etc.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "Device"}]}}, @@ -327,12 +358,12 @@ class NWBFileGeneral(ConfiguredBaseModel): intracellular_ephys: Optional[GeneralIntracellularEphys] = Field( None, description="""Metadata related to intracellular electrophysiology.""" ) - optogenetics: Optional[List[OptogeneticStimulusSite]] = Field( + optogenetics: Optional[Dict[str, OptogeneticStimulusSite]] = Field( None, description="""Metadata describing optogenetic stimuluation.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "OptogeneticStimulusSite"}]}}, ) - optophysiology: Optional[List[ImagingPlane]] = Field( + optophysiology: Optional[Dict[str, ImagingPlane]] = Field( None, description="""Metadata related to optophysiology.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "ImagingPlane"}]}}, @@ -372,7 +403,7 @@ class GeneralExtracellularEphys(ConfiguredBaseModel): } }, ) - electrode_group: Optional[List[ElectrodeGroup]] = Field( + electrode_group: Optional[Dict[str, ElectrodeGroup]] = Field( None, description="""Physical group of electrodes.""" ) electrodes: Optional[ExtracellularEphysElectrodes] = Field( @@ -393,7 +424,7 @@ class ExtracellularEphysElectrodes(DynamicTable): "linkml_meta": {"equals_string": "electrodes", "ifabsent": "string(electrodes)"} }, ) - x: VectorData[Optional[NDArray[Any, float]]] = Field( + x: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""x coordinate of the channel location in the brain (+x is posterior).""", json_schema_extra={ @@ -402,7 +433,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - y: VectorData[Optional[NDArray[Any, float]]] = Field( + y: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""y coordinate of the channel location in the brain (+y is inferior).""", json_schema_extra={ @@ -411,7 +442,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - z: VectorData[Optional[NDArray[Any, float]]] = Field( + z: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""z coordinate of the channel location in the brain (+z is right).""", json_schema_extra={ @@ -420,7 +451,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - imp: VectorData[Optional[NDArray[Any, float]]] = Field( + imp: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""Impedance of the channel, in ohms.""", json_schema_extra={ @@ -438,7 +469,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - filtering: VectorData[Optional[NDArray[Any, str]]] = Field( + filtering: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""Description of hardware filtering, including the filter name and frequency cutoffs.""", json_schema_extra={ @@ -447,8 +478,14 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - group: List[ElectrodeGroup] = Field( - ..., description="""Reference to the ElectrodeGroup this electrode is a part of.""" + group: VectorData[NDArray[Any, ElectrodeGroup]] = Field( + ..., + description="""Reference to the ElectrodeGroup this electrode is a part of.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) group_name: VectorData[NDArray[Any, str]] = Field( ..., @@ -459,7 +496,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_x: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_x: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""x coordinate in electrode group""", json_schema_extra={ @@ -468,7 +505,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_y: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_y: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""y coordinate in electrode group""", json_schema_extra={ @@ -477,7 +514,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - rel_z: VectorData[Optional[NDArray[Any, float]]] = Field( + rel_z: Optional[VectorData[NDArray[Any, float]]] = Field( None, description="""z coordinate in electrode group""", json_schema_extra={ @@ -486,7 +523,7 @@ class ExtracellularEphysElectrodes(DynamicTable): } }, ) - reference: VectorData[Optional[NDArray[Any, str]]] = Field( + reference: Optional[VectorData[NDArray[Any, str]]] = Field( None, description="""Description of the reference electrode and/or reference scheme used for this electrode, e.g., \"stainless steel skull screw\" or \"online common average referencing\".""", json_schema_extra={ @@ -500,14 +537,11 @@ class ExtracellularEphysElectrodes(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class GeneralIntracellularEphys(ConfiguredBaseModel): @@ -530,7 +564,7 @@ class GeneralIntracellularEphys(ConfiguredBaseModel): None, description="""[DEPRECATED] Use IntracellularElectrode.filtering instead. Description of filtering used. Includes filtering type and parameters, frequency fall-off, etc. If this changes between TimeSeries, filter description should be stored as a text attribute for each TimeSeries.""", ) - intracellular_electrode: Optional[List[IntracellularElectrode]] = Field( + intracellular_electrode: Optional[Dict[str, IntracellularElectrode]] = Field( None, description="""An intracellular electrode.""" ) sweep_table: Optional[SweepTable] = Field( @@ -582,7 +616,7 @@ class NWBFileIntervals(ConfiguredBaseModel): invalid_times: Optional[TimeIntervals] = Field( None, description="""Time intervals that should be removed from analysis.""" ) - time_intervals: Optional[List[TimeIntervals]] = Field( + time_intervals: Optional[Dict[str, TimeIntervals]] = Field( None, description="""Optional additional table(s) for describing other experimental time intervals.""", ) diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_icephys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_icephys.py index 9e72812..c1818b4 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_icephys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_icephys.py @@ -31,6 +31,7 @@ from ...hdmf_common.v1_8_0.hdmf_common_table import ( AlignedDynamicTable, DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -44,7 +45,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -63,6 +64,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -887,8 +919,14 @@ class SweepTable(DynamicTable): } }, ) - series: List[PatchClampSeries] = Field( - ..., description="""The PatchClampSeries with the sweep number in that row.""" + series: VectorData[NDArray[Any, PatchClampSeries]] = Field( + ..., + description="""The PatchClampSeries with the sweep number in that row.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) series_index: Named[VectorIndex] = Field( ..., @@ -907,14 +945,11 @@ class SweepTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class IntracellularElectrodesTable(DynamicTable): @@ -937,21 +972,24 @@ class IntracellularElectrodesTable(DynamicTable): } }, ) - electrode: List[IntracellularElectrode] = Field( - ..., description="""Column for storing the reference to the intracellular electrode.""" + electrode: VectorData[NDArray[Any, IntracellularElectrode]] = Field( + ..., + description="""Column for storing the reference to the intracellular electrode.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) colnames: List[str] = Field( ..., description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class IntracellularStimuliTable(DynamicTable): @@ -986,7 +1024,7 @@ class IntracellularStimuliTable(DynamicTable): } }, ) - stimulus_template: Named[Optional[TimeSeriesReferenceVectorData]] = Field( + stimulus_template: Optional[Named[TimeSeriesReferenceVectorData]] = Field( None, description="""Column storing the reference to the stimulus template for the recording (rows).""", json_schema_extra={ @@ -1002,14 +1040,11 @@ class IntracellularStimuliTable(DynamicTable): ..., description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class IntracellularResponsesTable(DynamicTable): @@ -1048,14 +1083,11 @@ class IntracellularResponsesTable(DynamicTable): ..., description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class IntracellularRecordingsTable(AlignedDynamicTable): @@ -1107,21 +1139,18 @@ class IntracellularRecordingsTable(AlignedDynamicTable): responses: IntracellularResponsesTable = Field( ..., description="""Table for storing intracellular response related metadata.""" ) - value: Optional[List[DynamicTable]] = Field( + value: Optional[Dict[str, DynamicTable]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "DynamicTable"}]}} ) colnames: List[str] = Field( ..., description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class SimultaneousRecordingsTable(DynamicTable): @@ -1163,14 +1192,11 @@ class SimultaneousRecordingsTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class SimultaneousRecordingsTableRecordings(DynamicTableRegion): @@ -1251,14 +1277,11 @@ class SequentialRecordingsTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class SequentialRecordingsTableSimultaneousRecordings(DynamicTableRegion): @@ -1330,14 +1353,11 @@ class RepetitionsTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class RepetitionsTableSequentialRecordings(DynamicTableRegion): @@ -1411,14 +1431,11 @@ class ExperimentalConditionsTable(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class ExperimentalConditionsTableRepetitions(DynamicTableRegion): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_image.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_image.py index d98ffe1..6e97172 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_image.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_image.py @@ -29,7 +29,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -48,6 +48,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -91,17 +122,16 @@ class GrayscaleImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "x"}, {"alias": "y"}]}} + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBImage(Image): @@ -114,17 +144,24 @@ class RGBImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 3 r_g_b"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b", "exact_cardinality": 3}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class RGBAImage(Image): @@ -137,17 +174,24 @@ class RGBAImage(Image): ) name: str = Field(...) + value: Optional[NDArray[Shape["* x, * y, 4 r_g_b_a"], float]] = Field( + None, + json_schema_extra={ + "linkml_meta": { + "array": { + "dimensions": [ + {"alias": "x"}, + {"alias": "y"}, + {"alias": "r_g_b_a", "exact_cardinality": 4}, + ] + } + } + }, + ) resolution: Optional[float] = Field( None, description="""Pixel resolution of the image, in pixels per centimeter.""" ) description: Optional[str] = Field(None, description="""Description of the image.""") - value: Optional[ - Union[ - NDArray[Shape["* x, * y"], float], - NDArray[Shape["* x, * y, 3 r_g_b"], float], - NDArray[Shape["* x, * y, 4 r_g_b_a"], float], - ] - ] = Field(None) class ImageSeries(TimeSeries): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_misc.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_misc.py index 31c081b..1eb2c3a 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_misc.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_misc.py @@ -24,6 +24,7 @@ from ...core.v2_7_0.core_nwb_ecephys import ElectrodeGroup from ...hdmf_common.v1_8_0.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -37,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +57,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -312,7 +344,7 @@ class DecompositionSeries(TimeSeries): ..., description="""Data decomposed into frequency bands.""" ) metric: str = Field(..., description="""The metric used, e.g. phase, amplitude, power.""") - source_channels: Named[Optional[DynamicTableRegion]] = Field( + source_channels: Optional[Named[DynamicTableRegion]] = Field( None, description="""DynamicTableRegion pointer to the channels that this decomposition series was generated from.""", json_schema_extra={ @@ -455,14 +487,11 @@ class DecompositionSeriesBands(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class Units(DynamicTable): @@ -475,7 +504,7 @@ class Units(DynamicTable): ) name: str = Field("Units", json_schema_extra={"linkml_meta": {"ifabsent": "string(Units)"}}) - spike_times_index: Named[Optional[VectorIndex]] = Field( + spike_times_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the spike_times dataset.""", json_schema_extra={ @@ -490,7 +519,7 @@ class Units(DynamicTable): spike_times: Optional[UnitsSpikeTimes] = Field( None, description="""Spike times for each unit in seconds.""" ) - obs_intervals_index: Named[Optional[VectorIndex]] = Field( + obs_intervals_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the obs_intervals dataset.""", json_schema_extra={ @@ -502,7 +531,7 @@ class Units(DynamicTable): } }, ) - obs_intervals: VectorData[Optional[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( + obs_intervals: Optional[VectorData[NDArray[Shape["* num_intervals, 2 start_end"], float]]] = ( Field( None, description="""Observation intervals for each unit.""", @@ -518,7 +547,7 @@ class Units(DynamicTable): }, ) ) - electrodes_index: Named[Optional[VectorIndex]] = Field( + electrodes_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into electrodes.""", json_schema_extra={ @@ -530,7 +559,7 @@ class Units(DynamicTable): } }, ) - electrodes: Named[Optional[DynamicTableRegion]] = Field( + electrodes: Optional[Named[DynamicTableRegion]] = Field( None, description="""Electrode that each spike unit came from, specified using a DynamicTableRegion.""", json_schema_extra={ @@ -542,26 +571,32 @@ class Units(DynamicTable): } }, ) - electrode_group: Optional[List[ElectrodeGroup]] = Field( - None, description="""Electrode group that each spike unit came from.""" + electrode_group: Optional[VectorData[NDArray[Any, ElectrodeGroup]]] = Field( + None, + description="""Electrode group that each spike unit came from.""", + json_schema_extra={ + "linkml_meta": { + "array": {"maximum_number_dimensions": False, "minimum_number_dimensions": 1} + } + }, ) - waveform_mean: VectorData[ - Optional[ + waveform_mean: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], ] ] ] = Field(None, description="""Spike waveform mean for each spike unit.""") - waveform_sd: VectorData[ - Optional[ + waveform_sd: Optional[ + VectorData[ Union[ NDArray[Shape["* num_units, * num_samples"], float], NDArray[Shape["* num_units, * num_samples, * num_electrodes"], float], ] ] ] = Field(None, description="""Spike waveform standard deviation for each spike unit.""") - waveforms: VectorData[Optional[NDArray[Shape["* num_waveforms, * num_samples"], float]]] = ( + waveforms: Optional[VectorData[NDArray[Shape["* num_waveforms, * num_samples"], float]]] = ( Field( None, description="""Individual waveforms for each spike on each electrode. This is a doubly indexed column. The 'waveforms_index' column indexes which waveforms in this column belong to the same spike event for a given unit, where each waveform was recorded from a different electrode. The 'waveforms_index_index' column indexes the 'waveforms_index' column to indicate which spike events belong to a given unit. For example, if the 'waveforms_index_index' column has values [2, 5, 6], then the first 2 elements of the 'waveforms_index' column correspond to the 2 spike events of the first unit, the next 3 elements of the 'waveforms_index' column correspond to the 3 spike events of the second unit, and the next 1 element of the 'waveforms_index' column corresponds to the 1 spike event of the third unit. If the 'waveforms_index' column has values [3, 6, 8, 10, 12, 13], then the first 3 elements of the 'waveforms' column contain the 3 spike waveforms that were recorded from 3 different electrodes for the first spike time of the first unit. See https://nwb-schema.readthedocs.io/en/stable/format_description.html#doubly-ragged-arrays for a graphical representation of this example. When there is only one electrode for each unit (i.e., each spike time is associated with a single waveform), then the 'waveforms_index' column will have values 1, 2, ..., N, where N is the number of spike events. The number of electrodes for each spike event should be the same within a given unit. The 'electrodes' column should be used to indicate which electrodes are associated with each unit, and the order of the waveforms within a given unit x spike event should be in the same order as the electrodes referenced in the 'electrodes' column of this table. The number of samples for each waveform must be the same.""", @@ -572,7 +607,7 @@ class Units(DynamicTable): }, ) ) - waveforms_index: Named[Optional[VectorIndex]] = Field( + waveforms_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the waveforms dataset. One value for every spike event. See 'waveforms' for more detail.""", json_schema_extra={ @@ -584,7 +619,7 @@ class Units(DynamicTable): } }, ) - waveforms_index_index: Named[Optional[VectorIndex]] = Field( + waveforms_index_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into the waveforms_index dataset. One value for every unit (row in the table). See 'waveforms' for more detail.""", json_schema_extra={ @@ -601,14 +636,11 @@ class Units(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class UnitsSpikeTimes(VectorData): diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_ogen.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_ogen.py index 5eb87cf..626a28c 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_ogen.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_ogen.py @@ -28,7 +28,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -47,6 +47,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_ophys.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_ophys.py index 3f8d8eb..d462064 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_ophys.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_ophys.py @@ -31,6 +31,7 @@ from ...core.v2_7_0.core_nwb_image import ImageSeries, ImageSeriesExternalFile from ...hdmf_common.v1_8_0.hdmf_common_table import ( DynamicTable, DynamicTableRegion, + ElementIdentifiers, VectorData, VectorIndex, ) @@ -44,7 +45,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -63,6 +64,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -382,7 +414,7 @@ class DfOverF(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[RoiResponseSeries]] = Field( + value: Optional[Dict[str, RoiResponseSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "RoiResponseSeries"}]}} ) name: str = Field(...) @@ -397,7 +429,7 @@ class Fluorescence(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[RoiResponseSeries]] = Field( + value: Optional[Dict[str, RoiResponseSeries]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "RoiResponseSeries"}]}} ) name: str = Field(...) @@ -412,7 +444,7 @@ class ImageSegmentation(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[PlaneSegmentation]] = Field( + value: Optional[Dict[str, PlaneSegmentation]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "PlaneSegmentation"}]}} ) name: str = Field(...) @@ -428,11 +460,18 @@ class PlaneSegmentation(DynamicTable): ) name: str = Field(...) - image_mask: Optional[PlaneSegmentationImageMask] = Field( + image_mask: Optional[ + VectorData[ + Union[ + NDArray[Shape["* num_roi, * num_x, * num_y"], Any], + NDArray[Shape["* num_roi, * num_x, * num_y, * num_z"], Any], + ] + ] + ] = Field( None, description="""ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero.""", ) - pixel_mask_index: Named[Optional[VectorIndex]] = Field( + pixel_mask_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into pixel_mask.""", json_schema_extra={ @@ -448,7 +487,7 @@ class PlaneSegmentation(DynamicTable): None, description="""Pixel masks for each ROI: a list of indices and weights for the ROI. Pixel masks are concatenated and parsing of this dataset is maintained by the PlaneSegmentation""", ) - voxel_mask_index: Named[Optional[VectorIndex]] = Field( + voxel_mask_index: Optional[Named[VectorIndex]] = Field( None, description="""Index into voxel_mask.""", json_schema_extra={ @@ -464,7 +503,7 @@ class PlaneSegmentation(DynamicTable): None, description="""Voxel masks for each ROI: a list of indices and weights for the ROI. Voxel masks are concatenated and parsing of this dataset is maintained by the PlaneSegmentation""", ) - reference_images: Optional[List[ImageSeries]] = Field( + reference_images: Optional[Dict[str, ImageSeries]] = Field( None, description="""Image stacks that the segmentation masks apply to.""", json_schema_extra={"linkml_meta": {"any_of": [{"range": "ImageSeries"}]}}, @@ -483,38 +522,11 @@ class PlaneSegmentation(DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) - - -class PlaneSegmentationImageMask(VectorData): - """ - ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero. - """ - - linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta({"from_schema": "core.nwb.ophys"}) - - name: Literal["image_mask"] = Field( - "image_mask", - json_schema_extra={ - "linkml_meta": {"equals_string": "image_mask", "ifabsent": "string(image_mask)"} - }, - ) - description: str = Field(..., description="""Description of what these vectors represent.""") - value: Optional[ - Union[ - NDArray[Shape["* dim0"], Any], - NDArray[Shape["* dim0, * dim1"], Any], - NDArray[Shape["* dim0, * dim1, * dim2"], Any], - NDArray[Shape["* dim0, * dim1, * dim2, * dim3"], Any], - ] - ] = Field(None) class PlaneSegmentationPixelMask(VectorData): @@ -637,7 +649,7 @@ class ImagingPlane(NWBContainer): None, description="""Describes reference frame of origin_coords and grid_spacing. For example, this can be a text description of the anatomical location and orientation of the grid defined by origin_coords and grid_spacing or the vectors needed to transform or rotate the grid to a common anatomical axis (e.g., AP/DV/ML). This field is necessary to interpret origin_coords and grid_spacing. If origin_coords and grid_spacing are not present, then this field is not required. For example, if the microscope takes 10 x 10 x 2 images, where the first value of the data matrix (index (0, 0, 0)) corresponds to (-1.2, -0.6, -2) mm relative to bregma, the spacing between pixels is 0.2 mm in x, 0.2 mm in y and 0.5 mm in z, and larger numbers in x means more anterior, larger numbers in y means more rightward, and larger numbers in z means more ventral, then enter the following -- origin_coords = (-1.2, -0.6, -2) grid_spacing = (0.2, 0.2, 0.5) reference_frame = \"Origin coordinates are relative to bregma. First dimension corresponds to anterior-posterior axis (larger index = more anterior). Second dimension corresponds to medial-lateral axis (larger index = more rightward). Third dimension corresponds to dorsal-ventral axis (larger index = more ventral).\"""", ) - optical_channel: List[OpticalChannel] = Field( + optical_channel: Dict[str, OpticalChannel] = Field( ..., description="""An optical channel used to record from an imaging plane.""" ) device: Union[Device, str] = Field( @@ -751,7 +763,7 @@ class MotionCorrection(NWBDataInterface): {"from_schema": "core.nwb.ophys", "tree_root": True} ) - value: Optional[List[CorrectedImageStack]] = Field( + value: Optional[Dict[str, CorrectedImageStack]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "CorrectedImageStack"}]}} ) name: str = Field(...) @@ -794,7 +806,6 @@ DfOverF.model_rebuild() Fluorescence.model_rebuild() ImageSegmentation.model_rebuild() PlaneSegmentation.model_rebuild() -PlaneSegmentationImageMask.model_rebuild() PlaneSegmentationPixelMask.model_rebuild() PlaneSegmentationVoxelMask.model_rebuild() ImagingPlane.model_rebuild() diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_retinotopy.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_retinotopy.py index 909aaf3..26f2f92 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_retinotopy.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/core_nwb_retinotopy.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/namespace.py index 9256d2f..5747cde 100644 --- a/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/core/v2_7_0/namespace.py @@ -133,7 +133,6 @@ from ...core.v2_7_0.core_nwb_ophys import ( OnePhotonSeries, OpticalChannel, PlaneSegmentation, - PlaneSegmentationImageMask, PlaneSegmentationPixelMask, PlaneSegmentationVoxelMask, RoiResponseSeries, @@ -150,7 +149,7 @@ from ...core.v2_7_0.core_nwb_retinotopy import ( ImagingRetinotopyVasculatureImage, ) from ...hdmf_common.v1_8_0.hdmf_common_base import Container, Data, SimpleMultiContainer -from ...hdmf_common.v1_8_0.hdmf_common_sparse import CSRMatrix, CSRMatrixData +from ...hdmf_common.v1_8_0.hdmf_common_sparse import CSRMatrix from ...hdmf_common.v1_8_0.hdmf_common_table import ( AlignedDynamicTable, DynamicTable, @@ -179,7 +178,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -198,6 +197,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_0/hdmf_common_sparse.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_0/hdmf_common_sparse.py index c71894e..56af1b8 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_0/hdmf_common_sparse.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_0/hdmf_common_sparse.py @@ -20,7 +20,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -39,6 +39,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_0/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_0/hdmf_common_table.py index fb5ae6f..e52b294 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_0/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_0/hdmf_common_table.py @@ -44,7 +44,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -63,6 +63,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -96,7 +127,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -298,8 +329,11 @@ class DynamicTableMixin(BaseModel): NON_COLUMN_FIELDS: ClassVar[tuple[str]] = ( "id", "name", + "categories", "colnames", "description", + "hdf5_path", + "object_id", ) # overridden by subclass but implemented here for testing and typechecking purposes :) @@ -383,7 +417,7 @@ class DynamicTableMixin(BaseModel): # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -491,11 +525,14 @@ class DynamicTableMixin(BaseModel): if k not in cls.NON_COLUMN_FIELDS and not k.endswith("_index") and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] model["colnames"] = colnames else: # add any columns not explicitly given an order at the end colnames = model["colnames"].copy() + if isinstance(colnames, np.ndarray): + colnames = colnames.tolist() colnames.extend( [ k @@ -504,6 +541,7 @@ class DynamicTableMixin(BaseModel): and not k.endswith("_index") and k not in model["colnames"] and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] ) model["colnames"] = colnames @@ -522,17 +560,25 @@ class DynamicTableMixin(BaseModel): if isinstance(model, dict): for key, val in model.items(): - if key in cls.model_fields: + if key in cls.model_fields or key in cls.NON_COLUMN_FIELDS: continue if not isinstance(val, (VectorData, VectorIndex)): try: - if key.endswith("_index"): - model[key] = VectorIndex(name=key, description="", value=val) + to_cast = VectorIndex if key.endswith("_index") else VectorData + if isinstance(val, dict): + model[key] = to_cast(**val) else: - model[key] = VectorData(name=key, description="", value=val) + model[key] = to_cast(name=key, description="", value=val) except ValidationError as e: # pragma: no cover - raise ValidationError( - f"field {key} cannot be cast to VectorData from {val}" + raise ValidationError.from_exception_data( + title=f"field {key} cannot be cast to VectorData from {val}", + line_errors=[ + { + "type": "ValueError", + "loc": ("DynamicTableMixin", "cast_extra_columns"), + "input": val, + } + ], ) from e return model @@ -565,9 +611,9 @@ class DynamicTableMixin(BaseModel): """ Ensure that all columns are equal length """ - lengths = [len(v) for v in self._columns.values()] + [len(self.id)] + lengths = [len(v) for v in self._columns.values() if v is not None] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "DynamicTable columns are not of equal length! " f"Got colnames:\n{self.colnames}\nand lengths: {lengths}" ) return self @@ -617,10 +663,13 @@ class AlignedDynamicTableMixin(BaseModel): __pydantic_extra__: Dict[str, Union["DynamicTableMixin", "VectorDataMixin", "VectorIndexMixin"]] NON_CATEGORY_FIELDS: ClassVar[tuple[str]] = ( + "id", "name", "categories", "colnames", "description", + "hdf5_path", + "object_id", ) name: str = "aligned_table" @@ -650,28 +699,29 @@ class AlignedDynamicTableMixin(BaseModel): elif isinstance(item, tuple) and len(item) == 2 and isinstance(item[1], str): # get a slice of a single table return self._categories[item[1]][item[0]] - elif isinstance(item, (int, slice, Iterable)): + elif isinstance(item, (int, slice, Iterable, np.int_)): # get a slice of all the tables ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) - tables = [ids] + ids = pd.Index(data=ids, name="id") + tables = [] for category_name, category in self._categories.items(): table = category[item] if isinstance(table, pd.DataFrame): table = table.reset_index() + table.index = ids elif isinstance(table, np.ndarray): - table = pd.DataFrame({category_name: [table]}) + table = pd.DataFrame({category_name: [table]}, index=ids) elif isinstance(table, Iterable): - table = pd.DataFrame({category_name: table}) + table = pd.DataFrame({category_name: table}, index=ids) else: raise ValueError( f"Don't know how to construct category table for {category_name}" ) tables.append(table) - names = [self.name] + self.categories + # names = [self.name] + self.categories # construct below in case we need to support array indexing in the future else: raise ValueError( @@ -679,8 +729,7 @@ class AlignedDynamicTableMixin(BaseModel): "need an int, string, slice, ndarray, or tuple[int | slice, str]" ) - df = pd.concat(tables, axis=1, keys=names) - df.set_index((self.name, "id"), drop=True, inplace=True) + df = pd.concat(tables, axis=1, keys=self.categories) return df def __getattr__(self, item: str) -> Any: @@ -738,14 +787,19 @@ class AlignedDynamicTableMixin(BaseModel): model["categories"] = categories else: # add any columns not explicitly given an order at the end - categories = [ - k - for k in model - if k not in cls.NON_COLUMN_FIELDS - and not k.endswith("_index") - and k not in model["categories"] - ] - model["categories"].extend(categories) + categories = model["categories"].copy() + if isinstance(categories, np.ndarray): + categories = categories.tolist() + categories.extend( + [ + k + for k in model + if k not in cls.NON_CATEGORY_FIELDS + and not k.endswith("_index") + and k not in model["categories"] + ] + ) + model["categories"] = categories return model @model_validator(mode="after") @@ -779,12 +833,19 @@ class AlignedDynamicTableMixin(BaseModel): """ lengths = [len(v) for v in self._categories.values()] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "AlignedDynamicTableColumns are not of equal length! " f"Got colnames:\n{self.categories}\nand lengths: {lengths}" ) return self +class ElementIdentifiersMixin(VectorDataMixin): + """ + Mixin class for ElementIdentifiers - allow treating + as generic, and give general indexing methods from VectorData + """ + + linkml_meta = LinkMLMeta( { "annotations": { @@ -826,7 +887,7 @@ class Index(Data): ) -class VectorData(VectorDataMixin): +class VectorData(VectorDataMixin, ConfiguredBaseModel): """ An n-dimensional dataset representing a column of a DynamicTable. If used without an accompanying VectorIndex, first dimension is along the rows of the DynamicTable and each step along the first dimension is a cell of the larger table. VectorData can also be used to represent a ragged array if paired with a VectorIndex. This allows for storing arrays of varying length in a single cell of the DynamicTable by indexing into this VectorData. The first vector is at VectorData[0:VectorIndex(0)+1]. The second vector is at VectorData[VectorIndex(0)+1:VectorIndex(1)+1], and so on. """ @@ -839,7 +900,7 @@ class VectorData(VectorDataMixin): description: str = Field(..., description="""Description of what these vectors represent.""") -class VectorIndex(VectorIndexMixin): +class VectorIndex(VectorIndexMixin, ConfiguredBaseModel): """ Used with VectorData to encode a ragged array. An array of indices into the first dimension of the target VectorData, and forming a map between the rows of a DynamicTable and the indices of the VectorData. """ @@ -854,7 +915,7 @@ class VectorIndex(VectorIndexMixin): ) -class ElementIdentifiers(Data): +class ElementIdentifiers(ElementIdentifiersMixin, Data, ConfiguredBaseModel): """ A list of unique identifiers for values within a dataset, e.g. rows of a DynamicTable. """ @@ -866,9 +927,13 @@ class ElementIdentifiers(Data): name: str = Field( "element_id", json_schema_extra={"linkml_meta": {"ifabsent": "string(element_id)"}} ) + value: Optional[T] = Field( + None, + json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_elements"}]}}}, + ) -class DynamicTableRegion(DynamicTableRegionMixin, VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData, ConfiguredBaseModel): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ @@ -898,7 +963,7 @@ class Container(ConfiguredBaseModel): name: str = Field(...) -class DynamicTable(DynamicTableMixin): +class DynamicTable(DynamicTableMixin, ConfiguredBaseModel): """ A group containing multiple datasets that are aligned on the first dimension (Currently, this requirement if left up to APIs to check and enforce). Apart from a column that contains unique identifiers for each row there are no other required datasets. Users are free to add any number of VectorData objects here. Table functionality is already supported through compound types, which is analogous to storing an array-of-structs. DynamicTable can be thought of as a struct-of-arrays. This provides an alternative structure to choose from when optimizing storage for anticipated access patterns. Additionally, this type provides a way of creating a table without having to define a compound type up front. Although this convenience may be attractive, users should think carefully about how data will be accessed. DynamicTable is more appropriate for column-centric access, whereas a dataset with a compound type would be more appropriate for row-centric access. Finally, data size should also be taken into account. For small tables, performance loss may be an acceptable trade-off for the flexibility of a DynamicTable. For example, DynamicTable was originally developed for storing trial data and spike unit metadata. Both of these use cases are expected to produce relatively small tables, so the spatial locality of multiple datasets present in a DynamicTable is not expected to have a significant performance impact. Additionally, requirements of trial and unit metadata tables are sufficiently diverse that performance implications can be overlooked in favor of usability. """ @@ -913,14 +978,11 @@ class DynamicTable(DynamicTableMixin): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_0/namespace.py index 1a6e22f..dcc5707 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_0/namespace.py @@ -36,7 +36,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -55,6 +55,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_2/hdmf_common_sparse.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_2/hdmf_common_sparse.py index bd66832..8ce7f43 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_2/hdmf_common_sparse.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_2/hdmf_common_sparse.py @@ -20,7 +20,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -39,6 +39,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_2/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_2/hdmf_common_table.py index 50f164b..9065b81 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_2/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_2/hdmf_common_table.py @@ -44,7 +44,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -63,6 +63,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -96,7 +127,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -298,8 +329,11 @@ class DynamicTableMixin(BaseModel): NON_COLUMN_FIELDS: ClassVar[tuple[str]] = ( "id", "name", + "categories", "colnames", "description", + "hdf5_path", + "object_id", ) # overridden by subclass but implemented here for testing and typechecking purposes :) @@ -383,7 +417,7 @@ class DynamicTableMixin(BaseModel): # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -491,11 +525,14 @@ class DynamicTableMixin(BaseModel): if k not in cls.NON_COLUMN_FIELDS and not k.endswith("_index") and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] model["colnames"] = colnames else: # add any columns not explicitly given an order at the end colnames = model["colnames"].copy() + if isinstance(colnames, np.ndarray): + colnames = colnames.tolist() colnames.extend( [ k @@ -504,6 +541,7 @@ class DynamicTableMixin(BaseModel): and not k.endswith("_index") and k not in model["colnames"] and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] ) model["colnames"] = colnames @@ -522,17 +560,25 @@ class DynamicTableMixin(BaseModel): if isinstance(model, dict): for key, val in model.items(): - if key in cls.model_fields: + if key in cls.model_fields or key in cls.NON_COLUMN_FIELDS: continue if not isinstance(val, (VectorData, VectorIndex)): try: - if key.endswith("_index"): - model[key] = VectorIndex(name=key, description="", value=val) + to_cast = VectorIndex if key.endswith("_index") else VectorData + if isinstance(val, dict): + model[key] = to_cast(**val) else: - model[key] = VectorData(name=key, description="", value=val) + model[key] = to_cast(name=key, description="", value=val) except ValidationError as e: # pragma: no cover - raise ValidationError( - f"field {key} cannot be cast to VectorData from {val}" + raise ValidationError.from_exception_data( + title=f"field {key} cannot be cast to VectorData from {val}", + line_errors=[ + { + "type": "ValueError", + "loc": ("DynamicTableMixin", "cast_extra_columns"), + "input": val, + } + ], ) from e return model @@ -565,9 +611,9 @@ class DynamicTableMixin(BaseModel): """ Ensure that all columns are equal length """ - lengths = [len(v) for v in self._columns.values()] + [len(self.id)] + lengths = [len(v) for v in self._columns.values() if v is not None] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "DynamicTable columns are not of equal length! " f"Got colnames:\n{self.colnames}\nand lengths: {lengths}" ) return self @@ -617,10 +663,13 @@ class AlignedDynamicTableMixin(BaseModel): __pydantic_extra__: Dict[str, Union["DynamicTableMixin", "VectorDataMixin", "VectorIndexMixin"]] NON_CATEGORY_FIELDS: ClassVar[tuple[str]] = ( + "id", "name", "categories", "colnames", "description", + "hdf5_path", + "object_id", ) name: str = "aligned_table" @@ -650,28 +699,29 @@ class AlignedDynamicTableMixin(BaseModel): elif isinstance(item, tuple) and len(item) == 2 and isinstance(item[1], str): # get a slice of a single table return self._categories[item[1]][item[0]] - elif isinstance(item, (int, slice, Iterable)): + elif isinstance(item, (int, slice, Iterable, np.int_)): # get a slice of all the tables ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) - tables = [ids] + ids = pd.Index(data=ids, name="id") + tables = [] for category_name, category in self._categories.items(): table = category[item] if isinstance(table, pd.DataFrame): table = table.reset_index() + table.index = ids elif isinstance(table, np.ndarray): - table = pd.DataFrame({category_name: [table]}) + table = pd.DataFrame({category_name: [table]}, index=ids) elif isinstance(table, Iterable): - table = pd.DataFrame({category_name: table}) + table = pd.DataFrame({category_name: table}, index=ids) else: raise ValueError( f"Don't know how to construct category table for {category_name}" ) tables.append(table) - names = [self.name] + self.categories + # names = [self.name] + self.categories # construct below in case we need to support array indexing in the future else: raise ValueError( @@ -679,8 +729,7 @@ class AlignedDynamicTableMixin(BaseModel): "need an int, string, slice, ndarray, or tuple[int | slice, str]" ) - df = pd.concat(tables, axis=1, keys=names) - df.set_index((self.name, "id"), drop=True, inplace=True) + df = pd.concat(tables, axis=1, keys=self.categories) return df def __getattr__(self, item: str) -> Any: @@ -738,14 +787,19 @@ class AlignedDynamicTableMixin(BaseModel): model["categories"] = categories else: # add any columns not explicitly given an order at the end - categories = [ - k - for k in model - if k not in cls.NON_COLUMN_FIELDS - and not k.endswith("_index") - and k not in model["categories"] - ] - model["categories"].extend(categories) + categories = model["categories"].copy() + if isinstance(categories, np.ndarray): + categories = categories.tolist() + categories.extend( + [ + k + for k in model + if k not in cls.NON_CATEGORY_FIELDS + and not k.endswith("_index") + and k not in model["categories"] + ] + ) + model["categories"] = categories return model @model_validator(mode="after") @@ -779,12 +833,19 @@ class AlignedDynamicTableMixin(BaseModel): """ lengths = [len(v) for v in self._categories.values()] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "AlignedDynamicTableColumns are not of equal length! " f"Got colnames:\n{self.categories}\nand lengths: {lengths}" ) return self +class ElementIdentifiersMixin(VectorDataMixin): + """ + Mixin class for ElementIdentifiers - allow treating + as generic, and give general indexing methods from VectorData + """ + + linkml_meta = LinkMLMeta( { "annotations": { @@ -826,7 +887,7 @@ class Index(Data): ) -class VectorData(VectorDataMixin): +class VectorData(VectorDataMixin, ConfiguredBaseModel): """ An n-dimensional dataset representing a column of a DynamicTable. If used without an accompanying VectorIndex, first dimension is along the rows of the DynamicTable and each step along the first dimension is a cell of the larger table. VectorData can also be used to represent a ragged array if paired with a VectorIndex. This allows for storing arrays of varying length in a single cell of the DynamicTable by indexing into this VectorData. The first vector is at VectorData[0:VectorIndex(0)+1]. The second vector is at VectorData[VectorIndex(0)+1:VectorIndex(1)+1], and so on. """ @@ -839,7 +900,7 @@ class VectorData(VectorDataMixin): description: str = Field(..., description="""Description of what these vectors represent.""") -class VectorIndex(VectorIndexMixin): +class VectorIndex(VectorIndexMixin, ConfiguredBaseModel): """ Used with VectorData to encode a ragged array. An array of indices into the first dimension of the target VectorData, and forming a map between the rows of a DynamicTable and the indices of the VectorData. """ @@ -854,7 +915,7 @@ class VectorIndex(VectorIndexMixin): ) -class ElementIdentifiers(Data): +class ElementIdentifiers(ElementIdentifiersMixin, Data, ConfiguredBaseModel): """ A list of unique identifiers for values within a dataset, e.g. rows of a DynamicTable. """ @@ -866,9 +927,13 @@ class ElementIdentifiers(Data): name: str = Field( "element_id", json_schema_extra={"linkml_meta": {"ifabsent": "string(element_id)"}} ) + value: Optional[T] = Field( + None, + json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_elements"}]}}}, + ) -class DynamicTableRegion(DynamicTableRegionMixin, VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData, ConfiguredBaseModel): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ @@ -898,7 +963,7 @@ class Container(ConfiguredBaseModel): name: str = Field(...) -class DynamicTable(DynamicTableMixin): +class DynamicTable(DynamicTableMixin, ConfiguredBaseModel): """ A group containing multiple datasets that are aligned on the first dimension (Currently, this requirement if left up to APIs to check and enforce). Apart from a column that contains unique identifiers for each row there are no other required datasets. Users are free to add any number of VectorData objects here. Table functionality is already supported through compound types, which is analogous to storing an array-of-structs. DynamicTable can be thought of as a struct-of-arrays. This provides an alternative structure to choose from when optimizing storage for anticipated access patterns. Additionally, this type provides a way of creating a table without having to define a compound type up front. Although this convenience may be attractive, users should think carefully about how data will be accessed. DynamicTable is more appropriate for column-centric access, whereas a dataset with a compound type would be more appropriate for row-centric access. Finally, data size should also be taken into account. For small tables, performance loss may be an acceptable trade-off for the flexibility of a DynamicTable. For example, DynamicTable was originally developed for storing trial data and spike unit metadata. Both of these use cases are expected to produce relatively small tables, so the spatial locality of multiple datasets present in a DynamicTable is not expected to have a significant performance impact. Additionally, requirements of trial and unit metadata tables are sufficiently diverse that performance implications can be overlooked in favor of usability. """ @@ -913,14 +978,11 @@ class DynamicTable(DynamicTableMixin): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_2/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_2/namespace.py index 786e141..0f66985 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_2/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_2/namespace.py @@ -36,7 +36,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -55,6 +55,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_3/hdmf_common_sparse.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_3/hdmf_common_sparse.py index 09ea0f1..c0f7fcc 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_3/hdmf_common_sparse.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_3/hdmf_common_sparse.py @@ -20,7 +20,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -39,6 +39,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_3/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_3/hdmf_common_table.py index 0d16b55..749fab9 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_3/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_3/hdmf_common_table.py @@ -44,7 +44,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -63,6 +63,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -96,7 +127,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -298,8 +329,11 @@ class DynamicTableMixin(BaseModel): NON_COLUMN_FIELDS: ClassVar[tuple[str]] = ( "id", "name", + "categories", "colnames", "description", + "hdf5_path", + "object_id", ) # overridden by subclass but implemented here for testing and typechecking purposes :) @@ -383,7 +417,7 @@ class DynamicTableMixin(BaseModel): # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -491,11 +525,14 @@ class DynamicTableMixin(BaseModel): if k not in cls.NON_COLUMN_FIELDS and not k.endswith("_index") and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] model["colnames"] = colnames else: # add any columns not explicitly given an order at the end colnames = model["colnames"].copy() + if isinstance(colnames, np.ndarray): + colnames = colnames.tolist() colnames.extend( [ k @@ -504,6 +541,7 @@ class DynamicTableMixin(BaseModel): and not k.endswith("_index") and k not in model["colnames"] and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] ) model["colnames"] = colnames @@ -522,17 +560,25 @@ class DynamicTableMixin(BaseModel): if isinstance(model, dict): for key, val in model.items(): - if key in cls.model_fields: + if key in cls.model_fields or key in cls.NON_COLUMN_FIELDS: continue if not isinstance(val, (VectorData, VectorIndex)): try: - if key.endswith("_index"): - model[key] = VectorIndex(name=key, description="", value=val) + to_cast = VectorIndex if key.endswith("_index") else VectorData + if isinstance(val, dict): + model[key] = to_cast(**val) else: - model[key] = VectorData(name=key, description="", value=val) + model[key] = to_cast(name=key, description="", value=val) except ValidationError as e: # pragma: no cover - raise ValidationError( - f"field {key} cannot be cast to VectorData from {val}" + raise ValidationError.from_exception_data( + title=f"field {key} cannot be cast to VectorData from {val}", + line_errors=[ + { + "type": "ValueError", + "loc": ("DynamicTableMixin", "cast_extra_columns"), + "input": val, + } + ], ) from e return model @@ -565,9 +611,9 @@ class DynamicTableMixin(BaseModel): """ Ensure that all columns are equal length """ - lengths = [len(v) for v in self._columns.values()] + [len(self.id)] + lengths = [len(v) for v in self._columns.values() if v is not None] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "DynamicTable columns are not of equal length! " f"Got colnames:\n{self.colnames}\nand lengths: {lengths}" ) return self @@ -617,10 +663,13 @@ class AlignedDynamicTableMixin(BaseModel): __pydantic_extra__: Dict[str, Union["DynamicTableMixin", "VectorDataMixin", "VectorIndexMixin"]] NON_CATEGORY_FIELDS: ClassVar[tuple[str]] = ( + "id", "name", "categories", "colnames", "description", + "hdf5_path", + "object_id", ) name: str = "aligned_table" @@ -650,28 +699,29 @@ class AlignedDynamicTableMixin(BaseModel): elif isinstance(item, tuple) and len(item) == 2 and isinstance(item[1], str): # get a slice of a single table return self._categories[item[1]][item[0]] - elif isinstance(item, (int, slice, Iterable)): + elif isinstance(item, (int, slice, Iterable, np.int_)): # get a slice of all the tables ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) - tables = [ids] + ids = pd.Index(data=ids, name="id") + tables = [] for category_name, category in self._categories.items(): table = category[item] if isinstance(table, pd.DataFrame): table = table.reset_index() + table.index = ids elif isinstance(table, np.ndarray): - table = pd.DataFrame({category_name: [table]}) + table = pd.DataFrame({category_name: [table]}, index=ids) elif isinstance(table, Iterable): - table = pd.DataFrame({category_name: table}) + table = pd.DataFrame({category_name: table}, index=ids) else: raise ValueError( f"Don't know how to construct category table for {category_name}" ) tables.append(table) - names = [self.name] + self.categories + # names = [self.name] + self.categories # construct below in case we need to support array indexing in the future else: raise ValueError( @@ -679,8 +729,7 @@ class AlignedDynamicTableMixin(BaseModel): "need an int, string, slice, ndarray, or tuple[int | slice, str]" ) - df = pd.concat(tables, axis=1, keys=names) - df.set_index((self.name, "id"), drop=True, inplace=True) + df = pd.concat(tables, axis=1, keys=self.categories) return df def __getattr__(self, item: str) -> Any: @@ -738,14 +787,19 @@ class AlignedDynamicTableMixin(BaseModel): model["categories"] = categories else: # add any columns not explicitly given an order at the end - categories = [ - k - for k in model - if k not in cls.NON_COLUMN_FIELDS - and not k.endswith("_index") - and k not in model["categories"] - ] - model["categories"].extend(categories) + categories = model["categories"].copy() + if isinstance(categories, np.ndarray): + categories = categories.tolist() + categories.extend( + [ + k + for k in model + if k not in cls.NON_CATEGORY_FIELDS + and not k.endswith("_index") + and k not in model["categories"] + ] + ) + model["categories"] = categories return model @model_validator(mode="after") @@ -779,12 +833,19 @@ class AlignedDynamicTableMixin(BaseModel): """ lengths = [len(v) for v in self._categories.values()] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "AlignedDynamicTableColumns are not of equal length! " f"Got colnames:\n{self.categories}\nand lengths: {lengths}" ) return self +class ElementIdentifiersMixin(VectorDataMixin): + """ + Mixin class for ElementIdentifiers - allow treating + as generic, and give general indexing methods from VectorData + """ + + linkml_meta = LinkMLMeta( { "annotations": { @@ -826,7 +887,7 @@ class Index(Data): ) -class VectorData(VectorDataMixin): +class VectorData(VectorDataMixin, ConfiguredBaseModel): """ An n-dimensional dataset representing a column of a DynamicTable. If used without an accompanying VectorIndex, first dimension is along the rows of the DynamicTable and each step along the first dimension is a cell of the larger table. VectorData can also be used to represent a ragged array if paired with a VectorIndex. This allows for storing arrays of varying length in a single cell of the DynamicTable by indexing into this VectorData. The first vector is at VectorData[0:VectorIndex(0)+1]. The second vector is at VectorData[VectorIndex(0)+1:VectorIndex(1)+1], and so on. """ @@ -837,17 +898,10 @@ class VectorData(VectorDataMixin): name: str = Field(...) description: str = Field(..., description="""Description of what these vectors represent.""") - value: Optional[ - Union[ - NDArray[Shape["* dim0"], Any], - NDArray[Shape["* dim0, * dim1"], Any], - NDArray[Shape["* dim0, * dim1, * dim2"], Any], - NDArray[Shape["* dim0, * dim1, * dim2, * dim3"], Any], - ] - ] = Field(None) + value: Optional[T] = Field(None) -class VectorIndex(VectorIndexMixin): +class VectorIndex(VectorIndexMixin, ConfiguredBaseModel): """ Used with VectorData to encode a ragged array. An array of indices into the first dimension of the target VectorData, and forming a map between the rows of a DynamicTable and the indices of the VectorData. """ @@ -865,7 +919,7 @@ class VectorIndex(VectorIndexMixin): ) -class ElementIdentifiers(Data): +class ElementIdentifiers(ElementIdentifiersMixin, Data, ConfiguredBaseModel): """ A list of unique identifiers for values within a dataset, e.g. rows of a DynamicTable. """ @@ -877,9 +931,13 @@ class ElementIdentifiers(Data): name: str = Field( "element_id", json_schema_extra={"linkml_meta": {"ifabsent": "string(element_id)"}} ) + value: Optional[T] = Field( + None, + json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_elements"}]}}}, + ) -class DynamicTableRegion(DynamicTableRegionMixin, VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData, ConfiguredBaseModel): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ @@ -917,7 +975,7 @@ class Container(ConfiguredBaseModel): name: str = Field(...) -class DynamicTable(DynamicTableMixin): +class DynamicTable(DynamicTableMixin, ConfiguredBaseModel): """ A group containing multiple datasets that are aligned on the first dimension (Currently, this requirement if left up to APIs to check and enforce). Apart from a column that contains unique identifiers for each row there are no other required datasets. Users are free to add any number of VectorData objects here. Table functionality is already supported through compound types, which is analogous to storing an array-of-structs. DynamicTable can be thought of as a struct-of-arrays. This provides an alternative structure to choose from when optimizing storage for anticipated access patterns. Additionally, this type provides a way of creating a table without having to define a compound type up front. Although this convenience may be attractive, users should think carefully about how data will be accessed. DynamicTable is more appropriate for column-centric access, whereas a dataset with a compound type would be more appropriate for row-centric access. Finally, data size should also be taken into account. For small tables, performance loss may be an acceptable trade-off for the flexibility of a DynamicTable. For example, DynamicTable was originally developed for storing trial data and spike unit metadata. Both of these use cases are expected to produce relatively small tables, so the spatial locality of multiple datasets present in a DynamicTable is not expected to have a significant performance impact. Additionally, requirements of trial and unit metadata tables are sufficiently diverse that performance implications can be overlooked in favor of usability. """ @@ -932,14 +990,11 @@ class DynamicTable(DynamicTableMixin): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns of this dynamic table.""" - ) vector_index: Optional[List[VectorIndex]] = Field( None, description="""Indices for the vector columns of this dynamic table.""" ) diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_3/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_3/namespace.py index 1458d9b..c505d77 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_3/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_1_3/namespace.py @@ -36,7 +36,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -55,6 +55,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/hdmf_common_base.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/hdmf_common_base.py index aa2b460..656629d 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/hdmf_common_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/hdmf_common_base.py @@ -19,7 +19,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -38,6 +38,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/hdmf_common_sparse.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/hdmf_common_sparse.py index a0a70de..13824fe 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/hdmf_common_sparse.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/hdmf_common_sparse.py @@ -20,7 +20,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -39,6 +39,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/hdmf_common_table.py index 647bc23..fdd6bcc 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/hdmf_common_table.py @@ -46,7 +46,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -65,6 +65,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -98,7 +129,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -300,8 +331,11 @@ class DynamicTableMixin(BaseModel): NON_COLUMN_FIELDS: ClassVar[tuple[str]] = ( "id", "name", + "categories", "colnames", "description", + "hdf5_path", + "object_id", ) # overridden by subclass but implemented here for testing and typechecking purposes :) @@ -385,7 +419,7 @@ class DynamicTableMixin(BaseModel): # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -493,11 +527,14 @@ class DynamicTableMixin(BaseModel): if k not in cls.NON_COLUMN_FIELDS and not k.endswith("_index") and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] model["colnames"] = colnames else: # add any columns not explicitly given an order at the end colnames = model["colnames"].copy() + if isinstance(colnames, np.ndarray): + colnames = colnames.tolist() colnames.extend( [ k @@ -506,6 +543,7 @@ class DynamicTableMixin(BaseModel): and not k.endswith("_index") and k not in model["colnames"] and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] ) model["colnames"] = colnames @@ -524,17 +562,25 @@ class DynamicTableMixin(BaseModel): if isinstance(model, dict): for key, val in model.items(): - if key in cls.model_fields: + if key in cls.model_fields or key in cls.NON_COLUMN_FIELDS: continue if not isinstance(val, (VectorData, VectorIndex)): try: - if key.endswith("_index"): - model[key] = VectorIndex(name=key, description="", value=val) + to_cast = VectorIndex if key.endswith("_index") else VectorData + if isinstance(val, dict): + model[key] = to_cast(**val) else: - model[key] = VectorData(name=key, description="", value=val) + model[key] = to_cast(name=key, description="", value=val) except ValidationError as e: # pragma: no cover - raise ValidationError( - f"field {key} cannot be cast to VectorData from {val}" + raise ValidationError.from_exception_data( + title=f"field {key} cannot be cast to VectorData from {val}", + line_errors=[ + { + "type": "ValueError", + "loc": ("DynamicTableMixin", "cast_extra_columns"), + "input": val, + } + ], ) from e return model @@ -567,9 +613,9 @@ class DynamicTableMixin(BaseModel): """ Ensure that all columns are equal length """ - lengths = [len(v) for v in self._columns.values()] + [len(self.id)] + lengths = [len(v) for v in self._columns.values() if v is not None] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "DynamicTable columns are not of equal length! " f"Got colnames:\n{self.colnames}\nand lengths: {lengths}" ) return self @@ -619,10 +665,13 @@ class AlignedDynamicTableMixin(BaseModel): __pydantic_extra__: Dict[str, Union["DynamicTableMixin", "VectorDataMixin", "VectorIndexMixin"]] NON_CATEGORY_FIELDS: ClassVar[tuple[str]] = ( + "id", "name", "categories", "colnames", "description", + "hdf5_path", + "object_id", ) name: str = "aligned_table" @@ -652,28 +701,29 @@ class AlignedDynamicTableMixin(BaseModel): elif isinstance(item, tuple) and len(item) == 2 and isinstance(item[1], str): # get a slice of a single table return self._categories[item[1]][item[0]] - elif isinstance(item, (int, slice, Iterable)): + elif isinstance(item, (int, slice, Iterable, np.int_)): # get a slice of all the tables ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) - tables = [ids] + ids = pd.Index(data=ids, name="id") + tables = [] for category_name, category in self._categories.items(): table = category[item] if isinstance(table, pd.DataFrame): table = table.reset_index() + table.index = ids elif isinstance(table, np.ndarray): - table = pd.DataFrame({category_name: [table]}) + table = pd.DataFrame({category_name: [table]}, index=ids) elif isinstance(table, Iterable): - table = pd.DataFrame({category_name: table}) + table = pd.DataFrame({category_name: table}, index=ids) else: raise ValueError( f"Don't know how to construct category table for {category_name}" ) tables.append(table) - names = [self.name] + self.categories + # names = [self.name] + self.categories # construct below in case we need to support array indexing in the future else: raise ValueError( @@ -681,8 +731,7 @@ class AlignedDynamicTableMixin(BaseModel): "need an int, string, slice, ndarray, or tuple[int | slice, str]" ) - df = pd.concat(tables, axis=1, keys=names) - df.set_index((self.name, "id"), drop=True, inplace=True) + df = pd.concat(tables, axis=1, keys=self.categories) return df def __getattr__(self, item: str) -> Any: @@ -740,14 +789,19 @@ class AlignedDynamicTableMixin(BaseModel): model["categories"] = categories else: # add any columns not explicitly given an order at the end - categories = [ - k - for k in model - if k not in cls.NON_COLUMN_FIELDS - and not k.endswith("_index") - and k not in model["categories"] - ] - model["categories"].extend(categories) + categories = model["categories"].copy() + if isinstance(categories, np.ndarray): + categories = categories.tolist() + categories.extend( + [ + k + for k in model + if k not in cls.NON_CATEGORY_FIELDS + and not k.endswith("_index") + and k not in model["categories"] + ] + ) + model["categories"] = categories return model @model_validator(mode="after") @@ -781,12 +835,19 @@ class AlignedDynamicTableMixin(BaseModel): """ lengths = [len(v) for v in self._categories.values()] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "AlignedDynamicTableColumns are not of equal length! " f"Got colnames:\n{self.categories}\nand lengths: {lengths}" ) return self +class ElementIdentifiersMixin(VectorDataMixin): + """ + Mixin class for ElementIdentifiers - allow treating + as generic, and give general indexing methods from VectorData + """ + + linkml_meta = LinkMLMeta( { "annotations": { @@ -801,7 +862,7 @@ linkml_meta = LinkMLMeta( ) -class VectorData(VectorDataMixin): +class VectorData(VectorDataMixin, ConfiguredBaseModel): """ An n-dimensional dataset representing a column of a DynamicTable. If used without an accompanying VectorIndex, first dimension is along the rows of the DynamicTable and each step along the first dimension is a cell of the larger table. VectorData can also be used to represent a ragged array if paired with a VectorIndex. This allows for storing arrays of varying length in a single cell of the DynamicTable by indexing into this VectorData. The first vector is at VectorData[0:VectorIndex[0]]. The second vector is at VectorData[VectorIndex[0]:VectorIndex[1]], and so on. """ @@ -812,17 +873,10 @@ class VectorData(VectorDataMixin): name: str = Field(...) description: str = Field(..., description="""Description of what these vectors represent.""") - value: Optional[ - Union[ - NDArray[Shape["* dim0"], Any], - NDArray[Shape["* dim0, * dim1"], Any], - NDArray[Shape["* dim0, * dim1, * dim2"], Any], - NDArray[Shape["* dim0, * dim1, * dim2, * dim3"], Any], - ] - ] = Field(None) + value: Optional[T] = Field(None) -class VectorIndex(VectorIndexMixin): +class VectorIndex(VectorIndexMixin, ConfiguredBaseModel): """ Used with VectorData to encode a ragged array. An array of indices into the first dimension of the target VectorData, and forming a map between the rows of a DynamicTable and the indices of the VectorData. The name of the VectorIndex is expected to be the name of the target VectorData object followed by \"_index\". """ @@ -846,7 +900,7 @@ class VectorIndex(VectorIndexMixin): ] = Field(None) -class ElementIdentifiers(Data): +class ElementIdentifiers(ElementIdentifiersMixin, Data, ConfiguredBaseModel): """ A list of unique identifiers for values within a dataset, e.g. rows of a DynamicTable. """ @@ -858,9 +912,13 @@ class ElementIdentifiers(Data): name: str = Field( "element_id", json_schema_extra={"linkml_meta": {"ifabsent": "string(element_id)"}} ) + value: Optional[T] = Field( + None, + json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_elements"}]}}}, + ) -class DynamicTableRegion(DynamicTableRegionMixin, VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData, ConfiguredBaseModel): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ @@ -912,7 +970,7 @@ class VocabData(VectorData): ] = Field(None) -class DynamicTable(DynamicTableMixin): +class DynamicTable(DynamicTableMixin, ConfiguredBaseModel): """ A group containing multiple datasets that are aligned on the first dimension (Currently, this requirement if left up to APIs to check and enforce). These datasets represent different columns in the table. Apart from a column that contains unique identifiers for each row, there are no other required datasets. Users are free to add any number of custom VectorData objects (columns) here. DynamicTable also supports ragged array columns, where each element can be of a different size. To add a ragged array column, use a VectorIndex type to index the corresponding VectorData type. See documentation for VectorData and VectorIndex for more details. Unlike a compound data type, which is analogous to storing an array-of-structs, a DynamicTable can be thought of as a struct-of-arrays. This provides an alternative structure to choose from when optimizing storage for anticipated access patterns. Additionally, this type provides a way of creating a table without having to define a compound type up front. Although this convenience may be attractive, users should think carefully about how data will be accessed. DynamicTable is more appropriate for column-centric access, whereas a dataset with a compound type would be more appropriate for row-centric access. Finally, data size should also be taken into account. For small tables, performance loss may be an acceptable trade-off for the flexibility of a DynamicTable. """ @@ -927,14 +985,11 @@ class DynamicTable(DynamicTableMixin): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/namespace.py index f56d638..25e5651 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_0/namespace.py @@ -35,7 +35,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -54,6 +54,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/hdmf_common_base.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/hdmf_common_base.py index 7476a40..affaa59 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/hdmf_common_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/hdmf_common_base.py @@ -19,7 +19,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -38,6 +38,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/hdmf_common_sparse.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/hdmf_common_sparse.py index c2222d1..01484f3 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/hdmf_common_sparse.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/hdmf_common_sparse.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/hdmf_common_table.py index d1c0692..cc9029d 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/hdmf_common_table.py @@ -46,7 +46,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -65,6 +65,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -98,7 +129,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -300,8 +331,11 @@ class DynamicTableMixin(BaseModel): NON_COLUMN_FIELDS: ClassVar[tuple[str]] = ( "id", "name", + "categories", "colnames", "description", + "hdf5_path", + "object_id", ) # overridden by subclass but implemented here for testing and typechecking purposes :) @@ -385,7 +419,7 @@ class DynamicTableMixin(BaseModel): # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -493,11 +527,14 @@ class DynamicTableMixin(BaseModel): if k not in cls.NON_COLUMN_FIELDS and not k.endswith("_index") and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] model["colnames"] = colnames else: # add any columns not explicitly given an order at the end colnames = model["colnames"].copy() + if isinstance(colnames, np.ndarray): + colnames = colnames.tolist() colnames.extend( [ k @@ -506,6 +543,7 @@ class DynamicTableMixin(BaseModel): and not k.endswith("_index") and k not in model["colnames"] and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] ) model["colnames"] = colnames @@ -524,17 +562,25 @@ class DynamicTableMixin(BaseModel): if isinstance(model, dict): for key, val in model.items(): - if key in cls.model_fields: + if key in cls.model_fields or key in cls.NON_COLUMN_FIELDS: continue if not isinstance(val, (VectorData, VectorIndex)): try: - if key.endswith("_index"): - model[key] = VectorIndex(name=key, description="", value=val) + to_cast = VectorIndex if key.endswith("_index") else VectorData + if isinstance(val, dict): + model[key] = to_cast(**val) else: - model[key] = VectorData(name=key, description="", value=val) + model[key] = to_cast(name=key, description="", value=val) except ValidationError as e: # pragma: no cover - raise ValidationError( - f"field {key} cannot be cast to VectorData from {val}" + raise ValidationError.from_exception_data( + title=f"field {key} cannot be cast to VectorData from {val}", + line_errors=[ + { + "type": "ValueError", + "loc": ("DynamicTableMixin", "cast_extra_columns"), + "input": val, + } + ], ) from e return model @@ -567,9 +613,9 @@ class DynamicTableMixin(BaseModel): """ Ensure that all columns are equal length """ - lengths = [len(v) for v in self._columns.values()] + [len(self.id)] + lengths = [len(v) for v in self._columns.values() if v is not None] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "DynamicTable columns are not of equal length! " f"Got colnames:\n{self.colnames}\nand lengths: {lengths}" ) return self @@ -619,10 +665,13 @@ class AlignedDynamicTableMixin(BaseModel): __pydantic_extra__: Dict[str, Union["DynamicTableMixin", "VectorDataMixin", "VectorIndexMixin"]] NON_CATEGORY_FIELDS: ClassVar[tuple[str]] = ( + "id", "name", "categories", "colnames", "description", + "hdf5_path", + "object_id", ) name: str = "aligned_table" @@ -652,28 +701,29 @@ class AlignedDynamicTableMixin(BaseModel): elif isinstance(item, tuple) and len(item) == 2 and isinstance(item[1], str): # get a slice of a single table return self._categories[item[1]][item[0]] - elif isinstance(item, (int, slice, Iterable)): + elif isinstance(item, (int, slice, Iterable, np.int_)): # get a slice of all the tables ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) - tables = [ids] + ids = pd.Index(data=ids, name="id") + tables = [] for category_name, category in self._categories.items(): table = category[item] if isinstance(table, pd.DataFrame): table = table.reset_index() + table.index = ids elif isinstance(table, np.ndarray): - table = pd.DataFrame({category_name: [table]}) + table = pd.DataFrame({category_name: [table]}, index=ids) elif isinstance(table, Iterable): - table = pd.DataFrame({category_name: table}) + table = pd.DataFrame({category_name: table}, index=ids) else: raise ValueError( f"Don't know how to construct category table for {category_name}" ) tables.append(table) - names = [self.name] + self.categories + # names = [self.name] + self.categories # construct below in case we need to support array indexing in the future else: raise ValueError( @@ -681,8 +731,7 @@ class AlignedDynamicTableMixin(BaseModel): "need an int, string, slice, ndarray, or tuple[int | slice, str]" ) - df = pd.concat(tables, axis=1, keys=names) - df.set_index((self.name, "id"), drop=True, inplace=True) + df = pd.concat(tables, axis=1, keys=self.categories) return df def __getattr__(self, item: str) -> Any: @@ -740,14 +789,19 @@ class AlignedDynamicTableMixin(BaseModel): model["categories"] = categories else: # add any columns not explicitly given an order at the end - categories = [ - k - for k in model - if k not in cls.NON_COLUMN_FIELDS - and not k.endswith("_index") - and k not in model["categories"] - ] - model["categories"].extend(categories) + categories = model["categories"].copy() + if isinstance(categories, np.ndarray): + categories = categories.tolist() + categories.extend( + [ + k + for k in model + if k not in cls.NON_CATEGORY_FIELDS + and not k.endswith("_index") + and k not in model["categories"] + ] + ) + model["categories"] = categories return model @model_validator(mode="after") @@ -781,12 +835,19 @@ class AlignedDynamicTableMixin(BaseModel): """ lengths = [len(v) for v in self._categories.values()] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "AlignedDynamicTableColumns are not of equal length! " f"Got colnames:\n{self.categories}\nand lengths: {lengths}" ) return self +class ElementIdentifiersMixin(VectorDataMixin): + """ + Mixin class for ElementIdentifiers - allow treating + as generic, and give general indexing methods from VectorData + """ + + linkml_meta = LinkMLMeta( { "annotations": { @@ -801,7 +862,7 @@ linkml_meta = LinkMLMeta( ) -class VectorData(VectorDataMixin): +class VectorData(VectorDataMixin, ConfiguredBaseModel): """ An n-dimensional dataset representing a column of a DynamicTable. If used without an accompanying VectorIndex, first dimension is along the rows of the DynamicTable and each step along the first dimension is a cell of the larger table. VectorData can also be used to represent a ragged array if paired with a VectorIndex. This allows for storing arrays of varying length in a single cell of the DynamicTable by indexing into this VectorData. The first vector is at VectorData[0:VectorIndex[0]]. The second vector is at VectorData[VectorIndex[0]:VectorIndex[1]], and so on. """ @@ -812,17 +873,10 @@ class VectorData(VectorDataMixin): name: str = Field(...) description: str = Field(..., description="""Description of what these vectors represent.""") - value: Optional[ - Union[ - NDArray[Shape["* dim0"], Any], - NDArray[Shape["* dim0, * dim1"], Any], - NDArray[Shape["* dim0, * dim1, * dim2"], Any], - NDArray[Shape["* dim0, * dim1, * dim2, * dim3"], Any], - ] - ] = Field(None) + value: Optional[T] = Field(None) -class VectorIndex(VectorIndexMixin): +class VectorIndex(VectorIndexMixin, ConfiguredBaseModel): """ Used with VectorData to encode a ragged array. An array of indices into the first dimension of the target VectorData, and forming a map between the rows of a DynamicTable and the indices of the VectorData. The name of the VectorIndex is expected to be the name of the target VectorData object followed by \"_index\". """ @@ -846,7 +900,7 @@ class VectorIndex(VectorIndexMixin): ] = Field(None) -class ElementIdentifiers(Data): +class ElementIdentifiers(ElementIdentifiersMixin, Data, ConfiguredBaseModel): """ A list of unique identifiers for values within a dataset, e.g. rows of a DynamicTable. """ @@ -858,9 +912,13 @@ class ElementIdentifiers(Data): name: str = Field( "element_id", json_schema_extra={"linkml_meta": {"ifabsent": "string(element_id)"}} ) + value: Optional[T] = Field( + None, + json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_elements"}]}}}, + ) -class DynamicTableRegion(DynamicTableRegionMixin, VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData, ConfiguredBaseModel): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ @@ -912,7 +970,7 @@ class VocabData(VectorData): ] = Field(None) -class DynamicTable(DynamicTableMixin): +class DynamicTable(DynamicTableMixin, ConfiguredBaseModel): """ A group containing multiple datasets that are aligned on the first dimension (Currently, this requirement if left up to APIs to check and enforce). These datasets represent different columns in the table. Apart from a column that contains unique identifiers for each row, there are no other required datasets. Users are free to add any number of custom VectorData objects (columns) here. DynamicTable also supports ragged array columns, where each element can be of a different size. To add a ragged array column, use a VectorIndex type to index the corresponding VectorData type. See documentation for VectorData and VectorIndex for more details. Unlike a compound data type, which is analogous to storing an array-of-structs, a DynamicTable can be thought of as a struct-of-arrays. This provides an alternative structure to choose from when optimizing storage for anticipated access patterns. Additionally, this type provides a way of creating a table without having to define a compound type up front. Although this convenience may be attractive, users should think carefully about how data will be accessed. DynamicTable is more appropriate for column-centric access, whereas a dataset with a compound type would be more appropriate for row-centric access. Finally, data size should also be taken into account. For small tables, performance loss may be an acceptable trade-off for the flexibility of a DynamicTable. """ @@ -927,14 +985,11 @@ class DynamicTable(DynamicTableMixin): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/namespace.py index f8a0d6f..1338679 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_2_1/namespace.py @@ -35,7 +35,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -54,6 +54,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_base.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_base.py index d484652..a7ed66d 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_base.py @@ -19,7 +19,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -38,6 +38,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_resources.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_resources.py index 8f8f711..4d4850c 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_resources.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_resources.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_sparse.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_sparse.py index 4d93c70..2620eb6 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_sparse.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_sparse.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_table.py index ece0532..a55c212 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/hdmf_common_table.py @@ -46,7 +46,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -65,6 +65,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -98,7 +129,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -300,8 +331,11 @@ class DynamicTableMixin(BaseModel): NON_COLUMN_FIELDS: ClassVar[tuple[str]] = ( "id", "name", + "categories", "colnames", "description", + "hdf5_path", + "object_id", ) # overridden by subclass but implemented here for testing and typechecking purposes :) @@ -385,7 +419,7 @@ class DynamicTableMixin(BaseModel): # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -493,11 +527,14 @@ class DynamicTableMixin(BaseModel): if k not in cls.NON_COLUMN_FIELDS and not k.endswith("_index") and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] model["colnames"] = colnames else: # add any columns not explicitly given an order at the end colnames = model["colnames"].copy() + if isinstance(colnames, np.ndarray): + colnames = colnames.tolist() colnames.extend( [ k @@ -506,6 +543,7 @@ class DynamicTableMixin(BaseModel): and not k.endswith("_index") and k not in model["colnames"] and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] ) model["colnames"] = colnames @@ -524,17 +562,25 @@ class DynamicTableMixin(BaseModel): if isinstance(model, dict): for key, val in model.items(): - if key in cls.model_fields: + if key in cls.model_fields or key in cls.NON_COLUMN_FIELDS: continue if not isinstance(val, (VectorData, VectorIndex)): try: - if key.endswith("_index"): - model[key] = VectorIndex(name=key, description="", value=val) + to_cast = VectorIndex if key.endswith("_index") else VectorData + if isinstance(val, dict): + model[key] = to_cast(**val) else: - model[key] = VectorData(name=key, description="", value=val) + model[key] = to_cast(name=key, description="", value=val) except ValidationError as e: # pragma: no cover - raise ValidationError( - f"field {key} cannot be cast to VectorData from {val}" + raise ValidationError.from_exception_data( + title=f"field {key} cannot be cast to VectorData from {val}", + line_errors=[ + { + "type": "ValueError", + "loc": ("DynamicTableMixin", "cast_extra_columns"), + "input": val, + } + ], ) from e return model @@ -567,9 +613,9 @@ class DynamicTableMixin(BaseModel): """ Ensure that all columns are equal length """ - lengths = [len(v) for v in self._columns.values()] + [len(self.id)] + lengths = [len(v) for v in self._columns.values() if v is not None] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "DynamicTable columns are not of equal length! " f"Got colnames:\n{self.colnames}\nand lengths: {lengths}" ) return self @@ -619,10 +665,13 @@ class AlignedDynamicTableMixin(BaseModel): __pydantic_extra__: Dict[str, Union["DynamicTableMixin", "VectorDataMixin", "VectorIndexMixin"]] NON_CATEGORY_FIELDS: ClassVar[tuple[str]] = ( + "id", "name", "categories", "colnames", "description", + "hdf5_path", + "object_id", ) name: str = "aligned_table" @@ -652,28 +701,29 @@ class AlignedDynamicTableMixin(BaseModel): elif isinstance(item, tuple) and len(item) == 2 and isinstance(item[1], str): # get a slice of a single table return self._categories[item[1]][item[0]] - elif isinstance(item, (int, slice, Iterable)): + elif isinstance(item, (int, slice, Iterable, np.int_)): # get a slice of all the tables ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) - tables = [ids] + ids = pd.Index(data=ids, name="id") + tables = [] for category_name, category in self._categories.items(): table = category[item] if isinstance(table, pd.DataFrame): table = table.reset_index() + table.index = ids elif isinstance(table, np.ndarray): - table = pd.DataFrame({category_name: [table]}) + table = pd.DataFrame({category_name: [table]}, index=ids) elif isinstance(table, Iterable): - table = pd.DataFrame({category_name: table}) + table = pd.DataFrame({category_name: table}, index=ids) else: raise ValueError( f"Don't know how to construct category table for {category_name}" ) tables.append(table) - names = [self.name] + self.categories + # names = [self.name] + self.categories # construct below in case we need to support array indexing in the future else: raise ValueError( @@ -681,8 +731,7 @@ class AlignedDynamicTableMixin(BaseModel): "need an int, string, slice, ndarray, or tuple[int | slice, str]" ) - df = pd.concat(tables, axis=1, keys=names) - df.set_index((self.name, "id"), drop=True, inplace=True) + df = pd.concat(tables, axis=1, keys=self.categories) return df def __getattr__(self, item: str) -> Any: @@ -740,14 +789,19 @@ class AlignedDynamicTableMixin(BaseModel): model["categories"] = categories else: # add any columns not explicitly given an order at the end - categories = [ - k - for k in model - if k not in cls.NON_COLUMN_FIELDS - and not k.endswith("_index") - and k not in model["categories"] - ] - model["categories"].extend(categories) + categories = model["categories"].copy() + if isinstance(categories, np.ndarray): + categories = categories.tolist() + categories.extend( + [ + k + for k in model + if k not in cls.NON_CATEGORY_FIELDS + and not k.endswith("_index") + and k not in model["categories"] + ] + ) + model["categories"] = categories return model @model_validator(mode="after") @@ -781,12 +835,19 @@ class AlignedDynamicTableMixin(BaseModel): """ lengths = [len(v) for v in self._categories.values()] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "AlignedDynamicTableColumns are not of equal length! " f"Got colnames:\n{self.categories}\nand lengths: {lengths}" ) return self +class ElementIdentifiersMixin(VectorDataMixin): + """ + Mixin class for ElementIdentifiers - allow treating + as generic, and give general indexing methods from VectorData + """ + + linkml_meta = LinkMLMeta( { "annotations": { @@ -801,7 +862,7 @@ linkml_meta = LinkMLMeta( ) -class VectorData(VectorDataMixin): +class VectorData(VectorDataMixin, ConfiguredBaseModel): """ An n-dimensional dataset representing a column of a DynamicTable. If used without an accompanying VectorIndex, first dimension is along the rows of the DynamicTable and each step along the first dimension is a cell of the larger table. VectorData can also be used to represent a ragged array if paired with a VectorIndex. This allows for storing arrays of varying length in a single cell of the DynamicTable by indexing into this VectorData. The first vector is at VectorData[0:VectorIndex[0]]. The second vector is at VectorData[VectorIndex[0]:VectorIndex[1]], and so on. """ @@ -812,17 +873,10 @@ class VectorData(VectorDataMixin): name: str = Field(...) description: str = Field(..., description="""Description of what these vectors represent.""") - value: Optional[ - Union[ - NDArray[Shape["* dim0"], Any], - NDArray[Shape["* dim0, * dim1"], Any], - NDArray[Shape["* dim0, * dim1, * dim2"], Any], - NDArray[Shape["* dim0, * dim1, * dim2, * dim3"], Any], - ] - ] = Field(None) + value: Optional[T] = Field(None) -class VectorIndex(VectorIndexMixin): +class VectorIndex(VectorIndexMixin, ConfiguredBaseModel): """ Used with VectorData to encode a ragged array. An array of indices into the first dimension of the target VectorData, and forming a map between the rows of a DynamicTable and the indices of the VectorData. The name of the VectorIndex is expected to be the name of the target VectorData object followed by \"_index\". """ @@ -846,7 +900,7 @@ class VectorIndex(VectorIndexMixin): ] = Field(None) -class ElementIdentifiers(Data): +class ElementIdentifiers(ElementIdentifiersMixin, Data, ConfiguredBaseModel): """ A list of unique identifiers for values within a dataset, e.g. rows of a DynamicTable. """ @@ -858,9 +912,13 @@ class ElementIdentifiers(Data): name: str = Field( "element_id", json_schema_extra={"linkml_meta": {"ifabsent": "string(element_id)"}} ) + value: Optional[T] = Field( + None, + json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_elements"}]}}}, + ) -class DynamicTableRegion(DynamicTableRegionMixin, VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData, ConfiguredBaseModel): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ @@ -912,7 +970,7 @@ class VocabData(VectorData): ] = Field(None) -class DynamicTable(DynamicTableMixin): +class DynamicTable(DynamicTableMixin, ConfiguredBaseModel): """ A group containing multiple datasets that are aligned on the first dimension (Currently, this requirement if left up to APIs to check and enforce). These datasets represent different columns in the table. Apart from a column that contains unique identifiers for each row, there are no other required datasets. Users are free to add any number of custom VectorData objects (columns) here. DynamicTable also supports ragged array columns, where each element can be of a different size. To add a ragged array column, use a VectorIndex type to index the corresponding VectorData type. See documentation for VectorData and VectorIndex for more details. Unlike a compound data type, which is analogous to storing an array-of-structs, a DynamicTable can be thought of as a struct-of-arrays. This provides an alternative structure to choose from when optimizing storage for anticipated access patterns. Additionally, this type provides a way of creating a table without having to define a compound type up front. Although this convenience may be attractive, users should think carefully about how data will be accessed. DynamicTable is more appropriate for column-centric access, whereas a dataset with a compound type would be more appropriate for row-centric access. Finally, data size should also be taken into account. For small tables, performance loss may be an acceptable trade-off for the flexibility of a DynamicTable. """ @@ -927,14 +985,11 @@ class DynamicTable(DynamicTableMixin): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/namespace.py index dcb742c..040adf7 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_3_0/namespace.py @@ -37,7 +37,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -56,6 +56,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/hdmf_common_base.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/hdmf_common_base.py index f47e8ca..02d67bf 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/hdmf_common_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/hdmf_common_base.py @@ -19,7 +19,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -38,6 +38,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -103,7 +134,7 @@ class SimpleMultiContainer(Container): {"from_schema": "hdmf-common.base", "tree_root": True} ) - value: Optional[List[Container]] = Field( + value: Optional[Dict[str, Container]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "Container"}]}} ) name: str = Field(...) diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/hdmf_common_sparse.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/hdmf_common_sparse.py index af8cc73..ad70998 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/hdmf_common_sparse.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/hdmf_common_sparse.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -101,23 +132,15 @@ class CSRMatrix(Container): "linkml_meta": {"array": {"dimensions": [{"alias": "number_of_rows_in_the_matrix_1"}]}} }, ) - data: CSRMatrixData = Field(..., description="""The non-zero values in the matrix.""") - - -class CSRMatrixData(ConfiguredBaseModel): - """ - The non-zero values in the matrix. - """ - - linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta({"from_schema": "hdmf-common.sparse"}) - - name: Literal["data"] = Field( - "data", - json_schema_extra={"linkml_meta": {"equals_string": "data", "ifabsent": "string(data)"}}, + data: NDArray[Shape["* number_of_non_zero_values"], Any] = Field( + ..., + description="""The non-zero values in the matrix.""", + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "number_of_non_zero_values"}]}} + }, ) # Model rebuild # see https://pydantic-docs.helpmanual.io/usage/models/#rebuilding-a-model CSRMatrix.model_rebuild() -CSRMatrixData.model_rebuild() diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/hdmf_common_table.py index d1202ef..a730ec1 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/hdmf_common_table.py @@ -46,7 +46,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -65,6 +65,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -98,7 +129,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -300,8 +331,11 @@ class DynamicTableMixin(BaseModel): NON_COLUMN_FIELDS: ClassVar[tuple[str]] = ( "id", "name", + "categories", "colnames", "description", + "hdf5_path", + "object_id", ) # overridden by subclass but implemented here for testing and typechecking purposes :) @@ -385,7 +419,7 @@ class DynamicTableMixin(BaseModel): # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -493,11 +527,14 @@ class DynamicTableMixin(BaseModel): if k not in cls.NON_COLUMN_FIELDS and not k.endswith("_index") and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] model["colnames"] = colnames else: # add any columns not explicitly given an order at the end colnames = model["colnames"].copy() + if isinstance(colnames, np.ndarray): + colnames = colnames.tolist() colnames.extend( [ k @@ -506,6 +543,7 @@ class DynamicTableMixin(BaseModel): and not k.endswith("_index") and k not in model["colnames"] and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] ) model["colnames"] = colnames @@ -524,17 +562,25 @@ class DynamicTableMixin(BaseModel): if isinstance(model, dict): for key, val in model.items(): - if key in cls.model_fields: + if key in cls.model_fields or key in cls.NON_COLUMN_FIELDS: continue if not isinstance(val, (VectorData, VectorIndex)): try: - if key.endswith("_index"): - model[key] = VectorIndex(name=key, description="", value=val) + to_cast = VectorIndex if key.endswith("_index") else VectorData + if isinstance(val, dict): + model[key] = to_cast(**val) else: - model[key] = VectorData(name=key, description="", value=val) + model[key] = to_cast(name=key, description="", value=val) except ValidationError as e: # pragma: no cover - raise ValidationError( - f"field {key} cannot be cast to VectorData from {val}" + raise ValidationError.from_exception_data( + title=f"field {key} cannot be cast to VectorData from {val}", + line_errors=[ + { + "type": "ValueError", + "loc": ("DynamicTableMixin", "cast_extra_columns"), + "input": val, + } + ], ) from e return model @@ -567,9 +613,9 @@ class DynamicTableMixin(BaseModel): """ Ensure that all columns are equal length """ - lengths = [len(v) for v in self._columns.values()] + [len(self.id)] + lengths = [len(v) for v in self._columns.values() if v is not None] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "DynamicTable columns are not of equal length! " f"Got colnames:\n{self.colnames}\nand lengths: {lengths}" ) return self @@ -619,10 +665,13 @@ class AlignedDynamicTableMixin(BaseModel): __pydantic_extra__: Dict[str, Union["DynamicTableMixin", "VectorDataMixin", "VectorIndexMixin"]] NON_CATEGORY_FIELDS: ClassVar[tuple[str]] = ( + "id", "name", "categories", "colnames", "description", + "hdf5_path", + "object_id", ) name: str = "aligned_table" @@ -652,28 +701,29 @@ class AlignedDynamicTableMixin(BaseModel): elif isinstance(item, tuple) and len(item) == 2 and isinstance(item[1], str): # get a slice of a single table return self._categories[item[1]][item[0]] - elif isinstance(item, (int, slice, Iterable)): + elif isinstance(item, (int, slice, Iterable, np.int_)): # get a slice of all the tables ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) - tables = [ids] + ids = pd.Index(data=ids, name="id") + tables = [] for category_name, category in self._categories.items(): table = category[item] if isinstance(table, pd.DataFrame): table = table.reset_index() + table.index = ids elif isinstance(table, np.ndarray): - table = pd.DataFrame({category_name: [table]}) + table = pd.DataFrame({category_name: [table]}, index=ids) elif isinstance(table, Iterable): - table = pd.DataFrame({category_name: table}) + table = pd.DataFrame({category_name: table}, index=ids) else: raise ValueError( f"Don't know how to construct category table for {category_name}" ) tables.append(table) - names = [self.name] + self.categories + # names = [self.name] + self.categories # construct below in case we need to support array indexing in the future else: raise ValueError( @@ -681,8 +731,7 @@ class AlignedDynamicTableMixin(BaseModel): "need an int, string, slice, ndarray, or tuple[int | slice, str]" ) - df = pd.concat(tables, axis=1, keys=names) - df.set_index((self.name, "id"), drop=True, inplace=True) + df = pd.concat(tables, axis=1, keys=self.categories) return df def __getattr__(self, item: str) -> Any: @@ -740,14 +789,19 @@ class AlignedDynamicTableMixin(BaseModel): model["categories"] = categories else: # add any columns not explicitly given an order at the end - categories = [ - k - for k in model - if k not in cls.NON_COLUMN_FIELDS - and not k.endswith("_index") - and k not in model["categories"] - ] - model["categories"].extend(categories) + categories = model["categories"].copy() + if isinstance(categories, np.ndarray): + categories = categories.tolist() + categories.extend( + [ + k + for k in model + if k not in cls.NON_CATEGORY_FIELDS + and not k.endswith("_index") + and k not in model["categories"] + ] + ) + model["categories"] = categories return model @model_validator(mode="after") @@ -781,12 +835,19 @@ class AlignedDynamicTableMixin(BaseModel): """ lengths = [len(v) for v in self._categories.values()] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "AlignedDynamicTableColumns are not of equal length! " f"Got colnames:\n{self.categories}\nand lengths: {lengths}" ) return self +class ElementIdentifiersMixin(VectorDataMixin): + """ + Mixin class for ElementIdentifiers - allow treating + as generic, and give general indexing methods from VectorData + """ + + linkml_meta = LinkMLMeta( { "annotations": { @@ -801,7 +862,7 @@ linkml_meta = LinkMLMeta( ) -class VectorData(VectorDataMixin): +class VectorData(VectorDataMixin, ConfiguredBaseModel): """ An n-dimensional dataset representing a column of a DynamicTable. If used without an accompanying VectorIndex, first dimension is along the rows of the DynamicTable and each step along the first dimension is a cell of the larger table. VectorData can also be used to represent a ragged array if paired with a VectorIndex. This allows for storing arrays of varying length in a single cell of the DynamicTable by indexing into this VectorData. The first vector is at VectorData[0:VectorIndex[0]]. The second vector is at VectorData[VectorIndex[0]:VectorIndex[1]], and so on. """ @@ -812,17 +873,10 @@ class VectorData(VectorDataMixin): name: str = Field(...) description: str = Field(..., description="""Description of what these vectors represent.""") - value: Optional[ - Union[ - NDArray[Shape["* dim0"], Any], - NDArray[Shape["* dim0, * dim1"], Any], - NDArray[Shape["* dim0, * dim1, * dim2"], Any], - NDArray[Shape["* dim0, * dim1, * dim2, * dim3"], Any], - ] - ] = Field(None) + value: Optional[T] = Field(None) -class VectorIndex(VectorIndexMixin): +class VectorIndex(VectorIndexMixin, ConfiguredBaseModel): """ Used with VectorData to encode a ragged array. An array of indices into the first dimension of the target VectorData, and forming a map between the rows of a DynamicTable and the indices of the VectorData. The name of the VectorIndex is expected to be the name of the target VectorData object followed by \"_index\". """ @@ -846,7 +900,7 @@ class VectorIndex(VectorIndexMixin): ] = Field(None) -class ElementIdentifiers(Data): +class ElementIdentifiers(ElementIdentifiersMixin, Data, ConfiguredBaseModel): """ A list of unique identifiers for values within a dataset, e.g. rows of a DynamicTable. """ @@ -858,9 +912,13 @@ class ElementIdentifiers(Data): name: str = Field( "element_id", json_schema_extra={"linkml_meta": {"ifabsent": "string(element_id)"}} ) + value: Optional[T] = Field( + None, + json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_elements"}]}}}, + ) -class DynamicTableRegion(DynamicTableRegionMixin, VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData, ConfiguredBaseModel): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ @@ -886,7 +944,7 @@ class DynamicTableRegion(DynamicTableRegionMixin, VectorData): ] = Field(None) -class DynamicTable(DynamicTableMixin): +class DynamicTable(DynamicTableMixin, ConfiguredBaseModel): """ A group containing multiple datasets that are aligned on the first dimension (Currently, this requirement if left up to APIs to check and enforce). These datasets represent different columns in the table. Apart from a column that contains unique identifiers for each row, there are no other required datasets. Users are free to add any number of custom VectorData objects (columns) here. DynamicTable also supports ragged array columns, where each element can be of a different size. To add a ragged array column, use a VectorIndex type to index the corresponding VectorData type. See documentation for VectorData and VectorIndex for more details. Unlike a compound data type, which is analogous to storing an array-of-structs, a DynamicTable can be thought of as a struct-of-arrays. This provides an alternative structure to choose from when optimizing storage for anticipated access patterns. Additionally, this type provides a way of creating a table without having to define a compound type up front. Although this convenience may be attractive, users should think carefully about how data will be accessed. DynamicTable is more appropriate for column-centric access, whereas a dataset with a compound type would be more appropriate for row-centric access. Finally, data size should also be taken into account. For small tables, performance loss may be an acceptable trade-off for the flexibility of a DynamicTable. """ @@ -901,14 +959,11 @@ class DynamicTable(DynamicTableMixin): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/namespace.py index b110f2d..0a85a76 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_4_0/namespace.py @@ -11,7 +11,7 @@ import numpy as np from pydantic import BaseModel, ConfigDict, Field, RootModel, field_validator from ...hdmf_common.v1_4_0.hdmf_common_base import Container, Data, SimpleMultiContainer -from ...hdmf_common.v1_4_0.hdmf_common_sparse import CSRMatrix, CSRMatrixData +from ...hdmf_common.v1_4_0.hdmf_common_sparse import CSRMatrix from ...hdmf_common.v1_4_0.hdmf_common_table import ( DynamicTable, DynamicTableRegion, @@ -29,7 +29,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -48,6 +48,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/hdmf_common_base.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/hdmf_common_base.py index 2412f82..7c62f93 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/hdmf_common_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/hdmf_common_base.py @@ -19,7 +19,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -38,6 +38,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -103,7 +134,7 @@ class SimpleMultiContainer(Container): {"from_schema": "hdmf-common.base", "tree_root": True} ) - value: Optional[List[Container]] = Field( + value: Optional[Dict[str, Container]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "Container"}]}} ) name: str = Field(...) diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/hdmf_common_sparse.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/hdmf_common_sparse.py index 21258d8..d434cd9 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/hdmf_common_sparse.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/hdmf_common_sparse.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -101,23 +132,15 @@ class CSRMatrix(Container): "linkml_meta": {"array": {"dimensions": [{"alias": "number_of_rows_in_the_matrix_1"}]}} }, ) - data: CSRMatrixData = Field(..., description="""The non-zero values in the matrix.""") - - -class CSRMatrixData(ConfiguredBaseModel): - """ - The non-zero values in the matrix. - """ - - linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta({"from_schema": "hdmf-common.sparse"}) - - name: Literal["data"] = Field( - "data", - json_schema_extra={"linkml_meta": {"equals_string": "data", "ifabsent": "string(data)"}}, + data: NDArray[Shape["* number_of_non_zero_values"], Any] = Field( + ..., + description="""The non-zero values in the matrix.""", + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "number_of_non_zero_values"}]}} + }, ) # Model rebuild # see https://pydantic-docs.helpmanual.io/usage/models/#rebuilding-a-model CSRMatrix.model_rebuild() -CSRMatrixData.model_rebuild() diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/hdmf_common_table.py index c1b61d9..27a287c 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/hdmf_common_table.py @@ -46,7 +46,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -65,6 +65,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -98,7 +129,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -300,8 +331,11 @@ class DynamicTableMixin(BaseModel): NON_COLUMN_FIELDS: ClassVar[tuple[str]] = ( "id", "name", + "categories", "colnames", "description", + "hdf5_path", + "object_id", ) # overridden by subclass but implemented here for testing and typechecking purposes :) @@ -385,7 +419,7 @@ class DynamicTableMixin(BaseModel): # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -493,11 +527,14 @@ class DynamicTableMixin(BaseModel): if k not in cls.NON_COLUMN_FIELDS and not k.endswith("_index") and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] model["colnames"] = colnames else: # add any columns not explicitly given an order at the end colnames = model["colnames"].copy() + if isinstance(colnames, np.ndarray): + colnames = colnames.tolist() colnames.extend( [ k @@ -506,6 +543,7 @@ class DynamicTableMixin(BaseModel): and not k.endswith("_index") and k not in model["colnames"] and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] ) model["colnames"] = colnames @@ -524,17 +562,25 @@ class DynamicTableMixin(BaseModel): if isinstance(model, dict): for key, val in model.items(): - if key in cls.model_fields: + if key in cls.model_fields or key in cls.NON_COLUMN_FIELDS: continue if not isinstance(val, (VectorData, VectorIndex)): try: - if key.endswith("_index"): - model[key] = VectorIndex(name=key, description="", value=val) + to_cast = VectorIndex if key.endswith("_index") else VectorData + if isinstance(val, dict): + model[key] = to_cast(**val) else: - model[key] = VectorData(name=key, description="", value=val) + model[key] = to_cast(name=key, description="", value=val) except ValidationError as e: # pragma: no cover - raise ValidationError( - f"field {key} cannot be cast to VectorData from {val}" + raise ValidationError.from_exception_data( + title=f"field {key} cannot be cast to VectorData from {val}", + line_errors=[ + { + "type": "ValueError", + "loc": ("DynamicTableMixin", "cast_extra_columns"), + "input": val, + } + ], ) from e return model @@ -567,9 +613,9 @@ class DynamicTableMixin(BaseModel): """ Ensure that all columns are equal length """ - lengths = [len(v) for v in self._columns.values()] + [len(self.id)] + lengths = [len(v) for v in self._columns.values() if v is not None] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "DynamicTable columns are not of equal length! " f"Got colnames:\n{self.colnames}\nand lengths: {lengths}" ) return self @@ -619,10 +665,13 @@ class AlignedDynamicTableMixin(BaseModel): __pydantic_extra__: Dict[str, Union["DynamicTableMixin", "VectorDataMixin", "VectorIndexMixin"]] NON_CATEGORY_FIELDS: ClassVar[tuple[str]] = ( + "id", "name", "categories", "colnames", "description", + "hdf5_path", + "object_id", ) name: str = "aligned_table" @@ -652,28 +701,29 @@ class AlignedDynamicTableMixin(BaseModel): elif isinstance(item, tuple) and len(item) == 2 and isinstance(item[1], str): # get a slice of a single table return self._categories[item[1]][item[0]] - elif isinstance(item, (int, slice, Iterable)): + elif isinstance(item, (int, slice, Iterable, np.int_)): # get a slice of all the tables ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) - tables = [ids] + ids = pd.Index(data=ids, name="id") + tables = [] for category_name, category in self._categories.items(): table = category[item] if isinstance(table, pd.DataFrame): table = table.reset_index() + table.index = ids elif isinstance(table, np.ndarray): - table = pd.DataFrame({category_name: [table]}) + table = pd.DataFrame({category_name: [table]}, index=ids) elif isinstance(table, Iterable): - table = pd.DataFrame({category_name: table}) + table = pd.DataFrame({category_name: table}, index=ids) else: raise ValueError( f"Don't know how to construct category table for {category_name}" ) tables.append(table) - names = [self.name] + self.categories + # names = [self.name] + self.categories # construct below in case we need to support array indexing in the future else: raise ValueError( @@ -681,8 +731,7 @@ class AlignedDynamicTableMixin(BaseModel): "need an int, string, slice, ndarray, or tuple[int | slice, str]" ) - df = pd.concat(tables, axis=1, keys=names) - df.set_index((self.name, "id"), drop=True, inplace=True) + df = pd.concat(tables, axis=1, keys=self.categories) return df def __getattr__(self, item: str) -> Any: @@ -740,14 +789,19 @@ class AlignedDynamicTableMixin(BaseModel): model["categories"] = categories else: # add any columns not explicitly given an order at the end - categories = [ - k - for k in model - if k not in cls.NON_COLUMN_FIELDS - and not k.endswith("_index") - and k not in model["categories"] - ] - model["categories"].extend(categories) + categories = model["categories"].copy() + if isinstance(categories, np.ndarray): + categories = categories.tolist() + categories.extend( + [ + k + for k in model + if k not in cls.NON_CATEGORY_FIELDS + and not k.endswith("_index") + and k not in model["categories"] + ] + ) + model["categories"] = categories return model @model_validator(mode="after") @@ -781,12 +835,19 @@ class AlignedDynamicTableMixin(BaseModel): """ lengths = [len(v) for v in self._categories.values()] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "AlignedDynamicTableColumns are not of equal length! " f"Got colnames:\n{self.categories}\nand lengths: {lengths}" ) return self +class ElementIdentifiersMixin(VectorDataMixin): + """ + Mixin class for ElementIdentifiers - allow treating + as generic, and give general indexing methods from VectorData + """ + + linkml_meta = LinkMLMeta( { "annotations": { @@ -801,7 +862,7 @@ linkml_meta = LinkMLMeta( ) -class VectorData(VectorDataMixin): +class VectorData(VectorDataMixin, ConfiguredBaseModel): """ An n-dimensional dataset representing a column of a DynamicTable. If used without an accompanying VectorIndex, first dimension is along the rows of the DynamicTable and each step along the first dimension is a cell of the larger table. VectorData can also be used to represent a ragged array if paired with a VectorIndex. This allows for storing arrays of varying length in a single cell of the DynamicTable by indexing into this VectorData. The first vector is at VectorData[0:VectorIndex[0]]. The second vector is at VectorData[VectorIndex[0]:VectorIndex[1]], and so on. """ @@ -812,17 +873,10 @@ class VectorData(VectorDataMixin): name: str = Field(...) description: str = Field(..., description="""Description of what these vectors represent.""") - value: Optional[ - Union[ - NDArray[Shape["* dim0"], Any], - NDArray[Shape["* dim0, * dim1"], Any], - NDArray[Shape["* dim0, * dim1, * dim2"], Any], - NDArray[Shape["* dim0, * dim1, * dim2, * dim3"], Any], - ] - ] = Field(None) + value: Optional[T] = Field(None) -class VectorIndex(VectorIndexMixin): +class VectorIndex(VectorIndexMixin, ConfiguredBaseModel): """ Used with VectorData to encode a ragged array. An array of indices into the first dimension of the target VectorData, and forming a map between the rows of a DynamicTable and the indices of the VectorData. The name of the VectorIndex is expected to be the name of the target VectorData object followed by \"_index\". """ @@ -846,7 +900,7 @@ class VectorIndex(VectorIndexMixin): ] = Field(None) -class ElementIdentifiers(Data): +class ElementIdentifiers(ElementIdentifiersMixin, Data, ConfiguredBaseModel): """ A list of unique identifiers for values within a dataset, e.g. rows of a DynamicTable. """ @@ -858,9 +912,13 @@ class ElementIdentifiers(Data): name: str = Field( "element_id", json_schema_extra={"linkml_meta": {"ifabsent": "string(element_id)"}} ) + value: Optional[T] = Field( + None, + json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_elements"}]}}}, + ) -class DynamicTableRegion(DynamicTableRegionMixin, VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData, ConfiguredBaseModel): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ @@ -886,7 +944,7 @@ class DynamicTableRegion(DynamicTableRegionMixin, VectorData): ] = Field(None) -class DynamicTable(DynamicTableMixin): +class DynamicTable(DynamicTableMixin, ConfiguredBaseModel): """ A group containing multiple datasets that are aligned on the first dimension (Currently, this requirement if left up to APIs to check and enforce). These datasets represent different columns in the table. Apart from a column that contains unique identifiers for each row, there are no other required datasets. Users are free to add any number of custom VectorData objects (columns) here. DynamicTable also supports ragged array columns, where each element can be of a different size. To add a ragged array column, use a VectorIndex type to index the corresponding VectorData type. See documentation for VectorData and VectorIndex for more details. Unlike a compound data type, which is analogous to storing an array-of-structs, a DynamicTable can be thought of as a struct-of-arrays. This provides an alternative structure to choose from when optimizing storage for anticipated access patterns. Additionally, this type provides a way of creating a table without having to define a compound type up front. Although this convenience may be attractive, users should think carefully about how data will be accessed. DynamicTable is more appropriate for column-centric access, whereas a dataset with a compound type would be more appropriate for row-centric access. Finally, data size should also be taken into account. For small tables, performance loss may be an acceptable trade-off for the flexibility of a DynamicTable. """ @@ -901,14 +959,11 @@ class DynamicTable(DynamicTableMixin): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class AlignedDynamicTable(AlignedDynamicTableMixin, DynamicTable): @@ -920,7 +975,7 @@ class AlignedDynamicTable(AlignedDynamicTableMixin, DynamicTable): {"from_schema": "hdmf-common.table", "tree_root": True} ) - value: Optional[List[DynamicTable]] = Field( + value: Optional[Dict[str, DynamicTable]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "DynamicTable"}]}} ) name: str = Field(...) @@ -929,14 +984,11 @@ class AlignedDynamicTable(AlignedDynamicTableMixin, DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/namespace.py index d5c14d9..6e04fd0 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_0/namespace.py @@ -11,7 +11,7 @@ import numpy as np from pydantic import BaseModel, ConfigDict, Field, RootModel, field_validator from ...hdmf_common.v1_5_0.hdmf_common_base import Container, Data, SimpleMultiContainer -from ...hdmf_common.v1_5_0.hdmf_common_sparse import CSRMatrix, CSRMatrixData +from ...hdmf_common.v1_5_0.hdmf_common_sparse import CSRMatrix from ...hdmf_common.v1_5_0.hdmf_common_table import ( AlignedDynamicTable, DynamicTable, @@ -30,7 +30,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -49,6 +49,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/hdmf_common_base.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/hdmf_common_base.py index f3c24b4..df22948 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/hdmf_common_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/hdmf_common_base.py @@ -19,7 +19,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -38,6 +38,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -103,7 +134,7 @@ class SimpleMultiContainer(Container): {"from_schema": "hdmf-common.base", "tree_root": True} ) - value: Optional[List[Container]] = Field( + value: Optional[Dict[str, Container]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "Container"}]}} ) name: str = Field(...) diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/hdmf_common_sparse.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/hdmf_common_sparse.py index fce455b..4e921cc 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/hdmf_common_sparse.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/hdmf_common_sparse.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -101,23 +132,15 @@ class CSRMatrix(Container): "linkml_meta": {"array": {"dimensions": [{"alias": "number_of_rows_in_the_matrix_1"}]}} }, ) - data: CSRMatrixData = Field(..., description="""The non-zero values in the matrix.""") - - -class CSRMatrixData(ConfiguredBaseModel): - """ - The non-zero values in the matrix. - """ - - linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta({"from_schema": "hdmf-common.sparse"}) - - name: Literal["data"] = Field( - "data", - json_schema_extra={"linkml_meta": {"equals_string": "data", "ifabsent": "string(data)"}}, + data: NDArray[Shape["* number_of_non_zero_values"], Any] = Field( + ..., + description="""The non-zero values in the matrix.""", + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "number_of_non_zero_values"}]}} + }, ) # Model rebuild # see https://pydantic-docs.helpmanual.io/usage/models/#rebuilding-a-model CSRMatrix.model_rebuild() -CSRMatrixData.model_rebuild() diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/hdmf_common_table.py index 4a91362..3112a4f 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/hdmf_common_table.py @@ -46,7 +46,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -65,6 +65,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -98,7 +129,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -300,8 +331,11 @@ class DynamicTableMixin(BaseModel): NON_COLUMN_FIELDS: ClassVar[tuple[str]] = ( "id", "name", + "categories", "colnames", "description", + "hdf5_path", + "object_id", ) # overridden by subclass but implemented here for testing and typechecking purposes :) @@ -385,7 +419,7 @@ class DynamicTableMixin(BaseModel): # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -493,11 +527,14 @@ class DynamicTableMixin(BaseModel): if k not in cls.NON_COLUMN_FIELDS and not k.endswith("_index") and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] model["colnames"] = colnames else: # add any columns not explicitly given an order at the end colnames = model["colnames"].copy() + if isinstance(colnames, np.ndarray): + colnames = colnames.tolist() colnames.extend( [ k @@ -506,6 +543,7 @@ class DynamicTableMixin(BaseModel): and not k.endswith("_index") and k not in model["colnames"] and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] ) model["colnames"] = colnames @@ -524,17 +562,25 @@ class DynamicTableMixin(BaseModel): if isinstance(model, dict): for key, val in model.items(): - if key in cls.model_fields: + if key in cls.model_fields or key in cls.NON_COLUMN_FIELDS: continue if not isinstance(val, (VectorData, VectorIndex)): try: - if key.endswith("_index"): - model[key] = VectorIndex(name=key, description="", value=val) + to_cast = VectorIndex if key.endswith("_index") else VectorData + if isinstance(val, dict): + model[key] = to_cast(**val) else: - model[key] = VectorData(name=key, description="", value=val) + model[key] = to_cast(name=key, description="", value=val) except ValidationError as e: # pragma: no cover - raise ValidationError( - f"field {key} cannot be cast to VectorData from {val}" + raise ValidationError.from_exception_data( + title=f"field {key} cannot be cast to VectorData from {val}", + line_errors=[ + { + "type": "ValueError", + "loc": ("DynamicTableMixin", "cast_extra_columns"), + "input": val, + } + ], ) from e return model @@ -567,9 +613,9 @@ class DynamicTableMixin(BaseModel): """ Ensure that all columns are equal length """ - lengths = [len(v) for v in self._columns.values()] + [len(self.id)] + lengths = [len(v) for v in self._columns.values() if v is not None] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "DynamicTable columns are not of equal length! " f"Got colnames:\n{self.colnames}\nand lengths: {lengths}" ) return self @@ -619,10 +665,13 @@ class AlignedDynamicTableMixin(BaseModel): __pydantic_extra__: Dict[str, Union["DynamicTableMixin", "VectorDataMixin", "VectorIndexMixin"]] NON_CATEGORY_FIELDS: ClassVar[tuple[str]] = ( + "id", "name", "categories", "colnames", "description", + "hdf5_path", + "object_id", ) name: str = "aligned_table" @@ -652,28 +701,29 @@ class AlignedDynamicTableMixin(BaseModel): elif isinstance(item, tuple) and len(item) == 2 and isinstance(item[1], str): # get a slice of a single table return self._categories[item[1]][item[0]] - elif isinstance(item, (int, slice, Iterable)): + elif isinstance(item, (int, slice, Iterable, np.int_)): # get a slice of all the tables ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) - tables = [ids] + ids = pd.Index(data=ids, name="id") + tables = [] for category_name, category in self._categories.items(): table = category[item] if isinstance(table, pd.DataFrame): table = table.reset_index() + table.index = ids elif isinstance(table, np.ndarray): - table = pd.DataFrame({category_name: [table]}) + table = pd.DataFrame({category_name: [table]}, index=ids) elif isinstance(table, Iterable): - table = pd.DataFrame({category_name: table}) + table = pd.DataFrame({category_name: table}, index=ids) else: raise ValueError( f"Don't know how to construct category table for {category_name}" ) tables.append(table) - names = [self.name] + self.categories + # names = [self.name] + self.categories # construct below in case we need to support array indexing in the future else: raise ValueError( @@ -681,8 +731,7 @@ class AlignedDynamicTableMixin(BaseModel): "need an int, string, slice, ndarray, or tuple[int | slice, str]" ) - df = pd.concat(tables, axis=1, keys=names) - df.set_index((self.name, "id"), drop=True, inplace=True) + df = pd.concat(tables, axis=1, keys=self.categories) return df def __getattr__(self, item: str) -> Any: @@ -740,14 +789,19 @@ class AlignedDynamicTableMixin(BaseModel): model["categories"] = categories else: # add any columns not explicitly given an order at the end - categories = [ - k - for k in model - if k not in cls.NON_COLUMN_FIELDS - and not k.endswith("_index") - and k not in model["categories"] - ] - model["categories"].extend(categories) + categories = model["categories"].copy() + if isinstance(categories, np.ndarray): + categories = categories.tolist() + categories.extend( + [ + k + for k in model + if k not in cls.NON_CATEGORY_FIELDS + and not k.endswith("_index") + and k not in model["categories"] + ] + ) + model["categories"] = categories return model @model_validator(mode="after") @@ -781,12 +835,19 @@ class AlignedDynamicTableMixin(BaseModel): """ lengths = [len(v) for v in self._categories.values()] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "AlignedDynamicTableColumns are not of equal length! " f"Got colnames:\n{self.categories}\nand lengths: {lengths}" ) return self +class ElementIdentifiersMixin(VectorDataMixin): + """ + Mixin class for ElementIdentifiers - allow treating + as generic, and give general indexing methods from VectorData + """ + + linkml_meta = LinkMLMeta( { "annotations": { @@ -801,7 +862,7 @@ linkml_meta = LinkMLMeta( ) -class VectorData(VectorDataMixin): +class VectorData(VectorDataMixin, ConfiguredBaseModel): """ An n-dimensional dataset representing a column of a DynamicTable. If used without an accompanying VectorIndex, first dimension is along the rows of the DynamicTable and each step along the first dimension is a cell of the larger table. VectorData can also be used to represent a ragged array if paired with a VectorIndex. This allows for storing arrays of varying length in a single cell of the DynamicTable by indexing into this VectorData. The first vector is at VectorData[0:VectorIndex[0]]. The second vector is at VectorData[VectorIndex[0]:VectorIndex[1]], and so on. """ @@ -812,17 +873,10 @@ class VectorData(VectorDataMixin): name: str = Field(...) description: str = Field(..., description="""Description of what these vectors represent.""") - value: Optional[ - Union[ - NDArray[Shape["* dim0"], Any], - NDArray[Shape["* dim0, * dim1"], Any], - NDArray[Shape["* dim0, * dim1, * dim2"], Any], - NDArray[Shape["* dim0, * dim1, * dim2, * dim3"], Any], - ] - ] = Field(None) + value: Optional[T] = Field(None) -class VectorIndex(VectorIndexMixin): +class VectorIndex(VectorIndexMixin, ConfiguredBaseModel): """ Used with VectorData to encode a ragged array. An array of indices into the first dimension of the target VectorData, and forming a map between the rows of a DynamicTable and the indices of the VectorData. The name of the VectorIndex is expected to be the name of the target VectorData object followed by \"_index\". """ @@ -846,7 +900,7 @@ class VectorIndex(VectorIndexMixin): ] = Field(None) -class ElementIdentifiers(Data): +class ElementIdentifiers(ElementIdentifiersMixin, Data, ConfiguredBaseModel): """ A list of unique identifiers for values within a dataset, e.g. rows of a DynamicTable. """ @@ -858,9 +912,13 @@ class ElementIdentifiers(Data): name: str = Field( "element_id", json_schema_extra={"linkml_meta": {"ifabsent": "string(element_id)"}} ) + value: Optional[T] = Field( + None, + json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_elements"}]}}}, + ) -class DynamicTableRegion(DynamicTableRegionMixin, VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData, ConfiguredBaseModel): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ @@ -886,7 +944,7 @@ class DynamicTableRegion(DynamicTableRegionMixin, VectorData): ] = Field(None) -class DynamicTable(DynamicTableMixin): +class DynamicTable(DynamicTableMixin, ConfiguredBaseModel): """ A group containing multiple datasets that are aligned on the first dimension (Currently, this requirement if left up to APIs to check and enforce). These datasets represent different columns in the table. Apart from a column that contains unique identifiers for each row, there are no other required datasets. Users are free to add any number of custom VectorData objects (columns) here. DynamicTable also supports ragged array columns, where each element can be of a different size. To add a ragged array column, use a VectorIndex type to index the corresponding VectorData type. See documentation for VectorData and VectorIndex for more details. Unlike a compound data type, which is analogous to storing an array-of-structs, a DynamicTable can be thought of as a struct-of-arrays. This provides an alternative structure to choose from when optimizing storage for anticipated access patterns. Additionally, this type provides a way of creating a table without having to define a compound type up front. Although this convenience may be attractive, users should think carefully about how data will be accessed. DynamicTable is more appropriate for column-centric access, whereas a dataset with a compound type would be more appropriate for row-centric access. Finally, data size should also be taken into account. For small tables, performance loss may be an acceptable trade-off for the flexibility of a DynamicTable. """ @@ -901,14 +959,11 @@ class DynamicTable(DynamicTableMixin): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class AlignedDynamicTable(AlignedDynamicTableMixin, DynamicTable): @@ -920,7 +975,7 @@ class AlignedDynamicTable(AlignedDynamicTableMixin, DynamicTable): {"from_schema": "hdmf-common.table", "tree_root": True} ) - value: Optional[List[DynamicTable]] = Field( + value: Optional[Dict[str, DynamicTable]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "DynamicTable"}]}} ) name: str = Field(...) @@ -929,14 +984,11 @@ class AlignedDynamicTable(AlignedDynamicTableMixin, DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/namespace.py index 442efe7..fa4ea72 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_5_1/namespace.py @@ -11,7 +11,7 @@ import numpy as np from pydantic import BaseModel, ConfigDict, Field, RootModel, field_validator from ...hdmf_common.v1_5_1.hdmf_common_base import Container, Data, SimpleMultiContainer -from ...hdmf_common.v1_5_1.hdmf_common_sparse import CSRMatrix, CSRMatrixData +from ...hdmf_common.v1_5_1.hdmf_common_sparse import CSRMatrix from ...hdmf_common.v1_5_1.hdmf_common_table import ( AlignedDynamicTable, DynamicTable, @@ -30,7 +30,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -49,6 +49,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/hdmf_common_base.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/hdmf_common_base.py index d53c4b5..57c9079 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/hdmf_common_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/hdmf_common_base.py @@ -19,7 +19,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -38,6 +38,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -103,7 +134,7 @@ class SimpleMultiContainer(Container): {"from_schema": "hdmf-common.base", "tree_root": True} ) - value: Optional[List[Container]] = Field( + value: Optional[Dict[str, Container]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "Container"}]}} ) name: str = Field(...) diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/hdmf_common_sparse.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/hdmf_common_sparse.py index d60f430..73a6043 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/hdmf_common_sparse.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/hdmf_common_sparse.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -101,23 +132,15 @@ class CSRMatrix(Container): "linkml_meta": {"array": {"dimensions": [{"alias": "number_of_rows_in_the_matrix_1"}]}} }, ) - data: CSRMatrixData = Field(..., description="""The non-zero values in the matrix.""") - - -class CSRMatrixData(ConfiguredBaseModel): - """ - The non-zero values in the matrix. - """ - - linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta({"from_schema": "hdmf-common.sparse"}) - - name: Literal["data"] = Field( - "data", - json_schema_extra={"linkml_meta": {"equals_string": "data", "ifabsent": "string(data)"}}, + data: NDArray[Shape["* number_of_non_zero_values"], Any] = Field( + ..., + description="""The non-zero values in the matrix.""", + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "number_of_non_zero_values"}]}} + }, ) # Model rebuild # see https://pydantic-docs.helpmanual.io/usage/models/#rebuilding-a-model CSRMatrix.model_rebuild() -CSRMatrixData.model_rebuild() diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/hdmf_common_table.py index da5a37f..0759b51 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/hdmf_common_table.py @@ -46,7 +46,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -65,6 +65,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -98,7 +129,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -300,8 +331,11 @@ class DynamicTableMixin(BaseModel): NON_COLUMN_FIELDS: ClassVar[tuple[str]] = ( "id", "name", + "categories", "colnames", "description", + "hdf5_path", + "object_id", ) # overridden by subclass but implemented here for testing and typechecking purposes :) @@ -385,7 +419,7 @@ class DynamicTableMixin(BaseModel): # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -493,11 +527,14 @@ class DynamicTableMixin(BaseModel): if k not in cls.NON_COLUMN_FIELDS and not k.endswith("_index") and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] model["colnames"] = colnames else: # add any columns not explicitly given an order at the end colnames = model["colnames"].copy() + if isinstance(colnames, np.ndarray): + colnames = colnames.tolist() colnames.extend( [ k @@ -506,6 +543,7 @@ class DynamicTableMixin(BaseModel): and not k.endswith("_index") and k not in model["colnames"] and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] ) model["colnames"] = colnames @@ -524,17 +562,25 @@ class DynamicTableMixin(BaseModel): if isinstance(model, dict): for key, val in model.items(): - if key in cls.model_fields: + if key in cls.model_fields or key in cls.NON_COLUMN_FIELDS: continue if not isinstance(val, (VectorData, VectorIndex)): try: - if key.endswith("_index"): - model[key] = VectorIndex(name=key, description="", value=val) + to_cast = VectorIndex if key.endswith("_index") else VectorData + if isinstance(val, dict): + model[key] = to_cast(**val) else: - model[key] = VectorData(name=key, description="", value=val) + model[key] = to_cast(name=key, description="", value=val) except ValidationError as e: # pragma: no cover - raise ValidationError( - f"field {key} cannot be cast to VectorData from {val}" + raise ValidationError.from_exception_data( + title=f"field {key} cannot be cast to VectorData from {val}", + line_errors=[ + { + "type": "ValueError", + "loc": ("DynamicTableMixin", "cast_extra_columns"), + "input": val, + } + ], ) from e return model @@ -567,9 +613,9 @@ class DynamicTableMixin(BaseModel): """ Ensure that all columns are equal length """ - lengths = [len(v) for v in self._columns.values()] + [len(self.id)] + lengths = [len(v) for v in self._columns.values() if v is not None] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "DynamicTable columns are not of equal length! " f"Got colnames:\n{self.colnames}\nand lengths: {lengths}" ) return self @@ -619,10 +665,13 @@ class AlignedDynamicTableMixin(BaseModel): __pydantic_extra__: Dict[str, Union["DynamicTableMixin", "VectorDataMixin", "VectorIndexMixin"]] NON_CATEGORY_FIELDS: ClassVar[tuple[str]] = ( + "id", "name", "categories", "colnames", "description", + "hdf5_path", + "object_id", ) name: str = "aligned_table" @@ -652,28 +701,29 @@ class AlignedDynamicTableMixin(BaseModel): elif isinstance(item, tuple) and len(item) == 2 and isinstance(item[1], str): # get a slice of a single table return self._categories[item[1]][item[0]] - elif isinstance(item, (int, slice, Iterable)): + elif isinstance(item, (int, slice, Iterable, np.int_)): # get a slice of all the tables ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) - tables = [ids] + ids = pd.Index(data=ids, name="id") + tables = [] for category_name, category in self._categories.items(): table = category[item] if isinstance(table, pd.DataFrame): table = table.reset_index() + table.index = ids elif isinstance(table, np.ndarray): - table = pd.DataFrame({category_name: [table]}) + table = pd.DataFrame({category_name: [table]}, index=ids) elif isinstance(table, Iterable): - table = pd.DataFrame({category_name: table}) + table = pd.DataFrame({category_name: table}, index=ids) else: raise ValueError( f"Don't know how to construct category table for {category_name}" ) tables.append(table) - names = [self.name] + self.categories + # names = [self.name] + self.categories # construct below in case we need to support array indexing in the future else: raise ValueError( @@ -681,8 +731,7 @@ class AlignedDynamicTableMixin(BaseModel): "need an int, string, slice, ndarray, or tuple[int | slice, str]" ) - df = pd.concat(tables, axis=1, keys=names) - df.set_index((self.name, "id"), drop=True, inplace=True) + df = pd.concat(tables, axis=1, keys=self.categories) return df def __getattr__(self, item: str) -> Any: @@ -740,14 +789,19 @@ class AlignedDynamicTableMixin(BaseModel): model["categories"] = categories else: # add any columns not explicitly given an order at the end - categories = [ - k - for k in model - if k not in cls.NON_COLUMN_FIELDS - and not k.endswith("_index") - and k not in model["categories"] - ] - model["categories"].extend(categories) + categories = model["categories"].copy() + if isinstance(categories, np.ndarray): + categories = categories.tolist() + categories.extend( + [ + k + for k in model + if k not in cls.NON_CATEGORY_FIELDS + and not k.endswith("_index") + and k not in model["categories"] + ] + ) + model["categories"] = categories return model @model_validator(mode="after") @@ -781,12 +835,19 @@ class AlignedDynamicTableMixin(BaseModel): """ lengths = [len(v) for v in self._categories.values()] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "AlignedDynamicTableColumns are not of equal length! " f"Got colnames:\n{self.categories}\nand lengths: {lengths}" ) return self +class ElementIdentifiersMixin(VectorDataMixin): + """ + Mixin class for ElementIdentifiers - allow treating + as generic, and give general indexing methods from VectorData + """ + + linkml_meta = LinkMLMeta( { "annotations": { @@ -801,7 +862,7 @@ linkml_meta = LinkMLMeta( ) -class VectorData(VectorDataMixin): +class VectorData(VectorDataMixin, ConfiguredBaseModel): """ An n-dimensional dataset representing a column of a DynamicTable. If used without an accompanying VectorIndex, first dimension is along the rows of the DynamicTable and each step along the first dimension is a cell of the larger table. VectorData can also be used to represent a ragged array if paired with a VectorIndex. This allows for storing arrays of varying length in a single cell of the DynamicTable by indexing into this VectorData. The first vector is at VectorData[0:VectorIndex[0]]. The second vector is at VectorData[VectorIndex[0]:VectorIndex[1]], and so on. """ @@ -812,17 +873,10 @@ class VectorData(VectorDataMixin): name: str = Field(...) description: str = Field(..., description="""Description of what these vectors represent.""") - value: Optional[ - Union[ - NDArray[Shape["* dim0"], Any], - NDArray[Shape["* dim0, * dim1"], Any], - NDArray[Shape["* dim0, * dim1, * dim2"], Any], - NDArray[Shape["* dim0, * dim1, * dim2, * dim3"], Any], - ] - ] = Field(None) + value: Optional[T] = Field(None) -class VectorIndex(VectorIndexMixin): +class VectorIndex(VectorIndexMixin, ConfiguredBaseModel): """ Used with VectorData to encode a ragged array. An array of indices into the first dimension of the target VectorData, and forming a map between the rows of a DynamicTable and the indices of the VectorData. The name of the VectorIndex is expected to be the name of the target VectorData object followed by \"_index\". """ @@ -846,7 +900,7 @@ class VectorIndex(VectorIndexMixin): ] = Field(None) -class ElementIdentifiers(Data): +class ElementIdentifiers(ElementIdentifiersMixin, Data, ConfiguredBaseModel): """ A list of unique identifiers for values within a dataset, e.g. rows of a DynamicTable. """ @@ -858,9 +912,13 @@ class ElementIdentifiers(Data): name: str = Field( "element_id", json_schema_extra={"linkml_meta": {"ifabsent": "string(element_id)"}} ) + value: Optional[T] = Field( + None, + json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_elements"}]}}}, + ) -class DynamicTableRegion(DynamicTableRegionMixin, VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData, ConfiguredBaseModel): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ @@ -886,7 +944,7 @@ class DynamicTableRegion(DynamicTableRegionMixin, VectorData): ] = Field(None) -class DynamicTable(DynamicTableMixin): +class DynamicTable(DynamicTableMixin, ConfiguredBaseModel): """ A group containing multiple datasets that are aligned on the first dimension (Currently, this requirement if left up to APIs to check and enforce). These datasets represent different columns in the table. Apart from a column that contains unique identifiers for each row, there are no other required datasets. Users are free to add any number of custom VectorData objects (columns) here. DynamicTable also supports ragged array columns, where each element can be of a different size. To add a ragged array column, use a VectorIndex type to index the corresponding VectorData type. See documentation for VectorData and VectorIndex for more details. Unlike a compound data type, which is analogous to storing an array-of-structs, a DynamicTable can be thought of as a struct-of-arrays. This provides an alternative structure to choose from when optimizing storage for anticipated access patterns. Additionally, this type provides a way of creating a table without having to define a compound type up front. Although this convenience may be attractive, users should think carefully about how data will be accessed. DynamicTable is more appropriate for column-centric access, whereas a dataset with a compound type would be more appropriate for row-centric access. Finally, data size should also be taken into account. For small tables, performance loss may be an acceptable trade-off for the flexibility of a DynamicTable. """ @@ -901,14 +959,11 @@ class DynamicTable(DynamicTableMixin): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class AlignedDynamicTable(AlignedDynamicTableMixin, DynamicTable): @@ -920,7 +975,7 @@ class AlignedDynamicTable(AlignedDynamicTableMixin, DynamicTable): {"from_schema": "hdmf-common.table", "tree_root": True} ) - value: Optional[List[DynamicTable]] = Field( + value: Optional[Dict[str, DynamicTable]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "DynamicTable"}]}} ) name: str = Field(...) @@ -929,14 +984,11 @@ class AlignedDynamicTable(AlignedDynamicTableMixin, DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/namespace.py index f9dd0a8..981e600 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_6_0/namespace.py @@ -11,7 +11,7 @@ import numpy as np from pydantic import BaseModel, ConfigDict, Field, RootModel, field_validator from ...hdmf_common.v1_6_0.hdmf_common_base import Container, Data, SimpleMultiContainer -from ...hdmf_common.v1_6_0.hdmf_common_sparse import CSRMatrix, CSRMatrixData +from ...hdmf_common.v1_6_0.hdmf_common_sparse import CSRMatrix from ...hdmf_common.v1_6_0.hdmf_common_table import ( AlignedDynamicTable, DynamicTable, @@ -30,7 +30,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -49,6 +49,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/hdmf_common_base.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/hdmf_common_base.py index 7872584..e785e04 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/hdmf_common_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/hdmf_common_base.py @@ -19,7 +19,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -38,6 +38,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -103,7 +134,7 @@ class SimpleMultiContainer(Container): {"from_schema": "hdmf-common.base", "tree_root": True} ) - value: Optional[List[Container]] = Field( + value: Optional[Dict[str, Container]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "Container"}]}} ) name: str = Field(...) diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/hdmf_common_sparse.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/hdmf_common_sparse.py index e21dca7..b2fe190 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/hdmf_common_sparse.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/hdmf_common_sparse.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -101,23 +132,15 @@ class CSRMatrix(Container): "linkml_meta": {"array": {"dimensions": [{"alias": "number_of_rows_in_the_matrix_1"}]}} }, ) - data: CSRMatrixData = Field(..., description="""The non-zero values in the matrix.""") - - -class CSRMatrixData(ConfiguredBaseModel): - """ - The non-zero values in the matrix. - """ - - linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta({"from_schema": "hdmf-common.sparse"}) - - name: Literal["data"] = Field( - "data", - json_schema_extra={"linkml_meta": {"equals_string": "data", "ifabsent": "string(data)"}}, + data: NDArray[Shape["* number_of_non_zero_values"], Any] = Field( + ..., + description="""The non-zero values in the matrix.""", + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "number_of_non_zero_values"}]}} + }, ) # Model rebuild # see https://pydantic-docs.helpmanual.io/usage/models/#rebuilding-a-model CSRMatrix.model_rebuild() -CSRMatrixData.model_rebuild() diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/hdmf_common_table.py index 9638faa..e805fe7 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/hdmf_common_table.py @@ -46,7 +46,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -65,6 +65,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -98,7 +129,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -300,8 +331,11 @@ class DynamicTableMixin(BaseModel): NON_COLUMN_FIELDS: ClassVar[tuple[str]] = ( "id", "name", + "categories", "colnames", "description", + "hdf5_path", + "object_id", ) # overridden by subclass but implemented here for testing and typechecking purposes :) @@ -385,7 +419,7 @@ class DynamicTableMixin(BaseModel): # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -493,11 +527,14 @@ class DynamicTableMixin(BaseModel): if k not in cls.NON_COLUMN_FIELDS and not k.endswith("_index") and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] model["colnames"] = colnames else: # add any columns not explicitly given an order at the end colnames = model["colnames"].copy() + if isinstance(colnames, np.ndarray): + colnames = colnames.tolist() colnames.extend( [ k @@ -506,6 +543,7 @@ class DynamicTableMixin(BaseModel): and not k.endswith("_index") and k not in model["colnames"] and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] ) model["colnames"] = colnames @@ -524,17 +562,25 @@ class DynamicTableMixin(BaseModel): if isinstance(model, dict): for key, val in model.items(): - if key in cls.model_fields: + if key in cls.model_fields or key in cls.NON_COLUMN_FIELDS: continue if not isinstance(val, (VectorData, VectorIndex)): try: - if key.endswith("_index"): - model[key] = VectorIndex(name=key, description="", value=val) + to_cast = VectorIndex if key.endswith("_index") else VectorData + if isinstance(val, dict): + model[key] = to_cast(**val) else: - model[key] = VectorData(name=key, description="", value=val) + model[key] = to_cast(name=key, description="", value=val) except ValidationError as e: # pragma: no cover - raise ValidationError( - f"field {key} cannot be cast to VectorData from {val}" + raise ValidationError.from_exception_data( + title=f"field {key} cannot be cast to VectorData from {val}", + line_errors=[ + { + "type": "ValueError", + "loc": ("DynamicTableMixin", "cast_extra_columns"), + "input": val, + } + ], ) from e return model @@ -567,9 +613,9 @@ class DynamicTableMixin(BaseModel): """ Ensure that all columns are equal length """ - lengths = [len(v) for v in self._columns.values()] + [len(self.id)] + lengths = [len(v) for v in self._columns.values() if v is not None] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "DynamicTable columns are not of equal length! " f"Got colnames:\n{self.colnames}\nand lengths: {lengths}" ) return self @@ -619,10 +665,13 @@ class AlignedDynamicTableMixin(BaseModel): __pydantic_extra__: Dict[str, Union["DynamicTableMixin", "VectorDataMixin", "VectorIndexMixin"]] NON_CATEGORY_FIELDS: ClassVar[tuple[str]] = ( + "id", "name", "categories", "colnames", "description", + "hdf5_path", + "object_id", ) name: str = "aligned_table" @@ -652,28 +701,29 @@ class AlignedDynamicTableMixin(BaseModel): elif isinstance(item, tuple) and len(item) == 2 and isinstance(item[1], str): # get a slice of a single table return self._categories[item[1]][item[0]] - elif isinstance(item, (int, slice, Iterable)): + elif isinstance(item, (int, slice, Iterable, np.int_)): # get a slice of all the tables ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) - tables = [ids] + ids = pd.Index(data=ids, name="id") + tables = [] for category_name, category in self._categories.items(): table = category[item] if isinstance(table, pd.DataFrame): table = table.reset_index() + table.index = ids elif isinstance(table, np.ndarray): - table = pd.DataFrame({category_name: [table]}) + table = pd.DataFrame({category_name: [table]}, index=ids) elif isinstance(table, Iterable): - table = pd.DataFrame({category_name: table}) + table = pd.DataFrame({category_name: table}, index=ids) else: raise ValueError( f"Don't know how to construct category table for {category_name}" ) tables.append(table) - names = [self.name] + self.categories + # names = [self.name] + self.categories # construct below in case we need to support array indexing in the future else: raise ValueError( @@ -681,8 +731,7 @@ class AlignedDynamicTableMixin(BaseModel): "need an int, string, slice, ndarray, or tuple[int | slice, str]" ) - df = pd.concat(tables, axis=1, keys=names) - df.set_index((self.name, "id"), drop=True, inplace=True) + df = pd.concat(tables, axis=1, keys=self.categories) return df def __getattr__(self, item: str) -> Any: @@ -740,14 +789,19 @@ class AlignedDynamicTableMixin(BaseModel): model["categories"] = categories else: # add any columns not explicitly given an order at the end - categories = [ - k - for k in model - if k not in cls.NON_COLUMN_FIELDS - and not k.endswith("_index") - and k not in model["categories"] - ] - model["categories"].extend(categories) + categories = model["categories"].copy() + if isinstance(categories, np.ndarray): + categories = categories.tolist() + categories.extend( + [ + k + for k in model + if k not in cls.NON_CATEGORY_FIELDS + and not k.endswith("_index") + and k not in model["categories"] + ] + ) + model["categories"] = categories return model @model_validator(mode="after") @@ -781,12 +835,19 @@ class AlignedDynamicTableMixin(BaseModel): """ lengths = [len(v) for v in self._categories.values()] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "AlignedDynamicTableColumns are not of equal length! " f"Got colnames:\n{self.categories}\nand lengths: {lengths}" ) return self +class ElementIdentifiersMixin(VectorDataMixin): + """ + Mixin class for ElementIdentifiers - allow treating + as generic, and give general indexing methods from VectorData + """ + + linkml_meta = LinkMLMeta( { "annotations": { @@ -801,7 +862,7 @@ linkml_meta = LinkMLMeta( ) -class VectorData(VectorDataMixin): +class VectorData(VectorDataMixin, ConfiguredBaseModel): """ An n-dimensional dataset representing a column of a DynamicTable. If used without an accompanying VectorIndex, first dimension is along the rows of the DynamicTable and each step along the first dimension is a cell of the larger table. VectorData can also be used to represent a ragged array if paired with a VectorIndex. This allows for storing arrays of varying length in a single cell of the DynamicTable by indexing into this VectorData. The first vector is at VectorData[0:VectorIndex[0]]. The second vector is at VectorData[VectorIndex[0]:VectorIndex[1]], and so on. """ @@ -812,17 +873,10 @@ class VectorData(VectorDataMixin): name: str = Field(...) description: str = Field(..., description="""Description of what these vectors represent.""") - value: Optional[ - Union[ - NDArray[Shape["* dim0"], Any], - NDArray[Shape["* dim0, * dim1"], Any], - NDArray[Shape["* dim0, * dim1, * dim2"], Any], - NDArray[Shape["* dim0, * dim1, * dim2, * dim3"], Any], - ] - ] = Field(None) + value: Optional[T] = Field(None) -class VectorIndex(VectorIndexMixin): +class VectorIndex(VectorIndexMixin, ConfiguredBaseModel): """ Used with VectorData to encode a ragged array. An array of indices into the first dimension of the target VectorData, and forming a map between the rows of a DynamicTable and the indices of the VectorData. The name of the VectorIndex is expected to be the name of the target VectorData object followed by \"_index\". """ @@ -846,7 +900,7 @@ class VectorIndex(VectorIndexMixin): ] = Field(None) -class ElementIdentifiers(Data): +class ElementIdentifiers(ElementIdentifiersMixin, Data, ConfiguredBaseModel): """ A list of unique identifiers for values within a dataset, e.g. rows of a DynamicTable. """ @@ -858,9 +912,13 @@ class ElementIdentifiers(Data): name: str = Field( "element_id", json_schema_extra={"linkml_meta": {"ifabsent": "string(element_id)"}} ) + value: Optional[T] = Field( + None, + json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_elements"}]}}}, + ) -class DynamicTableRegion(DynamicTableRegionMixin, VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData, ConfiguredBaseModel): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ @@ -886,7 +944,7 @@ class DynamicTableRegion(DynamicTableRegionMixin, VectorData): ] = Field(None) -class DynamicTable(DynamicTableMixin): +class DynamicTable(DynamicTableMixin, ConfiguredBaseModel): """ A group containing multiple datasets that are aligned on the first dimension (Currently, this requirement if left up to APIs to check and enforce). These datasets represent different columns in the table. Apart from a column that contains unique identifiers for each row, there are no other required datasets. Users are free to add any number of custom VectorData objects (columns) here. DynamicTable also supports ragged array columns, where each element can be of a different size. To add a ragged array column, use a VectorIndex type to index the corresponding VectorData type. See documentation for VectorData and VectorIndex for more details. Unlike a compound data type, which is analogous to storing an array-of-structs, a DynamicTable can be thought of as a struct-of-arrays. This provides an alternative structure to choose from when optimizing storage for anticipated access patterns. Additionally, this type provides a way of creating a table without having to define a compound type up front. Although this convenience may be attractive, users should think carefully about how data will be accessed. DynamicTable is more appropriate for column-centric access, whereas a dataset with a compound type would be more appropriate for row-centric access. Finally, data size should also be taken into account. For small tables, performance loss may be an acceptable trade-off for the flexibility of a DynamicTable. """ @@ -901,14 +959,11 @@ class DynamicTable(DynamicTableMixin): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class AlignedDynamicTable(AlignedDynamicTableMixin, DynamicTable): @@ -920,7 +975,7 @@ class AlignedDynamicTable(AlignedDynamicTableMixin, DynamicTable): {"from_schema": "hdmf-common.table", "tree_root": True} ) - value: Optional[List[DynamicTable]] = Field( + value: Optional[Dict[str, DynamicTable]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "DynamicTable"}]}} ) name: str = Field(...) @@ -929,14 +984,11 @@ class AlignedDynamicTable(AlignedDynamicTableMixin, DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/namespace.py index a3f0b14..4aaa46d 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_7_0/namespace.py @@ -11,7 +11,7 @@ import numpy as np from pydantic import BaseModel, ConfigDict, Field, RootModel, field_validator from ...hdmf_common.v1_7_0.hdmf_common_base import Container, Data, SimpleMultiContainer -from ...hdmf_common.v1_7_0.hdmf_common_sparse import CSRMatrix, CSRMatrixData +from ...hdmf_common.v1_7_0.hdmf_common_sparse import CSRMatrix from ...hdmf_common.v1_7_0.hdmf_common_table import ( AlignedDynamicTable, DynamicTable, @@ -30,7 +30,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -49,6 +49,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/hdmf_common_base.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/hdmf_common_base.py index 8b4d98f..7731368 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/hdmf_common_base.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/hdmf_common_base.py @@ -19,7 +19,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -38,6 +38,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -103,7 +134,7 @@ class SimpleMultiContainer(Container): {"from_schema": "hdmf-common.base", "tree_root": True} ) - value: Optional[List[Container]] = Field( + value: Optional[Dict[str, Container]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "Container"}]}} ) name: str = Field(...) diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/hdmf_common_sparse.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/hdmf_common_sparse.py index 4da904e..7a3e72c 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/hdmf_common_sparse.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/hdmf_common_sparse.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -101,23 +132,15 @@ class CSRMatrix(Container): "linkml_meta": {"array": {"dimensions": [{"alias": "number_of_rows_in_the_matrix_1"}]}} }, ) - data: CSRMatrixData = Field(..., description="""The non-zero values in the matrix.""") - - -class CSRMatrixData(ConfiguredBaseModel): - """ - The non-zero values in the matrix. - """ - - linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta({"from_schema": "hdmf-common.sparse"}) - - name: Literal["data"] = Field( - "data", - json_schema_extra={"linkml_meta": {"equals_string": "data", "ifabsent": "string(data)"}}, + data: NDArray[Shape["* number_of_non_zero_values"], Any] = Field( + ..., + description="""The non-zero values in the matrix.""", + json_schema_extra={ + "linkml_meta": {"array": {"dimensions": [{"alias": "number_of_non_zero_values"}]}} + }, ) # Model rebuild # see https://pydantic-docs.helpmanual.io/usage/models/#rebuilding-a-model CSRMatrix.model_rebuild() -CSRMatrixData.model_rebuild() diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/hdmf_common_table.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/hdmf_common_table.py index b62156a..8f0d610 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/hdmf_common_table.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/hdmf_common_table.py @@ -46,7 +46,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -65,6 +65,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} @@ -98,7 +129,7 @@ class VectorDataMixin(BaseModel, Generic[T]): # redefined in `VectorData`, but included here for testing and type checking value: Optional[T] = None - def __init__(self, value: Optional[NDArray] = None, **kwargs): + def __init__(self, value: Optional[T] = None, **kwargs): if value is not None and "value" not in kwargs: kwargs["value"] = value super().__init__(**kwargs) @@ -300,8 +331,11 @@ class DynamicTableMixin(BaseModel): NON_COLUMN_FIELDS: ClassVar[tuple[str]] = ( "id", "name", + "categories", "colnames", "description", + "hdf5_path", + "object_id", ) # overridden by subclass but implemented here for testing and typechecking purposes :) @@ -385,7 +419,7 @@ class DynamicTableMixin(BaseModel): # cast to DF if not isinstance(index, Iterable): index = [index] - index = pd.Index(data=index) + index = pd.Index(data=index, name="id") return pd.DataFrame(data, index=index) def _slice_range( @@ -493,11 +527,14 @@ class DynamicTableMixin(BaseModel): if k not in cls.NON_COLUMN_FIELDS and not k.endswith("_index") and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] model["colnames"] = colnames else: # add any columns not explicitly given an order at the end colnames = model["colnames"].copy() + if isinstance(colnames, np.ndarray): + colnames = colnames.tolist() colnames.extend( [ k @@ -506,6 +543,7 @@ class DynamicTableMixin(BaseModel): and not k.endswith("_index") and k not in model["colnames"] and not isinstance(model[k], VectorIndexMixin) + and model[k] is not None ] ) model["colnames"] = colnames @@ -524,17 +562,25 @@ class DynamicTableMixin(BaseModel): if isinstance(model, dict): for key, val in model.items(): - if key in cls.model_fields: + if key in cls.model_fields or key in cls.NON_COLUMN_FIELDS: continue if not isinstance(val, (VectorData, VectorIndex)): try: - if key.endswith("_index"): - model[key] = VectorIndex(name=key, description="", value=val) + to_cast = VectorIndex if key.endswith("_index") else VectorData + if isinstance(val, dict): + model[key] = to_cast(**val) else: - model[key] = VectorData(name=key, description="", value=val) + model[key] = to_cast(name=key, description="", value=val) except ValidationError as e: # pragma: no cover - raise ValidationError( - f"field {key} cannot be cast to VectorData from {val}" + raise ValidationError.from_exception_data( + title=f"field {key} cannot be cast to VectorData from {val}", + line_errors=[ + { + "type": "ValueError", + "loc": ("DynamicTableMixin", "cast_extra_columns"), + "input": val, + } + ], ) from e return model @@ -567,9 +613,9 @@ class DynamicTableMixin(BaseModel): """ Ensure that all columns are equal length """ - lengths = [len(v) for v in self._columns.values()] + [len(self.id)] + lengths = [len(v) for v in self._columns.values() if v is not None] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "DynamicTable columns are not of equal length! " f"Got colnames:\n{self.colnames}\nand lengths: {lengths}" ) return self @@ -619,10 +665,13 @@ class AlignedDynamicTableMixin(BaseModel): __pydantic_extra__: Dict[str, Union["DynamicTableMixin", "VectorDataMixin", "VectorIndexMixin"]] NON_CATEGORY_FIELDS: ClassVar[tuple[str]] = ( + "id", "name", "categories", "colnames", "description", + "hdf5_path", + "object_id", ) name: str = "aligned_table" @@ -652,28 +701,29 @@ class AlignedDynamicTableMixin(BaseModel): elif isinstance(item, tuple) and len(item) == 2 and isinstance(item[1], str): # get a slice of a single table return self._categories[item[1]][item[0]] - elif isinstance(item, (int, slice, Iterable)): + elif isinstance(item, (int, slice, Iterable, np.int_)): # get a slice of all the tables ids = self.id[item] if not isinstance(ids, Iterable): ids = pd.Series([ids]) - ids = pd.DataFrame({"id": ids}) - tables = [ids] + ids = pd.Index(data=ids, name="id") + tables = [] for category_name, category in self._categories.items(): table = category[item] if isinstance(table, pd.DataFrame): table = table.reset_index() + table.index = ids elif isinstance(table, np.ndarray): - table = pd.DataFrame({category_name: [table]}) + table = pd.DataFrame({category_name: [table]}, index=ids) elif isinstance(table, Iterable): - table = pd.DataFrame({category_name: table}) + table = pd.DataFrame({category_name: table}, index=ids) else: raise ValueError( f"Don't know how to construct category table for {category_name}" ) tables.append(table) - names = [self.name] + self.categories + # names = [self.name] + self.categories # construct below in case we need to support array indexing in the future else: raise ValueError( @@ -681,8 +731,7 @@ class AlignedDynamicTableMixin(BaseModel): "need an int, string, slice, ndarray, or tuple[int | slice, str]" ) - df = pd.concat(tables, axis=1, keys=names) - df.set_index((self.name, "id"), drop=True, inplace=True) + df = pd.concat(tables, axis=1, keys=self.categories) return df def __getattr__(self, item: str) -> Any: @@ -740,14 +789,19 @@ class AlignedDynamicTableMixin(BaseModel): model["categories"] = categories else: # add any columns not explicitly given an order at the end - categories = [ - k - for k in model - if k not in cls.NON_COLUMN_FIELDS - and not k.endswith("_index") - and k not in model["categories"] - ] - model["categories"].extend(categories) + categories = model["categories"].copy() + if isinstance(categories, np.ndarray): + categories = categories.tolist() + categories.extend( + [ + k + for k in model + if k not in cls.NON_CATEGORY_FIELDS + and not k.endswith("_index") + and k not in model["categories"] + ] + ) + model["categories"] = categories return model @model_validator(mode="after") @@ -781,12 +835,19 @@ class AlignedDynamicTableMixin(BaseModel): """ lengths = [len(v) for v in self._categories.values()] + [len(self.id)] assert all([length == lengths[0] for length in lengths]), ( - "Columns are not of equal length! " + "AlignedDynamicTableColumns are not of equal length! " f"Got colnames:\n{self.categories}\nand lengths: {lengths}" ) return self +class ElementIdentifiersMixin(VectorDataMixin): + """ + Mixin class for ElementIdentifiers - allow treating + as generic, and give general indexing methods from VectorData + """ + + linkml_meta = LinkMLMeta( { "annotations": { @@ -801,7 +862,7 @@ linkml_meta = LinkMLMeta( ) -class VectorData(VectorDataMixin): +class VectorData(VectorDataMixin, ConfiguredBaseModel): """ An n-dimensional dataset representing a column of a DynamicTable. If used without an accompanying VectorIndex, first dimension is along the rows of the DynamicTable and each step along the first dimension is a cell of the larger table. VectorData can also be used to represent a ragged array if paired with a VectorIndex. This allows for storing arrays of varying length in a single cell of the DynamicTable by indexing into this VectorData. The first vector is at VectorData[0:VectorIndex[0]]. The second vector is at VectorData[VectorIndex[0]:VectorIndex[1]], and so on. """ @@ -812,17 +873,10 @@ class VectorData(VectorDataMixin): name: str = Field(...) description: str = Field(..., description="""Description of what these vectors represent.""") - value: Optional[ - Union[ - NDArray[Shape["* dim0"], Any], - NDArray[Shape["* dim0, * dim1"], Any], - NDArray[Shape["* dim0, * dim1, * dim2"], Any], - NDArray[Shape["* dim0, * dim1, * dim2, * dim3"], Any], - ] - ] = Field(None) + value: Optional[T] = Field(None) -class VectorIndex(VectorIndexMixin): +class VectorIndex(VectorIndexMixin, ConfiguredBaseModel): """ Used with VectorData to encode a ragged array. An array of indices into the first dimension of the target VectorData, and forming a map between the rows of a DynamicTable and the indices of the VectorData. The name of the VectorIndex is expected to be the name of the target VectorData object followed by \"_index\". """ @@ -846,7 +900,7 @@ class VectorIndex(VectorIndexMixin): ] = Field(None) -class ElementIdentifiers(Data): +class ElementIdentifiers(ElementIdentifiersMixin, Data, ConfiguredBaseModel): """ A list of unique identifiers for values within a dataset, e.g. rows of a DynamicTable. """ @@ -858,9 +912,13 @@ class ElementIdentifiers(Data): name: str = Field( "element_id", json_schema_extra={"linkml_meta": {"ifabsent": "string(element_id)"}} ) + value: Optional[T] = Field( + None, + json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_elements"}]}}}, + ) -class DynamicTableRegion(DynamicTableRegionMixin, VectorData): +class DynamicTableRegion(DynamicTableRegionMixin, VectorData, ConfiguredBaseModel): """ DynamicTableRegion provides a link from one table to an index or region of another. The `table` attribute is a link to another `DynamicTable`, indicating which table is referenced, and the data is int(s) indicating the row(s) (0-indexed) of the target array. `DynamicTableRegion`s can be used to associate rows with repeated meta-data without data duplication. They can also be used to create hierarchical relationships between multiple `DynamicTable`s. `DynamicTableRegion` objects may be paired with a `VectorIndex` object to create ragged references, so a single cell of a `DynamicTable` can reference many rows of another `DynamicTable`. """ @@ -886,7 +944,7 @@ class DynamicTableRegion(DynamicTableRegionMixin, VectorData): ] = Field(None) -class DynamicTable(DynamicTableMixin): +class DynamicTable(DynamicTableMixin, ConfiguredBaseModel): """ A group containing multiple datasets that are aligned on the first dimension (Currently, this requirement if left up to APIs to check and enforce). These datasets represent different columns in the table. Apart from a column that contains unique identifiers for each row, there are no other required datasets. Users are free to add any number of custom VectorData objects (columns) here. DynamicTable also supports ragged array columns, where each element can be of a different size. To add a ragged array column, use a VectorIndex type to index the corresponding VectorData type. See documentation for VectorData and VectorIndex for more details. Unlike a compound data type, which is analogous to storing an array-of-structs, a DynamicTable can be thought of as a struct-of-arrays. This provides an alternative structure to choose from when optimizing storage for anticipated access patterns. Additionally, this type provides a way of creating a table without having to define a compound type up front. Although this convenience may be attractive, users should think carefully about how data will be accessed. DynamicTable is more appropriate for column-centric access, whereas a dataset with a compound type would be more appropriate for row-centric access. Finally, data size should also be taken into account. For small tables, performance loss may be an acceptable trade-off for the flexibility of a DynamicTable. """ @@ -901,14 +959,11 @@ class DynamicTable(DynamicTableMixin): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) class AlignedDynamicTable(AlignedDynamicTableMixin, DynamicTable): @@ -920,7 +975,7 @@ class AlignedDynamicTable(AlignedDynamicTableMixin, DynamicTable): {"from_schema": "hdmf-common.table", "tree_root": True} ) - value: Optional[List[DynamicTable]] = Field( + value: Optional[Dict[str, DynamicTable]] = Field( None, json_schema_extra={"linkml_meta": {"any_of": [{"range": "DynamicTable"}]}} ) name: str = Field(...) @@ -929,14 +984,11 @@ class AlignedDynamicTable(AlignedDynamicTableMixin, DynamicTable): description="""The names of the columns in this table. This should be used to specify an order to the columns.""", ) description: str = Field(..., description="""Description of what is in this dynamic table.""") - id: VectorData[NDArray[Shape["* num_rows"], int]] = Field( + id: ElementIdentifiers = Field( ..., description="""Array of unique identifiers for the rows of this dynamic table.""", json_schema_extra={"linkml_meta": {"array": {"dimensions": [{"alias": "num_rows"}]}}}, ) - vector_data: Optional[List[VectorData]] = Field( - None, description="""Vector columns, including index columns, of this dynamic table.""" - ) # Model rebuild diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/namespace.py index 78035be..dd09b7f 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_common/v1_8_0/namespace.py @@ -11,7 +11,7 @@ import numpy as np from pydantic import BaseModel, ConfigDict, Field, RootModel, field_validator from ...hdmf_common.v1_8_0.hdmf_common_base import Container, Data, SimpleMultiContainer -from ...hdmf_common.v1_8_0.hdmf_common_sparse import CSRMatrix, CSRMatrixData +from ...hdmf_common.v1_8_0.hdmf_common_sparse import CSRMatrix from ...hdmf_common.v1_8_0.hdmf_common_table import ( AlignedDynamicTable, DynamicTable, @@ -30,7 +30,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -49,6 +49,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_1_0/hdmf_experimental_experimental.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_1_0/hdmf_experimental_experimental.py index c5848aa..ad617da 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_1_0/hdmf_experimental_experimental.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_1_0/hdmf_experimental_experimental.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_1_0/hdmf_experimental_resources.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_1_0/hdmf_experimental_resources.py index 5d0a97e..cda720e 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_1_0/hdmf_experimental_resources.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_1_0/hdmf_experimental_resources.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_1_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_1_0/namespace.py index 3389dc8..3429a1e 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_1_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_1_0/namespace.py @@ -11,7 +11,7 @@ import numpy as np from pydantic import BaseModel, ConfigDict, Field, RootModel, field_validator from ...hdmf_common.v1_4_0.hdmf_common_base import Container, Data, SimpleMultiContainer -from ...hdmf_common.v1_4_0.hdmf_common_sparse import CSRMatrix, CSRMatrixData +from ...hdmf_common.v1_4_0.hdmf_common_sparse import CSRMatrix from ...hdmf_common.v1_4_0.hdmf_common_table import ( DynamicTable, DynamicTableRegion, @@ -38,7 +38,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -57,6 +57,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_2_0/hdmf_experimental_experimental.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_2_0/hdmf_experimental_experimental.py index a840dc1..1a88edc 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_2_0/hdmf_experimental_experimental.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_2_0/hdmf_experimental_experimental.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_2_0/hdmf_experimental_resources.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_2_0/hdmf_experimental_resources.py index 8690772..8d5af36 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_2_0/hdmf_experimental_resources.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_2_0/hdmf_experimental_resources.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_2_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_2_0/namespace.py index 8b52b45..c697f83 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_2_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_2_0/namespace.py @@ -11,7 +11,7 @@ import numpy as np from pydantic import BaseModel, ConfigDict, Field, RootModel, field_validator from ...hdmf_common.v1_5_1.hdmf_common_base import Container, Data, SimpleMultiContainer -from ...hdmf_common.v1_5_1.hdmf_common_sparse import CSRMatrix, CSRMatrixData +from ...hdmf_common.v1_5_1.hdmf_common_sparse import CSRMatrix from ...hdmf_common.v1_5_1.hdmf_common_table import ( AlignedDynamicTable, DynamicTable, @@ -39,7 +39,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -58,6 +58,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_3_0/hdmf_experimental_experimental.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_3_0/hdmf_experimental_experimental.py index 4fdfb37..cbd0ad9 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_3_0/hdmf_experimental_experimental.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_3_0/hdmf_experimental_experimental.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_3_0/hdmf_experimental_resources.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_3_0/hdmf_experimental_resources.py index 1324db1..9f337fa 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_3_0/hdmf_experimental_resources.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_3_0/hdmf_experimental_resources.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_3_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_3_0/namespace.py index c4b9cd8..e1a12ca 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_3_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_3_0/namespace.py @@ -11,7 +11,7 @@ import numpy as np from pydantic import BaseModel, ConfigDict, Field, RootModel, field_validator from ...hdmf_common.v1_6_0.hdmf_common_base import Container, Data, SimpleMultiContainer -from ...hdmf_common.v1_6_0.hdmf_common_sparse import CSRMatrix, CSRMatrixData +from ...hdmf_common.v1_6_0.hdmf_common_sparse import CSRMatrix from ...hdmf_common.v1_6_0.hdmf_common_table import ( AlignedDynamicTable, DynamicTable, @@ -39,7 +39,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -58,6 +58,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_4_0/hdmf_experimental_experimental.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_4_0/hdmf_experimental_experimental.py index 43eab2a..0551cfd 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_4_0/hdmf_experimental_experimental.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_4_0/hdmf_experimental_experimental.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_4_0/hdmf_experimental_resources.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_4_0/hdmf_experimental_resources.py index 172b75b..09e6f05 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_4_0/hdmf_experimental_resources.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_4_0/hdmf_experimental_resources.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_4_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_4_0/namespace.py index 14fcb96..c904202 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_4_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_4_0/namespace.py @@ -11,7 +11,7 @@ import numpy as np from pydantic import BaseModel, ConfigDict, Field, RootModel, field_validator from ...hdmf_common.v1_7_0.hdmf_common_base import Container, Data, SimpleMultiContainer -from ...hdmf_common.v1_7_0.hdmf_common_sparse import CSRMatrix, CSRMatrixData +from ...hdmf_common.v1_7_0.hdmf_common_sparse import CSRMatrix from ...hdmf_common.v1_7_0.hdmf_common_table import ( AlignedDynamicTable, DynamicTable, @@ -40,7 +40,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -59,6 +59,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_5_0/hdmf_experimental_experimental.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_5_0/hdmf_experimental_experimental.py index a18965f..714ae52 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_5_0/hdmf_experimental_experimental.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_5_0/hdmf_experimental_experimental.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_5_0/hdmf_experimental_resources.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_5_0/hdmf_experimental_resources.py index 712c32e..d3132cd 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_5_0/hdmf_experimental_resources.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_5_0/hdmf_experimental_resources.py @@ -22,7 +22,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -41,6 +41,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_5_0/namespace.py b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_5_0/namespace.py index b4c0f3a..281e5b2 100644 --- a/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_5_0/namespace.py +++ b/nwb_models/src/nwb_models/models/pydantic/hdmf_experimental/v0_5_0/namespace.py @@ -11,7 +11,7 @@ import numpy as np from pydantic import BaseModel, ConfigDict, Field, RootModel, field_validator from ...hdmf_common.v1_8_0.hdmf_common_base import Container, Data, SimpleMultiContainer -from ...hdmf_common.v1_8_0.hdmf_common_sparse import CSRMatrix, CSRMatrixData +from ...hdmf_common.v1_8_0.hdmf_common_sparse import CSRMatrix from ...hdmf_common.v1_8_0.hdmf_common_table import ( AlignedDynamicTable, DynamicTable, @@ -40,7 +40,7 @@ class ConfiguredBaseModel(BaseModel): model_config = ConfigDict( validate_assignment=True, validate_default=True, - extra="forbid", + extra="allow", arbitrary_types_allowed=True, use_enum_values=True, strict=False, @@ -59,6 +59,37 @@ class ConfiguredBaseModel(BaseModel): else: raise KeyError("No value or data field to index from") + @field_validator("*", mode="wrap") + @classmethod + def coerce_value(cls, v: Any, handler) -> Any: + """Try to rescue instantiation by using the value field""" + try: + return handler(v) + except Exception as e1: + try: + return handler(v.value) + except AttributeError: + try: + return handler(v["value"]) + except (IndexError, KeyError, TypeError): + raise e1 + + @field_validator("*", mode="before") + @classmethod + def coerce_subclass(cls, v: Any, info) -> Any: + """Recast parent classes into child classes""" + if isinstance(v, BaseModel): + annotation = cls.model_fields[info.field_name].annotation + while hasattr(annotation, "__args__"): + annotation = annotation.__args__[0] + try: + if issubclass(annotation, type(v)) and annotation is not type(v): + v = annotation(**{**v.__dict__, **v.__pydantic_extra__}) + except TypeError: + # fine, annotation is a non-class type like a TypeVar + pass + return v + class LinkMLMeta(RootModel): root: Dict[str, Any] = {} diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_2_0/core.nwb.image.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_2_0/core.nwb.image.yaml index 6920484..9019ee0 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_2_0/core.nwb.image.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_2_0/core.nwb.image.yaml @@ -22,6 +22,13 @@ classes: name: name range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + range: numeric tree_root: true RGBImage: name: RGBImage @@ -32,6 +39,15 @@ classes: name: name range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b + exact_cardinality: 3 + range: numeric tree_root: true RGBAImage: name: RGBAImage @@ -42,6 +58,15 @@ classes: name: name range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b_a + exact_cardinality: 4 + range: numeric tree_root: true ImageSeries: name: ImageSeries diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_2_1/core.nwb.image.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_2_1/core.nwb.image.yaml index 7f406bc..785ebbb 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_2_1/core.nwb.image.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_2_1/core.nwb.image.yaml @@ -22,6 +22,13 @@ classes: name: name range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + range: numeric tree_root: true RGBImage: name: RGBImage @@ -32,6 +39,15 @@ classes: name: name range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b + exact_cardinality: 3 + range: numeric tree_root: true RGBAImage: name: RGBAImage @@ -42,6 +58,15 @@ classes: name: name range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b_a + exact_cardinality: 4 + range: numeric tree_root: true ImageSeries: name: ImageSeries diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_2_2/core.nwb.image.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_2_2/core.nwb.image.yaml index 1e11ca4..9b619e3 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_2_2/core.nwb.image.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_2_2/core.nwb.image.yaml @@ -22,6 +22,13 @@ classes: name: name range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + range: numeric tree_root: true RGBImage: name: RGBImage @@ -32,6 +39,15 @@ classes: name: name range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b + exact_cardinality: 3 + range: numeric tree_root: true RGBAImage: name: RGBAImage @@ -42,6 +58,15 @@ classes: name: name range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b_a + exact_cardinality: 4 + range: numeric tree_root: true ImageSeries: name: ImageSeries diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_2_4/core.nwb.image.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_2_4/core.nwb.image.yaml index 4beec01..1db1fba 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_2_4/core.nwb.image.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_2_4/core.nwb.image.yaml @@ -22,6 +22,13 @@ classes: name: name range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + range: numeric tree_root: true RGBImage: name: RGBImage @@ -32,6 +39,15 @@ classes: name: name range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b + exact_cardinality: 3 + range: numeric tree_root: true RGBAImage: name: RGBAImage @@ -42,6 +58,15 @@ classes: name: name range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b_a + exact_cardinality: 4 + range: numeric tree_root: true ImageSeries: name: ImageSeries diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_2_5/core.nwb.image.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_2_5/core.nwb.image.yaml index 4218d3b..3a7025a 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_2_5/core.nwb.image.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_2_5/core.nwb.image.yaml @@ -22,6 +22,13 @@ classes: name: name range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + range: numeric tree_root: true RGBImage: name: RGBImage @@ -32,6 +39,15 @@ classes: name: name range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b + exact_cardinality: 3 + range: numeric tree_root: true RGBAImage: name: RGBAImage @@ -42,6 +58,15 @@ classes: name: name range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b_a + exact_cardinality: 4 + range: numeric tree_root: true ImageSeries: name: ImageSeries diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.base.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.base.yaml index 24a4bbf..b11b02c 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.base.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.base.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -33,6 +34,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true resolution: @@ -74,6 +76,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -85,6 +88,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -95,6 +99,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -120,6 +125,7 @@ classes: range: TimeSeries__data required: true multivalued: false + inlined: true starting_time: name: starting_time description: Timestamp of the first sample in seconds. When timestamps are @@ -128,6 +134,7 @@ classes: range: TimeSeries__starting_time required: false multivalued: false + inlined: true timestamps: name: timestamps description: Timestamps for samples stored in data, in seconds, relative to @@ -171,6 +178,8 @@ classes: range: TimeSeries__sync required: false multivalued: false + inlined: true + inlined_as_list: true tree_root: true TimeSeries__data: name: TimeSeries__data @@ -181,6 +190,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -258,6 +268,7 @@ classes: name: name: name ifabsent: string(starting_time) + identifier: true range: string required: true equals_string: starting_time @@ -289,6 +300,7 @@ classes: name: name: name ifabsent: string(sync) + identifier: true range: string required: true equals_string: sync @@ -313,6 +325,7 @@ classes: name: name: name ifabsent: string(Images) + identifier: true range: string required: true description: diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.behavior.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.behavior.yaml index 1b74203..f63c218 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.behavior.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.behavior.yaml @@ -29,6 +29,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -38,6 +39,7 @@ classes: range: SpatialSeries__data required: true multivalued: false + inlined: true reference_frame: name: reference_frame description: Description defining what exactly 'straight-ahead' means. @@ -53,6 +55,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.device.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.device.yaml index 7881fcf..8d60d56 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.device.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.device.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.ecephys.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.ecephys.yaml index 2f3dd97..7a93461 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.ecephys.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.ecephys.yaml @@ -25,6 +25,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true filtering: @@ -71,6 +72,7 @@ classes: range: DynamicTableRegion required: true multivalued: false + inlined: true channel_conversion: name: channel_conversion description: Channel-specific conversion factor. Multiply the data in the @@ -103,6 +105,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -143,6 +146,7 @@ classes: name: name: name ifabsent: string(FeatureExtraction) + identifier: true range: string required: true description: @@ -189,6 +193,7 @@ classes: range: DynamicTableRegion required: true multivalued: false + inlined: true tree_root: true EventDetection: name: EventDetection @@ -198,6 +203,7 @@ classes: name: name: name ifabsent: string(EventDetection) + identifier: true range: string required: true detection_method: @@ -236,6 +242,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ElectricalSeries - range: string @@ -297,6 +304,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -317,6 +325,7 @@ classes: range: ElectrodeGroup__position required: false multivalued: false + inlined: true device: name: device annotations: @@ -325,6 +334,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string @@ -336,6 +346,7 @@ classes: name: name: name ifabsent: string(position) + identifier: true range: string required: true equals_string: position @@ -376,6 +387,7 @@ classes: name: name: name ifabsent: string(ClusterWaveforms) + identifier: true range: string required: true waveform_filtering: @@ -416,6 +428,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Clustering - range: string @@ -429,6 +442,7 @@ classes: name: name: name ifabsent: string(Clustering) + identifier: true range: string required: true description: diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.epoch.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.epoch.yaml index ce14120..4eb778d 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.epoch.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.epoch.yaml @@ -22,6 +22,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true start_time: @@ -64,12 +65,14 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true timeseries: name: timeseries description: An index into a TimeSeries object. range: TimeIntervals__timeseries required: false multivalued: false + inlined: true timeseries_index: name: timeseries_index annotations: @@ -83,6 +86,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true tree_root: true TimeIntervals__timeseries: name: TimeIntervals__timeseries @@ -92,6 +96,7 @@ classes: name: name: name ifabsent: string(timeseries) + identifier: true range: string required: true equals_string: timeseries @@ -122,3 +127,4 @@ classes: range: TimeSeries required: false multivalued: false + inlined: true diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.file.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.file.yaml index 0b76f4f..a3eb463 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.file.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.file.yaml @@ -28,6 +28,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true notes: @@ -45,6 +46,7 @@ classes: name: name: name ifabsent: string(root) + identifier: true range: string required: true equals_string: root @@ -184,6 +186,8 @@ classes: range: NWBFile__stimulus required: true multivalued: false + inlined: true + inlined_as_list: true general: name: general description: Experimental metadata, including protocol, notes and description @@ -204,6 +208,8 @@ classes: range: NWBFile__general required: true multivalued: false + inlined: true + inlined_as_list: true intervals: name: intervals description: Experimental intervals, whether that be logically distinct sub-experiments @@ -213,12 +219,16 @@ classes: range: NWBFile__intervals required: false multivalued: false + inlined: true + inlined_as_list: true units: name: units description: Data about sorted spike units. range: Units required: false multivalued: false + inlined: true + inlined_as_list: false tree_root: true NWBFile__stimulus: name: NWBFile__stimulus @@ -238,6 +248,7 @@ classes: name: name: name ifabsent: string(stimulus) + identifier: true range: string required: true equals_string: stimulus @@ -280,6 +291,7 @@ classes: name: name: name ifabsent: string(general) + identifier: true range: string required: true equals_string: general @@ -375,6 +387,7 @@ classes: range: general__source_script required: false multivalued: false + inlined: true stimulus: name: stimulus description: Notes about stimuli, such as how and where they were presented. @@ -402,6 +415,8 @@ classes: range: LabMetaData required: false multivalued: true + inlined: true + inlined_as_list: false devices: name: devices description: Description of hardware devices used during experiment, e.g., @@ -418,18 +433,24 @@ classes: range: Subject required: false multivalued: false + inlined: true + inlined_as_list: false extracellular_ephys: name: extracellular_ephys description: Metadata related to extracellular electrophysiology. range: general__extracellular_ephys required: false multivalued: false + inlined: true + inlined_as_list: true intracellular_ephys: name: intracellular_ephys description: Metadata related to intracellular electrophysiology. range: general__intracellular_ephys required: false multivalued: false + inlined: true + inlined_as_list: true optogenetics: name: optogenetics description: Metadata describing optogenetic stimuluation. @@ -454,6 +475,7 @@ classes: name: name: name ifabsent: string(source_script) + identifier: true range: string required: true equals_string: source_script @@ -473,6 +495,7 @@ classes: name: name: name ifabsent: string(extracellular_ephys) + identifier: true range: string required: true equals_string: extracellular_ephys @@ -482,12 +505,16 @@ classes: range: ElectrodeGroup required: false multivalued: true + inlined: true + inlined_as_list: false electrodes: name: electrodes description: A table of all electrodes (i.e. channels) used for recording. range: extracellular_ephys__electrodes required: false multivalued: false + inlined: true + inlined_as_list: true extracellular_ephys__electrodes: name: extracellular_ephys__electrodes description: A table of all electrodes (i.e. channels) used for recording. @@ -496,6 +523,7 @@ classes: name: name: name ifabsent: string(electrodes) + identifier: true range: string required: true equals_string: electrodes @@ -559,9 +587,13 @@ classes: group: name: group description: Reference to the ElectrodeGroup this electrode is a part of. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: ElectrodeGroup required: true - multivalued: true + multivalued: false + inlined: true group_name: name: group_name description: Name of the ElectrodeGroup this electrode is a part of. @@ -614,6 +646,7 @@ classes: name: name: name ifabsent: string(intracellular_ephys) + identifier: true range: string required: true equals_string: intracellular_ephys @@ -631,12 +664,16 @@ classes: range: IntracellularElectrode required: false multivalued: true + inlined: true + inlined_as_list: false sweep_table: name: sweep_table description: The table which groups different PatchClampSeries together. range: SweepTable required: false multivalued: false + inlined: true + inlined_as_list: false NWBFile__intervals: name: NWBFile__intervals description: Experimental intervals, whether that be logically distinct sub-experiments @@ -646,6 +683,7 @@ classes: name: name: name ifabsent: string(intervals) + identifier: true range: string required: true equals_string: intervals @@ -656,18 +694,24 @@ classes: range: TimeIntervals required: false multivalued: false + inlined: true + inlined_as_list: false trials: name: trials description: Repeated experimental events that have a logical grouping. range: TimeIntervals required: false multivalued: false + inlined: true + inlined_as_list: false invalid_times: name: invalid_times description: Time intervals that should be removed from analysis. range: TimeIntervals required: false multivalued: false + inlined: true + inlined_as_list: false time_intervals: name: time_intervals description: Optional additional table(s) for describing other experimental @@ -675,6 +719,8 @@ classes: range: TimeIntervals required: false multivalued: true + inlined: true + inlined_as_list: false LabMetaData: name: LabMetaData description: Lab-specific meta-data. @@ -682,6 +728,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -692,6 +739,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true age: diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.icephys.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.icephys.yaml index d93bb52..26823be 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.icephys.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.icephys.yaml @@ -23,6 +23,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true stimulus_description: @@ -41,6 +42,7 @@ classes: range: PatchClampSeries__data required: true multivalued: false + inlined: true gain: name: gain description: Gain of the recording, in units Volt/Amp (v-clamp) or Volt/Volt @@ -56,6 +58,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: IntracellularElectrode - range: string @@ -67,6 +70,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -92,6 +96,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -100,6 +105,7 @@ classes: range: CurrentClampSeries__data required: true multivalued: false + inlined: true bias_current: name: bias_current description: Bias current, in amps. @@ -126,6 +132,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -152,6 +159,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true stimulus_description: @@ -188,6 +196,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -196,6 +205,7 @@ classes: range: CurrentClampStimulusSeries__data required: true multivalued: false + inlined: true tree_root: true CurrentClampStimulusSeries__data: name: CurrentClampStimulusSeries__data @@ -204,6 +214,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -229,6 +240,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -237,48 +249,56 @@ classes: range: VoltageClampSeries__data required: true multivalued: false + inlined: true capacitance_fast: name: capacitance_fast description: Fast capacitance, in farads. range: VoltageClampSeries__capacitance_fast required: false multivalued: false + inlined: true capacitance_slow: name: capacitance_slow description: Slow capacitance, in farads. range: VoltageClampSeries__capacitance_slow required: false multivalued: false + inlined: true resistance_comp_bandwidth: name: resistance_comp_bandwidth description: Resistance compensation bandwidth, in hertz. range: VoltageClampSeries__resistance_comp_bandwidth required: false multivalued: false + inlined: true resistance_comp_correction: name: resistance_comp_correction description: Resistance compensation correction, in percent. range: VoltageClampSeries__resistance_comp_correction required: false multivalued: false + inlined: true resistance_comp_prediction: name: resistance_comp_prediction description: Resistance compensation prediction, in percent. range: VoltageClampSeries__resistance_comp_prediction required: false multivalued: false + inlined: true whole_cell_capacitance_comp: name: whole_cell_capacitance_comp description: Whole cell capacitance compensation, in farads. range: VoltageClampSeries__whole_cell_capacitance_comp required: false multivalued: false + inlined: true whole_cell_series_resistance_comp: name: whole_cell_series_resistance_comp description: Whole cell series resistance compensation, in ohms. range: VoltageClampSeries__whole_cell_series_resistance_comp required: false multivalued: false + inlined: true tree_root: true VoltageClampSeries__data: name: VoltageClampSeries__data @@ -287,6 +307,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -310,6 +331,7 @@ classes: name: name: name ifabsent: string(capacitance_fast) + identifier: true range: string required: true equals_string: capacitance_fast @@ -331,6 +353,7 @@ classes: name: name: name ifabsent: string(capacitance_slow) + identifier: true range: string required: true equals_string: capacitance_slow @@ -352,6 +375,7 @@ classes: name: name: name ifabsent: string(resistance_comp_bandwidth) + identifier: true range: string required: true equals_string: resistance_comp_bandwidth @@ -374,6 +398,7 @@ classes: name: name: name ifabsent: string(resistance_comp_correction) + identifier: true range: string required: true equals_string: resistance_comp_correction @@ -396,6 +421,7 @@ classes: name: name: name ifabsent: string(resistance_comp_prediction) + identifier: true range: string required: true equals_string: resistance_comp_prediction @@ -418,6 +444,7 @@ classes: name: name: name ifabsent: string(whole_cell_capacitance_comp) + identifier: true range: string required: true equals_string: whole_cell_capacitance_comp @@ -440,6 +467,7 @@ classes: name: name: name ifabsent: string(whole_cell_series_resistance_comp) + identifier: true range: string required: true equals_string: whole_cell_series_resistance_comp @@ -462,6 +490,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -470,6 +499,7 @@ classes: range: VoltageClampStimulusSeries__data required: true multivalued: false + inlined: true tree_root: true VoltageClampStimulusSeries__data: name: VoltageClampStimulusSeries__data @@ -478,6 +508,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -501,6 +532,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -555,6 +587,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string @@ -566,6 +599,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true sweep_number: @@ -580,9 +614,13 @@ classes: series: name: series description: The PatchClampSeries with the sweep number in that row. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: PatchClampSeries required: true - multivalued: true + multivalued: false + inlined: true series_index: name: series_index annotations: @@ -596,4 +634,5 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.image.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.image.yaml index bbbcfce..adfab1b 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.image.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.image.yaml @@ -21,8 +21,16 @@ classes: attributes: name: name: name + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + range: numeric tree_root: true RGBImage: name: RGBImage @@ -31,8 +39,18 @@ classes: attributes: name: name: name + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b + exact_cardinality: 3 + range: numeric tree_root: true RGBAImage: name: RGBAImage @@ -41,8 +59,18 @@ classes: attributes: name: name: name + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b_a + exact_cardinality: 4 + range: numeric tree_root: true ImageSeries: name: ImageSeries @@ -56,6 +84,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -95,6 +124,7 @@ classes: range: ImageSeries__external_file required: false multivalued: false + inlined: true format: name: format description: Format of image. If this is 'external', then the attribute 'external_file' @@ -112,6 +142,7 @@ classes: value: link required: false multivalued: false + inlined: true any_of: - range: Device - range: string @@ -126,6 +157,7 @@ classes: name: name: name ifabsent: string(external_file) + identifier: true range: string required: true equals_string: external_file @@ -163,6 +195,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true masked_imageseries: @@ -173,6 +206,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImageSeries - range: string @@ -188,6 +222,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true distance: @@ -250,6 +285,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -269,6 +305,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImageSeries - range: string diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.misc.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.misc.yaml index 89d5ee0..c2323b8 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.misc.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.misc.yaml @@ -30,6 +30,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -38,6 +39,7 @@ classes: range: AbstractFeatureSeries__data required: true multivalued: false + inlined: true feature_units: name: feature_units description: Units of each feature. @@ -64,6 +66,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -96,6 +99,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -121,6 +125,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -140,6 +145,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -148,6 +154,7 @@ classes: range: DecompositionSeries__data required: true multivalued: false + inlined: true metric: name: metric description: The metric used, e.g. phase, amplitude, power. @@ -168,6 +175,7 @@ classes: range: DynamicTableRegion required: false multivalued: false + inlined: true bands: name: bands description: Table for describing the bands that this series was generated @@ -175,6 +183,8 @@ classes: range: DecompositionSeries__bands required: true multivalued: false + inlined: true + inlined_as_list: true source_timeseries: name: source_timeseries annotations: @@ -183,6 +193,7 @@ classes: value: link required: false multivalued: false + inlined: true any_of: - range: TimeSeries - range: string @@ -194,6 +205,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -222,6 +234,7 @@ classes: name: name: name ifabsent: string(bands) + identifier: true range: string required: true equals_string: bands @@ -273,6 +286,7 @@ classes: name: name: name ifabsent: string(Units) + identifier: true range: string required: true spike_times_index: @@ -288,12 +302,14 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true spike_times: name: spike_times description: Spike times for each unit. range: Units__spike_times required: false multivalued: false + inlined: true obs_intervals_index: name: obs_intervals_index annotations: @@ -307,6 +323,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true obs_intervals: name: obs_intervals description: Observation intervals for each unit. @@ -331,6 +348,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true electrodes: name: electrodes annotations: @@ -344,12 +362,17 @@ classes: range: DynamicTableRegion required: false multivalued: false + inlined: true electrode_group: name: electrode_group description: Electrode group that each spike unit came from. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: ElectrodeGroup required: false - multivalued: true + multivalued: false + inlined: true waveform_mean: name: waveform_mean description: Spike waveform mean for each spike unit. @@ -428,6 +451,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true waveforms_index_index: name: waveforms_index_index annotations: @@ -442,6 +466,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true tree_root: true Units__spike_times: name: Units__spike_times @@ -451,6 +476,7 @@ classes: name: name: name ifabsent: string(spike_times) + identifier: true range: string required: true equals_string: spike_times diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.ogen.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.ogen.yaml index 3148b98..0dc7be0 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.ogen.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.ogen.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -40,6 +41,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: OptogeneticStimulusSite - range: string @@ -51,6 +53,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -81,6 +84,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.ophys.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.ophys.yaml index e9c680d..40860fc 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.ophys.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.ophys.yaml @@ -23,6 +23,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true pmt_gain: @@ -60,6 +61,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImagingPlane - range: string @@ -72,6 +74,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -102,6 +105,7 @@ classes: range: DynamicTableRegion required: true multivalued: false + inlined: true tree_root: true DfOverF: name: DfOverF @@ -156,15 +160,28 @@ classes: attributes: name: name: name + identifier: true range: string required: true image_mask: name: image_mask description: ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero. - range: PlaneSegmentation__image_mask + range: AnyType required: false multivalued: false + any_of: + - array: + dimensions: + - alias: num_roi + - alias: num_x + - alias: num_y + - array: + dimensions: + - alias: num_roi + - alias: num_x + - alias: num_y + - alias: num_z pixel_mask_index: name: pixel_mask_index annotations: @@ -178,6 +195,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true pixel_mask: name: pixel_mask description: 'Pixel masks for each ROI: a list of indices and weights for @@ -186,6 +204,7 @@ classes: range: PlaneSegmentation__pixel_mask required: false multivalued: false + inlined: true voxel_mask_index: name: voxel_mask_index annotations: @@ -199,6 +218,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true voxel_mask: name: voxel_mask description: 'Voxel masks for each ROI: a list of indices and weights for @@ -207,6 +227,7 @@ classes: range: PlaneSegmentation__voxel_mask required: false multivalued: false + inlined: true reference_images: name: reference_images description: Image stacks that the segmentation masks apply to. @@ -223,22 +244,11 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImagingPlane - range: string tree_root: true - PlaneSegmentation__image_mask: - name: PlaneSegmentation__image_mask - description: ROI masks for each ROI. Each image mask is the size of the original - imaging plane (or volume) and members of the ROI are finite non-zero. - is_a: VectorData - attributes: - name: - name: name - ifabsent: string(image_mask) - range: string - required: true - equals_string: image_mask PlaneSegmentation__pixel_mask: name: PlaneSegmentation__pixel_mask description: 'Pixel masks for each ROI: a list of indices and weights for the @@ -249,6 +259,7 @@ classes: name: name: name ifabsent: string(pixel_mask) + identifier: true range: string required: true equals_string: pixel_mask @@ -286,6 +297,7 @@ classes: name: name: name ifabsent: string(voxel_mask) + identifier: true range: string required: true equals_string: voxel_mask @@ -328,6 +340,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -371,6 +384,7 @@ classes: range: ImagingPlane__manifold required: false multivalued: false + inlined: true origin_coords: name: origin_coords description: Physical location of the first element of the imaging plane (0, @@ -379,6 +393,7 @@ classes: range: ImagingPlane__origin_coords required: false multivalued: false + inlined: true grid_spacing: name: grid_spacing description: Space between pixels in (x, y) or voxels in (x, y, z) directions, @@ -387,6 +402,7 @@ classes: range: ImagingPlane__grid_spacing required: false multivalued: false + inlined: true reference_frame: name: reference_frame description: Describes reference frame of origin_coords and grid_spacing. @@ -415,6 +431,8 @@ classes: range: OpticalChannel required: true multivalued: true + inlined: true + inlined_as_list: false device: name: device annotations: @@ -423,6 +441,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string @@ -436,6 +455,7 @@ classes: name: name: name ifabsent: string(manifold) + identifier: true range: string required: true equals_string: manifold @@ -485,6 +505,7 @@ classes: name: name: name ifabsent: string(origin_coords) + identifier: true range: string required: true equals_string: origin_coords @@ -515,6 +536,7 @@ classes: name: name: name ifabsent: string(grid_spacing) + identifier: true range: string required: true equals_string: grid_spacing @@ -543,6 +565,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -579,6 +602,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true corrected: @@ -587,6 +611,8 @@ classes: range: ImageSeries required: true multivalued: false + inlined: true + inlined_as_list: false xy_translation: name: xy_translation description: Stores the x,y delta necessary to align each frame to the common @@ -594,6 +620,8 @@ classes: range: TimeSeries required: true multivalued: false + inlined: true + inlined_as_list: false original: name: original annotations: @@ -602,6 +630,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImageSeries - range: string diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.retinotopy.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.retinotopy.yaml index cc06e90..97007ea 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.retinotopy.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_3_0/core.nwb.retinotopy.yaml @@ -29,6 +29,7 @@ classes: name: name: name ifabsent: string(ImagingRetinotopy) + identifier: true range: string required: true axis_1_phase_map: @@ -37,6 +38,7 @@ classes: range: ImagingRetinotopy__axis_1_phase_map required: true multivalued: false + inlined: true axis_1_power_map: name: axis_1_power_map description: Power response on the first measured axis. Response is scaled @@ -44,12 +46,14 @@ classes: range: ImagingRetinotopy__axis_1_power_map required: false multivalued: false + inlined: true axis_2_phase_map: name: axis_2_phase_map description: Phase response to stimulus on the second measured axis. range: ImagingRetinotopy__axis_2_phase_map required: true multivalued: false + inlined: true axis_2_power_map: name: axis_2_power_map description: Power response on the second measured axis. Response is scaled @@ -57,6 +61,7 @@ classes: range: ImagingRetinotopy__axis_2_power_map required: false multivalued: false + inlined: true axis_descriptions: name: axis_descriptions description: Two-element array describing the contents of the two response @@ -76,6 +81,7 @@ classes: range: ImagingRetinotopy__focal_depth_image required: false multivalued: false + inlined: true sign_map: name: sign_map description: Sine of the angle between the direction of the gradient in axis_1 @@ -83,6 +89,7 @@ classes: range: ImagingRetinotopy__sign_map required: false multivalued: false + inlined: true vasculature_image: name: vasculature_image description: 'Gray-scale anatomical image of cortical surface. Array structure: @@ -90,6 +97,7 @@ classes: range: ImagingRetinotopy__vasculature_image required: true multivalued: false + inlined: true tree_root: true ImagingRetinotopy__axis_1_phase_map: name: ImagingRetinotopy__axis_1_phase_map @@ -98,6 +106,7 @@ classes: name: name: name ifabsent: string(axis_1_phase_map) + identifier: true range: string required: true equals_string: axis_1_phase_map @@ -134,6 +143,7 @@ classes: name: name: name ifabsent: string(axis_1_power_map) + identifier: true range: string required: true equals_string: axis_1_power_map @@ -169,6 +179,7 @@ classes: name: name: name ifabsent: string(axis_2_phase_map) + identifier: true range: string required: true equals_string: axis_2_phase_map @@ -205,6 +216,7 @@ classes: name: name: name ifabsent: string(axis_2_power_map) + identifier: true range: string required: true equals_string: axis_2_power_map @@ -241,6 +253,7 @@ classes: name: name: name ifabsent: string(focal_depth_image) + identifier: true range: string required: true equals_string: focal_depth_image @@ -288,6 +301,7 @@ classes: name: name: name ifabsent: string(sign_map) + identifier: true range: string required: true equals_string: sign_map @@ -319,6 +333,7 @@ classes: name: name: name ifabsent: string(vasculature_image) + identifier: true range: string required: true equals_string: vasculature_image diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.base.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.base.yaml index 8369707..1bfb911 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.base.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.base.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -34,6 +35,7 @@ classes: name: name: name ifabsent: string(timeseries) + identifier: true range: string required: true idx_start: @@ -63,6 +65,7 @@ classes: range: TimeSeries required: true multivalued: false + inlined: true tree_root: true Image: name: Image @@ -73,6 +76,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true resolution: @@ -114,6 +118,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -125,6 +130,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -135,6 +141,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -160,6 +167,7 @@ classes: range: TimeSeries__data required: true multivalued: false + inlined: true starting_time: name: starting_time description: Timestamp of the first sample in seconds. When timestamps are @@ -168,6 +176,7 @@ classes: range: TimeSeries__starting_time required: false multivalued: false + inlined: true timestamps: name: timestamps description: Timestamps for samples stored in data, in seconds, relative to @@ -211,6 +220,8 @@ classes: range: TimeSeries__sync required: false multivalued: false + inlined: true + inlined_as_list: true tree_root: true TimeSeries__data: name: TimeSeries__data @@ -221,6 +232,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -298,6 +310,7 @@ classes: name: name: name ifabsent: string(starting_time) + identifier: true range: string required: true equals_string: starting_time @@ -329,6 +342,7 @@ classes: name: name: name ifabsent: string(sync) + identifier: true range: string required: true equals_string: sync @@ -353,6 +367,7 @@ classes: name: name: name ifabsent: string(Images) + identifier: true range: string required: true description: diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.behavior.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.behavior.yaml index ba29236..47aa752 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.behavior.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.behavior.yaml @@ -29,6 +29,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -38,6 +39,7 @@ classes: range: SpatialSeries__data required: true multivalued: false + inlined: true reference_frame: name: reference_frame description: Description defining what exactly 'straight-ahead' means. @@ -53,6 +55,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.device.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.device.yaml index fc320af..307f846 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.device.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.device.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.ecephys.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.ecephys.yaml index f0eccd6..4d8e539 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.ecephys.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.ecephys.yaml @@ -25,6 +25,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true filtering: @@ -71,6 +72,7 @@ classes: range: DynamicTableRegion required: true multivalued: false + inlined: true channel_conversion: name: channel_conversion description: Channel-specific conversion factor. Multiply the data in the @@ -103,6 +105,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -143,6 +146,7 @@ classes: name: name: name ifabsent: string(FeatureExtraction) + identifier: true range: string required: true description: @@ -189,6 +193,7 @@ classes: range: DynamicTableRegion required: true multivalued: false + inlined: true tree_root: true EventDetection: name: EventDetection @@ -198,6 +203,7 @@ classes: name: name: name ifabsent: string(EventDetection) + identifier: true range: string required: true detection_method: @@ -236,6 +242,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ElectricalSeries - range: string @@ -297,6 +304,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -317,6 +325,7 @@ classes: range: ElectrodeGroup__position required: false multivalued: false + inlined: true device: name: device annotations: @@ -325,6 +334,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string @@ -336,6 +346,7 @@ classes: name: name: name ifabsent: string(position) + identifier: true range: string required: true equals_string: position @@ -376,6 +387,7 @@ classes: name: name: name ifabsent: string(ClusterWaveforms) + identifier: true range: string required: true waveform_filtering: @@ -416,6 +428,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Clustering - range: string @@ -429,6 +442,7 @@ classes: name: name: name ifabsent: string(Clustering) + identifier: true range: string required: true description: diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.epoch.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.epoch.yaml index c3fb2cb..e264a54 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.epoch.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.epoch.yaml @@ -22,6 +22,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true start_time: @@ -64,12 +65,14 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true timeseries: name: timeseries description: An index into a TimeSeries object. range: TimeIntervals__timeseries required: false multivalued: false + inlined: true timeseries_index: name: timeseries_index annotations: @@ -83,6 +86,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true tree_root: true TimeIntervals__timeseries: name: TimeIntervals__timeseries @@ -92,6 +96,7 @@ classes: name: name: name ifabsent: string(timeseries) + identifier: true range: string required: true equals_string: timeseries @@ -122,3 +127,4 @@ classes: range: TimeSeries required: false multivalued: false + inlined: true diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.file.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.file.yaml index 13bf8a1..f81b157 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.file.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.file.yaml @@ -28,6 +28,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true notes: @@ -45,6 +46,7 @@ classes: name: name: name ifabsent: string(root) + identifier: true range: string required: true equals_string: root @@ -184,6 +186,8 @@ classes: range: NWBFile__stimulus required: true multivalued: false + inlined: true + inlined_as_list: true general: name: general description: Experimental metadata, including protocol, notes and description @@ -204,6 +208,8 @@ classes: range: NWBFile__general required: true multivalued: false + inlined: true + inlined_as_list: true intervals: name: intervals description: Experimental intervals, whether that be logically distinct sub-experiments @@ -213,12 +219,16 @@ classes: range: NWBFile__intervals required: false multivalued: false + inlined: true + inlined_as_list: true units: name: units description: Data about sorted spike units. range: Units required: false multivalued: false + inlined: true + inlined_as_list: false tree_root: true NWBFile__stimulus: name: NWBFile__stimulus @@ -238,6 +248,7 @@ classes: name: name: name ifabsent: string(stimulus) + identifier: true range: string required: true equals_string: stimulus @@ -280,6 +291,7 @@ classes: name: name: name ifabsent: string(general) + identifier: true range: string required: true equals_string: general @@ -375,6 +387,7 @@ classes: range: general__source_script required: false multivalued: false + inlined: true stimulus: name: stimulus description: Notes about stimuli, such as how and where they were presented. @@ -402,6 +415,8 @@ classes: range: LabMetaData required: false multivalued: true + inlined: true + inlined_as_list: false devices: name: devices description: Description of hardware devices used during experiment, e.g., @@ -418,18 +433,24 @@ classes: range: Subject required: false multivalued: false + inlined: true + inlined_as_list: false extracellular_ephys: name: extracellular_ephys description: Metadata related to extracellular electrophysiology. range: general__extracellular_ephys required: false multivalued: false + inlined: true + inlined_as_list: true intracellular_ephys: name: intracellular_ephys description: Metadata related to intracellular electrophysiology. range: general__intracellular_ephys required: false multivalued: false + inlined: true + inlined_as_list: true optogenetics: name: optogenetics description: Metadata describing optogenetic stimuluation. @@ -454,6 +475,7 @@ classes: name: name: name ifabsent: string(source_script) + identifier: true range: string required: true equals_string: source_script @@ -473,6 +495,7 @@ classes: name: name: name ifabsent: string(extracellular_ephys) + identifier: true range: string required: true equals_string: extracellular_ephys @@ -482,12 +505,16 @@ classes: range: ElectrodeGroup required: false multivalued: true + inlined: true + inlined_as_list: false electrodes: name: electrodes description: A table of all electrodes (i.e. channels) used for recording. range: extracellular_ephys__electrodes required: false multivalued: false + inlined: true + inlined_as_list: true extracellular_ephys__electrodes: name: extracellular_ephys__electrodes description: A table of all electrodes (i.e. channels) used for recording. @@ -496,6 +523,7 @@ classes: name: name: name ifabsent: string(electrodes) + identifier: true range: string required: true equals_string: electrodes @@ -559,9 +587,13 @@ classes: group: name: group description: Reference to the ElectrodeGroup this electrode is a part of. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: ElectrodeGroup required: true - multivalued: true + multivalued: false + inlined: true group_name: name: group_name description: Name of the ElectrodeGroup this electrode is a part of. @@ -614,6 +646,7 @@ classes: name: name: name ifabsent: string(intracellular_ephys) + identifier: true range: string required: true equals_string: intracellular_ephys @@ -632,6 +665,8 @@ classes: range: IntracellularElectrode required: false multivalued: true + inlined: true + inlined_as_list: false sweep_table: name: sweep_table description: '[DEPRECATED] Table used to group different PatchClampSeries. @@ -641,6 +676,8 @@ classes: range: SweepTable required: false multivalued: false + inlined: true + inlined_as_list: false intracellular_recordings: name: intracellular_recordings description: A table to group together a stimulus and response from a single @@ -658,6 +695,8 @@ classes: range: IntracellularRecordingsTable required: false multivalued: false + inlined: true + inlined_as_list: false simultaneous_recordings: name: simultaneous_recordings description: A table for grouping different intracellular recordings from @@ -666,6 +705,8 @@ classes: range: SimultaneousRecordingsTable required: false multivalued: false + inlined: true + inlined_as_list: false sequential_recordings: name: sequential_recordings description: A table for grouping different sequential recordings from the @@ -675,6 +716,8 @@ classes: range: SequentialRecordingsTable required: false multivalued: false + inlined: true + inlined_as_list: false repetitions: name: repetitions description: A table for grouping different sequential intracellular recordings @@ -684,6 +727,8 @@ classes: range: RepetitionsTable required: false multivalued: false + inlined: true + inlined_as_list: false experimental_conditions: name: experimental_conditions description: A table for grouping different intracellular recording repetitions @@ -691,6 +736,8 @@ classes: range: ExperimentalConditionsTable required: false multivalued: false + inlined: true + inlined_as_list: false NWBFile__intervals: name: NWBFile__intervals description: Experimental intervals, whether that be logically distinct sub-experiments @@ -700,6 +747,7 @@ classes: name: name: name ifabsent: string(intervals) + identifier: true range: string required: true equals_string: intervals @@ -710,18 +758,24 @@ classes: range: TimeIntervals required: false multivalued: false + inlined: true + inlined_as_list: false trials: name: trials description: Repeated experimental events that have a logical grouping. range: TimeIntervals required: false multivalued: false + inlined: true + inlined_as_list: false invalid_times: name: invalid_times description: Time intervals that should be removed from analysis. range: TimeIntervals required: false multivalued: false + inlined: true + inlined_as_list: false time_intervals: name: time_intervals description: Optional additional table(s) for describing other experimental @@ -729,6 +783,8 @@ classes: range: TimeIntervals required: false multivalued: true + inlined: true + inlined_as_list: false LabMetaData: name: LabMetaData description: Lab-specific meta-data. @@ -736,6 +792,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -746,6 +803,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true age: diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.icephys.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.icephys.yaml index 346751e..d3a808f 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.icephys.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.icephys.yaml @@ -23,6 +23,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true stimulus_description: @@ -41,6 +42,7 @@ classes: range: PatchClampSeries__data required: true multivalued: false + inlined: true gain: name: gain description: Gain of the recording, in units Volt/Amp (v-clamp) or Volt/Volt @@ -56,6 +58,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: IntracellularElectrode - range: string @@ -67,6 +70,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -92,6 +96,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -100,6 +105,7 @@ classes: range: CurrentClampSeries__data required: true multivalued: false + inlined: true bias_current: name: bias_current description: Bias current, in amps. @@ -126,6 +132,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -152,6 +159,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true stimulus_description: @@ -188,6 +196,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -196,6 +205,7 @@ classes: range: CurrentClampStimulusSeries__data required: true multivalued: false + inlined: true tree_root: true CurrentClampStimulusSeries__data: name: CurrentClampStimulusSeries__data @@ -204,6 +214,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -229,6 +240,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -237,48 +249,56 @@ classes: range: VoltageClampSeries__data required: true multivalued: false + inlined: true capacitance_fast: name: capacitance_fast description: Fast capacitance, in farads. range: VoltageClampSeries__capacitance_fast required: false multivalued: false + inlined: true capacitance_slow: name: capacitance_slow description: Slow capacitance, in farads. range: VoltageClampSeries__capacitance_slow required: false multivalued: false + inlined: true resistance_comp_bandwidth: name: resistance_comp_bandwidth description: Resistance compensation bandwidth, in hertz. range: VoltageClampSeries__resistance_comp_bandwidth required: false multivalued: false + inlined: true resistance_comp_correction: name: resistance_comp_correction description: Resistance compensation correction, in percent. range: VoltageClampSeries__resistance_comp_correction required: false multivalued: false + inlined: true resistance_comp_prediction: name: resistance_comp_prediction description: Resistance compensation prediction, in percent. range: VoltageClampSeries__resistance_comp_prediction required: false multivalued: false + inlined: true whole_cell_capacitance_comp: name: whole_cell_capacitance_comp description: Whole cell capacitance compensation, in farads. range: VoltageClampSeries__whole_cell_capacitance_comp required: false multivalued: false + inlined: true whole_cell_series_resistance_comp: name: whole_cell_series_resistance_comp description: Whole cell series resistance compensation, in ohms. range: VoltageClampSeries__whole_cell_series_resistance_comp required: false multivalued: false + inlined: true tree_root: true VoltageClampSeries__data: name: VoltageClampSeries__data @@ -287,6 +307,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -310,6 +331,7 @@ classes: name: name: name ifabsent: string(capacitance_fast) + identifier: true range: string required: true equals_string: capacitance_fast @@ -331,6 +353,7 @@ classes: name: name: name ifabsent: string(capacitance_slow) + identifier: true range: string required: true equals_string: capacitance_slow @@ -352,6 +375,7 @@ classes: name: name: name ifabsent: string(resistance_comp_bandwidth) + identifier: true range: string required: true equals_string: resistance_comp_bandwidth @@ -374,6 +398,7 @@ classes: name: name: name ifabsent: string(resistance_comp_correction) + identifier: true range: string required: true equals_string: resistance_comp_correction @@ -396,6 +421,7 @@ classes: name: name: name ifabsent: string(resistance_comp_prediction) + identifier: true range: string required: true equals_string: resistance_comp_prediction @@ -418,6 +444,7 @@ classes: name: name: name ifabsent: string(whole_cell_capacitance_comp) + identifier: true range: string required: true equals_string: whole_cell_capacitance_comp @@ -440,6 +467,7 @@ classes: name: name: name ifabsent: string(whole_cell_series_resistance_comp) + identifier: true range: string required: true equals_string: whole_cell_series_resistance_comp @@ -462,6 +490,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -470,6 +499,7 @@ classes: range: VoltageClampStimulusSeries__data required: true multivalued: false + inlined: true tree_root: true VoltageClampStimulusSeries__data: name: VoltageClampStimulusSeries__data @@ -478,6 +508,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -501,6 +532,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -555,6 +587,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string @@ -569,6 +602,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true sweep_number: @@ -583,9 +617,13 @@ classes: series: name: series description: The PatchClampSeries with the sweep number in that row. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: PatchClampSeries required: true - multivalued: true + multivalued: false + inlined: true series_index: name: series_index annotations: @@ -599,6 +637,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true IntracellularElectrodesTable: name: IntracellularElectrodesTable @@ -607,6 +646,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -619,9 +659,13 @@ classes: electrode: name: electrode description: Column for storing the reference to the intracellular electrode. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: IntracellularElectrode required: true - multivalued: true + multivalued: false + inlined: true tree_root: true IntracellularStimuliTable: name: IntracellularStimuliTable @@ -630,6 +674,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -653,6 +698,7 @@ classes: range: TimeSeriesReferenceVectorData required: true multivalued: false + inlined: true tree_root: true IntracellularResponsesTable: name: IntracellularResponsesTable @@ -661,6 +707,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -684,6 +731,7 @@ classes: range: TimeSeriesReferenceVectorData required: true multivalued: false + inlined: true tree_root: true IntracellularRecordingsTable: name: IntracellularRecordingsTable @@ -703,6 +751,7 @@ classes: name: name: name ifabsent: string(intracellular_recordings) + identifier: true range: string required: true equals_string: intracellular_recordings @@ -724,18 +773,24 @@ classes: range: IntracellularElectrodesTable required: true multivalued: false + inlined: true + inlined_as_list: false stimuli: name: stimuli description: Table for storing intracellular stimulus related metadata. range: IntracellularStimuliTable required: true multivalued: false + inlined: true + inlined_as_list: false responses: name: responses description: Table for storing intracellular response related metadata. range: IntracellularResponsesTable required: true multivalued: false + inlined: true + inlined_as_list: false tree_root: true SimultaneousRecordingsTable: name: SimultaneousRecordingsTable @@ -747,6 +802,7 @@ classes: name: name: name ifabsent: string(simultaneous_recordings) + identifier: true range: string required: true equals_string: simultaneous_recordings @@ -757,6 +813,7 @@ classes: range: SimultaneousRecordingsTable__recordings required: true multivalued: false + inlined: true recordings_index: name: recordings_index annotations: @@ -770,6 +827,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true SimultaneousRecordingsTable__recordings: name: SimultaneousRecordingsTable__recordings @@ -780,6 +838,7 @@ classes: name: name: name ifabsent: string(recordings) + identifier: true range: string required: true equals_string: recordings @@ -790,6 +849,7 @@ classes: to fix the type of table that can be referenced here. range: IntracellularRecordingsTable required: true + inlined: true SequentialRecordingsTable: name: SequentialRecordingsTable description: A table for grouping different sequential recordings from the SimultaneousRecordingsTable @@ -801,6 +861,7 @@ classes: name: name: name ifabsent: string(sequential_recordings) + identifier: true range: string required: true equals_string: sequential_recordings @@ -811,6 +872,7 @@ classes: range: SequentialRecordingsTable__simultaneous_recordings required: true multivalued: false + inlined: true simultaneous_recordings_index: name: simultaneous_recordings_index annotations: @@ -824,6 +886,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true stimulus_type: name: stimulus_type description: The type of stimulus used for the sequential recording. @@ -843,6 +906,7 @@ classes: name: name: name ifabsent: string(simultaneous_recordings) + identifier: true range: string required: true equals_string: simultaneous_recordings @@ -853,6 +917,7 @@ classes: to fix the type of table that can be referenced here. range: SimultaneousRecordingsTable required: true + inlined: true RepetitionsTable: name: RepetitionsTable description: A table for grouping different sequential intracellular recordings @@ -864,6 +929,7 @@ classes: name: name: name ifabsent: string(repetitions) + identifier: true range: string required: true equals_string: repetitions @@ -874,6 +940,7 @@ classes: range: RepetitionsTable__sequential_recordings required: true multivalued: false + inlined: true sequential_recordings_index: name: sequential_recordings_index annotations: @@ -887,6 +954,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true RepetitionsTable__sequential_recordings: name: RepetitionsTable__sequential_recordings @@ -897,6 +965,7 @@ classes: name: name: name ifabsent: string(sequential_recordings) + identifier: true range: string required: true equals_string: sequential_recordings @@ -907,6 +976,7 @@ classes: to fix the type of table that can be referenced here. range: SequentialRecordingsTable required: true + inlined: true ExperimentalConditionsTable: name: ExperimentalConditionsTable description: A table for grouping different intracellular recording repetitions @@ -916,6 +986,7 @@ classes: name: name: name ifabsent: string(experimental_conditions) + identifier: true range: string required: true equals_string: experimental_conditions @@ -925,6 +996,7 @@ classes: range: ExperimentalConditionsTable__repetitions required: true multivalued: false + inlined: true repetitions_index: name: repetitions_index annotations: @@ -938,6 +1010,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true ExperimentalConditionsTable__repetitions: name: ExperimentalConditionsTable__repetitions @@ -947,6 +1020,7 @@ classes: name: name: name ifabsent: string(repetitions) + identifier: true range: string required: true equals_string: repetitions @@ -957,3 +1031,4 @@ classes: to fix the type of table that can be referenced here. range: RepetitionsTable required: true + inlined: true diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.image.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.image.yaml index ac28a30..fec75ec 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.image.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.image.yaml @@ -21,8 +21,16 @@ classes: attributes: name: name: name + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + range: numeric tree_root: true RGBImage: name: RGBImage @@ -31,8 +39,18 @@ classes: attributes: name: name: name + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b + exact_cardinality: 3 + range: numeric tree_root: true RGBAImage: name: RGBAImage @@ -41,8 +59,18 @@ classes: attributes: name: name: name + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b_a + exact_cardinality: 4 + range: numeric tree_root: true ImageSeries: name: ImageSeries @@ -56,6 +84,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -96,6 +125,7 @@ classes: range: ImageSeries__external_file required: false multivalued: false + inlined: true format: name: format description: Format of image. If this is 'external', then the attribute 'external_file' @@ -113,6 +143,7 @@ classes: value: link required: false multivalued: false + inlined: true any_of: - range: Device - range: string @@ -127,6 +158,7 @@ classes: name: name: name ifabsent: string(external_file) + identifier: true range: string required: true equals_string: external_file @@ -164,6 +196,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true masked_imageseries: @@ -174,6 +207,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImageSeries - range: string @@ -189,6 +223,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true distance: @@ -251,6 +286,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -270,6 +306,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImageSeries - range: string diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.misc.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.misc.yaml index 97927d6..ec02fc4 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.misc.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.misc.yaml @@ -30,6 +30,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -38,6 +39,7 @@ classes: range: AbstractFeatureSeries__data required: true multivalued: false + inlined: true feature_units: name: feature_units description: Units of each feature. @@ -64,6 +66,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -96,6 +99,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -121,6 +125,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -140,6 +145,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -148,6 +154,7 @@ classes: range: DecompositionSeries__data required: true multivalued: false + inlined: true metric: name: metric description: The metric used, e.g. phase, amplitude, power. @@ -168,6 +175,7 @@ classes: range: DynamicTableRegion required: false multivalued: false + inlined: true bands: name: bands description: Table for describing the bands that this series was generated @@ -175,6 +183,8 @@ classes: range: DecompositionSeries__bands required: true multivalued: false + inlined: true + inlined_as_list: true source_timeseries: name: source_timeseries annotations: @@ -183,6 +193,7 @@ classes: value: link required: false multivalued: false + inlined: true any_of: - range: TimeSeries - range: string @@ -194,6 +205,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -222,6 +234,7 @@ classes: name: name: name ifabsent: string(bands) + identifier: true range: string required: true equals_string: bands @@ -273,6 +286,7 @@ classes: name: name: name ifabsent: string(Units) + identifier: true range: string required: true spike_times_index: @@ -288,12 +302,14 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true spike_times: name: spike_times description: Spike times for each unit. range: Units__spike_times required: false multivalued: false + inlined: true obs_intervals_index: name: obs_intervals_index annotations: @@ -307,6 +323,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true obs_intervals: name: obs_intervals description: Observation intervals for each unit. @@ -331,6 +348,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true electrodes: name: electrodes annotations: @@ -344,12 +362,17 @@ classes: range: DynamicTableRegion required: false multivalued: false + inlined: true electrode_group: name: electrode_group description: Electrode group that each spike unit came from. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: ElectrodeGroup required: false - multivalued: true + multivalued: false + inlined: true waveform_mean: name: waveform_mean description: Spike waveform mean for each spike unit. @@ -428,6 +451,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true waveforms_index_index: name: waveforms_index_index annotations: @@ -442,6 +466,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true tree_root: true Units__spike_times: name: Units__spike_times @@ -451,6 +476,7 @@ classes: name: name: name ifabsent: string(spike_times) + identifier: true range: string required: true equals_string: spike_times diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.ogen.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.ogen.yaml index 1add778..cbe1a6d 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.ogen.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.ogen.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -40,6 +41,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: OptogeneticStimulusSite - range: string @@ -51,6 +53,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -81,6 +84,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.ophys.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.ophys.yaml index c6215f1..aec8547 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.ophys.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.ophys.yaml @@ -23,6 +23,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true pmt_gain: @@ -60,6 +61,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImagingPlane - range: string @@ -72,6 +74,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -102,6 +105,7 @@ classes: range: DynamicTableRegion required: true multivalued: false + inlined: true tree_root: true DfOverF: name: DfOverF @@ -156,15 +160,28 @@ classes: attributes: name: name: name + identifier: true range: string required: true image_mask: name: image_mask description: ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero. - range: PlaneSegmentation__image_mask + range: AnyType required: false multivalued: false + any_of: + - array: + dimensions: + - alias: num_roi + - alias: num_x + - alias: num_y + - array: + dimensions: + - alias: num_roi + - alias: num_x + - alias: num_y + - alias: num_z pixel_mask_index: name: pixel_mask_index annotations: @@ -178,6 +195,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true pixel_mask: name: pixel_mask description: 'Pixel masks for each ROI: a list of indices and weights for @@ -186,6 +204,7 @@ classes: range: PlaneSegmentation__pixel_mask required: false multivalued: false + inlined: true voxel_mask_index: name: voxel_mask_index annotations: @@ -199,6 +218,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true voxel_mask: name: voxel_mask description: 'Voxel masks for each ROI: a list of indices and weights for @@ -207,6 +227,7 @@ classes: range: PlaneSegmentation__voxel_mask required: false multivalued: false + inlined: true reference_images: name: reference_images description: Image stacks that the segmentation masks apply to. @@ -223,22 +244,11 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImagingPlane - range: string tree_root: true - PlaneSegmentation__image_mask: - name: PlaneSegmentation__image_mask - description: ROI masks for each ROI. Each image mask is the size of the original - imaging plane (or volume) and members of the ROI are finite non-zero. - is_a: VectorData - attributes: - name: - name: name - ifabsent: string(image_mask) - range: string - required: true - equals_string: image_mask PlaneSegmentation__pixel_mask: name: PlaneSegmentation__pixel_mask description: 'Pixel masks for each ROI: a list of indices and weights for the @@ -249,6 +259,7 @@ classes: name: name: name ifabsent: string(pixel_mask) + identifier: true range: string required: true equals_string: pixel_mask @@ -286,6 +297,7 @@ classes: name: name: name ifabsent: string(voxel_mask) + identifier: true range: string required: true equals_string: voxel_mask @@ -328,6 +340,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -371,6 +384,7 @@ classes: range: ImagingPlane__manifold required: false multivalued: false + inlined: true origin_coords: name: origin_coords description: Physical location of the first element of the imaging plane (0, @@ -379,6 +393,7 @@ classes: range: ImagingPlane__origin_coords required: false multivalued: false + inlined: true grid_spacing: name: grid_spacing description: Space between pixels in (x, y) or voxels in (x, y, z) directions, @@ -387,6 +402,7 @@ classes: range: ImagingPlane__grid_spacing required: false multivalued: false + inlined: true reference_frame: name: reference_frame description: Describes reference frame of origin_coords and grid_spacing. @@ -415,6 +431,8 @@ classes: range: OpticalChannel required: true multivalued: true + inlined: true + inlined_as_list: false device: name: device annotations: @@ -423,6 +441,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string @@ -436,6 +455,7 @@ classes: name: name: name ifabsent: string(manifold) + identifier: true range: string required: true equals_string: manifold @@ -485,6 +505,7 @@ classes: name: name: name ifabsent: string(origin_coords) + identifier: true range: string required: true equals_string: origin_coords @@ -515,6 +536,7 @@ classes: name: name: name ifabsent: string(grid_spacing) + identifier: true range: string required: true equals_string: grid_spacing @@ -543,6 +565,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -579,6 +602,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true corrected: @@ -587,6 +611,8 @@ classes: range: ImageSeries required: true multivalued: false + inlined: true + inlined_as_list: false xy_translation: name: xy_translation description: Stores the x,y delta necessary to align each frame to the common @@ -594,6 +620,8 @@ classes: range: TimeSeries required: true multivalued: false + inlined: true + inlined_as_list: false original: name: original annotations: @@ -602,6 +630,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImageSeries - range: string diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.retinotopy.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.retinotopy.yaml index f433f10..f30f06f 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.retinotopy.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_4_0/core.nwb.retinotopy.yaml @@ -29,6 +29,7 @@ classes: name: name: name ifabsent: string(ImagingRetinotopy) + identifier: true range: string required: true axis_1_phase_map: @@ -37,6 +38,7 @@ classes: range: ImagingRetinotopy__axis_1_phase_map required: true multivalued: false + inlined: true axis_1_power_map: name: axis_1_power_map description: Power response on the first measured axis. Response is scaled @@ -44,12 +46,14 @@ classes: range: ImagingRetinotopy__axis_1_power_map required: false multivalued: false + inlined: true axis_2_phase_map: name: axis_2_phase_map description: Phase response to stimulus on the second measured axis. range: ImagingRetinotopy__axis_2_phase_map required: true multivalued: false + inlined: true axis_2_power_map: name: axis_2_power_map description: Power response on the second measured axis. Response is scaled @@ -57,6 +61,7 @@ classes: range: ImagingRetinotopy__axis_2_power_map required: false multivalued: false + inlined: true axis_descriptions: name: axis_descriptions description: Two-element array describing the contents of the two response @@ -76,6 +81,7 @@ classes: range: ImagingRetinotopy__focal_depth_image required: false multivalued: false + inlined: true sign_map: name: sign_map description: Sine of the angle between the direction of the gradient in axis_1 @@ -83,6 +89,7 @@ classes: range: ImagingRetinotopy__sign_map required: false multivalued: false + inlined: true vasculature_image: name: vasculature_image description: 'Gray-scale anatomical image of cortical surface. Array structure: @@ -90,6 +97,7 @@ classes: range: ImagingRetinotopy__vasculature_image required: true multivalued: false + inlined: true tree_root: true ImagingRetinotopy__axis_1_phase_map: name: ImagingRetinotopy__axis_1_phase_map @@ -98,6 +106,7 @@ classes: name: name: name ifabsent: string(axis_1_phase_map) + identifier: true range: string required: true equals_string: axis_1_phase_map @@ -134,6 +143,7 @@ classes: name: name: name ifabsent: string(axis_1_power_map) + identifier: true range: string required: true equals_string: axis_1_power_map @@ -169,6 +179,7 @@ classes: name: name: name ifabsent: string(axis_2_phase_map) + identifier: true range: string required: true equals_string: axis_2_phase_map @@ -205,6 +216,7 @@ classes: name: name: name ifabsent: string(axis_2_power_map) + identifier: true range: string required: true equals_string: axis_2_power_map @@ -241,6 +253,7 @@ classes: name: name: name ifabsent: string(focal_depth_image) + identifier: true range: string required: true equals_string: focal_depth_image @@ -288,6 +301,7 @@ classes: name: name: name ifabsent: string(sign_map) + identifier: true range: string required: true equals_string: sign_map @@ -319,6 +333,7 @@ classes: name: name: name ifabsent: string(vasculature_image) + identifier: true range: string required: true equals_string: vasculature_image diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.base.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.base.yaml index 373ff4d..547dd4c 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.base.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.base.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -34,6 +35,7 @@ classes: name: name: name ifabsent: string(timeseries) + identifier: true range: string required: true idx_start: @@ -63,6 +65,7 @@ classes: range: TimeSeries required: true multivalued: false + inlined: true tree_root: true Image: name: Image @@ -73,6 +76,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true resolution: @@ -113,6 +117,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true value: @@ -125,6 +130,8 @@ classes: range: Image required: true multivalued: true + inlined: true + inlined_as_list: true tree_root: true NWBContainer: name: NWBContainer @@ -134,6 +141,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -145,6 +153,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -155,6 +164,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -180,6 +190,7 @@ classes: range: TimeSeries__data required: true multivalued: false + inlined: true starting_time: name: starting_time description: Timestamp of the first sample in seconds. When timestamps are @@ -188,6 +199,7 @@ classes: range: TimeSeries__starting_time required: false multivalued: false + inlined: true timestamps: name: timestamps description: Timestamps for samples stored in data, in seconds, relative to @@ -231,6 +243,8 @@ classes: range: TimeSeries__sync required: false multivalued: false + inlined: true + inlined_as_list: true tree_root: true TimeSeries__data: name: TimeSeries__data @@ -241,6 +255,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -327,6 +342,7 @@ classes: name: name: name ifabsent: string(starting_time) + identifier: true range: string required: true equals_string: starting_time @@ -358,6 +374,7 @@ classes: name: name: name ifabsent: string(sync) + identifier: true range: string required: true equals_string: sync @@ -384,6 +401,7 @@ classes: name: name: name ifabsent: string(Images) + identifier: true range: string required: true description: @@ -413,4 +431,5 @@ classes: range: ImageReferences required: false multivalued: false + inlined: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.behavior.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.behavior.yaml index fd2c46f..94ff5f8 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.behavior.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.behavior.yaml @@ -29,6 +29,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -38,6 +39,7 @@ classes: range: SpatialSeries__data required: true multivalued: false + inlined: true reference_frame: name: reference_frame description: Description defining what exactly 'straight-ahead' means. @@ -53,6 +55,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.device.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.device.yaml index 3f1acc9..d2ec1a5 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.device.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.device.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.ecephys.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.ecephys.yaml index ce256eb..b611d74 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.ecephys.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.ecephys.yaml @@ -25,6 +25,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true filtering: @@ -71,6 +72,7 @@ classes: range: DynamicTableRegion required: true multivalued: false + inlined: true channel_conversion: name: channel_conversion description: Channel-specific conversion factor. Multiply the data in the @@ -103,6 +105,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -143,6 +146,7 @@ classes: name: name: name ifabsent: string(FeatureExtraction) + identifier: true range: string required: true description: @@ -189,6 +193,7 @@ classes: range: DynamicTableRegion required: true multivalued: false + inlined: true tree_root: true EventDetection: name: EventDetection @@ -198,6 +203,7 @@ classes: name: name: name ifabsent: string(EventDetection) + identifier: true range: string required: true detection_method: @@ -236,6 +242,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ElectricalSeries - range: string @@ -297,6 +304,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -317,6 +325,7 @@ classes: range: ElectrodeGroup__position required: false multivalued: false + inlined: true device: name: device annotations: @@ -325,6 +334,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string @@ -336,6 +346,7 @@ classes: name: name: name ifabsent: string(position) + identifier: true range: string required: true equals_string: position @@ -376,6 +387,7 @@ classes: name: name: name ifabsent: string(ClusterWaveforms) + identifier: true range: string required: true waveform_filtering: @@ -416,6 +428,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Clustering - range: string @@ -429,6 +442,7 @@ classes: name: name: name ifabsent: string(Clustering) + identifier: true range: string required: true description: diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.epoch.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.epoch.yaml index 3764b00..9857394 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.epoch.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.epoch.yaml @@ -22,6 +22,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true start_time: @@ -64,6 +65,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true timeseries: name: timeseries annotations: @@ -77,6 +79,7 @@ classes: range: TimeSeriesReferenceVectorData required: false multivalued: false + inlined: true timeseries_index: name: timeseries_index annotations: @@ -90,4 +93,5 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.file.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.file.yaml index f468049..01ef5b5 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.file.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.file.yaml @@ -28,6 +28,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true notes: @@ -45,6 +46,7 @@ classes: name: name: name ifabsent: string(root) + identifier: true range: string required: true equals_string: root @@ -184,6 +186,8 @@ classes: range: NWBFile__stimulus required: true multivalued: false + inlined: true + inlined_as_list: true general: name: general description: Experimental metadata, including protocol, notes and description @@ -204,6 +208,8 @@ classes: range: NWBFile__general required: true multivalued: false + inlined: true + inlined_as_list: true intervals: name: intervals description: Experimental intervals, whether that be logically distinct sub-experiments @@ -213,12 +219,16 @@ classes: range: NWBFile__intervals required: false multivalued: false + inlined: true + inlined_as_list: true units: name: units description: Data about sorted spike units. range: Units required: false multivalued: false + inlined: true + inlined_as_list: false tree_root: true NWBFile__stimulus: name: NWBFile__stimulus @@ -238,6 +248,7 @@ classes: name: name: name ifabsent: string(stimulus) + identifier: true range: string required: true equals_string: stimulus @@ -281,6 +292,7 @@ classes: name: name: name ifabsent: string(general) + identifier: true range: string required: true equals_string: general @@ -376,6 +388,7 @@ classes: range: general__source_script required: false multivalued: false + inlined: true stimulus: name: stimulus description: Notes about stimuli, such as how and where they were presented. @@ -403,6 +416,8 @@ classes: range: LabMetaData required: false multivalued: true + inlined: true + inlined_as_list: false devices: name: devices description: Description of hardware devices used during experiment, e.g., @@ -419,18 +434,24 @@ classes: range: Subject required: false multivalued: false + inlined: true + inlined_as_list: false extracellular_ephys: name: extracellular_ephys description: Metadata related to extracellular electrophysiology. range: general__extracellular_ephys required: false multivalued: false + inlined: true + inlined_as_list: true intracellular_ephys: name: intracellular_ephys description: Metadata related to intracellular electrophysiology. range: general__intracellular_ephys required: false multivalued: false + inlined: true + inlined_as_list: true optogenetics: name: optogenetics description: Metadata describing optogenetic stimuluation. @@ -455,6 +476,7 @@ classes: name: name: name ifabsent: string(source_script) + identifier: true range: string required: true equals_string: source_script @@ -474,6 +496,7 @@ classes: name: name: name ifabsent: string(extracellular_ephys) + identifier: true range: string required: true equals_string: extracellular_ephys @@ -483,12 +506,16 @@ classes: range: ElectrodeGroup required: false multivalued: true + inlined: true + inlined_as_list: false electrodes: name: electrodes description: A table of all electrodes (i.e. channels) used for recording. range: extracellular_ephys__electrodes required: false multivalued: false + inlined: true + inlined_as_list: true extracellular_ephys__electrodes: name: extracellular_ephys__electrodes description: A table of all electrodes (i.e. channels) used for recording. @@ -497,6 +524,7 @@ classes: name: name: name ifabsent: string(electrodes) + identifier: true range: string required: true equals_string: electrodes @@ -560,9 +588,13 @@ classes: group: name: group description: Reference to the ElectrodeGroup this electrode is a part of. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: ElectrodeGroup required: true - multivalued: true + multivalued: false + inlined: true group_name: name: group_name description: Name of the ElectrodeGroup this electrode is a part of. @@ -617,6 +649,7 @@ classes: name: name: name ifabsent: string(intracellular_ephys) + identifier: true range: string required: true equals_string: intracellular_ephys @@ -635,6 +668,8 @@ classes: range: IntracellularElectrode required: false multivalued: true + inlined: true + inlined_as_list: false sweep_table: name: sweep_table description: '[DEPRECATED] Table used to group different PatchClampSeries. @@ -644,6 +679,8 @@ classes: range: SweepTable required: false multivalued: false + inlined: true + inlined_as_list: false intracellular_recordings: name: intracellular_recordings description: A table to group together a stimulus and response from a single @@ -661,6 +698,8 @@ classes: range: IntracellularRecordingsTable required: false multivalued: false + inlined: true + inlined_as_list: false simultaneous_recordings: name: simultaneous_recordings description: A table for grouping different intracellular recordings from @@ -669,6 +708,8 @@ classes: range: SimultaneousRecordingsTable required: false multivalued: false + inlined: true + inlined_as_list: false sequential_recordings: name: sequential_recordings description: A table for grouping different sequential recordings from the @@ -678,6 +719,8 @@ classes: range: SequentialRecordingsTable required: false multivalued: false + inlined: true + inlined_as_list: false repetitions: name: repetitions description: A table for grouping different sequential intracellular recordings @@ -687,6 +730,8 @@ classes: range: RepetitionsTable required: false multivalued: false + inlined: true + inlined_as_list: false experimental_conditions: name: experimental_conditions description: A table for grouping different intracellular recording repetitions @@ -694,6 +739,8 @@ classes: range: ExperimentalConditionsTable required: false multivalued: false + inlined: true + inlined_as_list: false NWBFile__intervals: name: NWBFile__intervals description: Experimental intervals, whether that be logically distinct sub-experiments @@ -703,6 +750,7 @@ classes: name: name: name ifabsent: string(intervals) + identifier: true range: string required: true equals_string: intervals @@ -713,18 +761,24 @@ classes: range: TimeIntervals required: false multivalued: false + inlined: true + inlined_as_list: false trials: name: trials description: Repeated experimental events that have a logical grouping. range: TimeIntervals required: false multivalued: false + inlined: true + inlined_as_list: false invalid_times: name: invalid_times description: Time intervals that should be removed from analysis. range: TimeIntervals required: false multivalued: false + inlined: true + inlined_as_list: false time_intervals: name: time_intervals description: Optional additional table(s) for describing other experimental @@ -732,6 +786,8 @@ classes: range: TimeIntervals required: false multivalued: true + inlined: true + inlined_as_list: false LabMetaData: name: LabMetaData description: Lab-specific meta-data. @@ -739,6 +795,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -749,6 +806,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true age: diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.icephys.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.icephys.yaml index bdd9dd5..257b07b 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.icephys.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.icephys.yaml @@ -23,6 +23,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true stimulus_description: @@ -41,6 +42,7 @@ classes: range: PatchClampSeries__data required: true multivalued: false + inlined: true gain: name: gain description: Gain of the recording, in units Volt/Amp (v-clamp) or Volt/Volt @@ -56,6 +58,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: IntracellularElectrode - range: string @@ -67,6 +70,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -92,6 +96,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -100,6 +105,7 @@ classes: range: CurrentClampSeries__data required: true multivalued: false + inlined: true bias_current: name: bias_current description: Bias current, in amps. @@ -126,6 +132,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -153,6 +160,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true stimulus_description: @@ -189,6 +197,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -197,6 +206,7 @@ classes: range: CurrentClampStimulusSeries__data required: true multivalued: false + inlined: true tree_root: true CurrentClampStimulusSeries__data: name: CurrentClampStimulusSeries__data @@ -205,6 +215,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -231,6 +242,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -239,48 +251,56 @@ classes: range: VoltageClampSeries__data required: true multivalued: false + inlined: true capacitance_fast: name: capacitance_fast description: Fast capacitance, in farads. range: VoltageClampSeries__capacitance_fast required: false multivalued: false + inlined: true capacitance_slow: name: capacitance_slow description: Slow capacitance, in farads. range: VoltageClampSeries__capacitance_slow required: false multivalued: false + inlined: true resistance_comp_bandwidth: name: resistance_comp_bandwidth description: Resistance compensation bandwidth, in hertz. range: VoltageClampSeries__resistance_comp_bandwidth required: false multivalued: false + inlined: true resistance_comp_correction: name: resistance_comp_correction description: Resistance compensation correction, in percent. range: VoltageClampSeries__resistance_comp_correction required: false multivalued: false + inlined: true resistance_comp_prediction: name: resistance_comp_prediction description: Resistance compensation prediction, in percent. range: VoltageClampSeries__resistance_comp_prediction required: false multivalued: false + inlined: true whole_cell_capacitance_comp: name: whole_cell_capacitance_comp description: Whole cell capacitance compensation, in farads. range: VoltageClampSeries__whole_cell_capacitance_comp required: false multivalued: false + inlined: true whole_cell_series_resistance_comp: name: whole_cell_series_resistance_comp description: Whole cell series resistance compensation, in ohms. range: VoltageClampSeries__whole_cell_series_resistance_comp required: false multivalued: false + inlined: true tree_root: true VoltageClampSeries__data: name: VoltageClampSeries__data @@ -289,6 +309,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -313,6 +334,7 @@ classes: name: name: name ifabsent: string(capacitance_fast) + identifier: true range: string required: true equals_string: capacitance_fast @@ -334,6 +356,7 @@ classes: name: name: name ifabsent: string(capacitance_slow) + identifier: true range: string required: true equals_string: capacitance_slow @@ -355,6 +378,7 @@ classes: name: name: name ifabsent: string(resistance_comp_bandwidth) + identifier: true range: string required: true equals_string: resistance_comp_bandwidth @@ -377,6 +401,7 @@ classes: name: name: name ifabsent: string(resistance_comp_correction) + identifier: true range: string required: true equals_string: resistance_comp_correction @@ -399,6 +424,7 @@ classes: name: name: name ifabsent: string(resistance_comp_prediction) + identifier: true range: string required: true equals_string: resistance_comp_prediction @@ -421,6 +447,7 @@ classes: name: name: name ifabsent: string(whole_cell_capacitance_comp) + identifier: true range: string required: true equals_string: whole_cell_capacitance_comp @@ -443,6 +470,7 @@ classes: name: name: name ifabsent: string(whole_cell_series_resistance_comp) + identifier: true range: string required: true equals_string: whole_cell_series_resistance_comp @@ -465,6 +493,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -473,6 +502,7 @@ classes: range: VoltageClampStimulusSeries__data required: true multivalued: false + inlined: true tree_root: true VoltageClampStimulusSeries__data: name: VoltageClampStimulusSeries__data @@ -481,6 +511,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -505,6 +536,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true cell_id: @@ -565,6 +597,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string @@ -579,6 +612,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true sweep_number: @@ -593,9 +627,13 @@ classes: series: name: series description: The PatchClampSeries with the sweep number in that row. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: PatchClampSeries required: true - multivalued: true + multivalued: false + inlined: true series_index: name: series_index annotations: @@ -609,6 +647,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true IntracellularElectrodesTable: name: IntracellularElectrodesTable @@ -617,6 +656,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -629,9 +669,13 @@ classes: electrode: name: electrode description: Column for storing the reference to the intracellular electrode. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: IntracellularElectrode required: true - multivalued: true + multivalued: false + inlined: true tree_root: true IntracellularStimuliTable: name: IntracellularStimuliTable @@ -640,6 +684,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -663,6 +708,7 @@ classes: range: TimeSeriesReferenceVectorData required: true multivalued: false + inlined: true tree_root: true IntracellularResponsesTable: name: IntracellularResponsesTable @@ -671,6 +717,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -694,6 +741,7 @@ classes: range: TimeSeriesReferenceVectorData required: true multivalued: false + inlined: true tree_root: true IntracellularRecordingsTable: name: IntracellularRecordingsTable @@ -713,6 +761,7 @@ classes: name: name: name ifabsent: string(intracellular_recordings) + identifier: true range: string required: true equals_string: intracellular_recordings @@ -734,18 +783,24 @@ classes: range: IntracellularElectrodesTable required: true multivalued: false + inlined: true + inlined_as_list: false stimuli: name: stimuli description: Table for storing intracellular stimulus related metadata. range: IntracellularStimuliTable required: true multivalued: false + inlined: true + inlined_as_list: false responses: name: responses description: Table for storing intracellular response related metadata. range: IntracellularResponsesTable required: true multivalued: false + inlined: true + inlined_as_list: false tree_root: true SimultaneousRecordingsTable: name: SimultaneousRecordingsTable @@ -757,6 +812,7 @@ classes: name: name: name ifabsent: string(simultaneous_recordings) + identifier: true range: string required: true equals_string: simultaneous_recordings @@ -767,6 +823,7 @@ classes: range: SimultaneousRecordingsTable__recordings required: true multivalued: false + inlined: true recordings_index: name: recordings_index annotations: @@ -780,6 +837,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true SimultaneousRecordingsTable__recordings: name: SimultaneousRecordingsTable__recordings @@ -790,6 +848,7 @@ classes: name: name: name ifabsent: string(recordings) + identifier: true range: string required: true equals_string: recordings @@ -800,6 +859,7 @@ classes: to fix the type of table that can be referenced here. range: IntracellularRecordingsTable required: true + inlined: true SequentialRecordingsTable: name: SequentialRecordingsTable description: A table for grouping different sequential recordings from the SimultaneousRecordingsTable @@ -811,6 +871,7 @@ classes: name: name: name ifabsent: string(sequential_recordings) + identifier: true range: string required: true equals_string: sequential_recordings @@ -821,6 +882,7 @@ classes: range: SequentialRecordingsTable__simultaneous_recordings required: true multivalued: false + inlined: true simultaneous_recordings_index: name: simultaneous_recordings_index annotations: @@ -834,6 +896,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true stimulus_type: name: stimulus_type description: The type of stimulus used for the sequential recording. @@ -853,6 +916,7 @@ classes: name: name: name ifabsent: string(simultaneous_recordings) + identifier: true range: string required: true equals_string: simultaneous_recordings @@ -863,6 +927,7 @@ classes: to fix the type of table that can be referenced here. range: SimultaneousRecordingsTable required: true + inlined: true RepetitionsTable: name: RepetitionsTable description: A table for grouping different sequential intracellular recordings @@ -874,6 +939,7 @@ classes: name: name: name ifabsent: string(repetitions) + identifier: true range: string required: true equals_string: repetitions @@ -884,6 +950,7 @@ classes: range: RepetitionsTable__sequential_recordings required: true multivalued: false + inlined: true sequential_recordings_index: name: sequential_recordings_index annotations: @@ -897,6 +964,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true RepetitionsTable__sequential_recordings: name: RepetitionsTable__sequential_recordings @@ -907,6 +975,7 @@ classes: name: name: name ifabsent: string(sequential_recordings) + identifier: true range: string required: true equals_string: sequential_recordings @@ -917,6 +986,7 @@ classes: to fix the type of table that can be referenced here. range: SequentialRecordingsTable required: true + inlined: true ExperimentalConditionsTable: name: ExperimentalConditionsTable description: A table for grouping different intracellular recording repetitions @@ -926,6 +996,7 @@ classes: name: name: name ifabsent: string(experimental_conditions) + identifier: true range: string required: true equals_string: experimental_conditions @@ -935,6 +1006,7 @@ classes: range: ExperimentalConditionsTable__repetitions required: true multivalued: false + inlined: true repetitions_index: name: repetitions_index annotations: @@ -948,6 +1020,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true ExperimentalConditionsTable__repetitions: name: ExperimentalConditionsTable__repetitions @@ -957,6 +1030,7 @@ classes: name: name: name ifabsent: string(repetitions) + identifier: true range: string required: true equals_string: repetitions @@ -967,3 +1041,4 @@ classes: to fix the type of table that can be referenced here. range: RepetitionsTable required: true + inlined: true diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.image.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.image.yaml index 0f6efd9..dd4d2f4 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.image.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.image.yaml @@ -21,8 +21,16 @@ classes: attributes: name: name: name + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + range: numeric tree_root: true RGBImage: name: RGBImage @@ -31,8 +39,18 @@ classes: attributes: name: name: name + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b + exact_cardinality: 3 + range: numeric tree_root: true RGBAImage: name: RGBAImage @@ -41,8 +59,18 @@ classes: attributes: name: name: name + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b_a + exact_cardinality: 4 + range: numeric tree_root: true ImageSeries: name: ImageSeries @@ -56,6 +84,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -96,6 +125,7 @@ classes: range: ImageSeries__external_file required: false multivalued: false + inlined: true format: name: format description: Format of image. If this is 'external', then the attribute 'external_file' @@ -113,6 +143,7 @@ classes: value: link required: false multivalued: false + inlined: true any_of: - range: Device - range: string @@ -127,6 +158,7 @@ classes: name: name: name ifabsent: string(external_file) + identifier: true range: string required: true equals_string: external_file @@ -164,6 +196,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true masked_imageseries: @@ -174,6 +207,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImageSeries - range: string @@ -189,6 +223,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true distance: @@ -252,6 +287,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -272,6 +308,7 @@ classes: value: link required: false multivalued: false + inlined: true any_of: - range: ImageSeries - range: string @@ -283,6 +320,7 @@ classes: value: link required: false multivalued: false + inlined: true any_of: - range: Images - range: string diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.misc.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.misc.yaml index f663994..5bfeb44 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.misc.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.misc.yaml @@ -30,6 +30,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -38,6 +39,7 @@ classes: range: AbstractFeatureSeries__data required: true multivalued: false + inlined: true feature_units: name: feature_units description: Units of each feature. @@ -64,6 +66,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -96,6 +99,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -121,6 +125,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -140,6 +145,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -148,6 +154,7 @@ classes: range: DecompositionSeries__data required: true multivalued: false + inlined: true metric: name: metric description: The metric used, e.g. phase, amplitude, power. @@ -168,6 +175,7 @@ classes: range: DynamicTableRegion required: false multivalued: false + inlined: true bands: name: bands description: Table for describing the bands that this series was generated @@ -175,6 +183,8 @@ classes: range: DecompositionSeries__bands required: true multivalued: false + inlined: true + inlined_as_list: true source_timeseries: name: source_timeseries annotations: @@ -183,6 +193,7 @@ classes: value: link required: false multivalued: false + inlined: true any_of: - range: TimeSeries - range: string @@ -194,6 +205,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -222,6 +234,7 @@ classes: name: name: name ifabsent: string(bands) + identifier: true range: string required: true equals_string: bands @@ -273,6 +286,7 @@ classes: name: name: name ifabsent: string(Units) + identifier: true range: string required: true spike_times_index: @@ -288,12 +302,14 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true spike_times: name: spike_times description: Spike times for each unit. range: Units__spike_times required: false multivalued: false + inlined: true obs_intervals_index: name: obs_intervals_index annotations: @@ -307,6 +323,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true obs_intervals: name: obs_intervals description: Observation intervals for each unit. @@ -331,6 +348,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true electrodes: name: electrodes annotations: @@ -344,12 +362,17 @@ classes: range: DynamicTableRegion required: false multivalued: false + inlined: true electrode_group: name: electrode_group description: Electrode group that each spike unit came from. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: ElectrodeGroup required: false - multivalued: true + multivalued: false + inlined: true waveform_mean: name: waveform_mean description: Spike waveform mean for each spike unit. @@ -428,6 +451,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true waveforms_index_index: name: waveforms_index_index annotations: @@ -442,6 +466,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true tree_root: true Units__spike_times: name: Units__spike_times @@ -451,6 +476,7 @@ classes: name: name: name ifabsent: string(spike_times) + identifier: true range: string required: true equals_string: spike_times diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.ogen.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.ogen.yaml index adadc3e..8c6b076 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.ogen.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.ogen.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -40,6 +41,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: OptogeneticStimulusSite - range: string @@ -51,6 +53,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -81,6 +84,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.ophys.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.ophys.yaml index 9cd8b1e..17bb442 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.ophys.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.ophys.yaml @@ -23,6 +23,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true pmt_gain: @@ -60,6 +61,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImagingPlane - range: string @@ -72,6 +74,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -102,6 +105,7 @@ classes: range: DynamicTableRegion required: true multivalued: false + inlined: true tree_root: true DfOverF: name: DfOverF @@ -156,15 +160,28 @@ classes: attributes: name: name: name + identifier: true range: string required: true image_mask: name: image_mask description: ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero. - range: PlaneSegmentation__image_mask + range: AnyType required: false multivalued: false + any_of: + - array: + dimensions: + - alias: num_roi + - alias: num_x + - alias: num_y + - array: + dimensions: + - alias: num_roi + - alias: num_x + - alias: num_y + - alias: num_z pixel_mask_index: name: pixel_mask_index annotations: @@ -178,6 +195,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true pixel_mask: name: pixel_mask description: 'Pixel masks for each ROI: a list of indices and weights for @@ -186,6 +204,7 @@ classes: range: PlaneSegmentation__pixel_mask required: false multivalued: false + inlined: true voxel_mask_index: name: voxel_mask_index annotations: @@ -199,6 +218,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true voxel_mask: name: voxel_mask description: 'Voxel masks for each ROI: a list of indices and weights for @@ -207,6 +227,7 @@ classes: range: PlaneSegmentation__voxel_mask required: false multivalued: false + inlined: true reference_images: name: reference_images description: Image stacks that the segmentation masks apply to. @@ -223,22 +244,11 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImagingPlane - range: string tree_root: true - PlaneSegmentation__image_mask: - name: PlaneSegmentation__image_mask - description: ROI masks for each ROI. Each image mask is the size of the original - imaging plane (or volume) and members of the ROI are finite non-zero. - is_a: VectorData - attributes: - name: - name: name - ifabsent: string(image_mask) - range: string - required: true - equals_string: image_mask PlaneSegmentation__pixel_mask: name: PlaneSegmentation__pixel_mask description: 'Pixel masks for each ROI: a list of indices and weights for the @@ -249,6 +259,7 @@ classes: name: name: name ifabsent: string(pixel_mask) + identifier: true range: string required: true equals_string: pixel_mask @@ -286,6 +297,7 @@ classes: name: name: name ifabsent: string(voxel_mask) + identifier: true range: string required: true equals_string: voxel_mask @@ -328,6 +340,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -371,6 +384,7 @@ classes: range: ImagingPlane__manifold required: false multivalued: false + inlined: true origin_coords: name: origin_coords description: Physical location of the first element of the imaging plane (0, @@ -379,6 +393,7 @@ classes: range: ImagingPlane__origin_coords required: false multivalued: false + inlined: true grid_spacing: name: grid_spacing description: Space between pixels in (x, y) or voxels in (x, y, z) directions, @@ -387,6 +402,7 @@ classes: range: ImagingPlane__grid_spacing required: false multivalued: false + inlined: true reference_frame: name: reference_frame description: Describes reference frame of origin_coords and grid_spacing. @@ -415,6 +431,8 @@ classes: range: OpticalChannel required: true multivalued: true + inlined: true + inlined_as_list: false device: name: device annotations: @@ -423,6 +441,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string @@ -436,6 +455,7 @@ classes: name: name: name ifabsent: string(manifold) + identifier: true range: string required: true equals_string: manifold @@ -485,6 +505,7 @@ classes: name: name: name ifabsent: string(origin_coords) + identifier: true range: string required: true equals_string: origin_coords @@ -515,6 +536,7 @@ classes: name: name: name ifabsent: string(grid_spacing) + identifier: true range: string required: true equals_string: grid_spacing @@ -543,6 +565,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -579,6 +602,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true corrected: @@ -587,6 +611,8 @@ classes: range: ImageSeries required: true multivalued: false + inlined: true + inlined_as_list: false xy_translation: name: xy_translation description: Stores the x,y delta necessary to align each frame to the common @@ -594,6 +620,8 @@ classes: range: TimeSeries required: true multivalued: false + inlined: true + inlined_as_list: false original: name: original annotations: @@ -602,6 +630,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImageSeries - range: string diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.retinotopy.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.retinotopy.yaml index 3a624b1..26b6ed6 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.retinotopy.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_5_0/core.nwb.retinotopy.yaml @@ -29,6 +29,7 @@ classes: name: name: name ifabsent: string(ImagingRetinotopy) + identifier: true range: string required: true axis_1_phase_map: @@ -37,6 +38,7 @@ classes: range: ImagingRetinotopy__axis_1_phase_map required: true multivalued: false + inlined: true axis_1_power_map: name: axis_1_power_map description: Power response on the first measured axis. Response is scaled @@ -44,12 +46,14 @@ classes: range: ImagingRetinotopy__axis_1_power_map required: false multivalued: false + inlined: true axis_2_phase_map: name: axis_2_phase_map description: Phase response to stimulus on the second measured axis. range: ImagingRetinotopy__axis_2_phase_map required: true multivalued: false + inlined: true axis_2_power_map: name: axis_2_power_map description: Power response on the second measured axis. Response is scaled @@ -57,6 +61,7 @@ classes: range: ImagingRetinotopy__axis_2_power_map required: false multivalued: false + inlined: true axis_descriptions: name: axis_descriptions description: Two-element array describing the contents of the two response @@ -76,6 +81,7 @@ classes: range: ImagingRetinotopy__focal_depth_image required: false multivalued: false + inlined: true sign_map: name: sign_map description: Sine of the angle between the direction of the gradient in axis_1 @@ -83,6 +89,7 @@ classes: range: ImagingRetinotopy__sign_map required: false multivalued: false + inlined: true vasculature_image: name: vasculature_image description: 'Gray-scale anatomical image of cortical surface. Array structure: @@ -90,6 +97,7 @@ classes: range: ImagingRetinotopy__vasculature_image required: true multivalued: false + inlined: true tree_root: true ImagingRetinotopy__axis_1_phase_map: name: ImagingRetinotopy__axis_1_phase_map @@ -98,6 +106,7 @@ classes: name: name: name ifabsent: string(axis_1_phase_map) + identifier: true range: string required: true equals_string: axis_1_phase_map @@ -134,6 +143,7 @@ classes: name: name: name ifabsent: string(axis_1_power_map) + identifier: true range: string required: true equals_string: axis_1_power_map @@ -169,6 +179,7 @@ classes: name: name: name ifabsent: string(axis_2_phase_map) + identifier: true range: string required: true equals_string: axis_2_phase_map @@ -205,6 +216,7 @@ classes: name: name: name ifabsent: string(axis_2_power_map) + identifier: true range: string required: true equals_string: axis_2_power_map @@ -241,6 +253,7 @@ classes: name: name: name ifabsent: string(focal_depth_image) + identifier: true range: string required: true equals_string: focal_depth_image @@ -288,6 +301,7 @@ classes: name: name: name ifabsent: string(sign_map) + identifier: true range: string required: true equals_string: sign_map @@ -319,6 +333,7 @@ classes: name: name: name ifabsent: string(vasculature_image) + identifier: true range: string required: true equals_string: vasculature_image diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.base.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.base.yaml index bae736e..9aeec32 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.base.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.base.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -34,6 +35,7 @@ classes: name: name: name ifabsent: string(timeseries) + identifier: true range: string required: true idx_start: @@ -63,6 +65,7 @@ classes: range: TimeSeries required: true multivalued: false + inlined: true tree_root: true Image: name: Image @@ -73,6 +76,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true resolution: @@ -113,6 +117,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true value: @@ -125,6 +130,8 @@ classes: range: Image required: true multivalued: true + inlined: true + inlined_as_list: true tree_root: true NWBContainer: name: NWBContainer @@ -134,6 +141,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -145,6 +153,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -155,6 +164,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -180,6 +190,7 @@ classes: range: TimeSeries__data required: true multivalued: false + inlined: true starting_time: name: starting_time description: Timestamp of the first sample in seconds. When timestamps are @@ -188,6 +199,7 @@ classes: range: TimeSeries__starting_time required: false multivalued: false + inlined: true timestamps: name: timestamps description: Timestamps for samples stored in data, in seconds, relative to @@ -231,6 +243,8 @@ classes: range: TimeSeries__sync required: false multivalued: false + inlined: true + inlined_as_list: true tree_root: true TimeSeries__data: name: TimeSeries__data @@ -241,6 +255,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -327,6 +342,7 @@ classes: name: name: name ifabsent: string(starting_time) + identifier: true range: string required: true equals_string: starting_time @@ -358,6 +374,7 @@ classes: name: name: name ifabsent: string(sync) + identifier: true range: string required: true equals_string: sync @@ -384,6 +401,7 @@ classes: name: name: name ifabsent: string(Images) + identifier: true range: string required: true description: @@ -413,4 +431,5 @@ classes: range: ImageReferences required: false multivalued: false + inlined: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.behavior.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.behavior.yaml index 0f6f89e..9d96389 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.behavior.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.behavior.yaml @@ -29,6 +29,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -38,6 +39,7 @@ classes: range: SpatialSeries__data required: true multivalued: false + inlined: true reference_frame: name: reference_frame description: Description defining what exactly 'straight-ahead' means. @@ -53,6 +55,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.device.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.device.yaml index 4dd254b..f41ac54 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.device.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.device.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.ecephys.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.ecephys.yaml index d63fc10..6fba341 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.ecephys.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.ecephys.yaml @@ -25,6 +25,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true filtering: @@ -71,6 +72,7 @@ classes: range: DynamicTableRegion required: true multivalued: false + inlined: true channel_conversion: name: channel_conversion description: Channel-specific conversion factor. Multiply the data in the @@ -103,6 +105,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -143,6 +146,7 @@ classes: name: name: name ifabsent: string(FeatureExtraction) + identifier: true range: string required: true description: @@ -189,6 +193,7 @@ classes: range: DynamicTableRegion required: true multivalued: false + inlined: true tree_root: true EventDetection: name: EventDetection @@ -198,6 +203,7 @@ classes: name: name: name ifabsent: string(EventDetection) + identifier: true range: string required: true detection_method: @@ -236,6 +242,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ElectricalSeries - range: string @@ -297,6 +304,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -317,6 +325,7 @@ classes: range: ElectrodeGroup__position required: false multivalued: false + inlined: true device: name: device annotations: @@ -325,6 +334,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string @@ -336,6 +346,7 @@ classes: name: name: name ifabsent: string(position) + identifier: true range: string required: true equals_string: position @@ -376,6 +387,7 @@ classes: name: name: name ifabsent: string(ClusterWaveforms) + identifier: true range: string required: true waveform_filtering: @@ -416,6 +428,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Clustering - range: string @@ -429,6 +442,7 @@ classes: name: name: name ifabsent: string(Clustering) + identifier: true range: string required: true description: diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.epoch.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.epoch.yaml index fb0df61..0a9685b 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.epoch.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.epoch.yaml @@ -22,6 +22,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true start_time: @@ -64,6 +65,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true timeseries: name: timeseries annotations: @@ -77,6 +79,7 @@ classes: range: TimeSeriesReferenceVectorData required: false multivalued: false + inlined: true timeseries_index: name: timeseries_index annotations: @@ -90,4 +93,5 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.file.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.file.yaml index f5d5d49..481256f 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.file.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.file.yaml @@ -28,6 +28,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true notes: @@ -45,6 +46,7 @@ classes: name: name: name ifabsent: string(root) + identifier: true range: string required: true equals_string: root @@ -184,6 +186,8 @@ classes: range: NWBFile__stimulus required: true multivalued: false + inlined: true + inlined_as_list: true general: name: general description: Experimental metadata, including protocol, notes and description @@ -204,6 +208,8 @@ classes: range: NWBFile__general required: true multivalued: false + inlined: true + inlined_as_list: true intervals: name: intervals description: Experimental intervals, whether that be logically distinct sub-experiments @@ -213,12 +219,16 @@ classes: range: NWBFile__intervals required: false multivalued: false + inlined: true + inlined_as_list: true units: name: units description: Data about sorted spike units. range: Units required: false multivalued: false + inlined: true + inlined_as_list: false tree_root: true NWBFile__stimulus: name: NWBFile__stimulus @@ -238,6 +248,7 @@ classes: name: name: name ifabsent: string(stimulus) + identifier: true range: string required: true equals_string: stimulus @@ -281,6 +292,7 @@ classes: name: name: name ifabsent: string(general) + identifier: true range: string required: true equals_string: general @@ -376,6 +388,7 @@ classes: range: general__source_script required: false multivalued: false + inlined: true stimulus: name: stimulus description: Notes about stimuli, such as how and where they were presented. @@ -403,6 +416,8 @@ classes: range: LabMetaData required: false multivalued: true + inlined: true + inlined_as_list: false devices: name: devices description: Description of hardware devices used during experiment, e.g., @@ -419,18 +434,24 @@ classes: range: Subject required: false multivalued: false + inlined: true + inlined_as_list: false extracellular_ephys: name: extracellular_ephys description: Metadata related to extracellular electrophysiology. range: general__extracellular_ephys required: false multivalued: false + inlined: true + inlined_as_list: true intracellular_ephys: name: intracellular_ephys description: Metadata related to intracellular electrophysiology. range: general__intracellular_ephys required: false multivalued: false + inlined: true + inlined_as_list: true optogenetics: name: optogenetics description: Metadata describing optogenetic stimuluation. @@ -455,6 +476,7 @@ classes: name: name: name ifabsent: string(source_script) + identifier: true range: string required: true equals_string: source_script @@ -474,6 +496,7 @@ classes: name: name: name ifabsent: string(extracellular_ephys) + identifier: true range: string required: true equals_string: extracellular_ephys @@ -483,12 +506,16 @@ classes: range: ElectrodeGroup required: false multivalued: true + inlined: true + inlined_as_list: false electrodes: name: electrodes description: A table of all electrodes (i.e. channels) used for recording. range: extracellular_ephys__electrodes required: false multivalued: false + inlined: true + inlined_as_list: true extracellular_ephys__electrodes: name: extracellular_ephys__electrodes description: A table of all electrodes (i.e. channels) used for recording. @@ -497,6 +524,7 @@ classes: name: name: name ifabsent: string(electrodes) + identifier: true range: string required: true equals_string: electrodes @@ -560,9 +588,13 @@ classes: group: name: group description: Reference to the ElectrodeGroup this electrode is a part of. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: ElectrodeGroup required: true - multivalued: true + multivalued: false + inlined: true group_name: name: group_name description: Name of the ElectrodeGroup this electrode is a part of. @@ -617,6 +649,7 @@ classes: name: name: name ifabsent: string(intracellular_ephys) + identifier: true range: string required: true equals_string: intracellular_ephys @@ -635,6 +668,8 @@ classes: range: IntracellularElectrode required: false multivalued: true + inlined: true + inlined_as_list: false sweep_table: name: sweep_table description: '[DEPRECATED] Table used to group different PatchClampSeries. @@ -644,6 +679,8 @@ classes: range: SweepTable required: false multivalued: false + inlined: true + inlined_as_list: false intracellular_recordings: name: intracellular_recordings description: A table to group together a stimulus and response from a single @@ -661,6 +698,8 @@ classes: range: IntracellularRecordingsTable required: false multivalued: false + inlined: true + inlined_as_list: false simultaneous_recordings: name: simultaneous_recordings description: A table for grouping different intracellular recordings from @@ -669,6 +708,8 @@ classes: range: SimultaneousRecordingsTable required: false multivalued: false + inlined: true + inlined_as_list: false sequential_recordings: name: sequential_recordings description: A table for grouping different sequential recordings from the @@ -678,6 +719,8 @@ classes: range: SequentialRecordingsTable required: false multivalued: false + inlined: true + inlined_as_list: false repetitions: name: repetitions description: A table for grouping different sequential intracellular recordings @@ -687,6 +730,8 @@ classes: range: RepetitionsTable required: false multivalued: false + inlined: true + inlined_as_list: false experimental_conditions: name: experimental_conditions description: A table for grouping different intracellular recording repetitions @@ -694,6 +739,8 @@ classes: range: ExperimentalConditionsTable required: false multivalued: false + inlined: true + inlined_as_list: false NWBFile__intervals: name: NWBFile__intervals description: Experimental intervals, whether that be logically distinct sub-experiments @@ -703,6 +750,7 @@ classes: name: name: name ifabsent: string(intervals) + identifier: true range: string required: true equals_string: intervals @@ -713,18 +761,24 @@ classes: range: TimeIntervals required: false multivalued: false + inlined: true + inlined_as_list: false trials: name: trials description: Repeated experimental events that have a logical grouping. range: TimeIntervals required: false multivalued: false + inlined: true + inlined_as_list: false invalid_times: name: invalid_times description: Time intervals that should be removed from analysis. range: TimeIntervals required: false multivalued: false + inlined: true + inlined_as_list: false time_intervals: name: time_intervals description: Optional additional table(s) for describing other experimental @@ -732,6 +786,8 @@ classes: range: TimeIntervals required: false multivalued: true + inlined: true + inlined_as_list: false LabMetaData: name: LabMetaData description: Lab-specific meta-data. @@ -739,6 +795,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -749,6 +806,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true age: @@ -757,6 +815,7 @@ classes: range: Subject__age required: false multivalued: false + inlined: true date_of_birth: name: date_of_birth description: Date of birth of subject. Can be supplied instead of 'age'. @@ -815,6 +874,7 @@ classes: name: name: name ifabsent: string(age) + identifier: true range: string required: true equals_string: age diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.icephys.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.icephys.yaml index b3181bc..140e8c8 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.icephys.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.icephys.yaml @@ -23,6 +23,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true stimulus_description: @@ -41,6 +42,7 @@ classes: range: PatchClampSeries__data required: true multivalued: false + inlined: true gain: name: gain description: Gain of the recording, in units Volt/Amp (v-clamp) or Volt/Volt @@ -56,6 +58,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: IntracellularElectrode - range: string @@ -67,6 +70,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -92,6 +96,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -100,6 +105,7 @@ classes: range: CurrentClampSeries__data required: true multivalued: false + inlined: true bias_current: name: bias_current description: Bias current, in amps. @@ -126,6 +132,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -153,6 +160,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true stimulus_description: @@ -189,6 +197,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -197,6 +206,7 @@ classes: range: CurrentClampStimulusSeries__data required: true multivalued: false + inlined: true tree_root: true CurrentClampStimulusSeries__data: name: CurrentClampStimulusSeries__data @@ -205,6 +215,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -231,6 +242,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -239,48 +251,56 @@ classes: range: VoltageClampSeries__data required: true multivalued: false + inlined: true capacitance_fast: name: capacitance_fast description: Fast capacitance, in farads. range: VoltageClampSeries__capacitance_fast required: false multivalued: false + inlined: true capacitance_slow: name: capacitance_slow description: Slow capacitance, in farads. range: VoltageClampSeries__capacitance_slow required: false multivalued: false + inlined: true resistance_comp_bandwidth: name: resistance_comp_bandwidth description: Resistance compensation bandwidth, in hertz. range: VoltageClampSeries__resistance_comp_bandwidth required: false multivalued: false + inlined: true resistance_comp_correction: name: resistance_comp_correction description: Resistance compensation correction, in percent. range: VoltageClampSeries__resistance_comp_correction required: false multivalued: false + inlined: true resistance_comp_prediction: name: resistance_comp_prediction description: Resistance compensation prediction, in percent. range: VoltageClampSeries__resistance_comp_prediction required: false multivalued: false + inlined: true whole_cell_capacitance_comp: name: whole_cell_capacitance_comp description: Whole cell capacitance compensation, in farads. range: VoltageClampSeries__whole_cell_capacitance_comp required: false multivalued: false + inlined: true whole_cell_series_resistance_comp: name: whole_cell_series_resistance_comp description: Whole cell series resistance compensation, in ohms. range: VoltageClampSeries__whole_cell_series_resistance_comp required: false multivalued: false + inlined: true tree_root: true VoltageClampSeries__data: name: VoltageClampSeries__data @@ -289,6 +309,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -313,6 +334,7 @@ classes: name: name: name ifabsent: string(capacitance_fast) + identifier: true range: string required: true equals_string: capacitance_fast @@ -334,6 +356,7 @@ classes: name: name: name ifabsent: string(capacitance_slow) + identifier: true range: string required: true equals_string: capacitance_slow @@ -355,6 +378,7 @@ classes: name: name: name ifabsent: string(resistance_comp_bandwidth) + identifier: true range: string required: true equals_string: resistance_comp_bandwidth @@ -377,6 +401,7 @@ classes: name: name: name ifabsent: string(resistance_comp_correction) + identifier: true range: string required: true equals_string: resistance_comp_correction @@ -399,6 +424,7 @@ classes: name: name: name ifabsent: string(resistance_comp_prediction) + identifier: true range: string required: true equals_string: resistance_comp_prediction @@ -421,6 +447,7 @@ classes: name: name: name ifabsent: string(whole_cell_capacitance_comp) + identifier: true range: string required: true equals_string: whole_cell_capacitance_comp @@ -443,6 +470,7 @@ classes: name: name: name ifabsent: string(whole_cell_series_resistance_comp) + identifier: true range: string required: true equals_string: whole_cell_series_resistance_comp @@ -465,6 +493,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -473,6 +502,7 @@ classes: range: VoltageClampStimulusSeries__data required: true multivalued: false + inlined: true tree_root: true VoltageClampStimulusSeries__data: name: VoltageClampStimulusSeries__data @@ -481,6 +511,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -505,6 +536,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true cell_id: @@ -565,6 +597,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string @@ -579,6 +612,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true sweep_number: @@ -593,9 +627,13 @@ classes: series: name: series description: The PatchClampSeries with the sweep number in that row. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: PatchClampSeries required: true - multivalued: true + multivalued: false + inlined: true series_index: name: series_index annotations: @@ -609,6 +647,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true IntracellularElectrodesTable: name: IntracellularElectrodesTable @@ -617,6 +656,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -629,9 +669,13 @@ classes: electrode: name: electrode description: Column for storing the reference to the intracellular electrode. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: IntracellularElectrode required: true - multivalued: true + multivalued: false + inlined: true tree_root: true IntracellularStimuliTable: name: IntracellularStimuliTable @@ -640,6 +684,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -663,6 +708,7 @@ classes: range: TimeSeriesReferenceVectorData required: true multivalued: false + inlined: true tree_root: true IntracellularResponsesTable: name: IntracellularResponsesTable @@ -671,6 +717,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -694,6 +741,7 @@ classes: range: TimeSeriesReferenceVectorData required: true multivalued: false + inlined: true tree_root: true IntracellularRecordingsTable: name: IntracellularRecordingsTable @@ -713,6 +761,7 @@ classes: name: name: name ifabsent: string(intracellular_recordings) + identifier: true range: string required: true equals_string: intracellular_recordings @@ -734,18 +783,24 @@ classes: range: IntracellularElectrodesTable required: true multivalued: false + inlined: true + inlined_as_list: false stimuli: name: stimuli description: Table for storing intracellular stimulus related metadata. range: IntracellularStimuliTable required: true multivalued: false + inlined: true + inlined_as_list: false responses: name: responses description: Table for storing intracellular response related metadata. range: IntracellularResponsesTable required: true multivalued: false + inlined: true + inlined_as_list: false tree_root: true SimultaneousRecordingsTable: name: SimultaneousRecordingsTable @@ -757,6 +812,7 @@ classes: name: name: name ifabsent: string(simultaneous_recordings) + identifier: true range: string required: true equals_string: simultaneous_recordings @@ -767,6 +823,7 @@ classes: range: SimultaneousRecordingsTable__recordings required: true multivalued: false + inlined: true recordings_index: name: recordings_index annotations: @@ -780,6 +837,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true SimultaneousRecordingsTable__recordings: name: SimultaneousRecordingsTable__recordings @@ -790,6 +848,7 @@ classes: name: name: name ifabsent: string(recordings) + identifier: true range: string required: true equals_string: recordings @@ -800,6 +859,7 @@ classes: to fix the type of table that can be referenced here. range: IntracellularRecordingsTable required: true + inlined: true SequentialRecordingsTable: name: SequentialRecordingsTable description: A table for grouping different sequential recordings from the SimultaneousRecordingsTable @@ -811,6 +871,7 @@ classes: name: name: name ifabsent: string(sequential_recordings) + identifier: true range: string required: true equals_string: sequential_recordings @@ -821,6 +882,7 @@ classes: range: SequentialRecordingsTable__simultaneous_recordings required: true multivalued: false + inlined: true simultaneous_recordings_index: name: simultaneous_recordings_index annotations: @@ -834,6 +896,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true stimulus_type: name: stimulus_type description: The type of stimulus used for the sequential recording. @@ -853,6 +916,7 @@ classes: name: name: name ifabsent: string(simultaneous_recordings) + identifier: true range: string required: true equals_string: simultaneous_recordings @@ -863,6 +927,7 @@ classes: to fix the type of table that can be referenced here. range: SimultaneousRecordingsTable required: true + inlined: true RepetitionsTable: name: RepetitionsTable description: A table for grouping different sequential intracellular recordings @@ -874,6 +939,7 @@ classes: name: name: name ifabsent: string(repetitions) + identifier: true range: string required: true equals_string: repetitions @@ -884,6 +950,7 @@ classes: range: RepetitionsTable__sequential_recordings required: true multivalued: false + inlined: true sequential_recordings_index: name: sequential_recordings_index annotations: @@ -897,6 +964,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true RepetitionsTable__sequential_recordings: name: RepetitionsTable__sequential_recordings @@ -907,6 +975,7 @@ classes: name: name: name ifabsent: string(sequential_recordings) + identifier: true range: string required: true equals_string: sequential_recordings @@ -917,6 +986,7 @@ classes: to fix the type of table that can be referenced here. range: SequentialRecordingsTable required: true + inlined: true ExperimentalConditionsTable: name: ExperimentalConditionsTable description: A table for grouping different intracellular recording repetitions @@ -926,6 +996,7 @@ classes: name: name: name ifabsent: string(experimental_conditions) + identifier: true range: string required: true equals_string: experimental_conditions @@ -935,6 +1006,7 @@ classes: range: ExperimentalConditionsTable__repetitions required: true multivalued: false + inlined: true repetitions_index: name: repetitions_index annotations: @@ -948,6 +1020,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true ExperimentalConditionsTable__repetitions: name: ExperimentalConditionsTable__repetitions @@ -957,6 +1030,7 @@ classes: name: name: name ifabsent: string(repetitions) + identifier: true range: string required: true equals_string: repetitions @@ -967,3 +1041,4 @@ classes: to fix the type of table that can be referenced here. range: RepetitionsTable required: true + inlined: true diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.image.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.image.yaml index 45bd0a3..4406284 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.image.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.image.yaml @@ -21,8 +21,16 @@ classes: attributes: name: name: name + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + range: numeric tree_root: true RGBImage: name: RGBImage @@ -31,8 +39,18 @@ classes: attributes: name: name: name + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b + exact_cardinality: 3 + range: numeric tree_root: true RGBAImage: name: RGBAImage @@ -41,8 +59,18 @@ classes: attributes: name: name: name + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b_a + exact_cardinality: 4 + range: numeric tree_root: true ImageSeries: name: ImageSeries @@ -56,6 +84,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -96,6 +125,7 @@ classes: range: ImageSeries__external_file required: false multivalued: false + inlined: true format: name: format description: Format of image. If this is 'external', then the attribute 'external_file' @@ -113,6 +143,7 @@ classes: value: link required: false multivalued: false + inlined: true any_of: - range: Device - range: string @@ -127,6 +158,7 @@ classes: name: name: name ifabsent: string(external_file) + identifier: true range: string required: true equals_string: external_file @@ -164,6 +196,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true masked_imageseries: @@ -174,6 +207,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImageSeries - range: string @@ -189,6 +223,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true distance: @@ -252,6 +287,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -272,6 +308,7 @@ classes: value: link required: false multivalued: false + inlined: true any_of: - range: ImageSeries - range: string @@ -283,6 +320,7 @@ classes: value: link required: false multivalued: false + inlined: true any_of: - range: Images - range: string diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.misc.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.misc.yaml index 56f8824..ced8985 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.misc.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.misc.yaml @@ -30,6 +30,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -38,6 +39,7 @@ classes: range: AbstractFeatureSeries__data required: true multivalued: false + inlined: true feature_units: name: feature_units description: Units of each feature. @@ -64,6 +66,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -96,6 +99,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -121,6 +125,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -140,6 +145,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -148,6 +154,7 @@ classes: range: DecompositionSeries__data required: true multivalued: false + inlined: true metric: name: metric description: The metric used, e.g. phase, amplitude, power. @@ -168,6 +175,7 @@ classes: range: DynamicTableRegion required: false multivalued: false + inlined: true bands: name: bands description: Table for describing the bands that this series was generated @@ -175,6 +183,8 @@ classes: range: DecompositionSeries__bands required: true multivalued: false + inlined: true + inlined_as_list: true source_timeseries: name: source_timeseries annotations: @@ -183,6 +193,7 @@ classes: value: link required: false multivalued: false + inlined: true any_of: - range: TimeSeries - range: string @@ -194,6 +205,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -222,6 +234,7 @@ classes: name: name: name ifabsent: string(bands) + identifier: true range: string required: true equals_string: bands @@ -273,6 +286,7 @@ classes: name: name: name ifabsent: string(Units) + identifier: true range: string required: true spike_times_index: @@ -288,12 +302,14 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true spike_times: name: spike_times description: Spike times for each unit in seconds. range: Units__spike_times required: false multivalued: false + inlined: true obs_intervals_index: name: obs_intervals_index annotations: @@ -307,6 +323,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true obs_intervals: name: obs_intervals description: Observation intervals for each unit. @@ -331,6 +348,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true electrodes: name: electrodes annotations: @@ -344,12 +362,17 @@ classes: range: DynamicTableRegion required: false multivalued: false + inlined: true electrode_group: name: electrode_group description: Electrode group that each spike unit came from. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: ElectrodeGroup required: false - multivalued: true + multivalued: false + inlined: true waveform_mean: name: waveform_mean description: Spike waveform mean for each spike unit. @@ -428,6 +451,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true waveforms_index_index: name: waveforms_index_index annotations: @@ -442,6 +466,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true tree_root: true Units__spike_times: name: Units__spike_times @@ -451,6 +476,7 @@ classes: name: name: name ifabsent: string(spike_times) + identifier: true range: string required: true equals_string: spike_times diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.ogen.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.ogen.yaml index 93ab4af..b485822 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.ogen.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.ogen.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -40,6 +41,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: OptogeneticStimulusSite - range: string @@ -51,6 +53,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -81,6 +84,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.ophys.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.ophys.yaml index 730ece0..3da9ec5 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.ophys.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.ophys.yaml @@ -23,6 +23,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true pmt_gain: @@ -65,6 +66,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImagingPlane - range: string @@ -76,6 +78,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true pmt_gain: @@ -113,6 +116,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImagingPlane - range: string @@ -125,6 +129,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -155,6 +160,7 @@ classes: range: DynamicTableRegion required: true multivalued: false + inlined: true tree_root: true DfOverF: name: DfOverF @@ -209,15 +215,28 @@ classes: attributes: name: name: name + identifier: true range: string required: true image_mask: name: image_mask description: ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero. - range: PlaneSegmentation__image_mask + range: AnyType required: false multivalued: false + any_of: + - array: + dimensions: + - alias: num_roi + - alias: num_x + - alias: num_y + - array: + dimensions: + - alias: num_roi + - alias: num_x + - alias: num_y + - alias: num_z pixel_mask_index: name: pixel_mask_index annotations: @@ -231,6 +250,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true pixel_mask: name: pixel_mask description: 'Pixel masks for each ROI: a list of indices and weights for @@ -239,6 +259,7 @@ classes: range: PlaneSegmentation__pixel_mask required: false multivalued: false + inlined: true voxel_mask_index: name: voxel_mask_index annotations: @@ -252,6 +273,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true voxel_mask: name: voxel_mask description: 'Voxel masks for each ROI: a list of indices and weights for @@ -260,6 +282,7 @@ classes: range: PlaneSegmentation__voxel_mask required: false multivalued: false + inlined: true reference_images: name: reference_images description: Image stacks that the segmentation masks apply to. @@ -276,22 +299,11 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImagingPlane - range: string tree_root: true - PlaneSegmentation__image_mask: - name: PlaneSegmentation__image_mask - description: ROI masks for each ROI. Each image mask is the size of the original - imaging plane (or volume) and members of the ROI are finite non-zero. - is_a: VectorData - attributes: - name: - name: name - ifabsent: string(image_mask) - range: string - required: true - equals_string: image_mask PlaneSegmentation__pixel_mask: name: PlaneSegmentation__pixel_mask description: 'Pixel masks for each ROI: a list of indices and weights for the @@ -302,6 +314,7 @@ classes: name: name: name ifabsent: string(pixel_mask) + identifier: true range: string required: true equals_string: pixel_mask @@ -339,6 +352,7 @@ classes: name: name: name ifabsent: string(voxel_mask) + identifier: true range: string required: true equals_string: voxel_mask @@ -381,6 +395,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -424,6 +439,7 @@ classes: range: ImagingPlane__manifold required: false multivalued: false + inlined: true origin_coords: name: origin_coords description: Physical location of the first element of the imaging plane (0, @@ -432,6 +448,7 @@ classes: range: ImagingPlane__origin_coords required: false multivalued: false + inlined: true grid_spacing: name: grid_spacing description: Space between pixels in (x, y) or voxels in (x, y, z) directions, @@ -440,6 +457,7 @@ classes: range: ImagingPlane__grid_spacing required: false multivalued: false + inlined: true reference_frame: name: reference_frame description: Describes reference frame of origin_coords and grid_spacing. @@ -468,6 +486,8 @@ classes: range: OpticalChannel required: true multivalued: true + inlined: true + inlined_as_list: false device: name: device annotations: @@ -476,6 +496,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string @@ -489,6 +510,7 @@ classes: name: name: name ifabsent: string(manifold) + identifier: true range: string required: true equals_string: manifold @@ -538,6 +560,7 @@ classes: name: name: name ifabsent: string(origin_coords) + identifier: true range: string required: true equals_string: origin_coords @@ -568,6 +591,7 @@ classes: name: name: name ifabsent: string(grid_spacing) + identifier: true range: string required: true equals_string: grid_spacing @@ -596,6 +620,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -632,6 +657,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true corrected: @@ -640,6 +666,8 @@ classes: range: ImageSeries required: true multivalued: false + inlined: true + inlined_as_list: false xy_translation: name: xy_translation description: Stores the x,y delta necessary to align each frame to the common @@ -647,6 +675,8 @@ classes: range: TimeSeries required: true multivalued: false + inlined: true + inlined_as_list: false original: name: original annotations: @@ -655,6 +685,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImageSeries - range: string diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.retinotopy.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.retinotopy.yaml index dc790f3..c1fce82 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.retinotopy.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_6_0_alpha/core.nwb.retinotopy.yaml @@ -29,6 +29,7 @@ classes: name: name: name ifabsent: string(ImagingRetinotopy) + identifier: true range: string required: true axis_1_phase_map: @@ -37,6 +38,7 @@ classes: range: ImagingRetinotopy__axis_1_phase_map required: true multivalued: false + inlined: true axis_1_power_map: name: axis_1_power_map description: Power response on the first measured axis. Response is scaled @@ -44,12 +46,14 @@ classes: range: ImagingRetinotopy__axis_1_power_map required: false multivalued: false + inlined: true axis_2_phase_map: name: axis_2_phase_map description: Phase response to stimulus on the second measured axis. range: ImagingRetinotopy__axis_2_phase_map required: true multivalued: false + inlined: true axis_2_power_map: name: axis_2_power_map description: Power response on the second measured axis. Response is scaled @@ -57,6 +61,7 @@ classes: range: ImagingRetinotopy__axis_2_power_map required: false multivalued: false + inlined: true axis_descriptions: name: axis_descriptions description: Two-element array describing the contents of the two response @@ -76,6 +81,7 @@ classes: range: ImagingRetinotopy__focal_depth_image required: false multivalued: false + inlined: true sign_map: name: sign_map description: Sine of the angle between the direction of the gradient in axis_1 @@ -83,6 +89,7 @@ classes: range: ImagingRetinotopy__sign_map required: false multivalued: false + inlined: true vasculature_image: name: vasculature_image description: 'Gray-scale anatomical image of cortical surface. Array structure: @@ -90,6 +97,7 @@ classes: range: ImagingRetinotopy__vasculature_image required: true multivalued: false + inlined: true tree_root: true ImagingRetinotopy__axis_1_phase_map: name: ImagingRetinotopy__axis_1_phase_map @@ -98,6 +106,7 @@ classes: name: name: name ifabsent: string(axis_1_phase_map) + identifier: true range: string required: true equals_string: axis_1_phase_map @@ -134,6 +143,7 @@ classes: name: name: name ifabsent: string(axis_1_power_map) + identifier: true range: string required: true equals_string: axis_1_power_map @@ -169,6 +179,7 @@ classes: name: name: name ifabsent: string(axis_2_phase_map) + identifier: true range: string required: true equals_string: axis_2_phase_map @@ -205,6 +216,7 @@ classes: name: name: name ifabsent: string(axis_2_power_map) + identifier: true range: string required: true equals_string: axis_2_power_map @@ -241,6 +253,7 @@ classes: name: name: name ifabsent: string(focal_depth_image) + identifier: true range: string required: true equals_string: focal_depth_image @@ -288,6 +301,7 @@ classes: name: name: name ifabsent: string(sign_map) + identifier: true range: string required: true equals_string: sign_map @@ -319,6 +333,7 @@ classes: name: name: name ifabsent: string(vasculature_image) + identifier: true range: string required: true equals_string: vasculature_image diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.base.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.base.yaml index ca7cfe1..7c3450a 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.base.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.base.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -34,6 +35,7 @@ classes: name: name: name ifabsent: string(timeseries) + identifier: true range: string required: true idx_start: @@ -63,6 +65,7 @@ classes: range: TimeSeries required: true multivalued: false + inlined: true tree_root: true Image: name: Image @@ -73,6 +76,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true resolution: @@ -113,6 +117,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true value: @@ -125,6 +130,8 @@ classes: range: Image required: true multivalued: true + inlined: true + inlined_as_list: true tree_root: true NWBContainer: name: NWBContainer @@ -134,6 +141,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -145,6 +153,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -155,6 +164,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -180,6 +190,7 @@ classes: range: TimeSeries__data required: true multivalued: false + inlined: true starting_time: name: starting_time description: Timestamp of the first sample in seconds. When timestamps are @@ -188,6 +199,7 @@ classes: range: TimeSeries__starting_time required: false multivalued: false + inlined: true timestamps: name: timestamps description: Timestamps for samples stored in data, in seconds, relative to @@ -231,6 +243,8 @@ classes: range: TimeSeries__sync required: false multivalued: false + inlined: true + inlined_as_list: true tree_root: true TimeSeries__data: name: TimeSeries__data @@ -241,6 +255,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -327,6 +342,7 @@ classes: name: name: name ifabsent: string(starting_time) + identifier: true range: string required: true equals_string: starting_time @@ -358,6 +374,7 @@ classes: name: name: name ifabsent: string(sync) + identifier: true range: string required: true equals_string: sync @@ -384,6 +401,7 @@ classes: name: name: name ifabsent: string(Images) + identifier: true range: string required: true description: @@ -413,4 +431,5 @@ classes: range: ImageReferences required: false multivalued: false + inlined: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.behavior.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.behavior.yaml index f0b74b7..32ff4f8 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.behavior.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.behavior.yaml @@ -29,6 +29,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -38,6 +39,7 @@ classes: range: SpatialSeries__data required: true multivalued: false + inlined: true reference_frame: name: reference_frame description: Description defining what exactly 'straight-ahead' means. @@ -53,6 +55,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.device.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.device.yaml index ab2fc92..3116431 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.device.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.device.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.ecephys.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.ecephys.yaml index f2be1a3..71eadb4 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.ecephys.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.ecephys.yaml @@ -25,6 +25,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true filtering: @@ -71,6 +72,7 @@ classes: range: DynamicTableRegion required: true multivalued: false + inlined: true channel_conversion: name: channel_conversion description: Channel-specific conversion factor. Multiply the data in the @@ -103,6 +105,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -143,6 +146,7 @@ classes: name: name: name ifabsent: string(FeatureExtraction) + identifier: true range: string required: true description: @@ -189,6 +193,7 @@ classes: range: DynamicTableRegion required: true multivalued: false + inlined: true tree_root: true EventDetection: name: EventDetection @@ -198,6 +203,7 @@ classes: name: name: name ifabsent: string(EventDetection) + identifier: true range: string required: true detection_method: @@ -236,6 +242,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ElectricalSeries - range: string @@ -297,6 +304,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -317,6 +325,7 @@ classes: range: ElectrodeGroup__position required: false multivalued: false + inlined: true device: name: device annotations: @@ -325,6 +334,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string @@ -336,6 +346,7 @@ classes: name: name: name ifabsent: string(position) + identifier: true range: string required: true equals_string: position @@ -376,6 +387,7 @@ classes: name: name: name ifabsent: string(ClusterWaveforms) + identifier: true range: string required: true waveform_filtering: @@ -416,6 +428,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Clustering - range: string @@ -429,6 +442,7 @@ classes: name: name: name ifabsent: string(Clustering) + identifier: true range: string required: true description: diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.epoch.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.epoch.yaml index 1885024..471b87a 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.epoch.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.epoch.yaml @@ -22,6 +22,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true start_time: @@ -64,6 +65,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true timeseries: name: timeseries annotations: @@ -77,6 +79,7 @@ classes: range: TimeSeriesReferenceVectorData required: false multivalued: false + inlined: true timeseries_index: name: timeseries_index annotations: @@ -90,4 +93,5 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.file.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.file.yaml index 1b56d9d..a6b27f5 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.file.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.file.yaml @@ -28,6 +28,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true notes: @@ -45,6 +46,7 @@ classes: name: name: name ifabsent: string(root) + identifier: true range: string required: true equals_string: root @@ -184,6 +186,8 @@ classes: range: NWBFile__stimulus required: true multivalued: false + inlined: true + inlined_as_list: true general: name: general description: Experimental metadata, including protocol, notes and description @@ -204,6 +208,8 @@ classes: range: NWBFile__general required: true multivalued: false + inlined: true + inlined_as_list: true intervals: name: intervals description: Experimental intervals, whether that be logically distinct sub-experiments @@ -213,12 +219,16 @@ classes: range: NWBFile__intervals required: false multivalued: false + inlined: true + inlined_as_list: true units: name: units description: Data about sorted spike units. range: Units required: false multivalued: false + inlined: true + inlined_as_list: false tree_root: true NWBFile__stimulus: name: NWBFile__stimulus @@ -238,6 +248,7 @@ classes: name: name: name ifabsent: string(stimulus) + identifier: true range: string required: true equals_string: stimulus @@ -283,6 +294,7 @@ classes: name: name: name ifabsent: string(general) + identifier: true range: string required: true equals_string: general @@ -378,6 +390,7 @@ classes: range: general__source_script required: false multivalued: false + inlined: true stimulus: name: stimulus description: Notes about stimuli, such as how and where they were presented. @@ -405,6 +418,8 @@ classes: range: LabMetaData required: false multivalued: true + inlined: true + inlined_as_list: false devices: name: devices description: Description of hardware devices used during experiment, e.g., @@ -421,18 +436,24 @@ classes: range: Subject required: false multivalued: false + inlined: true + inlined_as_list: false extracellular_ephys: name: extracellular_ephys description: Metadata related to extracellular electrophysiology. range: general__extracellular_ephys required: false multivalued: false + inlined: true + inlined_as_list: true intracellular_ephys: name: intracellular_ephys description: Metadata related to intracellular electrophysiology. range: general__intracellular_ephys required: false multivalued: false + inlined: true + inlined_as_list: true optogenetics: name: optogenetics description: Metadata describing optogenetic stimuluation. @@ -457,6 +478,7 @@ classes: name: name: name ifabsent: string(source_script) + identifier: true range: string required: true equals_string: source_script @@ -476,6 +498,7 @@ classes: name: name: name ifabsent: string(extracellular_ephys) + identifier: true range: string required: true equals_string: extracellular_ephys @@ -485,12 +508,16 @@ classes: range: ElectrodeGroup required: false multivalued: true + inlined: true + inlined_as_list: false electrodes: name: electrodes description: A table of all electrodes (i.e. channels) used for recording. range: extracellular_ephys__electrodes required: false multivalued: false + inlined: true + inlined_as_list: true extracellular_ephys__electrodes: name: extracellular_ephys__electrodes description: A table of all electrodes (i.e. channels) used for recording. @@ -499,6 +526,7 @@ classes: name: name: name ifabsent: string(electrodes) + identifier: true range: string required: true equals_string: electrodes @@ -562,9 +590,13 @@ classes: group: name: group description: Reference to the ElectrodeGroup this electrode is a part of. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: ElectrodeGroup required: true - multivalued: true + multivalued: false + inlined: true group_name: name: group_name description: Name of the ElectrodeGroup this electrode is a part of. @@ -619,6 +651,7 @@ classes: name: name: name ifabsent: string(intracellular_ephys) + identifier: true range: string required: true equals_string: intracellular_ephys @@ -637,6 +670,8 @@ classes: range: IntracellularElectrode required: false multivalued: true + inlined: true + inlined_as_list: false sweep_table: name: sweep_table description: '[DEPRECATED] Table used to group different PatchClampSeries. @@ -646,6 +681,8 @@ classes: range: SweepTable required: false multivalued: false + inlined: true + inlined_as_list: false intracellular_recordings: name: intracellular_recordings description: A table to group together a stimulus and response from a single @@ -663,6 +700,8 @@ classes: range: IntracellularRecordingsTable required: false multivalued: false + inlined: true + inlined_as_list: false simultaneous_recordings: name: simultaneous_recordings description: A table for grouping different intracellular recordings from @@ -671,6 +710,8 @@ classes: range: SimultaneousRecordingsTable required: false multivalued: false + inlined: true + inlined_as_list: false sequential_recordings: name: sequential_recordings description: A table for grouping different sequential recordings from the @@ -680,6 +721,8 @@ classes: range: SequentialRecordingsTable required: false multivalued: false + inlined: true + inlined_as_list: false repetitions: name: repetitions description: A table for grouping different sequential intracellular recordings @@ -689,6 +732,8 @@ classes: range: RepetitionsTable required: false multivalued: false + inlined: true + inlined_as_list: false experimental_conditions: name: experimental_conditions description: A table for grouping different intracellular recording repetitions @@ -696,6 +741,8 @@ classes: range: ExperimentalConditionsTable required: false multivalued: false + inlined: true + inlined_as_list: false NWBFile__intervals: name: NWBFile__intervals description: Experimental intervals, whether that be logically distinct sub-experiments @@ -705,6 +752,7 @@ classes: name: name: name ifabsent: string(intervals) + identifier: true range: string required: true equals_string: intervals @@ -715,18 +763,24 @@ classes: range: TimeIntervals required: false multivalued: false + inlined: true + inlined_as_list: false trials: name: trials description: Repeated experimental events that have a logical grouping. range: TimeIntervals required: false multivalued: false + inlined: true + inlined_as_list: false invalid_times: name: invalid_times description: Time intervals that should be removed from analysis. range: TimeIntervals required: false multivalued: false + inlined: true + inlined_as_list: false time_intervals: name: time_intervals description: Optional additional table(s) for describing other experimental @@ -734,6 +788,8 @@ classes: range: TimeIntervals required: false multivalued: true + inlined: true + inlined_as_list: false LabMetaData: name: LabMetaData description: Lab-specific meta-data. @@ -741,6 +797,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -751,6 +808,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true age: @@ -759,6 +817,7 @@ classes: range: Subject__age required: false multivalued: false + inlined: true date_of_birth: name: date_of_birth description: Date of birth of subject. Can be supplied instead of 'age'. @@ -817,6 +876,7 @@ classes: name: name: name ifabsent: string(age) + identifier: true range: string required: true equals_string: age diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.icephys.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.icephys.yaml index 710ba36..a8662e7 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.icephys.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.icephys.yaml @@ -23,6 +23,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true stimulus_description: @@ -41,6 +42,7 @@ classes: range: PatchClampSeries__data required: true multivalued: false + inlined: true gain: name: gain description: Gain of the recording, in units Volt/Amp (v-clamp) or Volt/Volt @@ -56,6 +58,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: IntracellularElectrode - range: string @@ -67,6 +70,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -92,6 +96,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -100,6 +105,7 @@ classes: range: CurrentClampSeries__data required: true multivalued: false + inlined: true bias_current: name: bias_current description: Bias current, in amps. @@ -126,6 +132,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -153,6 +160,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true stimulus_description: @@ -189,6 +197,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -197,6 +206,7 @@ classes: range: CurrentClampStimulusSeries__data required: true multivalued: false + inlined: true tree_root: true CurrentClampStimulusSeries__data: name: CurrentClampStimulusSeries__data @@ -205,6 +215,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -231,6 +242,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -239,48 +251,56 @@ classes: range: VoltageClampSeries__data required: true multivalued: false + inlined: true capacitance_fast: name: capacitance_fast description: Fast capacitance, in farads. range: VoltageClampSeries__capacitance_fast required: false multivalued: false + inlined: true capacitance_slow: name: capacitance_slow description: Slow capacitance, in farads. range: VoltageClampSeries__capacitance_slow required: false multivalued: false + inlined: true resistance_comp_bandwidth: name: resistance_comp_bandwidth description: Resistance compensation bandwidth, in hertz. range: VoltageClampSeries__resistance_comp_bandwidth required: false multivalued: false + inlined: true resistance_comp_correction: name: resistance_comp_correction description: Resistance compensation correction, in percent. range: VoltageClampSeries__resistance_comp_correction required: false multivalued: false + inlined: true resistance_comp_prediction: name: resistance_comp_prediction description: Resistance compensation prediction, in percent. range: VoltageClampSeries__resistance_comp_prediction required: false multivalued: false + inlined: true whole_cell_capacitance_comp: name: whole_cell_capacitance_comp description: Whole cell capacitance compensation, in farads. range: VoltageClampSeries__whole_cell_capacitance_comp required: false multivalued: false + inlined: true whole_cell_series_resistance_comp: name: whole_cell_series_resistance_comp description: Whole cell series resistance compensation, in ohms. range: VoltageClampSeries__whole_cell_series_resistance_comp required: false multivalued: false + inlined: true tree_root: true VoltageClampSeries__data: name: VoltageClampSeries__data @@ -289,6 +309,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -313,6 +334,7 @@ classes: name: name: name ifabsent: string(capacitance_fast) + identifier: true range: string required: true equals_string: capacitance_fast @@ -334,6 +356,7 @@ classes: name: name: name ifabsent: string(capacitance_slow) + identifier: true range: string required: true equals_string: capacitance_slow @@ -355,6 +378,7 @@ classes: name: name: name ifabsent: string(resistance_comp_bandwidth) + identifier: true range: string required: true equals_string: resistance_comp_bandwidth @@ -377,6 +401,7 @@ classes: name: name: name ifabsent: string(resistance_comp_correction) + identifier: true range: string required: true equals_string: resistance_comp_correction @@ -399,6 +424,7 @@ classes: name: name: name ifabsent: string(resistance_comp_prediction) + identifier: true range: string required: true equals_string: resistance_comp_prediction @@ -421,6 +447,7 @@ classes: name: name: name ifabsent: string(whole_cell_capacitance_comp) + identifier: true range: string required: true equals_string: whole_cell_capacitance_comp @@ -443,6 +470,7 @@ classes: name: name: name ifabsent: string(whole_cell_series_resistance_comp) + identifier: true range: string required: true equals_string: whole_cell_series_resistance_comp @@ -465,6 +493,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -473,6 +502,7 @@ classes: range: VoltageClampStimulusSeries__data required: true multivalued: false + inlined: true tree_root: true VoltageClampStimulusSeries__data: name: VoltageClampStimulusSeries__data @@ -481,6 +511,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -505,6 +536,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true cell_id: @@ -565,6 +597,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string @@ -579,6 +612,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true sweep_number: @@ -593,9 +627,13 @@ classes: series: name: series description: The PatchClampSeries with the sweep number in that row. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: PatchClampSeries required: true - multivalued: true + multivalued: false + inlined: true series_index: name: series_index annotations: @@ -609,6 +647,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true IntracellularElectrodesTable: name: IntracellularElectrodesTable @@ -617,6 +656,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -629,9 +669,13 @@ classes: electrode: name: electrode description: Column for storing the reference to the intracellular electrode. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: IntracellularElectrode required: true - multivalued: true + multivalued: false + inlined: true tree_root: true IntracellularStimuliTable: name: IntracellularStimuliTable @@ -640,6 +684,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -663,6 +708,7 @@ classes: range: TimeSeriesReferenceVectorData required: true multivalued: false + inlined: true stimulus_template: name: stimulus_template annotations: @@ -677,6 +723,7 @@ classes: range: TimeSeriesReferenceVectorData required: false multivalued: false + inlined: true tree_root: true IntracellularResponsesTable: name: IntracellularResponsesTable @@ -685,6 +732,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -708,6 +756,7 @@ classes: range: TimeSeriesReferenceVectorData required: true multivalued: false + inlined: true tree_root: true IntracellularRecordingsTable: name: IntracellularRecordingsTable @@ -727,6 +776,7 @@ classes: name: name: name ifabsent: string(intracellular_recordings) + identifier: true range: string required: true equals_string: intracellular_recordings @@ -748,18 +798,24 @@ classes: range: IntracellularElectrodesTable required: true multivalued: false + inlined: true + inlined_as_list: false stimuli: name: stimuli description: Table for storing intracellular stimulus related metadata. range: IntracellularStimuliTable required: true multivalued: false + inlined: true + inlined_as_list: false responses: name: responses description: Table for storing intracellular response related metadata. range: IntracellularResponsesTable required: true multivalued: false + inlined: true + inlined_as_list: false tree_root: true SimultaneousRecordingsTable: name: SimultaneousRecordingsTable @@ -771,6 +827,7 @@ classes: name: name: name ifabsent: string(simultaneous_recordings) + identifier: true range: string required: true equals_string: simultaneous_recordings @@ -781,6 +838,7 @@ classes: range: SimultaneousRecordingsTable__recordings required: true multivalued: false + inlined: true recordings_index: name: recordings_index annotations: @@ -794,6 +852,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true SimultaneousRecordingsTable__recordings: name: SimultaneousRecordingsTable__recordings @@ -804,6 +863,7 @@ classes: name: name: name ifabsent: string(recordings) + identifier: true range: string required: true equals_string: recordings @@ -814,6 +874,7 @@ classes: to fix the type of table that can be referenced here. range: IntracellularRecordingsTable required: true + inlined: true SequentialRecordingsTable: name: SequentialRecordingsTable description: A table for grouping different sequential recordings from the SimultaneousRecordingsTable @@ -825,6 +886,7 @@ classes: name: name: name ifabsent: string(sequential_recordings) + identifier: true range: string required: true equals_string: sequential_recordings @@ -835,6 +897,7 @@ classes: range: SequentialRecordingsTable__simultaneous_recordings required: true multivalued: false + inlined: true simultaneous_recordings_index: name: simultaneous_recordings_index annotations: @@ -848,6 +911,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true stimulus_type: name: stimulus_type description: The type of stimulus used for the sequential recording. @@ -867,6 +931,7 @@ classes: name: name: name ifabsent: string(simultaneous_recordings) + identifier: true range: string required: true equals_string: simultaneous_recordings @@ -877,6 +942,7 @@ classes: to fix the type of table that can be referenced here. range: SimultaneousRecordingsTable required: true + inlined: true RepetitionsTable: name: RepetitionsTable description: A table for grouping different sequential intracellular recordings @@ -888,6 +954,7 @@ classes: name: name: name ifabsent: string(repetitions) + identifier: true range: string required: true equals_string: repetitions @@ -898,6 +965,7 @@ classes: range: RepetitionsTable__sequential_recordings required: true multivalued: false + inlined: true sequential_recordings_index: name: sequential_recordings_index annotations: @@ -911,6 +979,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true RepetitionsTable__sequential_recordings: name: RepetitionsTable__sequential_recordings @@ -921,6 +990,7 @@ classes: name: name: name ifabsent: string(sequential_recordings) + identifier: true range: string required: true equals_string: sequential_recordings @@ -931,6 +1001,7 @@ classes: to fix the type of table that can be referenced here. range: SequentialRecordingsTable required: true + inlined: true ExperimentalConditionsTable: name: ExperimentalConditionsTable description: A table for grouping different intracellular recording repetitions @@ -940,6 +1011,7 @@ classes: name: name: name ifabsent: string(experimental_conditions) + identifier: true range: string required: true equals_string: experimental_conditions @@ -949,6 +1021,7 @@ classes: range: ExperimentalConditionsTable__repetitions required: true multivalued: false + inlined: true repetitions_index: name: repetitions_index annotations: @@ -962,6 +1035,7 @@ classes: range: VectorIndex required: true multivalued: false + inlined: true tree_root: true ExperimentalConditionsTable__repetitions: name: ExperimentalConditionsTable__repetitions @@ -971,6 +1045,7 @@ classes: name: name: name ifabsent: string(repetitions) + identifier: true range: string required: true equals_string: repetitions @@ -981,3 +1056,4 @@ classes: to fix the type of table that can be referenced here. range: RepetitionsTable required: true + inlined: true diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.image.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.image.yaml index cac5d73..603c351 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.image.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.image.yaml @@ -21,8 +21,16 @@ classes: attributes: name: name: name + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + range: numeric tree_root: true RGBImage: name: RGBImage @@ -31,8 +39,18 @@ classes: attributes: name: name: name + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b + exact_cardinality: 3 + range: numeric tree_root: true RGBAImage: name: RGBAImage @@ -41,8 +59,18 @@ classes: attributes: name: name: name + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: x + - alias: y + - alias: r_g_b_a + exact_cardinality: 4 + range: numeric tree_root: true ImageSeries: name: ImageSeries @@ -56,6 +84,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -96,6 +125,7 @@ classes: range: ImageSeries__external_file required: false multivalued: false + inlined: true format: name: format description: Format of image. If this is 'external', then the attribute 'external_file' @@ -113,6 +143,7 @@ classes: value: link required: false multivalued: false + inlined: true any_of: - range: Device - range: string @@ -127,6 +158,7 @@ classes: name: name: name ifabsent: string(external_file) + identifier: true range: string required: true equals_string: external_file @@ -164,6 +196,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true masked_imageseries: @@ -174,6 +207,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImageSeries - range: string @@ -189,6 +223,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true distance: @@ -252,6 +287,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -272,6 +308,7 @@ classes: value: link required: false multivalued: false + inlined: true any_of: - range: ImageSeries - range: string @@ -283,6 +320,7 @@ classes: value: link required: false multivalued: false + inlined: true any_of: - range: Images - range: string diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.misc.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.misc.yaml index 9395fd9..b30070d 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.misc.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.misc.yaml @@ -30,6 +30,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -38,6 +39,7 @@ classes: range: AbstractFeatureSeries__data required: true multivalued: false + inlined: true feature_units: name: feature_units description: Units of each feature. @@ -64,6 +66,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -96,6 +99,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -121,6 +125,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -140,6 +145,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -148,6 +154,7 @@ classes: range: DecompositionSeries__data required: true multivalued: false + inlined: true metric: name: metric description: The metric used, e.g. phase, amplitude, power. @@ -168,6 +175,7 @@ classes: range: DynamicTableRegion required: false multivalued: false + inlined: true bands: name: bands description: Table for describing the bands that this series was generated @@ -175,6 +183,8 @@ classes: range: DecompositionSeries__bands required: true multivalued: false + inlined: true + inlined_as_list: true source_timeseries: name: source_timeseries annotations: @@ -183,6 +193,7 @@ classes: value: link required: false multivalued: false + inlined: true any_of: - range: TimeSeries - range: string @@ -194,6 +205,7 @@ classes: name: name: name ifabsent: string(data) + identifier: true range: string required: true equals_string: data @@ -222,6 +234,7 @@ classes: name: name: name ifabsent: string(bands) + identifier: true range: string required: true equals_string: bands @@ -273,6 +286,7 @@ classes: name: name: name ifabsent: string(Units) + identifier: true range: string required: true spike_times_index: @@ -288,12 +302,14 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true spike_times: name: spike_times description: Spike times for each unit in seconds. range: Units__spike_times required: false multivalued: false + inlined: true obs_intervals_index: name: obs_intervals_index annotations: @@ -307,6 +323,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true obs_intervals: name: obs_intervals description: Observation intervals for each unit. @@ -331,6 +348,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true electrodes: name: electrodes annotations: @@ -344,12 +362,17 @@ classes: range: DynamicTableRegion required: false multivalued: false + inlined: true electrode_group: name: electrode_group description: Electrode group that each spike unit came from. + array: + minimum_number_dimensions: 1 + maximum_number_dimensions: false range: ElectrodeGroup required: false - multivalued: true + multivalued: false + inlined: true waveform_mean: name: waveform_mean description: Spike waveform mean for each spike unit. @@ -428,6 +451,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true waveforms_index_index: name: waveforms_index_index annotations: @@ -442,6 +466,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true tree_root: true Units__spike_times: name: Units__spike_times @@ -451,6 +476,7 @@ classes: name: name: name ifabsent: string(spike_times) + identifier: true range: string required: true equals_string: spike_times diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.ogen.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.ogen.yaml index 085004d..9cc7b0d 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.ogen.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.ogen.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -47,6 +48,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: OptogeneticStimulusSite - range: string @@ -58,6 +60,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -88,6 +91,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.ophys.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.ophys.yaml index 8452b74..b5d3676 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.ophys.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.ophys.yaml @@ -23,6 +23,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true pmt_gain: @@ -65,6 +66,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImagingPlane - range: string @@ -76,6 +78,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true pmt_gain: @@ -113,6 +116,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImagingPlane - range: string @@ -125,6 +129,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true data: @@ -155,6 +160,7 @@ classes: range: DynamicTableRegion required: true multivalued: false + inlined: true tree_root: true DfOverF: name: DfOverF @@ -209,15 +215,28 @@ classes: attributes: name: name: name + identifier: true range: string required: true image_mask: name: image_mask description: ROI masks for each ROI. Each image mask is the size of the original imaging plane (or volume) and members of the ROI are finite non-zero. - range: PlaneSegmentation__image_mask + range: AnyType required: false multivalued: false + any_of: + - array: + dimensions: + - alias: num_roi + - alias: num_x + - alias: num_y + - array: + dimensions: + - alias: num_roi + - alias: num_x + - alias: num_y + - alias: num_z pixel_mask_index: name: pixel_mask_index annotations: @@ -231,6 +250,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true pixel_mask: name: pixel_mask description: 'Pixel masks for each ROI: a list of indices and weights for @@ -239,6 +259,7 @@ classes: range: PlaneSegmentation__pixel_mask required: false multivalued: false + inlined: true voxel_mask_index: name: voxel_mask_index annotations: @@ -252,6 +273,7 @@ classes: range: VectorIndex required: false multivalued: false + inlined: true voxel_mask: name: voxel_mask description: 'Voxel masks for each ROI: a list of indices and weights for @@ -260,6 +282,7 @@ classes: range: PlaneSegmentation__voxel_mask required: false multivalued: false + inlined: true reference_images: name: reference_images description: Image stacks that the segmentation masks apply to. @@ -276,22 +299,11 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImagingPlane - range: string tree_root: true - PlaneSegmentation__image_mask: - name: PlaneSegmentation__image_mask - description: ROI masks for each ROI. Each image mask is the size of the original - imaging plane (or volume) and members of the ROI are finite non-zero. - is_a: VectorData - attributes: - name: - name: name - ifabsent: string(image_mask) - range: string - required: true - equals_string: image_mask PlaneSegmentation__pixel_mask: name: PlaneSegmentation__pixel_mask description: 'Pixel masks for each ROI: a list of indices and weights for the @@ -302,6 +314,7 @@ classes: name: name: name ifabsent: string(pixel_mask) + identifier: true range: string required: true equals_string: pixel_mask @@ -339,6 +352,7 @@ classes: name: name: name ifabsent: string(voxel_mask) + identifier: true range: string required: true equals_string: voxel_mask @@ -381,6 +395,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -424,6 +439,7 @@ classes: range: ImagingPlane__manifold required: false multivalued: false + inlined: true origin_coords: name: origin_coords description: Physical location of the first element of the imaging plane (0, @@ -432,6 +448,7 @@ classes: range: ImagingPlane__origin_coords required: false multivalued: false + inlined: true grid_spacing: name: grid_spacing description: Space between pixels in (x, y) or voxels in (x, y, z) directions, @@ -440,6 +457,7 @@ classes: range: ImagingPlane__grid_spacing required: false multivalued: false + inlined: true reference_frame: name: reference_frame description: Describes reference frame of origin_coords and grid_spacing. @@ -468,6 +486,8 @@ classes: range: OpticalChannel required: true multivalued: true + inlined: true + inlined_as_list: false device: name: device annotations: @@ -476,6 +496,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: Device - range: string @@ -489,6 +510,7 @@ classes: name: name: name ifabsent: string(manifold) + identifier: true range: string required: true equals_string: manifold @@ -538,6 +560,7 @@ classes: name: name: name ifabsent: string(origin_coords) + identifier: true range: string required: true equals_string: origin_coords @@ -568,6 +591,7 @@ classes: name: name: name ifabsent: string(grid_spacing) + identifier: true range: string required: true equals_string: grid_spacing @@ -596,6 +620,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -632,6 +657,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true corrected: @@ -640,6 +666,8 @@ classes: range: ImageSeries required: true multivalued: false + inlined: true + inlined_as_list: false xy_translation: name: xy_translation description: Stores the x,y delta necessary to align each frame to the common @@ -647,6 +675,8 @@ classes: range: TimeSeries required: true multivalued: false + inlined: true + inlined_as_list: false original: name: original annotations: @@ -655,6 +685,7 @@ classes: value: link required: true multivalued: false + inlined: true any_of: - range: ImageSeries - range: string diff --git a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.retinotopy.yaml b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.retinotopy.yaml index 6416821..8cc1810 100644 --- a/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.retinotopy.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/core/v2_7_0/core.nwb.retinotopy.yaml @@ -29,6 +29,7 @@ classes: name: name: name ifabsent: string(ImagingRetinotopy) + identifier: true range: string required: true axis_1_phase_map: @@ -37,6 +38,7 @@ classes: range: ImagingRetinotopy__axis_1_phase_map required: true multivalued: false + inlined: true axis_1_power_map: name: axis_1_power_map description: Power response on the first measured axis. Response is scaled @@ -44,12 +46,14 @@ classes: range: ImagingRetinotopy__axis_1_power_map required: false multivalued: false + inlined: true axis_2_phase_map: name: axis_2_phase_map description: Phase response to stimulus on the second measured axis. range: ImagingRetinotopy__axis_2_phase_map required: true multivalued: false + inlined: true axis_2_power_map: name: axis_2_power_map description: Power response on the second measured axis. Response is scaled @@ -57,6 +61,7 @@ classes: range: ImagingRetinotopy__axis_2_power_map required: false multivalued: false + inlined: true axis_descriptions: name: axis_descriptions description: Two-element array describing the contents of the two response @@ -76,6 +81,7 @@ classes: range: ImagingRetinotopy__focal_depth_image required: false multivalued: false + inlined: true sign_map: name: sign_map description: Sine of the angle between the direction of the gradient in axis_1 @@ -83,6 +89,7 @@ classes: range: ImagingRetinotopy__sign_map required: false multivalued: false + inlined: true vasculature_image: name: vasculature_image description: 'Gray-scale anatomical image of cortical surface. Array structure: @@ -90,6 +97,7 @@ classes: range: ImagingRetinotopy__vasculature_image required: true multivalued: false + inlined: true tree_root: true ImagingRetinotopy__axis_1_phase_map: name: ImagingRetinotopy__axis_1_phase_map @@ -98,6 +106,7 @@ classes: name: name: name ifabsent: string(axis_1_phase_map) + identifier: true range: string required: true equals_string: axis_1_phase_map @@ -134,6 +143,7 @@ classes: name: name: name ifabsent: string(axis_1_power_map) + identifier: true range: string required: true equals_string: axis_1_power_map @@ -169,6 +179,7 @@ classes: name: name: name ifabsent: string(axis_2_phase_map) + identifier: true range: string required: true equals_string: axis_2_phase_map @@ -205,6 +216,7 @@ classes: name: name: name ifabsent: string(axis_2_power_map) + identifier: true range: string required: true equals_string: axis_2_power_map @@ -241,6 +253,7 @@ classes: name: name: name ifabsent: string(focal_depth_image) + identifier: true range: string required: true equals_string: focal_depth_image @@ -288,6 +301,7 @@ classes: name: name: name ifabsent: string(sign_map) + identifier: true range: string required: true equals_string: sign_map @@ -319,6 +333,7 @@ classes: name: name: name ifabsent: string(vasculature_image) + identifier: true range: string required: true equals_string: vasculature_image diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_1_0/hdmf-common.table.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_1_0/hdmf-common.table.yaml index 27a272c..a38fa1a 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_1_0/hdmf-common.table.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_1_0/hdmf-common.table.yaml @@ -86,6 +86,12 @@ classes: ifabsent: string(element_id) range: string required: true + value: + name: value + array: + dimensions: + - alias: num_elements + range: int tree_root: true DynamicTableRegion: name: DynamicTableRegion diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_1_2/hdmf-common.table.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_1_2/hdmf-common.table.yaml index fe82d7d..a668487 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_1_2/hdmf-common.table.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_1_2/hdmf-common.table.yaml @@ -86,6 +86,12 @@ classes: ifabsent: string(element_id) range: string required: true + value: + name: value + array: + dimensions: + - alias: num_elements + range: int tree_root: true DynamicTableRegion: name: DynamicTableRegion diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_1_3/hdmf-common.table.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_1_3/hdmf-common.table.yaml index 4285b03..1022ebf 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_1_3/hdmf-common.table.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_1_3/hdmf-common.table.yaml @@ -114,6 +114,12 @@ classes: ifabsent: string(element_id) range: string required: true + value: + name: value + array: + dimensions: + - alias: num_elements + range: int tree_root: true DynamicTableRegion: name: DynamicTableRegion diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_2_0/hdmf-common.table.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_2_0/hdmf-common.table.yaml index 7746e8e..e7f4d41 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_2_0/hdmf-common.table.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_2_0/hdmf-common.table.yaml @@ -87,6 +87,12 @@ classes: ifabsent: string(element_id) range: string required: true + value: + name: value + array: + dimensions: + - alias: num_elements + range: int tree_root: true DynamicTableRegion: name: DynamicTableRegion diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_2_1/hdmf-common.table.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_2_1/hdmf-common.table.yaml index 2ce11ab..b506697 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_2_1/hdmf-common.table.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_2_1/hdmf-common.table.yaml @@ -87,6 +87,12 @@ classes: ifabsent: string(element_id) range: string required: true + value: + name: value + array: + dimensions: + - alias: num_elements + range: int tree_root: true DynamicTableRegion: name: DynamicTableRegion diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_3_0/hdmf-common.table.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_3_0/hdmf-common.table.yaml index cae8e9e..94bb9f7 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_3_0/hdmf-common.table.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_3_0/hdmf-common.table.yaml @@ -87,6 +87,12 @@ classes: ifabsent: string(element_id) range: string required: true + value: + name: value + array: + dimensions: + - alias: num_elements + range: int tree_root: true DynamicTableRegion: name: DynamicTableRegion diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_4_0/hdmf-common.base.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_4_0/hdmf-common.base.yaml index 266d216..6ba8106 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_4_0/hdmf-common.base.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_4_0/hdmf-common.base.yaml @@ -18,6 +18,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -28,6 +29,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_4_0/hdmf-common.sparse.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_4_0/hdmf-common.sparse.yaml index 13b5f58..bbf0b5d 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_4_0/hdmf-common.sparse.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_4_0/hdmf-common.sparse.yaml @@ -22,6 +22,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true shape: @@ -52,17 +53,10 @@ classes: data: name: data description: The non-zero values in the matrix. - range: CSRMatrix__data + array: + dimensions: + - alias: number_of_non_zero_values + range: AnyType required: true multivalued: false tree_root: true - CSRMatrix__data: - name: CSRMatrix__data - description: The non-zero values in the matrix. - attributes: - name: - name: name - ifabsent: string(data) - range: string - required: true - equals_string: data diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_4_0/hdmf-common.table.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_4_0/hdmf-common.table.yaml index a88c85f..7e1a614 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_4_0/hdmf-common.table.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_4_0/hdmf-common.table.yaml @@ -27,6 +27,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -68,6 +69,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true target: @@ -75,6 +77,7 @@ classes: description: Reference to the target dataset that this index applies to. range: VectorData required: true + inlined: true tree_root: true ElementIdentifiers: name: ElementIdentifiers @@ -85,8 +88,15 @@ classes: name: name: name ifabsent: string(element_id) + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: num_elements + range: int tree_root: true DynamicTableRegion: name: DynamicTableRegion @@ -103,6 +113,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true table: @@ -111,6 +122,7 @@ classes: to. range: DynamicTable required: true + inlined: true description: name: description description: Description of what this table region points to. @@ -141,6 +153,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true colnames: @@ -170,4 +183,5 @@ classes: range: VectorData required: false multivalued: true + inlined: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_0/hdmf-common.base.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_0/hdmf-common.base.yaml index 36edb0a..91de7c2 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_0/hdmf-common.base.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_0/hdmf-common.base.yaml @@ -18,6 +18,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -28,6 +29,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_0/hdmf-common.sparse.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_0/hdmf-common.sparse.yaml index 24ea8fd..b256bbe 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_0/hdmf-common.sparse.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_0/hdmf-common.sparse.yaml @@ -22,6 +22,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true shape: @@ -52,17 +53,10 @@ classes: data: name: data description: The non-zero values in the matrix. - range: CSRMatrix__data + array: + dimensions: + - alias: number_of_non_zero_values + range: AnyType required: true multivalued: false tree_root: true - CSRMatrix__data: - name: CSRMatrix__data - description: The non-zero values in the matrix. - attributes: - name: - name: name - ifabsent: string(data) - range: string - required: true - equals_string: data diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_0/hdmf-common.table.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_0/hdmf-common.table.yaml index 2652e1c..f8adba6 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_0/hdmf-common.table.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_0/hdmf-common.table.yaml @@ -27,6 +27,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -68,6 +69,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true target: @@ -75,6 +77,7 @@ classes: description: Reference to the target dataset that this index applies to. range: VectorData required: true + inlined: true tree_root: true ElementIdentifiers: name: ElementIdentifiers @@ -85,8 +88,15 @@ classes: name: name: name ifabsent: string(element_id) + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: num_elements + range: int tree_root: true DynamicTableRegion: name: DynamicTableRegion @@ -103,6 +113,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true table: @@ -111,6 +122,7 @@ classes: to. range: DynamicTable required: true + inlined: true description: name: description description: Description of what this table region points to. @@ -141,6 +153,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true colnames: @@ -170,6 +183,7 @@ classes: range: VectorData required: false multivalued: true + inlined: true tree_root: true AlignedDynamicTable: name: AlignedDynamicTable diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_1/hdmf-common.base.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_1/hdmf-common.base.yaml index 8685dc7..4fd80e6 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_1/hdmf-common.base.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_1/hdmf-common.base.yaml @@ -18,6 +18,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -28,6 +29,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_1/hdmf-common.sparse.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_1/hdmf-common.sparse.yaml index 21654df..bde18eb 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_1/hdmf-common.sparse.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_1/hdmf-common.sparse.yaml @@ -22,6 +22,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true shape: @@ -52,17 +53,10 @@ classes: data: name: data description: The non-zero values in the matrix. - range: CSRMatrix__data + array: + dimensions: + - alias: number_of_non_zero_values + range: AnyType required: true multivalued: false tree_root: true - CSRMatrix__data: - name: CSRMatrix__data - description: The non-zero values in the matrix. - attributes: - name: - name: name - ifabsent: string(data) - range: string - required: true - equals_string: data diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_1/hdmf-common.table.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_1/hdmf-common.table.yaml index cb79bbe..52b119d 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_1/hdmf-common.table.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_5_1/hdmf-common.table.yaml @@ -27,6 +27,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -68,6 +69,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true target: @@ -75,6 +77,7 @@ classes: description: Reference to the target dataset that this index applies to. range: VectorData required: true + inlined: true tree_root: true ElementIdentifiers: name: ElementIdentifiers @@ -85,8 +88,15 @@ classes: name: name: name ifabsent: string(element_id) + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: num_elements + range: int tree_root: true DynamicTableRegion: name: DynamicTableRegion @@ -103,6 +113,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true table: @@ -111,6 +122,7 @@ classes: to. range: DynamicTable required: true + inlined: true description: name: description description: Description of what this table region points to. @@ -141,6 +153,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true colnames: @@ -170,6 +183,7 @@ classes: range: VectorData required: false multivalued: true + inlined: true tree_root: true AlignedDynamicTable: name: AlignedDynamicTable diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_6_0/hdmf-common.base.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_6_0/hdmf-common.base.yaml index 5ba5b73..beb539c 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_6_0/hdmf-common.base.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_6_0/hdmf-common.base.yaml @@ -18,6 +18,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -28,6 +29,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_6_0/hdmf-common.sparse.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_6_0/hdmf-common.sparse.yaml index 7ed736f..6085aa3 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_6_0/hdmf-common.sparse.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_6_0/hdmf-common.sparse.yaml @@ -22,6 +22,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true shape: @@ -52,17 +53,10 @@ classes: data: name: data description: The non-zero values in the matrix. - range: CSRMatrix__data + array: + dimensions: + - alias: number_of_non_zero_values + range: AnyType required: true multivalued: false tree_root: true - CSRMatrix__data: - name: CSRMatrix__data - description: The non-zero values in the matrix. - attributes: - name: - name: name - ifabsent: string(data) - range: string - required: true - equals_string: data diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_6_0/hdmf-common.table.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_6_0/hdmf-common.table.yaml index 5240bc3..85675e7 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_6_0/hdmf-common.table.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_6_0/hdmf-common.table.yaml @@ -27,6 +27,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -68,6 +69,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true target: @@ -75,6 +77,7 @@ classes: description: Reference to the target dataset that this index applies to. range: VectorData required: true + inlined: true tree_root: true ElementIdentifiers: name: ElementIdentifiers @@ -85,8 +88,15 @@ classes: name: name: name ifabsent: string(element_id) + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: num_elements + range: int tree_root: true DynamicTableRegion: name: DynamicTableRegion @@ -103,6 +113,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true table: @@ -111,6 +122,7 @@ classes: to. range: DynamicTable required: true + inlined: true description: name: description description: Description of what this table region points to. @@ -141,6 +153,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true colnames: @@ -170,6 +183,7 @@ classes: range: VectorData required: false multivalued: true + inlined: true tree_root: true AlignedDynamicTable: name: AlignedDynamicTable diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_7_0/hdmf-common.base.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_7_0/hdmf-common.base.yaml index 4ad36b7..f65f22b 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_7_0/hdmf-common.base.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_7_0/hdmf-common.base.yaml @@ -18,6 +18,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -28,6 +29,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_7_0/hdmf-common.sparse.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_7_0/hdmf-common.sparse.yaml index 6167b42..8974bc0 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_7_0/hdmf-common.sparse.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_7_0/hdmf-common.sparse.yaml @@ -22,6 +22,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true shape: @@ -52,17 +53,10 @@ classes: data: name: data description: The non-zero values in the matrix. - range: CSRMatrix__data + array: + dimensions: + - alias: number_of_non_zero_values + range: AnyType required: true multivalued: false tree_root: true - CSRMatrix__data: - name: CSRMatrix__data - description: The non-zero values in the matrix. - attributes: - name: - name: name - ifabsent: string(data) - range: string - required: true - equals_string: data diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_7_0/hdmf-common.table.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_7_0/hdmf-common.table.yaml index c20fdb3..9ffb97d 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_7_0/hdmf-common.table.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_7_0/hdmf-common.table.yaml @@ -27,6 +27,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -68,6 +69,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true target: @@ -75,6 +77,7 @@ classes: description: Reference to the target dataset that this index applies to. range: VectorData required: true + inlined: true tree_root: true ElementIdentifiers: name: ElementIdentifiers @@ -85,8 +88,15 @@ classes: name: name: name ifabsent: string(element_id) + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: num_elements + range: int tree_root: true DynamicTableRegion: name: DynamicTableRegion @@ -103,6 +113,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true table: @@ -111,6 +122,7 @@ classes: to. range: DynamicTable required: true + inlined: true description: name: description description: Description of what this table region points to. @@ -141,6 +153,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true colnames: @@ -170,6 +183,7 @@ classes: range: VectorData required: false multivalued: true + inlined: true tree_root: true AlignedDynamicTable: name: AlignedDynamicTable diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_8_0/hdmf-common.base.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_8_0/hdmf-common.base.yaml index bb9b324..ea83af3 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_8_0/hdmf-common.base.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_8_0/hdmf-common.base.yaml @@ -18,6 +18,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true @@ -28,6 +29,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_8_0/hdmf-common.sparse.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_8_0/hdmf-common.sparse.yaml index 842d1d6..9fd8ddd 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_8_0/hdmf-common.sparse.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_8_0/hdmf-common.sparse.yaml @@ -22,6 +22,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true shape: @@ -52,17 +53,10 @@ classes: data: name: data description: The non-zero values in the matrix. - range: CSRMatrix__data + array: + dimensions: + - alias: number_of_non_zero_values + range: AnyType required: true multivalued: false tree_root: true - CSRMatrix__data: - name: CSRMatrix__data - description: The non-zero values in the matrix. - attributes: - name: - name: name - ifabsent: string(data) - range: string - required: true - equals_string: data diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_8_0/hdmf-common.table.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_8_0/hdmf-common.table.yaml index cf14321..940f1b7 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_8_0/hdmf-common.table.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_common/v1_8_0/hdmf-common.table.yaml @@ -27,6 +27,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true description: @@ -68,6 +69,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true target: @@ -75,6 +77,7 @@ classes: description: Reference to the target dataset that this index applies to. range: VectorData required: true + inlined: true tree_root: true ElementIdentifiers: name: ElementIdentifiers @@ -85,8 +88,15 @@ classes: name: name: name ifabsent: string(element_id) + identifier: true range: string required: true + value: + name: value + array: + dimensions: + - alias: num_elements + range: int tree_root: true DynamicTableRegion: name: DynamicTableRegion @@ -103,6 +113,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true table: @@ -111,6 +122,7 @@ classes: to. range: DynamicTable required: true + inlined: true description: name: description description: Description of what this table region points to. @@ -141,6 +153,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true colnames: @@ -170,6 +183,7 @@ classes: range: VectorData required: false multivalued: true + inlined: true tree_root: true AlignedDynamicTable: name: AlignedDynamicTable diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_1_0/hdmf-experimental.experimental.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_1_0/hdmf-experimental.experimental.yaml index 064f647..2a10ba2 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_1_0/hdmf-experimental.experimental.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_1_0/hdmf-experimental.experimental.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true elements: @@ -29,4 +30,5 @@ classes: elements range: VectorData required: true + inlined: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_1_0/hdmf-experimental.resources.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_1_0/hdmf-experimental.resources.yaml index 05dc855..a962b8f 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_1_0/hdmf-experimental.resources.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_1_0/hdmf-experimental.resources.yaml @@ -22,6 +22,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true keys: @@ -31,18 +32,21 @@ classes: range: ExternalResources__keys required: true multivalued: false + inlined: true entities: name: entities description: A table for mapping user terms (i.e., keys) to resource entities. range: ExternalResources__entities required: true multivalued: false + inlined: true resources: name: resources description: A table for mapping user terms (i.e., keys) to resource entities. range: ExternalResources__resources required: true multivalued: false + inlined: true objects: name: objects description: A table for identifying which objects in a file contain references @@ -50,12 +54,14 @@ classes: range: ExternalResources__objects required: true multivalued: false + inlined: true object_keys: name: object_keys description: A table for identifying which objects use which keys. range: ExternalResources__object_keys required: true multivalued: false + inlined: true tree_root: true ExternalResources__keys: name: ExternalResources__keys @@ -66,6 +72,7 @@ classes: name: name: name ifabsent: string(keys) + identifier: true range: string required: true equals_string: keys @@ -86,6 +93,7 @@ classes: name: name: name ifabsent: string(entities) + identifier: true range: string required: true equals_string: entities @@ -130,6 +138,7 @@ classes: name: name: name ifabsent: string(resources) + identifier: true range: string required: true equals_string: resources @@ -158,6 +167,7 @@ classes: name: name: name ifabsent: string(objects) + identifier: true range: string required: true equals_string: objects @@ -186,6 +196,7 @@ classes: name: name: name ifabsent: string(object_keys) + identifier: true range: string required: true equals_string: object_keys diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_2_0/hdmf-experimental.experimental.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_2_0/hdmf-experimental.experimental.yaml index 94b3194..1f49508 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_2_0/hdmf-experimental.experimental.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_2_0/hdmf-experimental.experimental.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true elements: @@ -29,4 +30,5 @@ classes: elements range: VectorData required: true + inlined: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_2_0/hdmf-experimental.resources.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_2_0/hdmf-experimental.resources.yaml index a1b6ec0..89023ae 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_2_0/hdmf-experimental.resources.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_2_0/hdmf-experimental.resources.yaml @@ -22,6 +22,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true keys: @@ -31,18 +32,21 @@ classes: range: ExternalResources__keys required: true multivalued: false + inlined: true entities: name: entities description: A table for mapping user terms (i.e., keys) to resource entities. range: ExternalResources__entities required: true multivalued: false + inlined: true resources: name: resources description: A table for mapping user terms (i.e., keys) to resource entities. range: ExternalResources__resources required: true multivalued: false + inlined: true objects: name: objects description: A table for identifying which objects in a file contain references @@ -50,12 +54,14 @@ classes: range: ExternalResources__objects required: true multivalued: false + inlined: true object_keys: name: object_keys description: A table for identifying which objects use which keys. range: ExternalResources__object_keys required: true multivalued: false + inlined: true tree_root: true ExternalResources__keys: name: ExternalResources__keys @@ -66,6 +72,7 @@ classes: name: name: name ifabsent: string(keys) + identifier: true range: string required: true equals_string: keys @@ -86,6 +93,7 @@ classes: name: name: name ifabsent: string(entities) + identifier: true range: string required: true equals_string: entities @@ -130,6 +138,7 @@ classes: name: name: name ifabsent: string(resources) + identifier: true range: string required: true equals_string: resources @@ -158,6 +167,7 @@ classes: name: name: name ifabsent: string(objects) + identifier: true range: string required: true equals_string: objects @@ -198,6 +208,7 @@ classes: name: name: name ifabsent: string(object_keys) + identifier: true range: string required: true equals_string: object_keys diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_3_0/hdmf-experimental.experimental.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_3_0/hdmf-experimental.experimental.yaml index 4991b33..96f372c 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_3_0/hdmf-experimental.experimental.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_3_0/hdmf-experimental.experimental.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true elements: @@ -29,4 +30,5 @@ classes: elements range: VectorData required: true + inlined: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_3_0/hdmf-experimental.resources.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_3_0/hdmf-experimental.resources.yaml index ca25659..d0909a2 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_3_0/hdmf-experimental.resources.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_3_0/hdmf-experimental.resources.yaml @@ -22,6 +22,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true keys: @@ -31,18 +32,21 @@ classes: range: ExternalResources__keys required: true multivalued: false + inlined: true files: name: files description: A table for storing object ids of files used in external resources. range: ExternalResources__files required: true multivalued: false + inlined: true entities: name: entities description: A table for mapping user terms (i.e., keys) to resource entities. range: ExternalResources__entities required: true multivalued: false + inlined: true objects: name: objects description: A table for identifying which objects in a file contain references @@ -50,12 +54,14 @@ classes: range: ExternalResources__objects required: true multivalued: false + inlined: true object_keys: name: object_keys description: A table for identifying which objects use which keys. range: ExternalResources__object_keys required: true multivalued: false + inlined: true tree_root: true ExternalResources__keys: name: ExternalResources__keys @@ -66,6 +72,7 @@ classes: name: name: name ifabsent: string(keys) + identifier: true range: string required: true equals_string: keys @@ -86,6 +93,7 @@ classes: name: name: name ifabsent: string(files) + identifier: true range: string required: true equals_string: files @@ -106,6 +114,7 @@ classes: name: name: name ifabsent: string(entities) + identifier: true range: string required: true equals_string: entities @@ -144,6 +153,7 @@ classes: name: name: name ifabsent: string(objects) + identifier: true range: string required: true equals_string: objects @@ -201,6 +211,7 @@ classes: name: name: name ifabsent: string(object_keys) + identifier: true range: string required: true equals_string: object_keys diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_4_0/hdmf-experimental.experimental.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_4_0/hdmf-experimental.experimental.yaml index 6332939..28ae7e8 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_4_0/hdmf-experimental.experimental.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_4_0/hdmf-experimental.experimental.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true elements: @@ -29,4 +30,5 @@ classes: elements range: VectorData required: true + inlined: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_4_0/hdmf-experimental.resources.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_4_0/hdmf-experimental.resources.yaml index e2acf65..75f3938 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_4_0/hdmf-experimental.resources.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_4_0/hdmf-experimental.resources.yaml @@ -22,6 +22,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true keys: @@ -31,18 +32,21 @@ classes: range: ExternalResources__keys required: true multivalued: false + inlined: true files: name: files description: A table for storing object ids of files used in external resources. range: ExternalResources__files required: true multivalued: false + inlined: true entities: name: entities description: A table for mapping user terms (i.e., keys) to resource entities. range: ExternalResources__entities required: true multivalued: false + inlined: true objects: name: objects description: A table for identifying which objects in a file contain references @@ -50,18 +54,21 @@ classes: range: ExternalResources__objects required: true multivalued: false + inlined: true object_keys: name: object_keys description: A table for identifying which objects use which keys. range: ExternalResources__object_keys required: true multivalued: false + inlined: true entity_keys: name: entity_keys description: A table for identifying which keys use which entity. range: ExternalResources__entity_keys required: true multivalued: false + inlined: true tree_root: true ExternalResources__keys: name: ExternalResources__keys @@ -72,6 +79,7 @@ classes: name: name: name ifabsent: string(keys) + identifier: true range: string required: true equals_string: keys @@ -92,6 +100,7 @@ classes: name: name: name ifabsent: string(files) + identifier: true range: string required: true equals_string: files @@ -112,6 +121,7 @@ classes: name: name: name ifabsent: string(entities) + identifier: true range: string required: true equals_string: entities @@ -142,6 +152,7 @@ classes: name: name: name ifabsent: string(objects) + identifier: true range: string required: true equals_string: objects @@ -199,6 +210,7 @@ classes: name: name: name ifabsent: string(object_keys) + identifier: true range: string required: true equals_string: object_keys @@ -227,6 +239,7 @@ classes: name: name: name ifabsent: string(entity_keys) + identifier: true range: string required: true equals_string: entity_keys diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_5_0/hdmf-experimental.experimental.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_5_0/hdmf-experimental.experimental.yaml index c6cf1d4..208be72 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_5_0/hdmf-experimental.experimental.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_5_0/hdmf-experimental.experimental.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true elements: @@ -29,4 +30,5 @@ classes: elements range: VectorData required: true + inlined: true tree_root: true diff --git a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_5_0/hdmf-experimental.resources.yaml b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_5_0/hdmf-experimental.resources.yaml index 7478fe1..dcaf960 100644 --- a/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_5_0/hdmf-experimental.resources.yaml +++ b/nwb_models/src/nwb_models/schema/linkml/hdmf_experimental/v0_5_0/hdmf-experimental.resources.yaml @@ -21,6 +21,7 @@ classes: attributes: name: name: name + identifier: true range: string required: true keys: @@ -30,18 +31,21 @@ classes: range: HERD__keys required: true multivalued: false + inlined: true files: name: files description: A table for storing object ids of files used in external resources. range: HERD__files required: true multivalued: false + inlined: true entities: name: entities description: A table for mapping user terms (i.e., keys) to resource entities. range: HERD__entities required: true multivalued: false + inlined: true objects: name: objects description: A table for identifying which objects in a file contain references @@ -49,18 +53,21 @@ classes: range: HERD__objects required: true multivalued: false + inlined: true object_keys: name: object_keys description: A table for identifying which objects use which keys. range: HERD__object_keys required: true multivalued: false + inlined: true entity_keys: name: entity_keys description: A table for identifying which keys use which entity. range: HERD__entity_keys required: true multivalued: false + inlined: true tree_root: true HERD__keys: name: HERD__keys @@ -71,6 +78,7 @@ classes: name: name: name ifabsent: string(keys) + identifier: true range: string required: true equals_string: keys @@ -91,6 +99,7 @@ classes: name: name: name ifabsent: string(files) + identifier: true range: string required: true equals_string: files @@ -111,6 +120,7 @@ classes: name: name: name ifabsent: string(entities) + identifier: true range: string required: true equals_string: entities @@ -141,6 +151,7 @@ classes: name: name: name ifabsent: string(objects) + identifier: true range: string required: true equals_string: objects @@ -198,6 +209,7 @@ classes: name: name: name ifabsent: string(object_keys) + identifier: true range: string required: true equals_string: object_keys @@ -226,6 +238,7 @@ classes: name: name: name ifabsent: string(entity_keys) + identifier: true range: string required: true equals_string: entity_keys diff --git a/scripts/generate_core.py b/scripts/generate_core.py index 221aeaf..4aeb21a 100644 --- a/scripts/generate_core.py +++ b/scripts/generate_core.py @@ -171,17 +171,11 @@ def generate_versions( shutil.rmtree(tmp_dir / "linkml") shutil.rmtree(tmp_dir / "pydantic") - # import the most recent version of the schemaz we built - latest_version = sorted((pydantic_path / "core").glob("v*"), key=os.path.getmtime)[-1] - # make inits to use the schema! we don't usually do this in the # provider class because we directly import the files there. with open(pydantic_path / "__init__.py", "w") as initfile: initfile.write(" ") - with open(pydantic_path / "__init__.py", "w") as initfile: - initfile.write(f"from .pydantic.core.{latest_version.name}.namespace import *") - subprocess.run(["black", "."]) finally: @@ -228,6 +222,11 @@ def parser() -> ArgumentParser: ), action="store_true", ) + parser.add_argument( + "--debug", + help="Add annotations to generated schema that indicate how they were generated", + action="store_true", + ) parser.add_argument("--pdb", help="Launch debugger on an error", action="store_true") return parser @@ -235,6 +234,12 @@ def parser() -> ArgumentParser: def main(): args = parser().parse_args() + if args.debug: + os.environ["NWB_LINKML_DEBUG"] = "true" + else: + if "NWB_LINKML_DEBUG" in os.environ: + del os.environ["NWB_LINKML_DEBUG"] + tmp_dir = make_tmp_dir(clear=True) git_dir = tmp_dir / "git" git_dir.mkdir(exist_ok=True)