From b555ccb199cbc46461f806491c16e0011bfd47a1 Mon Sep 17 00:00:00 2001 From: sneakers-the-rat Date: Fri, 30 Aug 2024 00:39:10 -0700 Subject: [PATCH] testing nwb file, and experimenting with yaml format --- nwb_linkml/pdm.lock | 164 ++++++--- nwb_linkml/pyproject.toml | 1 + nwb_linkml/tests/data/test_nwb.yaml | 78 +++++ nwb_linkml/tests/fixtures.py | 444 ++++++++++++++++++++++++ nwb_linkml/tests/test_io/test_io_nwb.py | 17 + 5 files changed, 655 insertions(+), 49 deletions(-) create mode 100644 nwb_linkml/tests/data/test_nwb.yaml create mode 100644 nwb_linkml/tests/test_io/test_io_nwb.py diff --git a/nwb_linkml/pdm.lock b/nwb_linkml/pdm.lock index 7a1aca7..e4f43f3 100644 --- a/nwb_linkml/pdm.lock +++ b/nwb_linkml/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev", "plot", "tests"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:f219083028bd024c53bc55626c8b6088d6eb5c2ade56bd694a7a112098aa9bfc" +content_hash = "sha256:aaf3c34a5f39fc7db0c5dce91a0693eb78358a255d6b0a72f2e1f988eb7e899f" [[metadata.targets]] requires_python = ">=3.10,<3.13" @@ -549,7 +549,7 @@ name = "h5py" version = "3.11.0" requires_python = ">=3.8" summary = "Read and write HDF5 files from Python" -groups = ["default"] +groups = ["default", "dev", "tests"] dependencies = [ "numpy>=1.17.3", ] @@ -580,6 +580,26 @@ files = [ {file = "hbreader-0.9.1.tar.gz", hash = "sha256:d2c132f8ba6276d794c66224c3297cec25c8079d0a4cf019c061611e0a3b94fa"}, ] +[[package]] +name = "hdmf" +version = "3.14.3" +requires_python = ">=3.8" +summary = "A hierarchical data modeling framework for modern science data standards" +groups = ["dev", "tests"] +dependencies = [ + "h5py>=2.10", + "importlib-resources; python_version < \"3.9\"", + "jsonschema>=2.6.0", + "numpy>=1.18", + "pandas>=1.0.5", + "ruamel-yaml>=0.16", + "scipy>=1.4", +] +files = [ + {file = "hdmf-3.14.3-py3-none-any.whl", hash = "sha256:1417ccc0d336d535192b7a3db4c7354cbc15123f1ccb3cdd82e363308e78f9bc"}, + {file = "hdmf-3.14.3.tar.gz", hash = "sha256:e9548fc7bdbb534a2750092b6b9819df2ce50e27430866c3c32061a2306271cc"}, +] + [[package]] name = "idna" version = "3.8" @@ -751,7 +771,7 @@ name = "jsonschema" version = "4.23.0" requires_python = ">=3.8" summary = "An implementation of JSON Schema validation for Python" -groups = ["default"] +groups = ["default", "dev", "tests"] dependencies = [ "attrs>=22.2.0", "importlib-resources>=1.4.0; python_version < \"3.9\"", @@ -770,7 +790,7 @@ name = "jsonschema-specifications" version = "2023.12.1" requires_python = ">=3.8" summary = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" -groups = ["default"] +groups = ["default", "dev", "tests"] dependencies = [ "importlib-resources>=1.4.0; python_version < \"3.9\"", "referencing>=0.31.0", @@ -984,45 +1004,36 @@ files = [ [[package]] name = "numpy" -version = "2.1.0" -requires_python = ">=3.10" +version = "1.26.4" +requires_python = ">=3.9" summary = "Fundamental package for array computing in Python" -groups = ["default"] +groups = ["default", "dev", "tests"] files = [ - {file = "numpy-2.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6326ab99b52fafdcdeccf602d6286191a79fe2fda0ae90573c5814cd2b0bc1b8"}, - {file = "numpy-2.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0937e54c09f7a9a68da6889362ddd2ff584c02d015ec92672c099b61555f8911"}, - {file = "numpy-2.1.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:30014b234f07b5fec20f4146f69e13cfb1e33ee9a18a1879a0142fbb00d47673"}, - {file = "numpy-2.1.0-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:899da829b362ade41e1e7eccad2cf274035e1cb36ba73034946fccd4afd8606b"}, - {file = "numpy-2.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08801848a40aea24ce16c2ecde3b756f9ad756586fb2d13210939eb69b023f5b"}, - {file = "numpy-2.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:398049e237d1aae53d82a416dade04defed1a47f87d18d5bd615b6e7d7e41d1f"}, - {file = "numpy-2.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0abb3916a35d9090088a748636b2c06dc9a6542f99cd476979fb156a18192b84"}, - {file = "numpy-2.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:10e2350aea18d04832319aac0f887d5fcec1b36abd485d14f173e3e900b83e33"}, - {file = "numpy-2.1.0-cp310-cp310-win32.whl", hash = "sha256:f6b26e6c3b98adb648243670fddc8cab6ae17473f9dc58c51574af3e64d61211"}, - {file = "numpy-2.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:f505264735ee074250a9c78247ee8618292091d9d1fcc023290e9ac67e8f1afa"}, - {file = "numpy-2.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:76368c788ccb4f4782cf9c842b316140142b4cbf22ff8db82724e82fe1205dce"}, - {file = "numpy-2.1.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:f8e93a01a35be08d31ae33021e5268f157a2d60ebd643cfc15de6ab8e4722eb1"}, - {file = "numpy-2.1.0-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:9523f8b46485db6939bd069b28b642fec86c30909cea90ef550373787f79530e"}, - {file = "numpy-2.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54139e0eb219f52f60656d163cbe67c31ede51d13236c950145473504fa208cb"}, - {file = "numpy-2.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5ebbf9fbdabed208d4ecd2e1dfd2c0741af2f876e7ae522c2537d404ca895c3"}, - {file = "numpy-2.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:378cb4f24c7d93066ee4103204f73ed046eb88f9ad5bb2275bb9fa0f6a02bd36"}, - {file = "numpy-2.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8f699a709120b220dfe173f79c73cb2a2cab2c0b88dd59d7b49407d032b8ebd"}, - {file = "numpy-2.1.0-cp311-cp311-win32.whl", hash = "sha256:ffbd6faeb190aaf2b5e9024bac9622d2ee549b7ec89ef3a9373fa35313d44e0e"}, - {file = "numpy-2.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:0af3a5987f59d9c529c022c8c2a64805b339b7ef506509fba7d0556649b9714b"}, - {file = "numpy-2.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fe76d75b345dc045acdbc006adcb197cc680754afd6c259de60d358d60c93736"}, - {file = "numpy-2.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f358ea9e47eb3c2d6eba121ab512dfff38a88db719c38d1e67349af210bc7529"}, - {file = "numpy-2.1.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:dd94ce596bda40a9618324547cfaaf6650b1a24f5390350142499aa4e34e53d1"}, - {file = "numpy-2.1.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:b47c551c6724960479cefd7353656498b86e7232429e3a41ab83be4da1b109e8"}, - {file = "numpy-2.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0756a179afa766ad7cb6f036de622e8a8f16ffdd55aa31f296c870b5679d745"}, - {file = "numpy-2.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24003ba8ff22ea29a8c306e61d316ac74111cebf942afbf692df65509a05f111"}, - {file = "numpy-2.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b34fa5e3b5d6dc7e0a4243fa0f81367027cb6f4a7215a17852979634b5544ee0"}, - {file = "numpy-2.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c4f982715e65036c34897eb598d64aef15150c447be2cfc6643ec7a11af06574"}, - {file = "numpy-2.1.0-cp312-cp312-win32.whl", hash = "sha256:c4cd94dfefbefec3f8b544f61286584292d740e6e9d4677769bc76b8f41deb02"}, - {file = "numpy-2.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:a0cdef204199278f5c461a0bed6ed2e052998276e6d8ab2963d5b5c39a0500bc"}, - {file = "numpy-2.1.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:15ef8b2177eeb7e37dd5ef4016f30b7659c57c2c0b57a779f1d537ff33a72c7b"}, - {file = "numpy-2.1.0-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:e5f0642cdf4636198a4990de7a71b693d824c56a757862230454629cf62e323d"}, - {file = "numpy-2.1.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f15976718c004466406342789f31b6673776360f3b1e3c575f25302d7e789575"}, - {file = "numpy-2.1.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:6c1de77ded79fef664d5098a66810d4d27ca0224e9051906e634b3f7ead134c2"}, - {file = "numpy-2.1.0.tar.gz", hash = "sha256:7dc90da0081f7e1da49ec4e398ede6a8e9cc4f5ebe5f9e06b443ed889ee9aaa2"}, + {file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"}, + {file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2"}, + {file = "numpy-1.26.4-cp310-cp310-win32.whl", hash = "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07"}, + {file = "numpy-1.26.4-cp310-cp310-win_amd64.whl", hash = "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a"}, + {file = "numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20"}, + {file = "numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0"}, + {file = "numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110"}, + {file = "numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818"}, + {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"}, ] [[package]] @@ -1102,7 +1113,7 @@ name = "pandas" version = "2.2.2" requires_python = ">=3.9" summary = "Powerful data structures for data analysis, time series, and statistics" -groups = ["default"] +groups = ["default", "dev", "tests"] dependencies = [ "numpy>=1.22.4; python_version < \"3.11\"", "numpy>=1.23.2; python_version == \"3.11\"", @@ -1350,6 +1361,24 @@ files = [ {file = "PyJSG-0.11.10.tar.gz", hash = "sha256:4bd6e3ff2833fa2b395bbe803a2d72a5f0bab5b7285bccd0da1a1bc0aee88bfa"}, ] +[[package]] +name = "pynwb" +version = "2.8.1" +requires_python = ">=3.8" +summary = "Package for working with Neurodata stored in the NWB format." +groups = ["dev", "tests"] +dependencies = [ + "h5py>=2.10", + "hdmf>=3.14.0", + "numpy<2.0,>=1.18", + "pandas>=1.1.5", + "python-dateutil>=2.7.3", +] +files = [ + {file = "pynwb-2.8.1-py3-none-any.whl", hash = "sha256:f3c392652b26396e135cf6f1abd570d413c9eb7bf5bdb1a89d899852338fdf6c"}, + {file = "pynwb-2.8.1.tar.gz", hash = "sha256:498e4bc46a7b0a1331a0f754bac72ea7f9d10d1bba35af3c7be78a61bb1d104b"}, +] + [[package]] name = "pyparsing" version = "3.1.4" @@ -1469,7 +1498,7 @@ name = "python-dateutil" version = "2.9.0.post0" requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" summary = "Extensions to the standard Python datetime module" -groups = ["default"] +groups = ["default", "dev", "tests"] dependencies = [ "six>=1.5", ] @@ -1506,7 +1535,7 @@ files = [ name = "pytz" version = "2024.1" summary = "World timezone definitions, modern and historical" -groups = ["default"] +groups = ["default", "dev", "tests"] files = [ {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, @@ -1597,7 +1626,7 @@ name = "referencing" version = "0.35.1" requires_python = ">=3.8" summary = "JSON Referencing + Python" -groups = ["default"] +groups = ["default", "dev", "tests"] dependencies = [ "attrs>=22.2.0", "rpds-py>=0.7.0", @@ -1701,7 +1730,7 @@ name = "rpds-py" version = "0.20.0" requires_python = ">=3.8" summary = "Python bindings to Rust's persistent data structures (rpds)" -groups = ["default"] +groups = ["default", "dev", "tests"] files = [ {file = "rpds_py-0.20.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:3ad0fda1635f8439cde85c700f964b23ed5fc2d28016b32b9ee5fe30da5c84e2"}, {file = "rpds_py-0.20.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9bb4a0d90fdb03437c109a17eade42dfbf6190408f29b2744114d11586611d6f"}, @@ -1762,7 +1791,7 @@ name = "ruamel-yaml" version = "0.18.6" requires_python = ">=3.7" summary = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" -groups = ["default"] +groups = ["default", "dev", "tests"] dependencies = [ "ruamel-yaml-clib>=0.2.7; platform_python_implementation == \"CPython\" and python_version < \"3.13\"", ] @@ -1776,7 +1805,7 @@ name = "ruamel-yaml-clib" version = "0.2.8" requires_python = ">=3.6" summary = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" -groups = ["default"] +groups = ["default", "dev", "tests"] marker = "platform_python_implementation == \"CPython\" and python_version < \"3.13\"" files = [ {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b42169467c42b692c19cf539c38d4602069d8c1505e97b86387fcf7afb766e1d"}, @@ -1833,6 +1862,43 @@ files = [ {file = "ruff-0.6.2.tar.gz", hash = "sha256:239ee6beb9e91feb8e0ec384204a763f36cb53fb895a1a364618c6abb076b3be"}, ] +[[package]] +name = "scipy" +version = "1.14.1" +requires_python = ">=3.10" +summary = "Fundamental algorithms for scientific computing in Python" +groups = ["dev", "tests"] +dependencies = [ + "numpy<2.3,>=1.23.5", +] +files = [ + {file = "scipy-1.14.1-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:b28d2ca4add7ac16ae8bb6632a3c86e4b9e4d52d3e34267f6e1b0c1f8d87e389"}, + {file = "scipy-1.14.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:d0d2821003174de06b69e58cef2316a6622b60ee613121199cb2852a873f8cf3"}, + {file = "scipy-1.14.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:8bddf15838ba768bb5f5083c1ea012d64c9a444e16192762bd858f1e126196d0"}, + {file = "scipy-1.14.1-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:97c5dddd5932bd2a1a31c927ba5e1463a53b87ca96b5c9bdf5dfd6096e27efc3"}, + {file = "scipy-1.14.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2ff0a7e01e422c15739ecd64432743cf7aae2b03f3084288f399affcefe5222d"}, + {file = "scipy-1.14.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e32dced201274bf96899e6491d9ba3e9a5f6b336708656466ad0522d8528f69"}, + {file = "scipy-1.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8426251ad1e4ad903a4514712d2fa8fdd5382c978010d1c6f5f37ef286a713ad"}, + {file = "scipy-1.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:a49f6ed96f83966f576b33a44257d869756df6cf1ef4934f59dd58b25e0327e5"}, + {file = "scipy-1.14.1-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:2da0469a4ef0ecd3693761acbdc20f2fdeafb69e6819cc081308cc978153c675"}, + {file = "scipy-1.14.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:c0ee987efa6737242745f347835da2cc5bb9f1b42996a4d97d5c7ff7928cb6f2"}, + {file = "scipy-1.14.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3a1b111fac6baec1c1d92f27e76511c9e7218f1695d61b59e05e0fe04dc59617"}, + {file = "scipy-1.14.1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8475230e55549ab3f207bff11ebfc91c805dc3463ef62eda3ccf593254524ce8"}, + {file = "scipy-1.14.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:278266012eb69f4a720827bdd2dc54b2271c97d84255b2faaa8f161a158c3b37"}, + {file = "scipy-1.14.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fef8c87f8abfb884dac04e97824b61299880c43f4ce675dd2cbeadd3c9b466d2"}, + {file = "scipy-1.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b05d43735bb2f07d689f56f7b474788a13ed8adc484a85aa65c0fd931cf9ccd2"}, + {file = "scipy-1.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:716e389b694c4bb564b4fc0c51bc84d381735e0d39d3f26ec1af2556ec6aad94"}, + {file = "scipy-1.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:631f07b3734d34aced009aaf6fedfd0eb3498a97e581c3b1e5f14a04164a456d"}, + {file = "scipy-1.14.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:af29a935803cc707ab2ed7791c44288a682f9c8107bc00f0eccc4f92c08d6e07"}, + {file = "scipy-1.14.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:2843f2d527d9eebec9a43e6b406fb7266f3af25a751aa91d62ff416f54170bc5"}, + {file = "scipy-1.14.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:eb58ca0abd96911932f688528977858681a59d61a7ce908ffd355957f7025cfc"}, + {file = "scipy-1.14.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30ac8812c1d2aab7131a79ba62933a2a76f582d5dbbc695192453dae67ad6310"}, + {file = "scipy-1.14.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f9ea80f2e65bdaa0b7627fb00cbeb2daf163caa015e59b7516395fe3bd1e066"}, + {file = "scipy-1.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:edaf02b82cd7639db00dbff629995ef185c8df4c3ffa71a5562a595765a06ce1"}, + {file = "scipy-1.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2ff38e22128e6c03ff73b6bb0f85f897d2362f8c052e3b8ad00532198fbdae3f"}, + {file = "scipy-1.14.1.tar.gz", hash = "sha256:5a275584e726026a5699459aa72f828a610821006228e841b94275c4a7c08417"}, +] + [[package]] name = "setuptools" version = "74.0.0" @@ -2023,7 +2089,7 @@ name = "tzdata" version = "2024.1" requires_python = ">=2" summary = "Provider of IANA time zone data" -groups = ["default"] +groups = ["default", "dev", "tests"] files = [ {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, diff --git a/nwb_linkml/pyproject.toml b/nwb_linkml/pyproject.toml index 41cf80a..91aed24 100644 --- a/nwb_linkml/pyproject.toml +++ b/nwb_linkml/pyproject.toml @@ -44,6 +44,7 @@ tests = [ "pytest-cov<5.0.0,>=4.1.0", "sybil>=6.0.3", "requests-cache>=1.2.1", + "pynwb>=2.8.1", ] dev = [ "nwb-linkml[tests]", diff --git a/nwb_linkml/tests/data/test_nwb.yaml b/nwb_linkml/tests/data/test_nwb.yaml new file mode 100644 index 0000000..dfcb722 --- /dev/null +++ b/nwb_linkml/tests/data/test_nwb.yaml @@ -0,0 +1,78 @@ +# manually transcribed target version of nwb-linkml dataset +# matching the one created by fixtures.py:nwb_file +--- +id: my_dataset + +prefixes: + nwbfile: + - path: "test_nwb.nwb" + - hash: "blake2b:blahblahblahblah" + +imports: + core: + as: nwb + version: "2.7.0" + from: + - pypi: + package: nwb-models +--- + + hdmf-common: + as: hdmf + version: "1.8.0" + from: + - pypi: + package: nwb-models +--- + +extracellular_ephys: &ecephys + electrodes: + group: + - @shank{{i}} + - @shank{{i}} + - @shank{{i}} + # could have expression here like { range(3) } => i + # - ... { range(3) } => i + # or blank ... implies use expression from outer scope + - ... + shank{{i}}: + device: @general.devices.array + ...: { range(3) } => i + +# expands to +extracellular_ephys: + electrodes: + group: + - @shank0 + - @shank0 + - @shank0 + - @shank1 + - # etc. + shank0: + device: @general.devices.array + shank1: + device: @general.devices.array + # etc. + +data: !{{ nwb.NWBFile }} <== :nwbfile + file_create_date: [ 2024-01-01 ] + identifier: "1111-1111-1111-1111" + session_description: All that you touch, you change. + session_start_time: 2024-01-01T01:01:01 + general: + devices: + - Heka ITC-1600: + - Microscope: + - array: + description: old reliable + manufacturer: diy + extracellular_ephys: *ecephys + + experiment_description: All that you change, changes you. + experimenter: [ "Lauren Oya Olamina" ] + institution: Earthseed Research Institute + keywords: + - behavior + - belief + related_publications: doi:10.1016/j.neuron.2016.12.011 + diff --git a/nwb_linkml/tests/fixtures.py b/nwb_linkml/tests/fixtures.py index a38e3e0..c4a1c36 100644 --- a/nwb_linkml/tests/fixtures.py +++ b/nwb_linkml/tests/fixtures.py @@ -1,9 +1,12 @@ import shutil from dataclasses import dataclass, field +from datetime import datetime +from itertools import product from pathlib import Path from types import ModuleType from typing import Dict, Optional +import numpy as np import pytest from linkml_runtime.dumpers import yaml_dumper from linkml_runtime.linkml_model import ( @@ -13,6 +16,24 @@ from linkml_runtime.linkml_model import ( SlotDefinition, TypeDefinition, ) +from pynwb import NWBHDF5IO, NWBFile, TimeSeries +from pynwb.base import TimeSeriesReference, TimeSeriesReferenceVectorData +from pynwb.behavior import Position, SpatialSeries +from pynwb.core import DynamicTable, VectorData +from pynwb.ecephys import LFP, ElectricalSeries +from pynwb.file import Subject +from pynwb.icephys import VoltageClampSeries, VoltageClampStimulusSeries +from pynwb.image import ImageSeries +from pynwb.ophys import ( + CorrectedImageStack, + Fluorescence, + ImageSegmentation, + MotionCorrection, + OnePhotonSeries, + OpticalChannel, + RoiResponseSeries, + TwoPhotonSeries, +) from nwb_linkml.adapters.namespaces import NamespacesAdapter from nwb_linkml.io import schema as io @@ -27,6 +48,7 @@ __all__ = [ "linkml_schema", "linkml_schema_bare", "nwb_core_fixture", + "nwb_file", "nwb_schema", "tmp_output_dir", "tmp_output_dir_func", @@ -314,3 +336,425 @@ def nwb_schema() -> NWBSchemaTest: ], ) return NWBSchemaTest(datasets={"image": image}, groups={"images": images}) + + +@pytest.fixture(scope="session") +def nwb_file(tmp_output_dir) -> Path: + """ + NWB File created with pynwb that uses all the weird language features + + Borrowing code from pynwb docs in one humonogous fixture function + since there's not really a reason to + """ + generator = np.random.default_rng() + + nwb_path = tmp_output_dir / "test_nwb.nwb" + + nwbfile = NWBFile( + session_description="All that you touch, you change.", # required + identifier="1111-1111-1111-1111", # required + session_start_time=datetime(year=2024, month=1, day=1), # required + session_id="session_1234", # optional + experimenter=[ + "Lauren Oya Olamina", + ], # optional + institution="Earthseed Research Institute", # optional + experiment_description="All that you change, changes you.", # optional + keywords=["behavior", "belief"], # optional + related_publications="doi:10.1016/j.neuron.2016.12.011", # optional + ) + subject = Subject( + subject_id="001", + age="P90D", + description="mouse 5", + species="Mus musculus", + sex="M", + ) + nwbfile.subject = subject + + data = np.arange(100, 200, 10) + timestamps = np.arange(10.0) + time_series_with_timestamps = TimeSeries( + name="test_timeseries", + description="an example time series", + data=data, + unit="m", + timestamps=timestamps, + ) + nwbfile.add_acquisition(time_series_with_timestamps) + + position_data = np.array([np.linspace(0, 10, 50), np.linspace(0, 8, 50)]).T + position_timestamps = np.linspace(0, 50).astype(float) / 200 + + spatial_series_obj = SpatialSeries( + name="SpatialSeries", + description="(x,y) position in open field", + data=position_data, + timestamps=position_timestamps, + reference_frame="(0,0) is bottom left corner", + ) + # name is set to "Position" by default + position_obj = Position(spatial_series=spatial_series_obj) + behavior_module = nwbfile.create_processing_module( + name="behavior", description="processed behavioral data" + ) + behavior_module.add(position_obj) + + nwbfile.add_trial_column( + name="correct", + description="whether the trial was correct", + ) + nwbfile.add_trial(start_time=1.0, stop_time=5.0, correct=True) + nwbfile.add_trial(start_time=6.0, stop_time=10.0, correct=False) + + # -------------------------------------------------- + # Extracellular Ephys + # https://pynwb.readthedocs.io/en/latest/tutorials/domain/ecephys.html + # -------------------------------------------------- + device = nwbfile.create_device(name="array", description="old reliable", manufacturer="diy") + nwbfile.add_electrode_column(name="label", description="label of electrode") + + nshanks = 4 + nchannels_per_shank = 3 + electrode_counter = 0 + + for ishank in range(nshanks): + # create an electrode group for this shank + electrode_group = nwbfile.create_electrode_group( + name=f"shank{ishank}", + description=f"electrode group for shank {ishank}", + device=device, + location="brain area", + ) + # add electrodes to the electrode table + for ielec in range(nchannels_per_shank): + nwbfile.add_electrode( + group=electrode_group, + label=f"shank{ishank}elec{ielec}", + location="brain area", + ) + electrode_counter += 1 + all_table_region = nwbfile.create_electrode_table_region( + region=list(range(electrode_counter)), # reference row indices 0 to N-1 + description="all electrodes", + ) + raw_data = generator.standard_normal((50, 12)) + raw_electrical_series = ElectricalSeries( + name="ElectricalSeries", + description="Raw acquisition traces", + data=raw_data, + electrodes=all_table_region, + starting_time=0.0, + # timestamp of the first sample in seconds relative to the session start time + rate=20000.0, # in Hz + ) + nwbfile.add_acquisition(raw_electrical_series) + + # -------------------------------------------------- + # LFP + # -------------------------------------------------- + lfp_data = generator.standard_normal((50, 12)) + lfp_electrical_series = ElectricalSeries( + name="ElectricalSeries", + description="LFP data", + data=lfp_data, + electrodes=all_table_region, + starting_time=0.0, + rate=200.0, + ) + lfp = LFP(electrical_series=lfp_electrical_series) + ecephys_module = nwbfile.create_processing_module( + name="ecephys", description="processed extracellular electrophysiology data" + ) + ecephys_module.add(lfp) + + # Spike Times + nwbfile.add_unit_column(name="quality", description="sorting quality") + firing_rate = 20 + n_units = 10 + res = 1000 + duration = 20 + for _ in range(n_units): + spike_times = np.where(generator.random(res * duration) < (firing_rate / res))[0] / res + nwbfile.add_unit(spike_times=spike_times, quality="good") + + # -------------------------------------------------- + # Intracellular ephys + # -------------------------------------------------- + device = nwbfile.create_device(name="Heka ITC-1600") + electrode = nwbfile.create_icephys_electrode( + name="elec0", description="a mock intracellular electrode", device=device + ) + stimulus = VoltageClampStimulusSeries( + name="ccss", + data=[1, 2, 3, 4, 5], + starting_time=123.6, + rate=10e3, + electrode=electrode, + gain=0.02, + sweep_number=np.uint64(15), + ) + + # Create and icephys response + response = VoltageClampSeries( + name="vcs", + data=[0.1, 0.2, 0.3, 0.4, 0.5], + conversion=1e-12, + resolution=np.nan, + starting_time=123.6, + rate=20e3, + electrode=electrode, + gain=0.02, + capacitance_slow=100e-12, + resistance_comp_correction=70.0, + sweep_number=np.uint64(15), + ) + # we can also add stimulus template data as follows + rowindex = nwbfile.add_intracellular_recording( + electrode=electrode, stimulus=stimulus, response=response, id=10 + ) + + rowindex2 = nwbfile.add_intracellular_recording( + electrode=electrode, + stimulus=stimulus, + stimulus_start_index=1, + stimulus_index_count=3, + response=response, + response_start_index=2, + response_index_count=3, + id=11, + ) + rowindex3 = nwbfile.add_intracellular_recording(electrode=electrode, response=response, id=12) + + nwbfile.intracellular_recordings.add_column( + name="recording_tag", + data=["A1", "A2", "A3"], + description="String with a recording tag", + ) + location_column = VectorData( + name="location", + data=["Mordor", "Gondor", "Rohan"], + description="Recording location in Middle Earth", + ) + + lab_category = DynamicTable( + name="recording_lab_data", + description="category table for lab-specific recording metadata", + colnames=[ + "location", + ], + columns=[ + location_column, + ], + ) + # Add the table as a new category to our intracellular_recordings + nwbfile.intracellular_recordings.add_category(category=lab_category) + nwbfile.intracellular_recordings.add_column( + name="voltage_threshold", + data=[0.1, 0.12, 0.13], + description="Just an example column on the electrodes category table", + category="electrodes", + ) + stimulus_template = VoltageClampStimulusSeries( + name="ccst", + data=[0, 1, 2, 3, 4], + starting_time=0.0, + rate=10e3, + electrode=electrode, + gain=0.02, + ) + nwbfile.add_stimulus_template(stimulus_template) + + nwbfile.intracellular_recordings.add_column( + name="stimulus_template", + data=[ + TimeSeriesReference(0, 5, stimulus_template), + # (start_index, index_count, stimulus_template) + TimeSeriesReference(1, 3, stimulus_template), + TimeSeriesReference.empty(stimulus_template), + ], + # if there was no data for that recording, use empty reference + description=( + "Column storing the reference to the stimulus template for the recording (rows)." + ), + category="stimuli", + col_cls=TimeSeriesReferenceVectorData, + ) + + icephys_simultaneous_recordings = nwbfile.get_icephys_simultaneous_recordings() + icephys_simultaneous_recordings.add_column( + name="simultaneous_recording_tag", + description="A custom tag for simultaneous_recordings", + ) + simultaneous_index = nwbfile.add_icephys_simultaneous_recording( + recordings=[rowindex, rowindex2, rowindex3], + id=12, + simultaneous_recording_tag="LabTag1", + ) + repetition_index = nwbfile.add_icephys_repetition( + sequential_recordings=[simultaneous_index], id=17 + ) + nwbfile.add_icephys_experimental_condition(repetitions=[repetition_index], id=19) + nwbfile.icephys_experimental_conditions.add_column( + name="tag", + data=np.arange(1), + description="integer tag for a experimental condition", + ) + + # -------------------------------------------------- + # Calcium Imaging + # https://pynwb.readthedocs.io/en/latest/tutorials/domain/ophys.html + # -------------------------------------------------- + device = nwbfile.create_device( + name="Microscope", + description="My two-photon microscope", + manufacturer="The best microscope manufacturer", + ) + optical_channel = OpticalChannel( + name="OpticalChannel", + description="an optical channel", + emission_lambda=500.0, + ) + imaging_plane = nwbfile.create_imaging_plane( + name="ImagingPlane", + optical_channel=optical_channel, + imaging_rate=30.0, + description="a very interesting part of the brain", + device=device, + excitation_lambda=600.0, + indicator="GFP", + location="V1", + grid_spacing=[0.01, 0.01], + grid_spacing_unit="meters", + origin_coords=[1.0, 2.0, 3.0], + origin_coords_unit="meters", + ) + one_p_series = OnePhotonSeries( + name="OnePhotonSeries", + description="Raw 1p data", + data=np.ones((1000, 100, 100)), + imaging_plane=imaging_plane, + rate=1.0, + unit="normalized amplitude", + ) + nwbfile.add_acquisition(one_p_series) + two_p_series = TwoPhotonSeries( + name="TwoPhotonSeries", + description="Raw 2p data", + data=np.ones((1000, 100, 100)), + imaging_plane=imaging_plane, + rate=1.0, + unit="normalized amplitude", + ) + + nwbfile.add_acquisition(two_p_series) + + corrected = ImageSeries( + name="corrected", # this must be named "corrected" + description="A motion corrected image stack", + data=np.ones((1000, 100, 100)), + unit="na", + format="raw", + starting_time=0.0, + rate=1.0, + ) + + xy_translation = TimeSeries( + name="xy_translation", + description="x,y translation in pixels", + data=np.ones((1000, 2)), + unit="pixels", + starting_time=0.0, + rate=1.0, + ) + + corrected_image_stack = CorrectedImageStack( + corrected=corrected, + original=one_p_series, + xy_translation=xy_translation, + ) + + motion_correction = MotionCorrection(corrected_image_stacks=[corrected_image_stack]) + + ophys_module = nwbfile.create_processing_module( + name="ophys", description="optical physiology processed data" + ) + + ophys_module.add(motion_correction) + + img_seg = ImageSegmentation() + + ps = img_seg.create_plane_segmentation( + name="PlaneSegmentation", + description="output from segmenting my favorite imaging plane", + imaging_plane=imaging_plane, + reference_images=one_p_series, # optional + ) + + ophys_module.add(img_seg) + + for _ in range(30): + image_mask = np.zeros((100, 100)) + + # randomly generate example image masks + x = generator.integers(0, 95) + y = generator.integers(0, 95) + image_mask[x : x + 5, y : y + 5] = 1 + + # add image mask to plane segmentation + ps.add_roi(image_mask=image_mask) + + ps2 = img_seg.create_plane_segmentation( + name="PlaneSegmentation2", + description="output from segmenting my favorite imaging plane", + imaging_plane=imaging_plane, + reference_images=one_p_series, # optional + ) + + for _ in range(30): + # randomly generate example starting points for region + x = generator.integers(0, 95) + y = generator.integers(0, 95) + + # define an example 4 x 3 region of pixels of weight '1' + pixel_mask = [(ix, iy, 1) for ix in range(x, x + 4) for iy in range(y, y + 3)] + + # add pixel mask to plane segmentation + ps2.add_roi(pixel_mask=pixel_mask) + + ps3 = img_seg.create_plane_segmentation( + name="PlaneSegmentation3", + description="output from segmenting my favorite imaging plane", + imaging_plane=imaging_plane, + reference_images=one_p_series, # optional + ) + + for _ in range(30): + # randomly generate example starting points for region + x = generator.integers(0, 95) + y = generator.integers(0, 95) + z = generator.integers(0, 15) + + # define an example 4 x 3 x 2 voxel region of weight '0.5' + voxel_mask = [] + for ix, iy, iz in product(range(x, x + 4), range(y, y + 3), range(z, z + 2)): + voxel_mask.append((ix, iy, iz, 0.5)) + + # add voxel mask to plane segmentation + ps3.add_roi(voxel_mask=voxel_mask) + rt_region = ps.create_roi_table_region(region=[0, 1], description="the first of two ROIs") + roi_resp_series = RoiResponseSeries( + name="RoiResponseSeries", + description="Fluorescence responses for two ROIs", + data=np.ones((50, 2)), # 50 samples, 2 ROIs + rois=rt_region, + unit="lumens", + rate=30.0, + ) + fl = Fluorescence(roi_response_series=roi_resp_series) + ophys_module.add(fl) + + with NWBHDF5IO(nwb_path, "w") as io: + io.write(nwbfile) + + return nwb_path diff --git a/nwb_linkml/tests/test_io/test_io_nwb.py b/nwb_linkml/tests/test_io/test_io_nwb.py new file mode 100644 index 0000000..a6eb230 --- /dev/null +++ b/nwb_linkml/tests/test_io/test_io_nwb.py @@ -0,0 +1,17 @@ +""" +Placeholder test module to test reading from pynwb-generated NWB file +""" + + +def test_read_from_nwbfile(nwb_file): + """ + Read data from a pynwb HDF5 NWB file + """ + pass + + +def test_read_from_yaml(nwb_file): + """ + Read data from a yaml-fied NWB file + """ + pass