From 3f2be60ea68ecabb7a0feec6bb358f92e13dcf8f Mon Sep 17 00:00:00 2001 From: sneakers-the-rat Date: Mon, 11 Dec 2023 21:37:06 -0800 Subject: [PATCH] roadmap --- poetry.lock | 55 +++++-------------------- src/conf.py | 1 + src/index.md | 3 +- src/roadmap.md | 106 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 120 insertions(+), 45 deletions(-) create mode 100644 src/roadmap.md diff --git a/poetry.lock b/poetry.lock index 06a2206..bc04766 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,10 +1,9 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "alabaster" version = "0.7.13" description = "A configurable sidebar-enabled Sphinx theme" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -16,7 +15,6 @@ files = [ name = "babel" version = "2.12.1" description = "Internationalization utilities" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -28,7 +26,6 @@ files = [ name = "beautifulsoup4" version = "4.12.2" description = "Screen-scraping library" -category = "main" optional = false python-versions = ">=3.6.0" files = [ @@ -47,7 +44,6 @@ lxml = ["lxml"] name = "certifi" version = "2023.5.7" description = "Python package for providing Mozilla's CA Bundle." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -59,7 +55,6 @@ files = [ name = "charset-normalizer" version = "3.1.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -144,7 +139,6 @@ files = [ name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -156,7 +150,6 @@ files = [ name = "docutils" version = "0.19" description = "Docutils -- Python Documentation Utilities" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -168,7 +161,6 @@ files = [ name = "furo" version = "2023.5.20" description = "A clean customisable Sphinx documentation theme." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -186,7 +178,6 @@ sphinx-basic-ng = "*" name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -198,7 +189,6 @@ files = [ name = "imagesize" version = "1.4.1" description = "Getting image size from png/jpeg/jpeg2000/gif file" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -210,7 +200,6 @@ files = [ name = "jinja2" version = "3.1.2" description = "A very fast and expressive template engine." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -228,7 +217,6 @@ i18n = ["Babel (>=2.7)"] name = "latexcodec" version = "2.0.1" description = "A lexer and codec to work with LaTeX code in Python." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -243,7 +231,6 @@ six = ">=1.4.1" name = "linkify-it-py" version = "2.0.2" description = "Links recognition library with FULL unicode support." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -264,7 +251,6 @@ test = ["coverage", "pytest", "pytest-cov"] name = "livereload" version = "2.6.3" description = "Python LiveReload is an awesome tool for web developers" -category = "dev" optional = false python-versions = "*" files = [ @@ -280,7 +266,6 @@ tornado = {version = "*", markers = "python_version > \"2.7\""} name = "markdown-it-py" version = "3.0.0" description = "Python port of markdown-it. Markdown parsing, done right!" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -305,7 +290,6 @@ testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] name = "markupsafe" version = "2.1.3" description = "Safely add untrusted strings to HTML/XML markup." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -329,6 +313,16 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -365,7 +359,6 @@ files = [ name = "mdit-py-plugins" version = "0.4.0" description = "Collection of plugins for markdown-it-py" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -385,7 +378,6 @@ testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] name = "mdurl" version = "0.1.2" description = "Markdown URL utilities" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -397,7 +389,6 @@ files = [ name = "myst-parser" version = "2.0.0" description = "An extended [CommonMark](https://spec.commonmark.org/) compliant parser," -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -425,7 +416,6 @@ testing-docutils = ["pygments", "pytest (>=7,<8)", "pytest-param-files (>=0.3.4, name = "packaging" version = "23.1" description = "Core utilities for Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -437,7 +427,6 @@ files = [ name = "pybtex" version = "0.24.0" description = "A BibTeX-compatible bibliography processor in Python" -category = "main" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*" files = [ @@ -457,7 +446,6 @@ test = ["pytest"] name = "pybtex-docutils" version = "1.0.2" description = "A docutils backend for pybtex." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -473,7 +461,6 @@ pybtex = ">=0.16" name = "pygments" version = "2.15.1" description = "Pygments is a syntax highlighting package written in Python." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -488,7 +475,6 @@ plugins = ["importlib-metadata"] name = "pyyaml" version = "6.0" description = "YAML parser and emitter for Python" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -538,7 +524,6 @@ files = [ name = "requests" version = "2.31.0" description = "Python HTTP for Humans." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -560,7 +545,6 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -572,7 +556,6 @@ files = [ name = "snowballstemmer" version = "2.2.0" description = "This package provides 29 stemmers for 28 languages generated from Snowball algorithms." -category = "main" optional = false python-versions = "*" files = [ @@ -584,7 +567,6 @@ files = [ name = "soupsieve" version = "2.4.1" description = "A modern CSS selector implementation for Beautiful Soup." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -596,7 +578,6 @@ files = [ name = "sphinx" version = "6.2.1" description = "Python documentation generator" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -631,7 +612,6 @@ test = ["cython", "filelock", "html5lib", "pytest (>=4.6)"] name = "sphinx-autobuild" version = "2021.3.14" description = "Rebuild Sphinx documentation on changes, with live-reload in the browser." -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -651,7 +631,6 @@ test = ["pytest", "pytest-cov"] name = "sphinx-basic-ng" version = "1.0.0b1" description = "A modern skeleton for Sphinx themes." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -669,7 +648,6 @@ docs = ["furo", "ipython", "myst-parser", "sphinx-copybutton", "sphinx-inline-ta name = "sphinx-design" version = "0.4.1" description = "A sphinx extension for designing beautiful, view size responsive web components." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -693,7 +671,6 @@ theme-sbt = ["sphinx-book-theme (>=0.3.0,<0.4.0)"] name = "sphinxcontrib-applehelp" version = "1.0.4" description = "sphinxcontrib-applehelp is a Sphinx extension which outputs Apple help books" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -709,7 +686,6 @@ test = ["pytest"] name = "sphinxcontrib-bibtex" version = "2.5.0" description = "Sphinx extension for BibTeX style citations." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -727,7 +703,6 @@ Sphinx = ">=2.1" name = "sphinxcontrib-devhelp" version = "1.0.2" description = "sphinxcontrib-devhelp is a sphinx extension which outputs Devhelp document." -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -743,7 +718,6 @@ test = ["pytest"] name = "sphinxcontrib-htmlhelp" version = "2.0.1" description = "sphinxcontrib-htmlhelp is a sphinx extension which renders HTML help files" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -759,7 +733,6 @@ test = ["html5lib", "pytest"] name = "sphinxcontrib-jsmath" version = "1.0.1" description = "A sphinx extension which renders display math in HTML via JavaScript" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -774,7 +747,6 @@ test = ["flake8", "mypy", "pytest"] name = "sphinxcontrib-mermaid" version = "0.9.2" description = "Mermaid diagrams in yours Sphinx powered docs" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -786,7 +758,6 @@ files = [ name = "sphinxcontrib-qthelp" version = "1.0.3" description = "sphinxcontrib-qthelp is a sphinx extension which outputs QtHelp document." -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -802,7 +773,6 @@ test = ["pytest"] name = "sphinxcontrib-serializinghtml" version = "1.1.5" description = "sphinxcontrib-serializinghtml is a sphinx extension which outputs \"serialized\" HTML files (json and pickle)." -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -818,7 +788,6 @@ test = ["pytest"] name = "tornado" version = "6.3.2" description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." -category = "dev" optional = false python-versions = ">= 3.8" files = [ @@ -839,7 +808,6 @@ files = [ name = "uc-micro-py" version = "1.0.2" description = "Micro subset of unicode data files for linkify-it-py projects." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -854,7 +822,6 @@ test = ["coverage", "pytest", "pytest-cov"] name = "urllib3" version = "2.0.3" description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" optional = false python-versions = ">=3.7" files = [ diff --git a/src/conf.py b/src/conf.py index 13c8898..137631c 100644 --- a/src/conf.py +++ b/src/conf.py @@ -81,6 +81,7 @@ mermaid.initialize({ "theme": "dark" }) """ +mermaid_version="10.6.1" # "themeVariables": { # "darkMode": true, diff --git a/src/index.md b/src/index.md index e143c3e..9f1bdfc 100644 --- a/src/index.md +++ b/src/index.md @@ -8,13 +8,14 @@ This site describes the implementation of the p2p linked data protocol in {cite} **23-11-27** - Back at it again after some digressions into [chatbridge](https://git.jon-e.net/jonny/chatbridge) and [nwb-linkml](https://github.com/p2p-ld/nwb-linkml/) - gathering more information on storage and interchange formats for databases and triple stores before trying to prop up the first peers sharing graphs of NWB data. Still mostly populating the [Comparison](comparison) section as I take notes and before I restructure these docs. -**23-06-08** - Populating the [Comparison](comparison) section first to refresh myself on other projects, and starting to sketch diagrammatically in [Sketchpad](sketchpad). The rest of the pages are just stubs to keep track of ideas before fleshing them out. +**23-06-08** - Populating the [Comparison](comparison) section first to refresh myself on other projects, and starting to sketch diagrammatically in [Sketchpad](sketchpad). The rest of the pages are just stubs to keep track of ideas before fleshing them out. ```{toctree} :caption: Introduction :hidden: overview +roadmap comparison/index p2p_concepts out_of_scope diff --git a/src/roadmap.md b/src/roadmap.md new file mode 100644 index 0000000..5efebd6 --- /dev/null +++ b/src/roadmap.md @@ -0,0 +1,106 @@ +# Roadmap + +What things do we build and in what order? + +## Phase 1: Format Sandbox + +```{admonition} Goals +- Experimenting with needs for format translation +- Encoding of triple graphs + hashed binary files +``` + +In this phase we are using {index}`NWB` as a naturalistic development case of a complex format with an idiosyncratic implementation (ie. written in its own schema language, with its own I/O API, etc.) + +See [nwb-linkml](https://github.com/p2p-ld/nwb-linkml) + + +## Phase 2: Simple Transfer + +```{admonition} Goals +- Simplest possible peer model: self-sharing between devices owned by peer +- Share, query, and download data +``` + +A `peer` has a public `identity` that is represented by one or multiple `machines` that host the `graphs` associated with that identity. Each graph has some collection of `triples` which are identified relative to the IRI of the parent graph. A triple can refer to a `binary` file that is not stored in the graph itself by its hash, encoding, and other metadata needed to use the file. The (semantic) metadata that describes that `binary` file can point to it without needing to encode arrays/etc. in RDF-like graphs. + +```{mermaid} + +--- +config: + er: + layoutDirection: LR +--- +erDiagram + PEER { + string private_key + } + PEER ||--|{ IDENTITY : knownAs + IDENTITY { + string private_key + string public_key + string preferredName + } + IDENTITY ||--|{ GRAPH : hasFeed + IDENTITY }o--o{ IDENTITY : peerList + IDENTITY }|--|{ MACHINE : represents + MACHINE { + graph mirrorRules + } + MACHINE }|--o{ GRAPH: stores + + GRAPH { + IRI id + string name + capability read + capability write + } + GRAPH }|--|{ TRIPLE : contains + TRIPLE { + IRI relative_id + string subject + string predicate + string object + } + TRIPLE ||--o{ BINARY : indicates + BINARY { + string encoding + string hash + graph metadata + + } +``` + +The MVP for simple transfer is to have a single peer be able to transfer subgraphs and binary files between multiple machines keyed to their public identity. + + + +## Phase 3: Schema Manipulation + +```{admonition} Goals +- Make custom schema +- Declare and resolve sameness relations +- Dataset versioning +``` + +## Phase 4: Peer Federation + +```{admonition} Goals +- Ontology for determining access +- Pinning and mirroring +``` + + +## Phase 5: External Resources + + +```{admonition} Goals +- Treat external resources as peers +- HTTP, S3 mirroring +``` + +## Phase 6: Discoverability + +```{admonition} Goals +- Use external sources of identity to connect to peer +- Bootstrap servers +``` \ No newline at end of file