From 457083ce16b8670ba07b4f6ae8299f6ae86b70b1 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 20:52:21 -0500 Subject: [PATCH 1/2] Publish the schemas and guides as a GitHub Pages site gen-doc reference for the storage, extraction and paper-parse schemas, the guides and crosswalks staged with rewritten links, mkdocs-material, and a docs workflow that builds with --strict on pull requests and deploys on main. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/docs.yml | 47 +++++++++++++++++++ .gitignore | 1 + mkdocs.yml | 46 +++++++++++++++++++ pyproject.toml | 1 + tools/build_docs.py | 93 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 188 insertions(+) create mode 100644 .github/workflows/docs.yml create mode 100644 mkdocs.yml create mode 100644 tools/build_docs.py diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..b5e77cc --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,47 @@ +name: docs + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +concurrency: + group: pages + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install + run: pip install ".[docs]" + - name: Generate the schema reference and stage the guides + run: python tools/build_docs.py + - name: Build the site (broken links fail) + run: mkdocs build --strict + - uses: actions/upload-pages-artifact@v3 + if: github.event_name == 'push' + with: + path: build/site + + deploy: + if: github.event_name == 'push' + needs: build + runs-on: ubuntu-latest + permissions: + pages: write + id-token: write + contents: read + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore index d947398..c740af3 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ __pycache__/ build/ dist/ .pytest_cache/ +build/ diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..5c25314 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,46 @@ +site_name: study_schema +site_description: The contracts between the neurostuff pipeline repositories, generated from LinkML +site_url: https://neurostuff.github.io/study_schema/ +repo_url: https://github.com/neurostuff/study_schema +docs_dir: build/docs +site_dir: build/site +strict: true + +theme: + name: material + features: + - navigation.sections + - navigation.top + - search.suggest + - search.highlight + - content.code.copy + +plugins: + - search + +markdown_extensions: + - admonition + - tables + - toc: + permalink: true + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format + +nav: + - Home: index.md + - Guides: + - Schema tutorial: schema-tutorial.md + - Analysis entities: analysis-entities.md + - Representing models: representing-models.md + - Storage design notes: storage-schema-design-notes.md + - Extraction schema: extraction-readme.md + - Crosswalks: + - Standards crosswalk: standards-crosswalk.md + - ARS crosswalk: ars-crosswalk.md + - Reference: + - Storage: reference/neuroimaging-study-storage/index.md + - Extraction: reference/neuroimaging-study-extraction/index.md + - Paper parse: reference/neuroimaging-paper-parse/index.md diff --git a/pyproject.toml b/pyproject.toml index 50c6871..adc23dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ layouts = ["pyarty @ git+https://github.com/jdkent/pyarty.git@main"] parquet = ["pyarrow>=14"] generate = ["linkml>=1.12"] test = ["pytest>=7"] +docs = ["mkdocs>=1.6,<2", "mkdocs-material>=9.5", "linkml>=1.12"] [project.urls] Source = "https://github.com/neurostuff/study_schema" diff --git a/tools/build_docs.py b/tools/build_docs.py new file mode 100644 index 0000000..f287467 --- /dev/null +++ b/tools/build_docs.py @@ -0,0 +1,93 @@ +"""Assemble the documentation site sources into `build/docs/` for mkdocs. + +The reference pages are generated from the LinkML schemas with `gen-doc`, and the guides are +staged from the repository root, so nothing here is committed: + + python tools/build_docs.py + mkdocs build --strict + +The guides link to each other, to the schema directories and to files that are not part of +the site. Their links are rewritten while staging so the markdown on disk keeps working on +GitHub: schema directories point at the reference, other repository files at GitHub, and a +link to a file that is not in the repository is reduced to its text and reported. +""" + +from __future__ import annotations + +import re +import shutil +from pathlib import Path + +from linkml.generators.docgen import DocGenerator + +from study_schema import linkml + +ROOT = Path(__file__).resolve().parents[1] +BUILD = ROOT / "build" / "docs" +BLOB = "https://github.com/neurostuff/study_schema/blob/main/" +TREE = "https://github.com/neurostuff/study_schema/tree/main/" + +#: Top-level schemas; the evidence schema is imported by extraction and documented with it. +SCHEMAS = (linkml.STORAGE, linkml.EXTRACTION, linkml.PAPER_PARSE) + +GUIDES = ( + "schema-tutorial.md", + "analysis-entities.md", + "representing-models.md", + "storage-schema-design-notes.md", + "extraction-readme.md", + "standards-crosswalk.md", + "ars-crosswalk.md", +) + +LINK = re.compile(r"\[((?:[^\[\]]|\[[^\[\]]*\])*)\]\(([^)\s#]*)(#[^)\s]*)?\)") + + +def rewrite_links(markdown: str, source: Path, staged: set[str]) -> tuple[str, list[str]]: + dropped: list[str] = [] + + def replace(match: re.Match) -> str: + text, target, anchor = match.group(1), match.group(2), match.group(3) or "" + if not target or re.match(r"[a-z]+:", target): + return match.group(0) + path = (source.parent / target).resolve() + try: + relative = path.relative_to(ROOT).as_posix() + except ValueError: + relative = None + if relative == "README.md": + return f"[{text}](index.md{anchor})" + if relative in staged: + return match.group(0) + if relative and relative.startswith("src/study_schema/linkml/") and path.is_dir(): + return f"[{text}](reference/{path.name}/index.md{anchor})" + if relative and path.exists(): + return f"[{text}]({TREE if path.is_dir() else BLOB}{relative})" + dropped.append(f"{source.name}: {target}") + return text + + return LINK.sub(replace, markdown), dropped + + +def main() -> None: + if BUILD.parent.exists(): + shutil.rmtree(BUILD.parent) + BUILD.mkdir(parents=True) + + for schema in SCHEMAS: + DocGenerator(str(schema), mergeimports=True).serialize( + directory=str(BUILD / "reference" / schema.stem) + ) + + sources = {"README.md": "index.md", **{name: name for name in GUIDES}} + dropped: list[str] = [] + for name, target in sources.items(): + text, lost = rewrite_links((ROOT / name).read_text(), ROOT / name, set(sources)) + (BUILD / target).write_text(text) + dropped += lost + for entry in dropped: + print(f"link without a target, kept as text: {entry}") + + +if __name__ == "__main__": + main() From a22c8167401efa4043de10b9efac33926517090e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 20:56:25 -0500 Subject: [PATCH 2/2] Remove links to files that do not exist; stop hiding dead links in the docs build Co-Authored-By: Claude Sonnet 5.5 --- README.md | 2 -- extraction-readme.md | 9 +++------ representing-models.md | 3 +-- standards-crosswalk.md | 2 +- tools/build_docs.py | 19 ++++++------------- 5 files changed, 11 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 9e52237..a3330d5 100644 --- a/README.md +++ b/README.md @@ -446,8 +446,6 @@ the priority inventory from here. | [representing-models.md](representing-models.md) | How to put a reported analysis into the schema: which facts belong to the model and which to the contrast, where each class's job ends, and what to do when a paper does not divide things the way the schema does. Its YAML fragments and the worked records under `pondie/tests/fixtures/examples/` are checked on every test run | | [ars-crosswalk.md](ars-crosswalk.md) | Field-level comparison against CDISC's Analysis Results Standard — the closest peer this schema has, and so the main check on the model/contrast split. For design comparison, not an executable map | | [standards-crosswalk.md](standards-crosswalk.md) | The same reading against BIDS Stats Models and NIDM-Results — what each represents, where they are more expressive, and where this schema keeps something they cannot say. Companion to the ARS crosswalk and does not repeat it | -| [storage-schema-expressivity-probe.md](storage-schema-expressivity-probe.md) | Measured expressivity gaps against 25 corpus papers, with options | -| [multivariate-probe.md](multivariate-probe.md) | The same reading narrowed to decoding, RSA, PLS and searchlight work: what a meta-analyst cannot filter on today, ranked, with options. Companion to the probe above and numbered `M1`–`M8` so the two do not collide | # Ideation about LLM extraction workflow diff --git a/extraction-readme.md b/extraction-readme.md index da3d13c..305bfcd 100644 --- a/extraction-readme.md +++ b/extraction-readme.md @@ -6,8 +6,7 @@ express, and known limits worth knowing before someone rediscovers them. The schema files are the source of truth for *what* a field holds; this file covers *how* the pipeline is expected to behave around them. Reasoning behind the schema's shape is in -[storage-schema-design-notes.md](storage-schema-design-notes.md); measured expressivity gaps -are in [storage-schema-expressivity-probe.md](storage-schema-expressivity-probe.md). +[storage-schema-design-notes.md](storage-schema-design-notes.md). --- @@ -532,8 +531,7 @@ averaged [ +0.5 +0.5 -0.5 -0.5 ] They differ only in the rest cells, and that is exactly what the two records differ in: the simple effect has an unsigned `task` cell (restricting to it, so `rest` drops out) and the main effect has no condition cell at all (so both cells carry the diagnosis sign). The vector is recoverable from the -cells; [test_effect_kind_derivation.py](test_effect_kind_derivation.py) reconstructs all four of the -factorial example's. +cells. One factorial model therefore yields: @@ -761,8 +759,7 @@ for deciding which categories the enums are missing. Nothing in the repo aggrega ## 5. Known limits -Facts a paper may report that no field currently holds. Full evidence and options are in -[storage-schema-expressivity-probe.md](storage-schema-expressivity-probe.md); this is the short +Facts a paper may report that no field currently holds. This is the short list so an extractor is not left hunting for a slot. - **Non-imaging outcomes.** `MeasureFamily` covers imaging signals only, so an intervention diff --git a/representing-models.md b/representing-models.md index bf53882..76b1356 100644 --- a/representing-models.md +++ b/representing-models.md @@ -558,8 +558,7 @@ within HC" are the same two cells and the record cannot tell them apart. > Compared to the control group, RA patients showed enhanced activation in the left precuneus, left superior frontal gyrus and right cingulate gyrus during the rotation task, with left hemisphere dominance. **Referent** `Qa5HqrHq97Pm` (pmid 37559139) — a mental-rotation task in rheumatoid arthritis. This -one is **audited by hand**, and the audit is -[corrections/Qa5HqrHq97Pm.corrections.json](../corrections/Qa5HqrHq97Pm.corrections.json): 34 +one is **audited by hand**: 34 operations that take the record from five validator errors to none. Four of the five defects are worth knowing because they are not this paper's: diff --git a/standards-crosswalk.md b/standards-crosswalk.md index 0e42042..f8a57fe 100644 --- a/standards-crosswalk.md +++ b/standards-crosswalk.md @@ -53,7 +53,7 @@ scheme, so they are deliberately not crosswalked here, and no field in this sche [storage-schema-design-notes.md §"Conditions belong to a task"](storage-schema-design-notes.md) gives the reason. What a vocabulary can be checked against instead is whether a field exists whose free-text value a later normalization step could map onto it, which is a per-field audit rather -than a crosswalk; [onvoc-mapping-audit.md](onvoc-mapping-audit.md) is that audit for ONVOC. +than a crosswalk. --- diff --git a/tools/build_docs.py b/tools/build_docs.py index f287467..2a971e2 100644 --- a/tools/build_docs.py +++ b/tools/build_docs.py @@ -8,8 +8,8 @@ The guides link to each other, to the schema directories and to files that are not part of the site. Their links are rewritten while staging so the markdown on disk keeps working on -GitHub: schema directories point at the reference, other repository files at GitHub, and a -link to a file that is not in the repository is reduced to its text and reported. +GitHub: schema directories point at the reference and other repository files at GitHub. A +link to a file that does not exist is left alone, so `mkdocs build --strict` fails on it. """ from __future__ import annotations @@ -43,9 +43,7 @@ LINK = re.compile(r"\[((?:[^\[\]]|\[[^\[\]]*\])*)\]\(([^)\s#]*)(#[^)\s]*)?\)") -def rewrite_links(markdown: str, source: Path, staged: set[str]) -> tuple[str, list[str]]: - dropped: list[str] = [] - +def rewrite_links(markdown: str, source: Path, staged: set[str]) -> str: def replace(match: re.Match) -> str: text, target, anchor = match.group(1), match.group(2), match.group(3) or "" if not target or re.match(r"[a-z]+:", target): @@ -63,10 +61,9 @@ def replace(match: re.Match) -> str: return f"[{text}](reference/{path.name}/index.md{anchor})" if relative and path.exists(): return f"[{text}]({TREE if path.is_dir() else BLOB}{relative})" - dropped.append(f"{source.name}: {target}") - return text + return match.group(0) - return LINK.sub(replace, markdown), dropped + return LINK.sub(replace, markdown) def main() -> None: @@ -80,13 +77,9 @@ def main() -> None: ) sources = {"README.md": "index.md", **{name: name for name in GUIDES}} - dropped: list[str] = [] for name, target in sources.items(): - text, lost = rewrite_links((ROOT / name).read_text(), ROOT / name, set(sources)) + text = rewrite_links((ROOT / name).read_text(), ROOT / name, set(sources)) (BUILD / target).write_text(text) - dropped += lost - for entry in dropped: - print(f"link without a target, kept as text: {entry}") if __name__ == "__main__":