From 62c1fb21cb9f1dfd932424f6958826f22dde867f Mon Sep 17 00:00:00 2001 From: "Yu-Hsuan (Amy) Lin" Date: Thu, 12 Nov 2026 03:12:41 -0800 Subject: [PATCH] Add MkDocs Material docs site and GitHub Pages workflow for TPUMS. PiperOrigin-RevId: 1019962949 --- .github/workflows/docs.yml | 61 ++++ docs/javascripts/mathjax.js | 21 ++ docs/requirements.txt | 3 + mkdocs.yml | 70 +++++ mkdocs_hooks.py | 281 ++++++++++++++++++ tests/test_docs_site.py | 562 ++++++++++++++++++++++++++++++++++++ 6 files changed, 998 insertions(+) create mode 100644 .github/workflows/docs.yml create mode 100644 docs/javascripts/mathjax.js create mode 100644 docs/requirements.txt create mode 100644 mkdocs.yml create mode 100644 mkdocs_hooks.py create mode 100644 tests/test_docs_site.py diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000..cd7484d7 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,61 @@ +name: Deploy Documentation + +"on": + push: + branches: + - main + paths: + - "README.md" + - "docs/**" + - "mkdocs.yml" + - "mkdocs_hooks.py" + - ".github/workflows/docs.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: pages + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 + with: + python-version: "3.12" + + - name: Install documentation dependencies + run: | + python -m pip install --upgrade pip + pip install -r docs/requirements.txt + + - name: Build documentation site + run: mkdocs build --strict --site-dir _site + + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 + with: + path: _site + + deploy: + needs: build + runs-on: ubuntu-latest + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 diff --git a/docs/javascripts/mathjax.js b/docs/javascripts/mathjax.js new file mode 100644 index 00000000..242a868d --- /dev/null +++ b/docs/javascripts/mathjax.js @@ -0,0 +1,21 @@ +window.MathJax = { + tex: { + inlineMath: [["\\(", "\\)"]], + displayMath: [["\\[", "\\]"]], + processEscapes: true, + processEnvironments: true, + }, + options: { + ignoreHtmlClass: ".*|", + processHtmlClass: "arithmatex", + }, +}; + +document$.subscribe(() => { + if (window.MathJax && typeof window.MathJax.typesetPromise === "function") { + MathJax.startup.output.clearCache(); + MathJax.typesetClear(); + MathJax.texReset(); + MathJax.typesetPromise(); + } +}); diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 00000000..ef020cfa --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,3 @@ +mkdocs>=1.6,<2.0 +mkdocs-material>=9.5,<10.0 +pymdown-extensions>=10.0 diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 00000000..46fdb0c9 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,70 @@ +site_name: TPUMS (TPU Microbenchmark Suite) +site_description: Modular, high-fidelity JAX microbenchmarks and profiling framework for Cloud TPUs. +site_url: https://ai-hypercomputer.github.io/accelerator-microbenchmarks/ +repo_url: https://github.com/AI-Hypercomputer/accelerator-microbenchmarks +repo_name: AI-Hypercomputer/accelerator-microbenchmarks +docs_dir: docs +exclude_docs: requirements.txt +strict: true + +hooks: + - mkdocs_hooks.py + +theme: + name: material + palette: + - media: "(prefers-color-scheme: light)" + scheme: default + primary: indigo + accent: indigo + toggle: + icon: material/brightness-7 + name: Switch to dark mode + - media: "(prefers-color-scheme: dark)" + scheme: slate + primary: indigo + accent: indigo + toggle: + icon: material/brightness-4 + name: Switch to light mode + features: + - navigation.tabs + - navigation.top + - search.suggest + - search.highlight + - content.code.copy + - toc.follow + +nav: + - User Guide: index.md + - Benchmark Reference: BENCHMARKS.md + - Running on GKE: GKE.md + - Automation Pipeline: AUTOMATION.md + +markdown_extensions: + - admonition + - pymdownx.details + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format + - pymdownx.arithmatex: + generic: true + - pymdownx.highlight: + anchor_linenums: true + - pymdownx.inlinehilite + - pymdownx.tabbed: + alternate_style: true + - tables + - attr_list + - md_in_html + - toc: + permalink: true + slugify: !!python/object/apply:pymdownx.slugs.slugify + kwds: + case: lower + +extra_javascript: + - javascripts/mathjax.js + - https://unpkg.com/mathjax@3/es5/tex-mml-chtml.js diff --git a/mkdocs_hooks.py b/mkdocs_hooks.py new file mode 100644 index 00000000..bfa02d37 --- /dev/null +++ b/mkdocs_hooks.py @@ -0,0 +1,281 @@ +"""MkDocs build hooks for the TPUMS documentation site.""" + +from collections.abc import Mapping, Sequence +import os +import posixpath +import re +from typing import Any + +_FENCE_RE = re.compile( + r"(?m)^([ \t]*)(`{3,}|~{3,})[^\n]*\n[\s\S]*?^\1\2[ \t]*$" +) +_UNMASK_FENCE_RE = re.compile(r"(?m)^([ \t]*)\x00FENCE_(\d+)\x00") +_INLINE_CODE_RE = re.compile(r"(`+)([^`]*?)\1") +_UNMASK_INLINE_RE = re.compile(r"\x00INLINE_(\d+)\x00") +_MANUAL_TOC_RE = re.compile(r"(?m)^## Table of Contents\n[\s\S]*?^---\n+") +_LINK_RE = re.compile(r"(?[ \t]*\[!(NOTE|TIP|IMPORTANT|WARNING|CAUTION)\](?:[ \t]+(.+?))?[ \t]*$", + re.IGNORECASE, +) +_BOLD_NOTE_HEADER_RE = re.compile( + r"^>[ \t]*\*\*((Note|Tip|Important|Warning|Caution)[^*]*?):?\*\*:?[" + r" \t]*(.*)$", + re.IGNORECASE, +) +_BLOCKQUOTE_CONT_RE = re.compile(r"^>[ \t]?(.*)$") +_INLINE_MATH_RE = re.compile(r"(? tuple[str, list[str]]: + """Replaces fenced code blocks with indexed sentinel tokens.""" + fences: list[str] = [] + + def _store(match: re.Match[str]) -> str: + indent = match.group(1) + raw_fence = match.group(0) + if indent: + deindented = "\n".join( + line[len(indent) :] if line.startswith(indent) else line + for line in raw_fence.split("\n") + ) + else: + deindented = raw_fence + idx = len(fences) + fences.append(deindented) + return f"{indent}\x00FENCE_{idx}\x00" + + return _FENCE_RE.sub(_store, markdown), fences + + +def _unmask_fences(markdown: str, fences: Sequence[str]) -> str: + """Restores fenced code blocks, propagating any newly added indentation.""" + + def _restore(match: re.Match[str]) -> str: + indent = match.group(1) + fence = fences[int(match.group(2))] + if not indent: + return fence + return "\n".join( + (f"{indent}{line}" if line else "") for line in fence.split("\n") + ) + + return _UNMASK_FENCE_RE.sub(_restore, markdown) + + +def _mask_inline_code(markdown: str) -> tuple[str, list[str]]: + """Replaces inline backtick code spans with indexed sentinel tokens.""" + spans: list[str] = [] + + def _store(match: re.Match[str]) -> str: + idx = len(spans) + spans.append(match.group(0)) + return f"\x00INLINE_{idx}\x00" + + return _INLINE_CODE_RE.sub(_store, markdown), spans + + +def _unmask_inline_code(markdown: str, spans: Sequence[str]) -> str: + """Restores inline backtick code spans.""" + return _UNMASK_INLINE_RE.sub(lambda m: spans[int(m.group(1))], markdown) + + +def on_files(files: Any, config: Mapping[str, Any]) -> Any: + """Registers root README.md as virtual index.md for MkDocs builds.""" + if files.get_file_from_path("index.md") is not None: + return files + docs_dir = os.path.abspath(config["docs_dir"]) + readme_path = os.path.join(os.path.dirname(docs_dir), "README.md") + if not os.path.isfile(readme_path): + raise FileNotFoundError(f"Missing root README.md at {readme_path}") + file_cls = type(next(iter(files))) + files.append(file_cls.generated(config, "index.md", abs_src_path=readme_path)) + return files + + +def _rewrite_links( + markdown: str, + *, + src_uri: str, + docs_dir: str, + repo_root: str, + repo_url: str, +) -> str: + """Rewrites relative repository links for MkDocs pages and GitHub files.""" + page_docs_dir = os.path.dirname(os.path.join(docs_dir, src_uri)) + source_dir = repo_root if src_uri == "index.md" else page_docs_dir + readme_abs = os.path.join(repo_root, "README.md") + + def _replace(match: re.Match[str]) -> str: + label, raw_target = match.group(1), match.group(2) + if raw_target.startswith(("http://", "https://", "mailto:", "#")): + return match.group(0) + path_part, sep, fragment = raw_target.partition("#") + if not path_part: + return match.group(0) + anchor_suffix = f"#{fragment}" if sep else "" + had_trailing_slash = path_part.endswith("/") + target_abs = os.path.abspath( + os.path.normpath(os.path.join(source_dir, path_part)) + ) + + if target_abs == readme_abs: + rel_index = posixpath.relpath( + os.path.join(docs_dir, "index.md").replace(os.sep, "/"), + page_docs_dir.replace(os.sep, "/"), + ) + return f"{label}({rel_index}{anchor_suffix})" + + if os.path.commonpath([docs_dir, target_abs]) == docs_dir: + rel_doc = posixpath.relpath( + target_abs.replace(os.sep, "/"), + page_docs_dir.replace(os.sep, "/"), + ) + return f"{label}({rel_doc}{anchor_suffix})" + + if os.path.commonpath([repo_root, target_abs]) == repo_root: + rel_repo = posixpath.relpath( + target_abs.replace(os.sep, "/"), + repo_root.replace(os.sep, "/"), + ) + if os.path.isdir(target_abs) or had_trailing_slash: + slash = "/" if had_trailing_slash else "" + return f"{label}({repo_url}/tree/main/{rel_repo}{slash}{anchor_suffix})" + return f"{label}({repo_url}/blob/main/{rel_repo}{anchor_suffix})" + + return match.group(0) + + return _LINK_RE.sub(_replace, markdown) + + +def _normalize_list_indents(markdown: str) -> str: + """Normalizes 2-space and 4-space nested list hierarchies to 4-space steps.""" + lines = markdown.split("\n") + out: list[str] = [] + indent_stack: list[int] = [] + for line in lines: + if not line.strip(): + out.append(line) + continue + + marker_match = _LIST_MARKER_RE.match(line) + if marker_match: + raw_indent = len(marker_match.group(1)) + marker = marker_match.group(2) + rest = marker_match.group(3) + while indent_stack and raw_indent < indent_stack[-1]: + indent_stack.pop() + if not indent_stack: + if raw_indent != 0: + out.append(line) + continue + indent_stack = [0] + elif raw_indent > indent_stack[-1]: + indent_stack.append(raw_indent) + depth = len(indent_stack) - 1 + out.append(f"{' ' * depth}{marker}{rest}") + continue + + if not indent_stack: + out.append(line) + continue + + raw_indent = len(line) - len(line.lstrip(" ")) + if raw_indent == 0: + indent_stack = [] + out.append(line) + continue + + while len(indent_stack) > 1 and raw_indent <= indent_stack[-1]: + indent_stack.pop() + depth = len(indent_stack) - 1 + out.append(f"{' ' * (depth + 1)}{line.lstrip(' ')}") + return "\n".join(out) + + +def _convert_callouts(markdown: str) -> str: + """Converts GitHub alert and bold-note blockquotes into MkDocs admonitions.""" + lines = markdown.split("\n") + out: list[str] = [] + i = 0 + while i < len(lines): + line = lines[i] + gfm_match = _GFM_ALERT_HEADER_RE.match(line) + if gfm_match: + kind = gfm_match.group(1).lower() + custom_title = gfm_match.group(2) + header = f'!!! {kind} "{custom_title}"' if custom_title else f"!!! {kind}" + out.append(header) + i += 1 + while i < len(lines): + cont = _BLOCKQUOTE_CONT_RE.match(lines[i]) + if not cont: + break + body = cont.group(1) + out.append(f" {body}" if body else "") + i += 1 + continue + + bold_match = _BOLD_NOTE_HEADER_RE.match(line) + if bold_match: + title = bold_match.group(1).strip().rstrip(":") + kind = bold_match.group(2).lower() + first_body = bold_match.group(3).strip() + out.append(f'!!! {kind} "{title}"') + if first_body: + out.append(f" {first_body}") + i += 1 + while i < len(lines): + cont = _BLOCKQUOTE_CONT_RE.match(lines[i]) + if not cont: + break + body = cont.group(1) + out.append(f" {body}" if body else "") + i += 1 + continue + + out.append(line) + i += 1 + return "\n".join(out) + + +def _normalize_inline_math(markdown: str) -> str: + r"""Converts single-line `$$...$$` inline formulas to `\(...\)`.""" + return _INLINE_MATH_RE.sub(lambda m: f"\\({m.group(1).strip()}\\)", markdown) + + +def on_page_markdown( + markdown: str, + page: Any, + config: Mapping[str, Any], + files: Any, +) -> str: + """Transforms GitHub-flavored TPUMS Markdown pages for MkDocs Material.""" + del files + docs_dir = os.path.abspath(config["docs_dir"]) + repo_root = os.path.dirname(docs_dir) + repo_url = config["repo_url"].rstrip("/") + src_uri = page.file.src_uri + + hide = list(page.meta.get("hide", [])) + if "navigation" not in hide: + hide.append("navigation") + page.meta["hide"] = hide + + masked, fences = _mask_fences(markdown) + masked, inlines = _mask_inline_code(masked) + if src_uri == "index.md": + masked = _MANUAL_TOC_RE.sub("", masked, count=1) + masked = _rewrite_links( + masked, + src_uri=src_uri, + docs_dir=docs_dir, + repo_root=repo_root, + repo_url=repo_url, + ) + masked = _normalize_list_indents(masked) + masked = _convert_callouts(masked) + masked = _normalize_inline_math(masked) + unmasked = _unmask_inline_code(masked, inlines) + return _unmask_fences(unmasked, fences) diff --git a/tests/test_docs_site.py b/tests/test_docs_site.py new file mode 100644 index 00000000..e0d5951d --- /dev/null +++ b/tests/test_docs_site.py @@ -0,0 +1,562 @@ +"""Unit tests for the TPUMS MkDocs documentation site and build hooks.""" + +import dataclasses +import os +import re +import tempfile +from typing import Any + +from absl.testing import absltest +import mkdocs_hooks + +import yaml + + +class _MkDocsYamlLoader(yaml.SafeLoader): + """Safe YAML loader supporting MkDocs python tag references as strings.""" + + +_MkDocsYamlLoader.add_multi_constructor( + "tag:yaml.org,2002:python/", + lambda loader, suffix, node: suffix, +) + + +@dataclasses.dataclass(frozen=True) +class _FakeFile: + """Minimal MkDocs File double.""" + + src_uri: str + abs_src_path: str | None = None + + @classmethod + def generated( + cls, + config: Any, + src_uri: str, + *, + abs_src_path: str, + ) -> "_FakeFile": + """Constructs a virtual _FakeFile backed by abs_src_path.""" + del config + return cls(src_uri=src_uri, abs_src_path=abs_src_path) + + +class _FakeFiles: + """Minimal MkDocs Files collection double for testing on_files.""" + + def __init__(self, initial: list[_FakeFile]): + self._files = list(initial) + + def __iter__(self): + return iter(self._files) + + def __len__(self) -> int: + return len(self._files) + + def get_file_from_path(self, path: str) -> _FakeFile | None: + """Returns the _FakeFile matching path, or None if absent.""" + for f in self._files: + if f.src_uri == path: + return f + return None + + def append(self, file_obj: _FakeFile) -> None: + """Appends file_obj to the collection.""" + self._files.append(file_obj) + + +@dataclasses.dataclass(frozen=True) +class _FakePage: + """Minimal MkDocs Page double.""" + + file: _FakeFile + meta: dict[str, Any] = dataclasses.field(default_factory=dict) + + +def _make_page(src_uri: str, meta: dict[str, Any] | None = None) -> _FakePage: + return _FakePage( + file=_FakeFile(src_uri=src_uri), + meta={} if meta is None else meta, + ) + + +def _public_markdown(text: str) -> str: + """Returns public Markdown content with internal-only blocks removed.""" + return text + + +def _strip_fences(markdown: str) -> str: + """Strips fenced code blocks while preserving headings and prose.""" + return mkdocs_hooks._FENCE_RE.sub("", markdown) # pylint: disable=protected-access + + +def _strip_code(markdown: str) -> str: + """Strips fenced code blocks and inline code spans for link scans.""" + return re.sub(r"(`+)([^`]*?)\1", "", _strip_fences(markdown)) + + +def _github_heading_slug(heading_text: str) -> str: + """Computes a GitHub-compatible heading anchor slug.""" + cleaned = re.sub(r"<[^>]+>", "", heading_text) + cleaned = cleaned.replace("`", "") + cleaned = cleaned.strip().lower() + cleaned = re.sub(r"[^\w\s-]", "", cleaned) + return re.sub(r"\s", "-", cleaned) + + +def _collect_anchors(markdown: str) -> set[str]: + """Collects explicit `` anchors and heading slugs in `markdown`.""" + anchors = set(re.findall(r'', markdown)) + stripped = _strip_fences(markdown) + for match in re.finditer(r"(?m)^#{1,6}\s+(.+?)\s*$", stripped): + anchors.add(_github_heading_slug(match.group(1))) + return anchors + + +class DocsSiteTest(absltest.TestCase): + """Tests Markdown link integrity, MkDocs hooks, and site configuration.""" + + def setUp(self): + super().setUp() + self.repo_root = os.path.abspath( + os.path.join(os.path.dirname(__file__), "..") + ) + self.docs_dir = os.path.join(self.repo_root, "docs") + self.repo_url = ( + "https://github.com/AI-Hypercomputer/accelerator-microbenchmarks" + ) + self.hook_config: dict[str, Any] = { + "docs_dir": self.docs_dir, + "repo_url": self.repo_url, + } + + def _read_public_doc(self, rel_path: str) -> str: + full_path = os.path.join(self.repo_root, rel_path) + with open(full_path, "r", encoding="utf-8") as f: + return _public_markdown(f.read()) + + def test_public_markdown_links_and_anchors_resolve(self): + """Verifies every relative Markdown link and anchor in public docs resolves.""" + doc_paths = ( + "README.md", + "docs/BENCHMARKS.md", + "docs/GKE.md", + "docs/AUTOMATION.md", + ) + doc_contents = {p: self._read_public_doc(p) for p in doc_paths} + doc_anchors = { + os.path.abspath(os.path.join(self.repo_root, p)): _collect_anchors(text) + for p, text in doc_contents.items() + } + + link_re = re.compile(r"(?" + f" {target_abs}", + ) + if sep and target_abs.endswith(".md"): + if target_abs not in doc_anchors: + with open(target_abs, "r", encoding="utf-8") as f: + doc_anchors[target_abs] = _collect_anchors( + _public_markdown(f.read()) + ) + self.assertIn( + fragment, + doc_anchors[target_abs], + f"Missing target anchor #{fragment} for '{raw_target}' in" + f" {rel_path}", + ) + + def test_index_page_populates_from_readme_and_rewrites_links(self): + """Verifies index.md transforms root README.md, strips ToC, and rewrites links.""" + with tempfile.TemporaryDirectory() as tmpdir: + docs_dir = os.path.join(tmpdir, "docs") + os.makedirs(os.path.join(tmpdir, "configs", "tpu7x", "2x2x1")) + os.makedirs(os.path.join(tmpdir, "templates")) + os.makedirs(docs_dir) + with open( + os.path.join(tmpdir, "templates", "gke_job_template.yaml"), + "w", + encoding="utf-8", + ) as f: + f.write("kind: Job\n") + readme_src = self._read_public_doc("README.md") + ( + "\n\nExtra links: [Template](templates/gke_job_template.yaml) and" + " [Dir](configs/tpu7x/2x2x1/) and inline" + " `[CodeSpan](docs/BENCHMARKS.md)`.\n\n```markdown\n[FenceLink](configs/tpu7x/2x2x1/)\n```\n" + ) + + rendered = mkdocs_hooks.on_page_markdown( + readme_src, + page=_make_page("index.md"), + config={"docs_dir": docs_dir, "repo_url": self.repo_url}, + files=None, + ) + + self.assertIn("# TPUMS: TPU Microbenchmark Suite", rendered) + self.assertNotIn("## Table of Contents", rendered) + self.assertIn("(BENCHMARKS.md#21-matrix-multiplication-gemm)", rendered) + self.assertIn("(GKE.md)", rendered) + self.assertIn("(AUTOMATION.md)", rendered) + self.assertIn( + f"({self.repo_url}/blob/main/templates/gke_job_template.yaml)", + rendered, + ) + self.assertIn( + f"({self.repo_url}/tree/main/configs/tpu7x/2x2x1/)", + rendered, + ) + self.assertIn("`[CodeSpan](docs/BENCHMARKS.md)`", rendered) + self.assertIn("[FenceLink](configs/tpu7x/2x2x1/)", rendered) + + def test_subpage_relative_links_rewrite_to_index_and_github(self): + """Verifies docs/*.md links rewrite ../README.md and repo paths properly.""" + benchmarks_md = self._read_public_doc("docs/BENCHMARKS.md") + rendered_bench = mkdocs_hooks.on_page_markdown( + benchmarks_md, + page=_make_page("BENCHMARKS.md"), + config=self.hook_config, + files=None, + ) + self.assertIn("(index.md)", rendered_bench) + self.assertIn("(index.md#primary-metrics)", rendered_bench) + self.assertIn( + f"({self.repo_url}/blob/main/src/accelerator_microbenchmarks/core/dtypes.py)", + rendered_bench, + ) + + gke_md = self._read_public_doc("docs/GKE.md") + rendered_gke = mkdocs_hooks.on_page_markdown( + gke_md, + page=_make_page("GKE.md"), + config=self.hook_config, + files=None, + ) + self.assertIn("(AUTOMATION.md)", rendered_gke) + self.assertIn( + f"({self.repo_url}/tree/main/configs/tpu7x/2x2x1/)", + rendered_gke, + ) + self.assertIn( + f"({self.repo_url}/blob/main/templates/gke_job_template.yaml)", + rendered_gke, + ) + + def test_callouts_convert_to_admonitions_outside_code_fences(self): + """Verifies GitHub alerts and bold note callouts convert to admonitions.""" + sample = ( + "> [!IMPORTANT]\n" + "> First line with `code`.\n" + ">\n" + "> - Bullet line.\n\n" + "> [!WARNING]\n" + "> Watch out.\n\n" + "> **Note on Report Generation:** Automated pipeline runs end-to-end\n" + "> without extra steps.\n\n" + "```markdown\n" + "> [!TIP]\n" + "> Keep inside fence.\n" + "```\n" + ) + rendered = mkdocs_hooks.on_page_markdown( + sample, + page=_make_page("AUTOMATION.md"), + config=self.hook_config, + files=None, + ) + self.assertIn( + "!!! important\n First line with `code`.\n\n - Bullet line.", + rendered, + ) + self.assertIn("!!! warning\n Watch out.", rendered) + self.assertIn( + '!!! note "Note on Report Generation"\n' + " Automated pipeline runs end-to-end\n" + " without extra steps.", + rendered, + ) + self.assertIn("```markdown\n> [!TIP]\n> Keep inside fence.\n```", rendered) + + def test_inline_math_normalizes_without_corrupting_latex_or_code(self): + r"""Verifies inline $$...$$ formulas convert to \(...\) cleanly.""" + sample = ( + "Formula $$C = \\alpha (A \\times B) + \\beta C$$ in prose.\n\n| Op |" + " Formula |\n| :--- | :--- |\n| AR | $$2 \\times S \\times" + " \\frac{P}{R}$$ and $$C_{\\text{out}}$$ |\n\nInline code `$$x_1 +" + " x_2$$` stays untouched.\n\n$$\nE = mc^2\n$$\n\n```tex\n$$a \\times" + " b$$\n```\n" + ) + rendered = mkdocs_hooks.on_page_markdown( + sample, + page=_make_page("BENCHMARKS.md"), + config=self.hook_config, + files=None, + ) + self.assertIn(r"\(C = \alpha (A \times B) + \beta C\)", rendered) + self.assertIn( + r"| AR | \(2 \times S \times \frac{P}{R}\) and \(C_{\text{out}}\) |", + rendered, + ) + self.assertIn("`$$x_1 + x_2$$`", rendered) + self.assertIn("$$\nE = mc^2\n$$", rendered) + self.assertIn("```tex\n$$a \\times b$$\n```", rendered) + + def test_mkdocs_config_and_docs_requirements(self): + """Verifies mkdocs.yml configuration and docs/requirements.txt pins.""" + mkdocs_path = os.path.join(self.repo_root, "mkdocs.yml") + with open(mkdocs_path, "r", encoding="utf-8") as f: + cfg = yaml.load(f, Loader=_MkDocsYamlLoader) + self.assertIsInstance(cfg, dict) + self.assertTrue(cfg.get("strict")) + self.assertEqual(cfg.get("hooks"), ["mkdocs_hooks.py"]) + self.assertIn("requirements.txt", cfg.get("exclude_docs", "")) + extra_js = cfg.get("extra_javascript", []) + self.assertIn("javascripts/mathjax.js", extra_js) + self.assertIn("https://unpkg.com/mathjax@3/es5/tex-mml-chtml.js", extra_js) + self.assertEqual( + cfg.get("nav"), + [ + {"User Guide": "index.md"}, + {"Benchmark Reference": "BENCHMARKS.md"}, + {"Running on GKE": "GKE.md"}, + {"Automation Pipeline": "AUTOMATION.md"}, + ], + ) + extensions = cfg.get("markdown_extensions", []) + ext_names = { + ext if isinstance(ext, str) else next(iter(ext.keys())) + for ext in extensions + } + for required_ext in ( + "admonition", + "pymdownx.details", + "pymdownx.superfences", + "pymdownx.arithmatex", + "pymdownx.highlight", + "pymdownx.inlinehilite", + "pymdownx.tabbed", + "tables", + "attr_list", + "md_in_html", + "toc", + ): + self.assertIn(required_ext, ext_names) + + mathjax_path = os.path.join(self.docs_dir, "javascripts", "mathjax.js") + with open(mathjax_path, "r", encoding="utf-8") as f: + mathjax_js = f.read() + self.assertIn("window.MathJax", mathjax_js) + self.assertIn("arithmatex", mathjax_js) + + req_path = os.path.join(self.docs_dir, "requirements.txt") + self.assertTrue(os.path.isfile(req_path)) + with open(req_path, "r", encoding="utf-8") as f: + docs_deps = [line.strip() for line in f if line.strip()] + self.assertEqual( + docs_deps, + [ + "mkdocs>=1.6,<2.0", + "mkdocs-material>=9.5,<10.0", + "pymdown-extensions>=10.0", + ], + ) + + def test_github_pages_workflow_schema(self): + """Verifies .github/workflows/docs.yml security pinning and Pages build.""" + workflow_path = os.path.join( + self.repo_root, ".github", "workflows", "docs.yml" + ) + self.assertTrue(os.path.isfile(workflow_path)) + with open(workflow_path, "r", encoding="utf-8") as f: + raw_workflow = f.read() + workflow = yaml.safe_load(raw_workflow) + self.assertIsInstance(workflow, dict) + self.assertNotIn(True, workflow) + self.assertIn("on", workflow) + triggers = workflow["on"] + self.assertIn("push", triggers) + self.assertEqual(triggers["push"]["branches"], ["main"]) + self.assertIn("workflow_dispatch", triggers) + + self.assertEqual(workflow.get("permissions"), {"contents": "read"}) + self.assertEqual( + workflow.get("concurrency"), + {"group": "pages", "cancel-in-progress": False}, + ) + + jobs = workflow["jobs"] + build_job = jobs["build"] + build_steps = build_job["steps"] + uses_values: list[str] = [] + for step in build_steps: + if "uses" in step: + uses_values.append(step["uses"]) + + deploy_job = jobs["deploy"] + self.assertNotIn("if", deploy_job) + self.assertEqual(deploy_job.get("needs"), "build") + self.assertEqual( + deploy_job.get("permissions"), + {"pages": "write", "id-token": "write"}, + ) + for step in deploy_job["steps"]: + if "uses" in step: + uses_values.append(step["uses"]) + + self.assertNotEmpty(uses_values) + sha_re = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+@[0-9a-f]{40}$") + for uses_ref in uses_values: + self.assertRegex(uses_ref, sha_re) + self.assertNotIn("configure-pages", uses_ref) + + checkout_steps = [ + s + for s in build_steps + if s.get("uses", "").startswith("actions/checkout@") + ] + self.assertLen(checkout_steps, 1) + self.assertEqual( + checkout_steps[0].get("with", {}).get("persist-credentials"), False + ) + + run_commands = "\n".join(s["run"] for s in build_steps if "run" in s) + self.assertIn("pip install -r docs/requirements.txt", run_commands) + self.assertIn("mkdocs build --strict --site-dir _site", run_commands) + + def test_page_meta_hides_left_navigation_sidebar(self): + """Verifies on_page_markdown hides left navigation without dropping meta.""" + default_page = _make_page("BENCHMARKS.md") + mkdocs_hooks.on_page_markdown( + "# Heading\n", + page=default_page, + config=self.hook_config, + files=None, + ) + self.assertEqual(default_page.meta.get("hide"), ["navigation"]) + + existing_meta_page = _make_page("GKE.md", meta={"hide": ["toc"]}) + mkdocs_hooks.on_page_markdown( + "# Heading\n", + page=existing_meta_page, + config=self.hook_config, + files=None, + ) + self.assertEqual(existing_meta_page.meta.get("hide"), ["toc", "navigation"]) + + mkdocs_hooks.on_page_markdown( + "# Heading\n", + page=existing_meta_page, + config=self.hook_config, + files=None, + ) + self.assertEqual(existing_meta_page.meta.get("hide"), ["toc", "navigation"]) + + def test_nested_two_space_and_four_space_lists_normalize_cleanly(self): + """Verifies [0, 2, 4] and [0, 4, 8] nested lists normalize to 4-space steps.""" + two_space_sample = ( + "- **Collective Bus Bandwidth**:\n" + " - `xprof_bandwidth_per_chip_gb_s` — line one\n" + " wrapped continuation line two.\n" + "- **Discrete Value Sweep:**\n\n" + " ```yaml\n" + " benchmark:\n" + " cases:\n" + " - m: 1024\n" + " ```\n" + ) + rendered_two = mkdocs_hooks.on_page_markdown( + two_space_sample, + page=_make_page("BENCHMARKS.md"), + config=self.hook_config, + files=None, + ) + self.assertIn( + "- **Collective Bus Bandwidth**:\n" + " - `xprof_bandwidth_per_chip_gb_s` — line one\n" + " wrapped continuation line two.", + rendered_two, + ) + self.assertIn( + " ```yaml\n benchmark:\n cases:\n - m: 1024\n ```", + rendered_two, + ) + + four_space_sample = ( + "- **XProf Timing**:\n" + " 1. **Enable XProf**:\n" + " wrapped numbered continuation:\n" + " - **CLI**: pass `--xprof_timing`\n" + " wrapped sub-bullet continuation.\n" + ) + rendered_four = mkdocs_hooks.on_page_markdown( + four_space_sample, + page=_make_page("README.md"), + config=self.hook_config, + files=None, + ) + self.assertEqual( + rendered_four, + "- **XProf Timing**:\n" + " 1. **Enable XProf**:\n" + " wrapped numbered continuation:\n" + " - **CLI**: pass `--xprof_timing`\n" + " wrapped sub-bullet continuation.\n", + ) + + def test_on_files_registers_virtual_index_from_readme(self): + """Verifies on_files registers root README.md as virtual index.md.""" + with tempfile.TemporaryDirectory() as tmpdir: + docs_dir = os.path.join(tmpdir, "docs") + os.makedirs(docs_dir) + readme_path = os.path.join(tmpdir, "README.md") + with open(readme_path, "w", encoding="utf-8") as f: + f.write("# TPUMS\n") + + files = _FakeFiles([_FakeFile(src_uri="BENCHMARKS.md")]) + returned = mkdocs_hooks.on_files( + files, {"docs_dir": docs_dir, "repo_url": self.repo_url} + ) + self.assertIs(returned, files) + self.assertLen(files, 2) + index_file = files.get_file_from_path("index.md") + self.assertIsNotNone(index_file) + self.assertEqual(index_file.abs_src_path, readme_path) + + # Calling on_files again when index.md already exists is idempotent. + mkdocs_hooks.on_files( + files, {"docs_dir": docs_dir, "repo_url": self.repo_url} + ) + self.assertLen(files, 2) + + with tempfile.TemporaryDirectory() as empty_tmp: + empty_docs = os.path.join(empty_tmp, "docs") + os.makedirs(empty_docs) + missing_files = _FakeFiles([_FakeFile(src_uri="BENCHMARKS.md")]) + with self.assertRaises(FileNotFoundError): + mkdocs_hooks.on_files( + missing_files, + {"docs_dir": empty_docs, "repo_url": self.repo_url}, + ) + + +if __name__ == "__main__": + absltest.main()