Site renderer
Turns the executed notebooks into this integrated browser reader.
#!/usr/bin/env python3
"""Render the research and executed notebooks as a compact multi-page site.
Reader and research pages come directly from their Markdown sources. Notebook
pages come from the saved ``.ipynb`` artifacts, including exact Python cells
and outputs. Python files are published separately so long source listings do
not interrupt the reading flow. TeX is preserved and typeset by pinned MathJax
in the browser when network access permits.
"""
import argparse
import ast
import hashlib
import html
import json
import re
from pathlib import Path
from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple
try:
from scripts.python_source_renderer import PYTHON_SOURCE_CSS, render_python_source
except ModuleNotFoundError: # Direct execution from the scripts directory.
from python_source_renderer import PYTHON_SOURCE_CSS, render_python_source
ROOT = Path(__file__).resolve().parents[1]
NOTEBOOKS = ROOT / "notebooks"
READER_CSS = ROOT / "assets" / "reader.css"
SYSTEM_DESIGNS = ROOT / "research" / "system_designs.json"
DEFAULT_NOTEBOOK = NOTEBOOKS / "00_complete_rag_handbook.ipynb"
DEFAULT_OUTPUT = ROOT / "previews" / "index.html"
BUILD_PROVENANCE = ROOT / "previews" / "build_provenance.json"
PUBLIC_SITE_URL = "https://rag.babushkai.com/"
FIELD_PAGE_PATHS = tuple(sorted((ROOT / "research" / "field_notebook").glob("*.md")))
FIELD_ROUTE_SLUGS = (
"prologue",
"search",
"grounding",
"agents-memory-security",
"evaluation-production",
"epilogue",
)
RESEARCH_PAGE_ORDER = (
"README.md",
"field_map.md",
"mathematical_primer.md",
"chronology.md",
"frontier_2024_2026.md",
"corpus_and_indexing.md",
"retrieval_and_ranking.md",
"context_and_generation.md",
"training_and_optimization.md",
"structured_and_multimodal_rag.md",
"agents_memory_and_temporal.md",
"security_privacy_and_governance.md",
"evaluation_and_risks.md",
"production_systems.md",
"decision_guide.md",
"chronological_index.md",
"glossary.md",
"coverage_matrix.md",
)
FOCUSED_NOTEBOOK_ORDER = (
"01_rag_evolution.ipynb",
"04_corpus_chunking_and_indexes.ipynb",
"02_advanced_rag.ipynb",
"05_training_query_fusion_and_reranking.ipynb",
"06_structured_multimodal_and_graph_rag.ipynb",
"07_agents_memory_temporal_and_security.ipynb",
"03_evaluation_and_failure_analysis.ipynb",
"08_production_evaluation_and_cost.ipynb",
)
REFERENCE_MODULE_ORDER = (
"models.py",
"text.py",
"demo_data.py",
"ingestion.py",
"chunking.py",
"indexes.py",
"retrievers.py",
"rerankers.py",
"selection.py",
"context.py",
"generation.py",
"pipeline.py",
"structured.py",
"agentic.py",
"memory.py",
"temporal.py",
"security.py",
"training.py",
"evaluation.py",
"operations.py",
"__init__.py",
)
SCRIPT_ARTIFACTS = (
(
"Chronology builder",
"scripts/build_chronological_index.py",
"Sorts the primary-source registry into the dated evidence index.",
"python3 scripts/build_chronological_index.py",
),
(
"Notebook builder",
"scripts/build_curriculum_notebooks.py",
"Binds narrative, evidence, experiments, and source notes into one Jupyter manuscript.",
"python3 scripts/build_curriculum_notebooks.py",
),
(
"Notebook executor",
"scripts/execute_notebooks.py",
"Runs every Python cell and writes deterministic outputs into the notebooks.",
"PYTHONPATH=src python3 scripts/execute_notebooks.py --write",
),
(
"Site renderer",
"scripts/render_field_notebook_preview.py",
"Turns the executed notebooks into this integrated browser reader.",
"python3 scripts/render_field_notebook_preview.py",
),
(
"Python highlighter",
"scripts/python_source_renderer.py",
"Tokenizes Python with the standard library and emits safe, colored source HTML.",
"python3 -m unittest tests.test_python_source_renderer -v",
),
(
"Preview server",
"scripts/serve_notebook_site.py",
"Rebuilds, executes, renders, and serves the complete local artifact.",
"make serve",
),
(
"Research validator",
"scripts/validate_research.py",
"Checks sources, prose, execution, links, rendering, and generated artifacts.",
"make validate",
),
)
PRESERVED_HTML_TAGS = {
"a",
"abbr",
"article",
"aside",
"b",
"blockquote",
"br",
"circle",
"code",
"defs",
"desc",
"details",
"div",
"em",
"figcaption",
"figure",
"footer",
"foreignobject",
"g",
"h1",
"h2",
"h3",
"h4",
"h5",
"h6",
"header",
"hr",
"img",
"li",
"line",
"marker",
"nav",
"ol",
"p",
"path",
"polygon",
"polyline",
"pre",
"rect",
"section",
"small",
"span",
"strong",
"sub",
"summary",
"sup",
"svg",
"table",
"tbody",
"td",
"text",
"th",
"thead",
"title",
"tr",
"ul",
}
def cell_text(cell: Mapping[str, Any]) -> str:
source = cell.get("source", "")
return "".join(source) if isinstance(source, list) else str(source)
def sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def slugify(value: str) -> str:
slug = re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-")
return slug or "section"
def _math_markup(tex: str, display: bool = False, delimiter: str = "") -> str:
"""Wrap TeX for MathJax while preserving an auditable source expression."""
expression = tex.strip()
tag = "div" if display else "span"
kind = "equation-note math-display" if display else "math-inline"
opening, closing = (r"\[", r"\]") if display else (r"\(", r"\)")
source_delimiter = delimiter or opening
return (
f'<{tag} class="{kind}" data-math-delimiter="{html.escape(source_delimiter, quote=True)}" '
f'data-tex="{html.escape(expression, quote=True)}">'
f"{opening}{html.escape(expression)}{closing}</{tag}>"
)
def inline_markdown(value: str) -> str:
"""Render inline Markdown while protecting code, HTML, and TeX boundaries."""
protected: List[str] = []
def protect(rendered: str) -> str:
protected.append(rendered)
return f"@@PROTECTED{len(protected) - 1}@@"
value = re.sub(
r"`([^`]+)`",
lambda match: protect(f"<code>{html.escape(match.group(1))}</code>"),
value,
)
def preserve_tag(match: re.Match[str]) -> str:
name = re.match(r"</?\s*([a-zA-Z][\w-]*)", match.group(0))
if not name or name.group(1).lower() not in PRESERVED_HTML_TAGS:
return match.group(0)
return protect(match.group(0))
value = re.sub(r"<[^>]+>", preserve_tag, value)
value = re.sub(
r"\\\((.+?)\\\)",
lambda match: protect(_math_markup(match.group(1), delimiter=r"\(")),
value,
)
value = re.sub(
r"(?<![\\$])\$(?!\$)([^\s$](?:[^$\n]*?[^\s$])?)\$(?!\$)",
lambda match: protect(_math_markup(match.group(1), delimiter="$")),
value,
)
value = html.escape(value, quote=False)
value = re.sub(
r"!\[([^\]]*)\]\((https?://[^)]+|(?:\.\.?/)?[^)\s]+)\)",
r'<img src="\2" alt="\1">',
value,
)
value = re.sub(
r"\[([^\]]+)\]\((https?://[^)]+|mailto:[^)]+|#[^)]+|(?:\.\.?/)?[^)\s]+)\)",
r'<a href="\2">\1</a>',
value,
)
value = re.sub(r"\*\*([^*]+)\*\*", r"<strong>\1</strong>", value)
value = re.sub(r"(?<!\*)\*([^*]+)\*(?!\*)", r"<em>\1</em>", value)
for index, rendered in enumerate(protected):
value = value.replace(f"@@PROTECTED{index}@@", rendered)
return value
def _table_cells(line: str) -> List[str]:
return [cell.strip() for cell in line.strip().strip("|").split("|")]
def _render_table(rows: Sequence[str]) -> str:
cells = [_table_cells(row) for row in rows]
has_header = len(cells) > 1 and all(
re.fullmatch(r":?-{3,}:?", value.replace(" ", "")) is not None
for value in cells[1]
)
rendered: List[str] = ["<table>"]
body_start = 0
if has_header:
rendered.append("<thead><tr>")
rendered.extend(f"<th>{inline_markdown(value)}</th>" for value in cells[0])
rendered.append("</tr></thead>")
body_start = 2
rendered.append("<tbody>")
for row in cells[body_start:]:
rendered.append("<tr>")
rendered.extend(f"<td>{inline_markdown(value)}</td>" for value in row)
rendered.append("</tr>")
rendered.append("</tbody></table>")
return "".join(rendered)
def markdownish(value: str, id_prefix: str = "") -> str:
"""Convert the repository's block Markdown without external packages."""
rendered: List[str] = []
paragraph: List[str] = []
table_rows: List[str] = []
fence_lines: List[str] = []
equation_lines: List[str] = []
fence_marker = ""
in_equation = False
equation_end = ""
list_kind = ""
list_items: List[str] = []
html_block_lines: List[str] = []
html_block_tag = ""
html_block_depth = 0
def flush_paragraph() -> None:
if paragraph:
text = " ".join(part.strip() for part in paragraph)
rendered.append(f"<p>{inline_markdown(text)}</p>")
paragraph.clear()
def close_list() -> None:
nonlocal list_kind
if list_kind:
items = "".join(
f"<li>{inline_markdown(item)}</li>" for item in list_items
)
rendered.append(f"<{list_kind}>{items}</{list_kind}>")
list_kind = ""
list_items.clear()
def flush_table() -> None:
if table_rows:
rendered.append(_render_table(table_rows))
table_rows.clear()
def html_tag_delta(line: str, tag: str) -> int:
openings = len(re.findall(fr"<{tag}(?:\s|>)", line, flags=re.IGNORECASE))
closings = len(re.findall(fr"</{tag}\s*>", line, flags=re.IGNORECASE))
self_closing = len(
re.findall(fr"<{tag}(?:\s[^>]*)?/\s*>", line, flags=re.IGNORECASE)
)
return openings - closings - self_closing
for line in value.splitlines():
stripped = line.strip()
if html_block_tag:
html_block_lines.append(line)
html_block_depth += html_tag_delta(line, html_block_tag)
if html_block_depth <= 0:
rendered.append("\n".join(html_block_lines))
html_block_lines.clear()
html_block_tag = ""
html_block_depth = 0
continue
if fence_marker:
if stripped.startswith(fence_marker):
rendered.append(
'<pre class="markdown-code"><code>'
+ html.escape("\n".join(fence_lines))
+ "</code></pre>"
)
fence_lines.clear()
fence_marker = ""
else:
fence_lines.append(line)
continue
if stripped.startswith(("```", "~~~")):
flush_paragraph()
flush_table()
close_list()
fence_marker = stripped[:3]
continue
if in_equation:
if stripped == equation_end:
rendered.append(
_math_markup(
"\n".join(equation_lines),
display=True,
delimiter=r"\[" if equation_end == r"\]" else "$$",
)
)
equation_lines.clear()
in_equation = False
equation_end = ""
else:
equation_lines.append(line)
continue
if stripped in {r"\[", "$$"}:
flush_paragraph()
flush_table()
close_list()
in_equation = True
equation_end = r"\]" if stripped == r"\[" else "$$"
continue
if (
stripped.startswith(r"\[")
and stripped.endswith(r"\]")
and len(stripped) > 4
):
flush_paragraph()
flush_table()
close_list()
rendered.append(_math_markup(stripped[2:-2], display=True, delimiter=r"\["))
continue
if stripped.startswith("$$") and stripped.endswith("$$") and len(stripped) > 4:
flush_paragraph()
flush_table()
close_list()
rendered.append(_math_markup(stripped[2:-2], display=True, delimiter="$$"))
continue
if stripped.startswith("|") and stripped.endswith("|"):
flush_paragraph()
close_list()
table_rows.append(stripped)
continue
flush_table()
if not stripped:
flush_paragraph()
close_list()
continue
if stripped in {"---", "***", "___"}:
flush_paragraph()
close_list()
rendered.append("<hr>")
continue
heading = re.match(r"^(#{1,6})\s+(.+)$", stripped)
if heading:
flush_paragraph()
close_list()
level = len(heading.group(1))
title = heading.group(2)
plain_title = re.sub(r"<[^>]+>", "", title)
heading_id = slugify(
f"{id_prefix}-{plain_title}" if id_prefix else plain_title
)
rendered.append(
f'<h{level} id="{heading_id}">'
f"{inline_markdown(title)}</h{level}>"
)
continue
html_block = re.match(
r"^<(div|aside|svg|section|article|figure|table|details|nav|header|footer)\b",
stripped,
flags=re.IGNORECASE,
)
if html_block:
flush_paragraph()
close_list()
tag = html_block.group(1).lower()
depth = html_tag_delta(line, tag)
if depth > 0:
html_block_tag = tag
html_block_depth = depth
html_block_lines.append(line)
else:
rendered.append(line)
continue
if stripped.startswith("<"):
flush_paragraph()
close_list()
rendered.append(line)
continue
unordered = re.match(r"^[-*]\s+(.+)$", stripped)
ordered = re.match(r"^\d+\.\s+(.+)$", stripped)
if list_kind and line != line.lstrip() and not (unordered or ordered):
list_items[-1] = f"{list_items[-1]} {stripped}"
continue
if unordered or ordered:
flush_paragraph()
kind = "ul" if unordered else "ol"
if list_kind != kind:
close_list()
list_kind = kind
match = unordered or ordered
list_items.append(match.group(1))
continue
if stripped.startswith(">"):
flush_paragraph()
close_list()
rendered.append(f"<blockquote>{inline_markdown(stripped[1:].strip())}</blockquote>")
continue
paragraph.append(line)
flush_paragraph()
flush_table()
close_list()
if fence_marker:
rendered.append(
'<pre class="markdown-code"><code>'
+ html.escape("\n".join(fence_lines))
+ "</code></pre>"
)
if html_block_lines:
rendered.append("\n".join(html_block_lines))
if in_equation:
raise RuntimeError(f"unclosed display-math delimiter: expected {equation_end}")
return "\n".join(rendered)
def text_outputs(cell: Mapping[str, Any]) -> Iterable[Tuple[str, str]]:
for output in cell.get("outputs", []):
output_type = output.get("output_type", "output")
if output_type == "stream":
text = output.get("text", "")
yield "Saved output", "".join(text) if isinstance(text, list) else str(text)
elif "text/plain" in output.get("data", {}):
text = output["data"]["text/plain"]
yield "Saved result", "".join(text) if isinstance(text, list) else str(text)
elif output_type == "error":
yield "Execution error", f"{output.get('ename', 'Error')}: {output.get('evalue', '')}"
def preserve_preformatted_whitespace(value: str) -> str:
"""Encode end-of-line whitespace so HTML and Git preserve it exactly."""
def encode(match: re.Match[str]) -> str:
return "".join("	" if character == "\t" else " " for character in match.group())
return re.sub(r"[ \t]+(?=\r?\n|$)", encode, value)
def render_cell(
cell: Mapping[str, Any],
cell_index: int,
notebook_name: str,
raw_markdown: bool = False,
anchor: str = "",
aliases: Sequence[str] = (),
) -> str:
cell_type = cell.get("cell_type")
source = cell_text(cell)
identity = f"{slugify(notebook_name)}-cell-{cell_index}"
anchor_value = anchor or identity
alias_markup = "".join(
f'<span id="{html.escape(alias)}" class="anchor-alias" aria-hidden="true"></span>'
for alias in aliases
)
if cell_type == "markdown":
body = (
source
if raw_markdown
else _rewrite_publication_links(markdownish(source, identity))
)
return (
f'<section id="{anchor_value}" class="jp-Cell notebook-markdown" '
f'data-notebook-cell="{html.escape(notebook_name)}:{cell_index}" tabindex="-1">'
f'{alias_markup}<div class="jp-RenderedHTMLCommon">{body}</div></section>'
)
if cell_type == "code":
execution_count = cell.get("execution_count")
output = "".join(
'<div class="jp-OutputArea output-slip">'
f'<div class="output-label">{html.escape(label)}</div>'
f'<pre>{preserve_preformatted_whitespace(html.escape(text))}</pre></div>'
for label, text in text_outputs(cell)
)
return (
f'<section id="{anchor_value}" class="jp-Cell jp-CodeCell notebook-code workbench-note" '
f'data-notebook-cell="{html.escape(notebook_name)}:{cell_index}" '
f'data-code-cell="true" data-execution-count="{execution_count}" tabindex="-1">'
f'{alias_markup}'
'<div class="code-cell-head">'
'<span>Python</span>'
f'<span class="execution-badge">cell {cell_index} · executed</span>'
f'<button type="button" class="copy-code" data-copy-target="{identity}-source" '
f'aria-label="Copy Python cell {cell_index}">Copy</button>'
'</div>'
f'<div class="jp-InputArea"><pre tabindex="0"><code id="{identity}-source" '
f'class="python-source" data-highlighted-python="true">'
f'{preserve_preformatted_whitespace(render_python_source(source, compact=True))}'
f'</code></pre></div>{output}'
"</section>"
)
return ""
def _load_notebook(path: Path) -> Dict[str, Any]:
payload = json.loads(path.read_text(encoding="utf-8"))
for index, cell in enumerate(payload.get("cells", []), start=1):
if cell.get("cell_type") == "code" and cell.get("execution_count") is None:
raise RuntimeError(
f"{path.name}: code cell {index} is unexecuted; run "
"scripts/execute_notebooks.py --write before rendering"
)
if any(output.get("output_type") == "error" for output in cell.get("outputs", [])):
raise RuntimeError(f"{path.name}: code cell {index} contains an error output")
return payload
def _cover_parts(payload: Mapping[str, Any]) -> Tuple[str, str]:
source = cell_text(payload["cells"][0])
match = re.search(r"<style>\s*(.*?)\s*</style>", source, flags=re.DOTALL)
css = match.group(1) if match else ""
cover = re.sub(r"<style>.*?</style>\s*", "", source, count=1, flags=re.DOTALL)
return css, cover
def _notebook_title(payload: Mapping[str, Any], fallback: str) -> str:
source = cell_text(payload["cells"][0])
match = re.search(r"<h1>(.*?)</h1>", source, flags=re.DOTALL)
if not match:
return fallback
return html.unescape(re.sub(r"<[^>]+>", "", match.group(1))).strip()
def _notebook_stats(notebooks: Mapping[Path, Mapping[str, Any]]) -> Dict[str, int]:
cells = [cell for payload in notebooks.values() for cell in payload.get("cells", [])]
code = [cell for cell in cells if cell.get("cell_type") == "code"]
return {
"notebooks": len(notebooks),
"cells": len(cells),
"code_cells": len(code),
"executed_cells": sum(cell.get("execution_count") is not None for cell in code),
"output_cells": sum(bool(cell.get("outputs")) for cell in code),
}
def _source_slug(path: str) -> str:
source_path = Path(path)
if source_path.parts[:2] == ("src", "rag_evolution"):
stem = "package" if source_path.stem == "__init__" else source_path.stem
return f"rag-evolution-{stem.replace('_', '-')}"
return source_path.stem.replace("_", "-")
def _public_python_artifacts() -> Tuple[Tuple[str, str, str, str], ...]:
artifacts = list(SCRIPT_ARTIFACTS)
for name in REFERENCE_MODULE_ORDER:
path = f"src/rag_evolution/{name}"
source = (ROOT / path).read_text(encoding="utf-8")
try:
description = (ast.get_docstring(ast.parse(source)) or "").splitlines()[0]
except (SyntaxError, IndexError):
description = "RAG reference implementation module."
title = "Package index" if name == "__init__.py" else Path(name).stem.replace("_", " ").title()
artifacts.append((title, path, description, ""))
return tuple(artifacts)
def _system_design_families() -> Tuple[Mapping[str, Any], ...]:
"""Load and validate the compact, data-driven system-design catalog."""
payload = json.loads(SYSTEM_DESIGNS.read_text(encoding="utf-8"))
if payload.get("schema_version") != 1:
raise RuntimeError("unsupported system-design schema")
families = payload.get("families")
if set(payload) != {"schema_version", "families"}:
raise RuntimeError("system-design catalog has unreviewed fields")
if not isinstance(families, list) or len(families) != 12:
raise RuntimeError("system-design catalog must have exactly twelve families")
seen_slugs = set()
validated: List[Mapping[str, Any]] = []
for family in families:
if not isinstance(family, dict):
raise RuntimeError("system-design family is not an object")
if set(family) != {
"slug",
"title",
"category",
"summary",
"systems",
"lanes",
"reading",
}:
raise RuntimeError("system-design family has unreviewed fields")
slug = family.get("slug")
if not isinstance(slug, str) or slugify(slug) != slug:
raise RuntimeError(f"invalid system-design slug: {slug!r}")
if slug in seen_slugs:
raise RuntimeError(f"duplicate system-design slug: {slug}")
seen_slugs.add(slug)
for field in ("title", "category", "summary"):
if not isinstance(family.get(field), str) or not family[field].strip():
raise RuntimeError(f"{slug} has no {field}")
if "<" in family[field] or ">" in family[field]:
raise RuntimeError(f"{slug} contains raw markup in {field}")
systems = family.get("systems")
lanes = family.get("lanes")
reading = family.get("reading")
if not isinstance(systems, list) or not systems or not all(
isinstance(name, str)
and name.strip()
and "<" not in name
and ">" not in name
for name in systems
):
raise RuntimeError(f"{slug} has an invalid system inventory")
if len(systems) != len(set(systems)):
raise RuntimeError(f"{slug} repeats a system name")
if not isinstance(lanes, list) or not 2 <= len(lanes) <= 5:
raise RuntimeError(f"{slug} must have two to five diagram lanes")
assigned_systems: List[str] = []
for lane in lanes:
if not isinstance(lane, dict) or not str(lane.get("label", "")).strip():
raise RuntimeError(f"{slug} has an unlabeled diagram lane")
if set(lane) not in (
{"label", "steps", "systems"},
{"label", "steps", "systems", "feedback"},
):
raise RuntimeError(f"{slug} has an unreviewed diagram-lane field")
if "<" in lane["label"] or ">" in lane["label"]:
raise RuntimeError(f"{slug} contains raw markup in a lane label")
steps = lane.get("steps")
if not isinstance(steps, list) or not 3 <= len(steps) <= 6:
raise RuntimeError(f"{slug}/{lane.get('label')} has an invalid flow")
for step in steps:
if (
not isinstance(step, dict)
or set(step) != {"role", "label"}
or not all(
isinstance(step.get(field), str) and step[field].strip()
for field in ("role", "label")
)
or any("<" in step[field] or ">" in step[field] for field in ("role", "label"))
):
raise RuntimeError(f"{slug}/{lane.get('label')} has an invalid step")
feedback = lane.get("feedback", "")
if not isinstance(feedback, str) or "<" in feedback or ">" in feedback:
raise RuntimeError(f"{slug}/{lane.get('label')} has invalid feedback")
lane_systems = lane.get("systems")
if not isinstance(lane_systems, list) or not lane_systems or not all(
isinstance(name, str)
and name.strip()
and "<" not in name
and ">" not in name
for name in lane_systems
):
raise RuntimeError(f"{slug}/{lane.get('label')} has invalid systems")
if len(lane_systems) != len(set(lane_systems)):
raise RuntimeError(f"{slug}/{lane.get('label')} repeats a system")
assigned_systems.extend(lane_systems)
if len(assigned_systems) != len(systems) or set(assigned_systems) != set(systems):
raise RuntimeError(f"{slug} does not map every system to exactly one lane")
if not isinstance(reading, list) or not 1 <= len(reading) <= 3:
raise RuntimeError(f"{slug} has an invalid reading list")
for reference in reading:
if (
not isinstance(reference, dict)
or set(reference) != {"label", "href"}
or not isinstance(reference.get("label"), str)
or not reference["label"].strip()
or not isinstance(reference.get("href"), str)
or not reference["href"].startswith("/")
or reference["href"].startswith("//")
or not reference["href"].startswith(("/read/", "/research/"))
or any(character in reference["label"] for character in "<>")
):
raise RuntimeError(f"{slug} has an invalid reading reference")
validated.append(family)
return tuple(validated)
def _site_navigation(current: str = "") -> str:
def link(label: str, href: str, key: str) -> str:
current_value = ' aria-current="page"' if current == key else ""
return (
f'<a href="{html.escape(href)}"{current_value}>'
f"{html.escape(label)}</a>"
)
return f"""
<header class="site-nav">
<div class="site-nav-inner">
<a class="site-brand" href="/">The Evidence Path</a>
<nav class="site-links" aria-label="Publication sections">
{link("Reader", "/", "reader")}
{link("Systems", "/systems", "systems")}
{link("Research", "/research", "research")}
{link("Notebooks", "/notebooks", "notebooks")}
{link("Python", "/python", "python")}
{link("PDF", "/book", "pdf")}
</nav>
</div>
</header>
"""
def _source_page_document(path: str, title: str, description: str) -> str:
source_path = ROOT / path
source = source_path.read_text(encoding="utf-8")
digest = sha256(source_path)
slug = _source_slug(path)
rendered = preserve_preformatted_whitespace(render_python_source(source, compact=True))
document = f"""<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="description" content="Rendered Python source for {html.escape(title)}.">
<link rel="canonical" href="{PUBLIC_SITE_URL}python/{slug}">
<link rel="stylesheet" href="/assets/field_notebook.css">
<link rel="stylesheet" href="/assets/reader.css">
<title>{html.escape(title)} — The Evidence Path</title>
<style>{PYTHON_SOURCE_CSS}</style>
</head>
<body>
<a class="skip-link" href="#source">Skip to source</a>
{_site_navigation("python")}
<main id="source" class="source-document" tabindex="-1">
<header class="source-header">
<h1>{html.escape(title)}</h1>
<p class="source-description">{html.escape(description)}</p>
<p class="source-meta">{html.escape(path)} · {len(source.splitlines()):,} lines ·
sha256 {digest[:12]}…</p>
<p class="source-links"><a href="/{html.escape(path)}" download>Download raw .py</a>
· <a href="/python">All Python files</a></p>
</header>
<pre class="source-code" tabindex="0"><code class="python-source"
data-highlighted-python="true" data-source-path="{html.escape(path)}"
data-source-sha256="{digest}">{rendered}</code></pre>
</main>
<footer class="site-footer">The Evidence Path · exact source snapshot</footer>
</body>
</html>
"""
return "\n".join(line.rstrip() for line in document.splitlines()) + "\n"
def _source_index_document() -> str:
groups: Dict[str, List[str]] = {"Reference implementation": [], "Tooling": []}
for title, path, _, _ in _public_python_artifacts():
source_path = ROOT / path
line_count = len(source_path.read_text(encoding="utf-8").splitlines())
item = (
"<li>"
f'<a href="/python/{_source_slug(path)}">{html.escape(title)}</a>'
f"<small>{html.escape(path)} · {line_count:,} lines</small>"
"</li>"
)
group = "Reference implementation" if path.startswith("src/") else "Tooling"
groups[group].append(item)
sections = "".join(
f'<section><h2>{html.escape(label)}</h2><ul class="source-list">'
f"{''.join(groups[label])}</ul></section>"
for label in ("Reference implementation", "Tooling")
)
document = f"""<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="description" content="Python source used to build and verify The Evidence Path.">
<link rel="canonical" href="{PUBLIC_SITE_URL}python">
<link rel="stylesheet" href="/assets/field_notebook.css">
<link rel="stylesheet" href="/assets/reader.css">
<title>Python source — The Evidence Path</title>
</head>
<body>
{_site_navigation("python")}
<main class="source-index">
<h1>Python source</h1>
<p>The reference implementation and the tooling used to build and verify this publication.</p>
{sections}
</main>
<footer class="site-footer">The Evidence Path · {len(_public_python_artifacts())} verified source files</footer>
</body>
</html>
"""
return "\n".join(line.rstrip() for line in document.splitlines()) + "\n"
def _write_source_pages(output_directory: Path) -> Dict[str, str]:
source_directory = output_directory / "source"
source_directory.mkdir(parents=True, exist_ok=True)
pages: Dict[str, str] = {}
for title, path, description, _ in _public_python_artifacts():
name = f"{_source_slug(path)}.html"
page = source_directory / name
page.write_text(_source_page_document(path, title, description), encoding="utf-8")
pages[path] = page.relative_to(output_directory).as_posix()
(source_directory / "index.html").write_text(
_source_index_document(), encoding="utf-8"
)
return pages
def _site_css() -> str:
"""Return the external reader stylesheet for tests and source audits."""
return READER_CSS.read_text(encoding="utf-8")
def _site_script() -> str:
"""Add notebook copying and stable deep-link behavior."""
return r"""
document.addEventListener('DOMContentLoaded', () => {
const status = window.__mathStatus || {
label: 'Typesetting mathematics…', state: 'loading'
};
window.__setMathStatus(status.label, status.state);
});
const copyText = async (value) => {
if (navigator.clipboard && window.isSecureContext) {
await navigator.clipboard.writeText(value);
return;
}
const field = document.createElement('textarea');
field.value = value;
field.style.position = 'fixed';
field.style.opacity = '0';
document.body.appendChild(field);
field.select();
document.execCommand('copy');
field.remove();
};
document.querySelectorAll('.copy-code').forEach((button) => {
button.addEventListener('click', async () => {
const code = document.getElementById(button.dataset.copyTarget);
if (!code) return;
const original = button.textContent;
const status = document.getElementById('copy-status');
try {
await copyText(code.textContent);
button.textContent = 'Copied';
if (status) status.textContent = 'Python source copied.';
} catch (error) {
button.textContent = 'Copy failed';
if (status) status.textContent = 'Could not copy Python source.';
}
setTimeout(() => { button.textContent = original; }, 1400);
});
});
const revealHashTarget = () => {
if (!location.hash) return;
let identifier = '';
try { identifier = decodeURIComponent(location.hash.slice(1)); }
catch (error) { return; }
const target = document.getElementById(identifier);
if (!target) return;
requestAnimationFrame(() => target.scrollIntoView({block: 'start'}));
};
addEventListener('hashchange', revealHashTarget);
addEventListener('mathready', revealHashTarget);
revealHashTarget();
"""
def _page_slug(path: Path) -> str:
stem = re.sub(r"^\d+_", "", path.stem)
if path.name == "README.md":
return "method-and-scope"
return stem.replace("_", "-")
def _extract_document_title(value: str, fallback: str) -> Tuple[str, str]:
value = re.sub(r"\A\s*<!--.*?-->\s*", "", value, count=1, flags=re.DOTALL)
match = re.search(r"^#{1,2}\s+(.+?)\s*$", value, flags=re.MULTILINE)
if not match:
return fallback, value.strip()
title = re.sub(r"[*_`]", "", match.group(1)).strip() or fallback
body = (value[: match.start()] + value[match.end() :]).strip()
return title, body
def _word_count(value: str) -> int:
return len(re.findall(r"\b[\w'-]+\b", value, flags=re.UNICODE))
def _rewrite_publication_links(rendered: str) -> str:
"""Point source-relative references at their compact public equivalents."""
research_routes = {
name: f"/research/{_page_slug(Path(name))}" for name in RESEARCH_PAGE_ORDER
}
notebook_routes = {
name: f"/notebooks/view/{_page_slug(Path(name))}"
for name in FOCUSED_NOTEBOOK_ORDER
}
def research_link(match: re.Match[str]) -> str:
name = match.group("name")
fragment = match.group("fragment") or ""
return f'href="{research_routes.get(name, name)}{fragment}"'
rendered = re.sub(
r'href="(?:\.\./research/)?(?P<name>[A-Za-z0-9_]+\.md)'
r'(?P<fragment>#[^"]+)?"',
research_link,
rendered,
)
for name, route in notebook_routes.items():
rendered = rendered.replace(
f'href="../notebooks/{name}"', f'href="{route}"'
)
return (
rendered.replace('href="field_notebook"', 'href="/read/prologue"')
.replace('href="../research/field_notebook"', 'href="/read/prologue"')
.replace('href="sources.json"', 'href="/research/sources.json"')
.replace('href="../research/sources.json"', 'href="/research/sources.json"')
.replace(
'href="../notebooks/00_complete_rag_handbook.ipynb"',
'href="/notebooks/00_complete_rag_handbook.ipynb"',
)
)
def _math_head() -> str:
return r"""
<script>
window.__mathStatus = {label: 'Typesetting mathematics…', state: 'loading'};
window.__setMathStatus = (label, state) => {
window.__mathStatus = {label, state};
const status = document.getElementById('math-render-status');
if (status) { status.textContent = label; status.dataset.state = state; }
};
window.MathJax = {
tex: {
inlineMath: {'[+]': [['$', '$']]},
processEscapes: true,
processEnvironments: true,
tags: 'none'
},
options: {
skipHtmlTags: [
'script', 'noscript', 'style', 'textarea', 'pre', 'code',
'math', 'select', 'option', 'mjx-container'
],
ignoreHtmlClass: 'mathjax_ignore',
processHtmlClass: 'mathjax_process'
},
output: {
displayOverflow: 'linebreak',
linebreaks: {inline: true, width: '100%', lineleading: 0.2}
},
startup: {
pageReady: () => MathJax.startup.defaultPageReady().then(() => {
document.documentElement.dataset.mathReady = 'true';
window.__setMathStatus('Mathematics rendered', 'ready');
window.dispatchEvent(new Event('mathready'));
}).catch((error) => {
window.__setMathStatus('Math unavailable — TeX preserved', 'error');
throw error;
})
}
};
</script>
<script id="mathjax-runtime" defer
src="https://cdn.jsdelivr.net/npm/mathjax@4.1.3/tex-chtml.js"
onerror="window.__setMathStatus('Math unavailable — TeX preserved', 'error')"></script>
"""
def _page_head(
title: str,
description: str,
canonical_path: str,
*,
math: bool = False,
python: bool = False,
source_sha256: str = "",
) -> str:
digest_meta = (
f' <meta name="source-sha256" content="{html.escape(source_sha256)}">\n'
if source_sha256
else ""
)
python_css = f" <style>{PYTHON_SOURCE_CSS}</style>\n" if python else ""
math_runtime = _math_head() if math else ""
canonical = PUBLIC_SITE_URL.rstrip("/") + canonical_path
return f"""<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
{digest_meta} <meta name="description" content="{html.escape(description, quote=True)}">
<link rel="canonical" href="{html.escape(canonical, quote=True)}">
<meta property="og:type" content="article">
<meta property="og:title" content="{html.escape(title, quote=True)}">
<meta property="og:description" content="{html.escape(description, quote=True)}">
<meta property="og:url" content="{html.escape(canonical, quote=True)}">
<link rel="stylesheet" href="/assets/field_notebook.css">
<link rel="stylesheet" href="/assets/reader.css">
<title>{html.escape(title)} — The Evidence Path</title>
{python_css}{math_runtime}</head>"""
def _article_navigation(
previous: Optional[Tuple[str, str]],
following: Optional[Tuple[str, str]],
) -> str:
previous_link = (
f'<a rel="prev" href="{html.escape(previous[1])}">← {html.escape(previous[0])}</a>'
if previous
else "<span></span>"
)
following_link = (
f'<a rel="next" href="{html.escape(following[1])}">{html.escape(following[0])} →</a>'
if following
else "<span></span>"
)
return f'<nav class="article-navigation" aria-label="Reading order">{previous_link}{following_link}</nav>'
def _article_document(
*,
title: str,
source: str,
source_path: Path,
route: str,
current: str,
position: int,
total: int,
previous: Optional[Tuple[str, str]],
following: Optional[Tuple[str, str]],
) -> str:
description = f"{title}, part of The Evidence Path."
digest = sha256(source_path)
body = _rewrite_publication_links(markdownish(source))
document = f"""<!doctype html>
<html lang="en">
{_page_head(title, description, route, math=True, source_sha256=digest)}
<body>
<a class="skip-link" href="#article">Skip to article</a>
{_site_navigation(current)}
<main id="article" class="article-page" tabindex="-1">
<article>
<header class="article-header">
<h1>{html.escape(title)}</h1>
<p class="article-meta">{position} of {total} · {_word_count(source):,} words</p>
</header>
<div class="jp-RenderedHTMLCommon article-body">{body}</div>
</article>
{_article_navigation(previous, following)}
<span id="math-render-status" class="visually-hidden" data-state="loading"
aria-live="polite">Typesetting mathematics…</span>
</main>
<footer class="site-footer">The Evidence Path · evidence cutoff 9 August 2026</footer>
<script>{_site_script()}</script>
</body>
</html>
"""
return "\n".join(line.rstrip() for line in document.splitlines()) + "\n"
def _system_diagram_markup(family: Mapping[str, Any]) -> str:
slug = str(family["slug"])
lanes: List[str] = []
for index, lane in enumerate(family["lanes"], start=1):
lane_label = str(lane["label"])
lane_id = f"{slug}-lane-{index}"
steps = "".join(
"<li>"
f'<span class="system-step-role">{html.escape(str(step["role"]))}</span>'
f'<strong>{html.escape(str(step["label"]))}</strong>'
"</li>"
for step in lane["steps"]
)
feedback = str(lane.get("feedback", "")).strip()
feedback_markup = (
f'<p class="system-feedback"><span aria-hidden="true">↺</span> '
f"{html.escape(feedback)}</p>"
if feedback
else ""
)
lane_systems = "".join(
f"<li>{html.escape(str(name))}</li>" for name in lane["systems"]
)
lanes.append(
f'<section class="system-lane" aria-labelledby="{lane_id}" '
f'data-system-variant="{html.escape(lane_label, quote=True)}">'
f'<h2 id="{lane_id}">{html.escape(lane_label)}</h2>'
f'<div><ol class="system-flow" aria-label="{html.escape(lane_label, quote=True)} flow" '
f'style="--system-step-count: {len(lane["steps"])}">{steps}</ol>'
f'<ul class="system-lane-systems" aria-label="Systems mapped to {html.escape(lane_label, quote=True)}">'
f"{lane_systems}</ul>{feedback_markup}</div></section>"
)
caption_id = f"{slug}-diagram-caption"
return (
f'<figure class="system-design" data-system-diagram="{html.escape(slug)}" '
f'data-lane-count="{len(lanes)}" aria-labelledby="{caption_id}">'
f'<figcaption id="{caption_id}"><strong>{html.escape(str(family["title"]))}: architecture comparison</strong>'
"<span>Each lane reads from input to output.</span></figcaption>"
f'<div class="system-lanes">{"".join(lanes)}</div></figure>'
)
def _system_document(
family: Mapping[str, Any],
position: int,
total: int,
previous: Optional[Tuple[str, str]],
following: Optional[Tuple[str, str]],
) -> str:
slug = str(family["slug"])
title = str(family["title"])
category = str(family["category"])
summary = str(family["summary"])
systems = tuple(str(name) for name in family["systems"])
reading = "".join(
"<li>"
f'<a href="{html.escape(str(reference["href"]), quote=True)}">'
f'{html.escape(str(reference["label"]))}</a>'
"</li>"
for reference in family["reading"]
)
document = f"""<!doctype html>
<html lang="en">
{_page_head(title, summary, f"/systems/{slug}", source_sha256=sha256(SYSTEM_DESIGNS))}
<body>
<a class="skip-link" href="#system">Skip to system design</a>
{_site_navigation("systems")}
<main id="system" class="system-page" tabindex="-1">
<article>
<header class="article-header system-header">
<h1>{html.escape(title)}</h1>
<p class="article-meta">{position} of {total} · {html.escape(category.title())} · {len(systems)} named systems</p>
<p class="system-summary">{html.escape(summary)}</p>
</header>
{_system_diagram_markup(family)}
<section class="system-reading" aria-labelledby="{slug}-reading">
<h2 id="{slug}-reading">Read the full explanation</h2>
<ul>{reading}</ul>
</section>
</article>
{_article_navigation(previous, following)}
</main>
<footer class="site-footer">The Evidence Path · system architecture</footer>
</body>
</html>
"""
return "\n".join(line.rstrip() for line in document.splitlines()) + "\n"
def _collection_index_document(
*,
title: str,
intro: str,
route: str,
current: str,
records: Sequence[Mapping[str, Any]],
) -> str:
items = "".join(
"<li>"
f'<a href="{html.escape(str(record["route"]))}">'
f'<span>{html.escape(str(record["title"]))}</span>'
f'<small>{html.escape(str(record["meta"]))}</small></a>'
"</li>"
for record in records
)
document = f"""<!doctype html>
<html lang="en">
{_page_head(title, intro, route)}
<body>
{_site_navigation(current)}
<main class="listing-page">
<header class="listing-header">
<h1>{html.escape(title)}</h1>
<p>{html.escape(intro)}</p>
</header>
<ol class="publication-list">{items}</ol>
</main>
<footer class="site-footer">The Evidence Path · {len(records)} entries</footer>
</body>
</html>
"""
return "\n".join(line.rstrip() for line in document.splitlines()) + "\n"
def _home_document(field_records: Sequence[Mapping[str, Any]]) -> str:
entries = "".join(
"<li>"
f'<a href="{html.escape(str(record["route"]))}">'
f'<span>{html.escape(str(record["title"]))}</span>'
f'<small>{html.escape(str(record["meta"]))}</small></a>'
"</li>"
for record in field_records
)
document = f"""<!doctype html>
<html lang="en">
{_page_head(
"The Evidence Path",
"A field notebook of retrieval-augmented generation, from first principles to the 2026 frontier.",
"/",
)}
<body>
{_site_navigation("reader")}
<main class="home-page">
<header class="home-header">
<h1>The Evidence Path</h1>
<p>A field notebook of retrieval-augmented generation—from first principles to the 2026 frontier.</p>
</header>
<section aria-labelledby="reading-order-title">
<h2 id="reading-order-title">Reading order</h2>
<ol class="publication-list">{entries}</ol>
</section>
<nav class="collection-links" aria-label="Supporting material">
<a href="/systems"><span>Systems</span><small>{len(_system_design_families())} architecture comparisons</small></a>
<a href="/research"><span>Research</span><small>18 evidence and reference notes</small></a>
<a href="/notebooks"><span>Notebooks</span><small>8 focused executed notebooks</small></a>
<a href="/python"><span>Python</span><small>{len(_public_python_artifacts())} implementation and tooling files</small></a>
<a href="/book"><span>PDF</span><small>Compact reader's edition</small></a>
</nav>
</main>
<footer class="site-footer">The Evidence Path · evidence cutoff 9 August 2026</footer>
</body>
</html>
"""
return "\n".join(line.rstrip() for line in document.splitlines()) + "\n"
def _notebook_document(
path: Path,
position: int,
total: int,
previous: Optional[Tuple[str, str]],
following: Optional[Tuple[str, str]],
) -> str:
payload = _load_notebook(path)
title = _notebook_title(payload, path.stem.replace("_", " ").title())
_, cover = _cover_parts(payload)
rendered = [
render_cell(
{**payload["cells"][0], "source": cover},
1,
path.name,
raw_markdown=True,
anchor="notebook-cover",
)
]
rendered.extend(
render_cell(cell, index, path.name)
for index, cell in enumerate(payload.get("cells", [])[1:], start=2)
)
stats = _notebook_stats({path: payload})
slug = _page_slug(path)
description = f"Executed notebook: {title}."
manifest = json.dumps(
{
"notebook": path.name,
"sha256": sha256(path),
"stats": stats,
},
ensure_ascii=False,
sort_keys=True,
)
document = f"""<!doctype html>
<html lang="en">
{_page_head(title, description, f"/notebooks/view/{slug}", math=True, python=True, source_sha256=sha256(path))}
<body>
<a class="skip-link" href="#notebook">Skip to notebook</a>
{_site_navigation("notebooks")}
<main id="notebook" class="jp-Notebook notebook-publication" tabindex="-1">
{rendered[0]}
<div class="reader-tools notebook-tools">
<span>{position} of {total} · {stats['code_cells']} executed cells</span>
<a href="/notebooks/{html.escape(path.name)}" download>Download .ipynb</a>
</div>
{''.join(rendered[1:])}
{_article_navigation(previous, following)}
<p id="copy-status" class="visually-hidden" role="status" aria-live="polite"></p>
<span id="math-render-status" class="visually-hidden" data-state="loading"
aria-live="polite">Typesetting mathematics…</span>
</main>
<footer class="site-footer">The Evidence Path · executed notebook</footer>
<script id="artifact-manifest" type="application/json">{manifest}</script>
<script>{_site_script()}</script>
</body>
</html>
"""
return "\n".join(line.rstrip() for line in document.splitlines()) + "\n"
def _records_for_markdown(paths: Sequence[Path], route_prefix: str) -> List[Dict[str, Any]]:
records: List[Dict[str, Any]] = []
for path in paths:
value = path.read_text(encoding="utf-8")
title, body = _extract_document_title(value, path.stem.replace("_", " ").title())
records.append(
{
"path": path,
"title": title,
"body": body,
"slug": _page_slug(path),
"route": f"{route_prefix}/{_page_slug(path)}",
"meta": f"{_word_count(body):,} words",
}
)
return records
def _neighbor_links(
records: Sequence[Mapping[str, Any]], index: int
) -> Tuple[Optional[Tuple[str, str]], Optional[Tuple[str, str]]]:
previous = (
(str(records[index - 1]["title"]), str(records[index - 1]["route"]))
if index > 0
else None
)
following = (
(str(records[index + 1]["title"]), str(records[index + 1]["route"]))
if index + 1 < len(records)
else None
)
return previous, following
def _render_publication(notebook_path: Path, output_path: Path) -> Path:
"""Render one ordered publication as compact, source-bound pages."""
complete = _load_notebook(notebook_path)
output_directory = output_path.parent
output_directory.mkdir(parents=True, exist_ok=True)
field_records = _records_for_markdown(FIELD_PAGE_PATHS, "/read")
for record, slug in zip(field_records, FIELD_ROUTE_SLUGS):
record["slug"] = slug
record["route"] = f"/read/{slug}"
read_directory = output_directory / "read"
read_directory.mkdir(parents=True, exist_ok=True)
for index, record in enumerate(field_records):
previous, following = _neighbor_links(field_records, index)
page = _article_document(
title=str(record["title"]),
source=str(record["body"]),
source_path=record["path"],
route=str(record["route"]),
current="reader",
position=index + 1,
total=len(field_records),
previous=previous,
following=following,
)
(read_directory / f"{record['slug']}.html").write_text(
page, encoding="utf-8"
)
system_records: List[Dict[str, Any]] = []
for family in _system_design_families():
system_records.append(
{
**family,
"route": f"/systems/{family['slug']}",
"meta": f"{str(family['category']).title()} · {len(family['systems'])} systems",
}
)
systems_directory = output_directory / "systems"
systems_directory.mkdir(parents=True, exist_ok=True)
for index, record in enumerate(system_records):
previous, following = _neighbor_links(system_records, index)
page = _system_document(
record,
index + 1,
len(system_records),
previous,
following,
)
(systems_directory / f"{record['slug']}.html").write_text(
page, encoding="utf-8"
)
(systems_directory / "index.html").write_text(
_collection_index_document(
title="System designs",
intro=(
"Compact architecture comparisons for the named RAG systems, "
"grouped by shared information and control flow."
),
route="/systems",
current="systems",
records=system_records,
),
encoding="utf-8",
)
research_paths = tuple(ROOT / "research" / name for name in RESEARCH_PAGE_ORDER)
research_records = _records_for_markdown(research_paths, "/research")
research_directory = output_directory / "research"
research_directory.mkdir(parents=True, exist_ok=True)
for index, record in enumerate(research_records):
previous, following = _neighbor_links(research_records, index)
page = _article_document(
title=str(record["title"]),
source=str(record["body"]),
source_path=record["path"],
route=str(record["route"]),
current="research",
position=index + 1,
total=len(research_records),
previous=previous,
following=following,
)
(research_directory / f"{record['slug']}.html").write_text(
page, encoding="utf-8"
)
(research_directory / "index.html").write_text(
_collection_index_document(
title="Research",
intro="Evidence, methods, technical history, decisions, and reference material.",
route="/research",
current="research",
records=research_records,
),
encoding="utf-8",
)
notebook_paths = tuple(NOTEBOOKS / name for name in FOCUSED_NOTEBOOK_ORDER)
notebook_records: List[Dict[str, Any]] = []
for path in notebook_paths:
payload = _load_notebook(path)
stats = _notebook_stats({path: payload})
notebook_records.append(
{
"path": path,
"title": _notebook_title(payload, path.stem.replace("_", " ").title()),
"slug": _page_slug(path),
"route": f"/notebooks/view/{_page_slug(path)}",
"meta": f"{stats['code_cells']} executed cells",
}
)
notebook_directory = output_directory / "notebooks"
notebook_directory.mkdir(parents=True, exist_ok=True)
for index, record in enumerate(notebook_records):
previous, following = _neighbor_links(notebook_records, index)
page = _notebook_document(
record["path"],
index + 1,
len(notebook_records),
previous,
following,
)
(notebook_directory / f"{record['slug']}.html").write_text(
page, encoding="utf-8"
)
(notebook_directory / "index.html").write_text(
_collection_index_document(
title="Notebooks",
intro="Focused, executed experiments with their saved outputs and raw .ipynb files.",
route="/notebooks",
current="notebooks",
records=notebook_records,
),
encoding="utf-8",
)
source_pages = _write_source_pages(output_directory)
output_path.write_text(_home_document(field_records), encoding="utf-8")
site_manifest = {
"schema_version": 3,
"complete_notebook": {
"path": notebook_path.relative_to(ROOT).as_posix(),
"sha256": sha256(notebook_path),
"cells": len(complete.get("cells", [])),
},
"reader": {
record["path"].relative_to(ROOT).as_posix(): str(record["route"])
for record in field_records
},
"systems": {
"source": SYSTEM_DESIGNS.relative_to(ROOT).as_posix(),
"sha256": sha256(SYSTEM_DESIGNS),
"routes": {
str(record["slug"]): str(record["route"])
for record in system_records
},
},
"research": {
record["path"].relative_to(ROOT).as_posix(): str(record["route"])
for record in research_records
},
"notebooks": {
record["path"].relative_to(ROOT).as_posix(): str(record["route"])
for record in notebook_records
},
"python": source_pages,
}
(output_directory / "site_manifest.json").write_text(
json.dumps(site_manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
return output_path
# Keep ``render`` as the stable public entry point used by tests and scripts.
render = _render_publication
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--notebook", type=Path, default=DEFAULT_NOTEBOOK)
parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
arguments = parser.parse_args()
try:
path = render(arguments.notebook.resolve(), arguments.output.resolve())
except (OSError, RuntimeError, json.JSONDecodeError) as error:
parser.error(str(error))
print(path.relative_to(ROOT) if path.is_relative_to(ROOT) else path)
return 0
if __name__ == "__main__":
raise SystemExit(main())