#!/usr/bin/env python3 """Render the research and executed notebooks as a compact multi-page site. Reader and research pages come directly from their Markdown sources. Notebook pages come from the saved ``.ipynb`` artifacts, including exact Python cells and outputs. Python files are published separately so long source listings do not interrupt the reading flow. TeX is preserved and typeset by pinned MathJax in the browser when network access permits. """ import argparse import ast import hashlib import html import json import re from pathlib import Path from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple try: from scripts.python_source_renderer import PYTHON_SOURCE_CSS, render_python_source except ModuleNotFoundError: # Direct execution from the scripts directory. from python_source_renderer import PYTHON_SOURCE_CSS, render_python_source ROOT = Path(__file__).resolve().parents[1] NOTEBOOKS = ROOT / "notebooks" READER_CSS = ROOT / "assets" / "reader.css" DEFAULT_NOTEBOOK = NOTEBOOKS / "00_complete_rag_handbook.ipynb" DEFAULT_OUTPUT = ROOT / "previews" / "index.html" BUILD_PROVENANCE = ROOT / "previews" / "build_provenance.json" PUBLIC_SITE_URL = "https://rag.babushkai.com/" FIELD_PAGE_PATHS = tuple(sorted((ROOT / "research" / "field_notebook").glob("*.md"))) FIELD_ROUTE_SLUGS = ( "prologue", "search", "grounding", "agents-memory-security", "evaluation-production", "epilogue", ) RESEARCH_PAGE_ORDER = ( "README.md", "field_map.md", "mathematical_primer.md", "chronology.md", "frontier_2024_2026.md", "corpus_and_indexing.md", "retrieval_and_ranking.md", "context_and_generation.md", "training_and_optimization.md", "structured_and_multimodal_rag.md", "agents_memory_and_temporal.md", "security_privacy_and_governance.md", "evaluation_and_risks.md", "production_systems.md", "decision_guide.md", "chronological_index.md", "glossary.md", "coverage_matrix.md", ) FOCUSED_NOTEBOOK_ORDER = ( "01_rag_evolution.ipynb", "04_corpus_chunking_and_indexes.ipynb", "02_advanced_rag.ipynb", "05_training_query_fusion_and_reranking.ipynb", "06_structured_multimodal_and_graph_rag.ipynb", "07_agents_memory_temporal_and_security.ipynb", "03_evaluation_and_failure_analysis.ipynb", "08_production_evaluation_and_cost.ipynb", ) REFERENCE_MODULE_ORDER = ( "models.py", "text.py", "demo_data.py", "ingestion.py", "chunking.py", "indexes.py", "retrievers.py", "rerankers.py", "selection.py", "context.py", "generation.py", "pipeline.py", "structured.py", "agentic.py", "memory.py", "temporal.py", "security.py", "training.py", "evaluation.py", "operations.py", "__init__.py", ) SCRIPT_ARTIFACTS = ( ( "Chronology builder", "scripts/build_chronological_index.py", "Sorts the primary-source registry into the dated evidence index.", "python3 scripts/build_chronological_index.py", ), ( "Notebook builder", "scripts/build_curriculum_notebooks.py", "Binds narrative, evidence, experiments, and source notes into one Jupyter manuscript.", "python3 scripts/build_curriculum_notebooks.py", ), ( "Notebook executor", "scripts/execute_notebooks.py", "Runs every Python cell and writes deterministic outputs into the notebooks.", "PYTHONPATH=src python3 scripts/execute_notebooks.py --write", ), ( "Site renderer", "scripts/render_field_notebook_preview.py", "Turns the executed notebooks into this integrated browser reader.", "python3 scripts/render_field_notebook_preview.py", ), ( "Python highlighter", "scripts/python_source_renderer.py", "Tokenizes Python with the standard library and emits safe, colored source HTML.", "python3 -m unittest tests.test_python_source_renderer -v", ), ( "Preview server", "scripts/serve_notebook_site.py", "Rebuilds, executes, renders, and serves the complete local artifact.", "make serve", ), ( "Research validator", "scripts/validate_research.py", "Checks sources, prose, execution, links, rendering, and generated artifacts.", "make validate", ), ) PRESERVED_HTML_TAGS = { "a", "abbr", "article", "aside", "b", "blockquote", "br", "circle", "code", "defs", "desc", "details", "div", "em", "figcaption", "figure", "footer", "foreignobject", "g", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hr", "img", "li", "line", "marker", "nav", "ol", "p", "path", "polygon", "polyline", "pre", "rect", "section", "small", "span", "strong", "sub", "summary", "sup", "svg", "table", "tbody", "td", "text", "th", "thead", "title", "tr", "ul", } def cell_text(cell: Mapping[str, Any]) -> str: source = cell.get("source", "") return "".join(source) if isinstance(source, list) else str(source) def sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() def slugify(value: str) -> str: slug = re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") return slug or "section" def _math_markup(tex: str, display: bool = False, delimiter: str = "") -> str: """Wrap TeX for MathJax while preserving an auditable source expression.""" expression = tex.strip() tag = "div" if display else "span" kind = "equation-note math-display" if display else "math-inline" opening, closing = (r"\[", r"\]") if display else (r"\(", r"\)") source_delimiter = delimiter or opening return ( f'<{tag} class="{kind}" data-math-delimiter="{html.escape(source_delimiter, quote=True)}" ' f'data-tex="{html.escape(expression, quote=True)}">' f"{opening}{html.escape(expression)}{closing}" ) def inline_markdown(value: str) -> str: """Render inline Markdown while protecting code, HTML, and TeX boundaries.""" protected: List[str] = [] def protect(rendered: str) -> str: protected.append(rendered) return f"@@PROTECTED{len(protected) - 1}@@" value = re.sub( r"`([^`]+)`", lambda match: protect(f"{html.escape(match.group(1))}"), value, ) def preserve_tag(match: re.Match[str]) -> str: name = re.match(r"]+>", preserve_tag, value) value = re.sub( r"\\\((.+?)\\\)", lambda match: protect(_math_markup(match.group(1), delimiter=r"\(")), value, ) value = re.sub( r"(?', value, ) value = re.sub( r"\[([^\]]+)\]\((https?://[^)]+|mailto:[^)]+|#[^)]+|(?:\.\.?/)?[^)\s]+)\)", r'\1', value, ) value = re.sub(r"\*\*([^*]+)\*\*", r"\1", value) value = re.sub(r"(?\1", value) for index, rendered in enumerate(protected): value = value.replace(f"@@PROTECTED{index}@@", rendered) return value def _table_cells(line: str) -> List[str]: return [cell.strip() for cell in line.strip().strip("|").split("|")] def _render_table(rows: Sequence[str]) -> str: cells = [_table_cells(row) for row in rows] has_header = len(cells) > 1 and all( re.fullmatch(r":?-{3,}:?", value.replace(" ", "")) is not None for value in cells[1] ) rendered: List[str] = [""] body_start = 0 if has_header: rendered.append("") rendered.extend(f"" for value in cells[0]) rendered.append("") body_start = 2 rendered.append("") for row in cells[body_start:]: rendered.append("") rendered.extend(f"" for value in row) rendered.append("") rendered.append("
{inline_markdown(value)}
{inline_markdown(value)}
") return "".join(rendered) def markdownish(value: str, id_prefix: str = "") -> str: """Convert the repository's block Markdown without external packages.""" rendered: List[str] = [] paragraph: List[str] = [] table_rows: List[str] = [] fence_lines: List[str] = [] equation_lines: List[str] = [] fence_marker = "" in_equation = False equation_end = "" list_kind = "" list_items: List[str] = [] html_block_lines: List[str] = [] html_block_tag = "" html_block_depth = 0 def flush_paragraph() -> None: if paragraph: text = " ".join(part.strip() for part in paragraph) rendered.append(f"

{inline_markdown(text)}

") paragraph.clear() def close_list() -> None: nonlocal list_kind if list_kind: items = "".join( f"
  • {inline_markdown(item)}
    • " for item in list_items ) rendered.append(f"<{list_kind}>{items}") list_kind = "" list_items.clear() def flush_table() -> None: if table_rows: rendered.append(_render_table(table_rows)) table_rows.clear() def html_tag_delta(line: str, tag: str) -> int: openings = len(re.findall(fr"<{tag}(?:\s|>)", line, flags=re.IGNORECASE)) closings = len(re.findall(fr"", line, flags=re.IGNORECASE)) self_closing = len( re.findall(fr"<{tag}(?:\s[^>]*)?/\s*>", line, flags=re.IGNORECASE) ) return openings - closings - self_closing for line in value.splitlines(): stripped = line.strip() if html_block_tag: html_block_lines.append(line) html_block_depth += html_tag_delta(line, html_block_tag) if html_block_depth <= 0: rendered.append("\n".join(html_block_lines)) html_block_lines.clear() html_block_tag = "" html_block_depth = 0 continue if fence_marker: if stripped.startswith(fence_marker): rendered.append( '
    '
                    + html.escape("\n".join(fence_lines))
                    + "
    " ) fence_lines.clear() fence_marker = "" else: fence_lines.append(line) continue if stripped.startswith(("```", "~~~")): flush_paragraph() flush_table() close_list() fence_marker = stripped[:3] continue if in_equation: if stripped == equation_end: rendered.append( _math_markup( "\n".join(equation_lines), display=True, delimiter=r"\[" if equation_end == r"\]" else "$$", ) ) equation_lines.clear() in_equation = False equation_end = "" else: equation_lines.append(line) continue if stripped in {r"\[", "$$"}: flush_paragraph() flush_table() close_list() in_equation = True equation_end = r"\]" if stripped == r"\[" else "$$" continue if ( stripped.startswith(r"\[") and stripped.endswith(r"\]") and len(stripped) > 4 ): flush_paragraph() flush_table() close_list() rendered.append(_math_markup(stripped[2:-2], display=True, delimiter=r"\[")) continue if stripped.startswith("$$") and stripped.endswith("$$") and len(stripped) > 4: flush_paragraph() flush_table() close_list() rendered.append(_math_markup(stripped[2:-2], display=True, delimiter="$$")) continue if stripped.startswith("|") and stripped.endswith("|"): flush_paragraph() close_list() table_rows.append(stripped) continue flush_table() if not stripped: flush_paragraph() close_list() continue if stripped in {"---", "***", "___"}: flush_paragraph() close_list() rendered.append("
    ") continue heading = re.match(r"^(#{1,6})\s+(.+)$", stripped) if heading: flush_paragraph() close_list() level = len(heading.group(1)) title = heading.group(2) plain_title = re.sub(r"<[^>]+>", "", title) heading_id = slugify( f"{id_prefix}-{plain_title}" if id_prefix else plain_title ) rendered.append( f'' f"{inline_markdown(title)}" ) continue html_block = re.match( r"^<(div|aside|svg|section|article|figure|table|details|nav|header|footer)\b", stripped, flags=re.IGNORECASE, ) if html_block: flush_paragraph() close_list() tag = html_block.group(1).lower() depth = html_tag_delta(line, tag) if depth > 0: html_block_tag = tag html_block_depth = depth html_block_lines.append(line) else: rendered.append(line) continue if stripped.startswith("<"): flush_paragraph() close_list() rendered.append(line) continue unordered = re.match(r"^[-*]\s+(.+)$", stripped) ordered = re.match(r"^\d+\.\s+(.+)$", stripped) if list_kind and line != line.lstrip() and not (unordered or ordered): list_items[-1] = f"{list_items[-1]} {stripped}" continue if unordered or ordered: flush_paragraph() kind = "ul" if unordered else "ol" if list_kind != kind: close_list() list_kind = kind match = unordered or ordered list_items.append(match.group(1)) continue if stripped.startswith(">"): flush_paragraph() close_list() rendered.append(f"
    {inline_markdown(stripped[1:].strip())}
    ") continue paragraph.append(line) flush_paragraph() flush_table() close_list() if fence_marker: rendered.append( '
    '
            + html.escape("\n".join(fence_lines))
            + "
    " ) if html_block_lines: rendered.append("\n".join(html_block_lines)) if in_equation: raise RuntimeError(f"unclosed display-math delimiter: expected {equation_end}") return "\n".join(rendered) def text_outputs(cell: Mapping[str, Any]) -> Iterable[Tuple[str, str]]: for output in cell.get("outputs", []): output_type = output.get("output_type", "output") if output_type == "stream": text = output.get("text", "") yield "Saved output", "".join(text) if isinstance(text, list) else str(text) elif "text/plain" in output.get("data", {}): text = output["data"]["text/plain"] yield "Saved result", "".join(text) if isinstance(text, list) else str(text) elif output_type == "error": yield "Execution error", f"{output.get('ename', 'Error')}: {output.get('evalue', '')}" def preserve_preformatted_whitespace(value: str) -> str: """Encode end-of-line whitespace so HTML and Git preserve it exactly.""" def encode(match: re.Match[str]) -> str: return "".join(" " if character == "\t" else " " for character in match.group()) return re.sub(r"[ \t]+(?=\r?\n|$)", encode, value) def render_cell( cell: Mapping[str, Any], cell_index: int, notebook_name: str, raw_markdown: bool = False, anchor: str = "", aliases: Sequence[str] = (), ) -> str: cell_type = cell.get("cell_type") source = cell_text(cell) identity = f"{slugify(notebook_name)}-cell-{cell_index}" anchor_value = anchor or identity alias_markup = "".join( f'' for alias in aliases ) if cell_type == "markdown": body = ( source if raw_markdown else _rewrite_publication_links(markdownish(source, identity)) ) return ( f'
    ' f'{alias_markup}
    {body}
    ' ) if cell_type == "code": execution_count = cell.get("execution_count") output = "".join( '
    ' f'
    {html.escape(label)}
    ' f'
    {preserve_preformatted_whitespace(html.escape(text))}
    ' for label, text in text_outputs(cell) ) return ( f'
    ' f'{alias_markup}' '
    ' 'Python' f'cell {cell_index} · executed' f'' '
    ' f'
    '
            f'{preserve_preformatted_whitespace(render_python_source(source))}'
            f'
    {output}' "
    " ) return "" def _load_notebook(path: Path) -> Dict[str, Any]: payload = json.loads(path.read_text(encoding="utf-8")) for index, cell in enumerate(payload.get("cells", []), start=1): if cell.get("cell_type") == "code" and cell.get("execution_count") is None: raise RuntimeError( f"{path.name}: code cell {index} is unexecuted; run " "scripts/execute_notebooks.py --write before rendering" ) if any(output.get("output_type") == "error" for output in cell.get("outputs", [])): raise RuntimeError(f"{path.name}: code cell {index} contains an error output") return payload def _cover_parts(payload: Mapping[str, Any]) -> Tuple[str, str]: source = cell_text(payload["cells"][0]) match = re.search(r"", source, flags=re.DOTALL) css = match.group(1) if match else "" cover = re.sub(r"\s*", "", source, count=1, flags=re.DOTALL) return css, cover def _notebook_title(payload: Mapping[str, Any], fallback: str) -> str: source = cell_text(payload["cells"][0]) match = re.search(r"

    (.*?)

    ", source, flags=re.DOTALL) if not match: return fallback return html.unescape(re.sub(r"<[^>]+>", "", match.group(1))).strip() def _notebook_stats(notebooks: Mapping[Path, Mapping[str, Any]]) -> Dict[str, int]: cells = [cell for payload in notebooks.values() for cell in payload.get("cells", [])] code = [cell for cell in cells if cell.get("cell_type") == "code"] return { "notebooks": len(notebooks), "cells": len(cells), "code_cells": len(code), "executed_cells": sum(cell.get("execution_count") is not None for cell in code), "output_cells": sum(bool(cell.get("outputs")) for cell in code), } def _source_slug(path: str) -> str: source_path = Path(path) if source_path.parts[:2] == ("src", "rag_evolution"): stem = "package" if source_path.stem == "__init__" else source_path.stem return f"rag-evolution-{stem.replace('_', '-')}" return source_path.stem.replace("_", "-") def _public_python_artifacts() -> Tuple[Tuple[str, str, str, str], ...]: artifacts = list(SCRIPT_ARTIFACTS) for name in REFERENCE_MODULE_ORDER: path = f"src/rag_evolution/{name}" source = (ROOT / path).read_text(encoding="utf-8") try: description = (ast.get_docstring(ast.parse(source)) or "").splitlines()[0] except (SyntaxError, IndexError): description = "RAG reference implementation module." title = "Package index" if name == "__init__.py" else Path(name).stem.replace("_", " ").title() artifacts.append((title, path, description, "")) return tuple(artifacts) def _site_navigation(current: str = "") -> str: def link(label: str, href: str, key: str) -> str: current_value = ' aria-current="page"' if current == key else "" return ( f'' f"{html.escape(label)}" ) return f"""
    The Evidence Path
    {link("Reader", "/", "reader")} {link("Research", "/research", "research")} {link("Notebooks", "/notebooks", "notebooks")} {link("Python", "/python", "python")} {link("PDF", "/book", "pdf")}
    """ def _source_page_document(path: str, title: str, description: str) -> str: source_path = ROOT / path source = source_path.read_text(encoding="utf-8") digest = sha256(source_path) slug = _source_slug(path) rendered = preserve_preformatted_whitespace(render_python_source(source)) document = f""" {html.escape(title)} — The Evidence Path Skip to source {_site_navigation("python")}

    {html.escape(title)}

    {html.escape(description)}

    {html.escape(path)} · {len(source.splitlines()):,} lines · sha256 {digest[:12]}…

    Download raw .py · All Python files

    {rendered}
    The Evidence Path · exact source snapshot
    """ return "\n".join(line.rstrip() for line in document.splitlines()) + "\n" def _source_index_document() -> str: groups: Dict[str, List[str]] = {"Reference implementation": [], "Tooling": []} for title, path, _, _ in _public_python_artifacts(): source_path = ROOT / path line_count = len(source_path.read_text(encoding="utf-8").splitlines()) item = ( "
  • " f'{html.escape(title)}' f"{html.escape(path)} · {line_count:,} lines" "
    • " ) group = "Reference implementation" if path.startswith("src/") else "Tooling" groups[group].append(item) sections = "".join( f'

    {html.escape(label)}

      ' f"{''.join(groups[label])}
    " for label in ("Reference implementation", "Tooling") ) document = f""" Python source — The Evidence Path {_site_navigation("python")}

    Python source

    The reference implementation and the tooling used to build and verify this publication.

    {sections}
    The Evidence Path · {len(_public_python_artifacts())} verified source files
    """ return "\n".join(line.rstrip() for line in document.splitlines()) + "\n" def _write_source_pages(output_directory: Path) -> Dict[str, str]: source_directory = output_directory / "source" source_directory.mkdir(parents=True, exist_ok=True) pages: Dict[str, str] = {} for title, path, description, _ in _public_python_artifacts(): name = f"{_source_slug(path)}.html" page = source_directory / name page.write_text(_source_page_document(path, title, description), encoding="utf-8") pages[path] = page.relative_to(output_directory).as_posix() (source_directory / "index.html").write_text( _source_index_document(), encoding="utf-8" ) return pages def _site_css() -> str: """Return the external reader stylesheet for tests and source audits.""" return READER_CSS.read_text(encoding="utf-8") def _site_script() -> str: """Add notebook copying and stable deep-link behavior.""" return r""" document.addEventListener('DOMContentLoaded', () => { const status = window.__mathStatus || { label: 'Typesetting mathematics…', state: 'loading' }; window.__setMathStatus(status.label, status.state); }); const copyText = async (value) => { if (navigator.clipboard && window.isSecureContext) { await navigator.clipboard.writeText(value); return; } const field = document.createElement('textarea'); field.value = value; field.style.position = 'fixed'; field.style.opacity = '0'; document.body.appendChild(field); field.select(); document.execCommand('copy'); field.remove(); }; document.querySelectorAll('.copy-code').forEach((button) => { button.addEventListener('click', async () => { const code = document.getElementById(button.dataset.copyTarget); if (!code) return; const original = button.textContent; const status = document.getElementById('copy-status'); try { await copyText(code.textContent); button.textContent = 'Copied'; if (status) status.textContent = 'Python source copied.'; } catch (error) { button.textContent = 'Copy failed'; if (status) status.textContent = 'Could not copy Python source.'; } setTimeout(() => { button.textContent = original; }, 1400); }); }); const revealHashTarget = () => { if (!location.hash) return; let identifier = ''; try { identifier = decodeURIComponent(location.hash.slice(1)); } catch (error) { return; } const target = document.getElementById(identifier); if (!target) return; requestAnimationFrame(() => target.scrollIntoView({block: 'start'})); }; addEventListener('hashchange', revealHashTarget); addEventListener('mathready', revealHashTarget); revealHashTarget(); """ def _page_slug(path: Path) -> str: stem = re.sub(r"^\d+_", "", path.stem) if path.name == "README.md": return "method-and-scope" return stem.replace("_", "-") def _extract_document_title(value: str, fallback: str) -> Tuple[str, str]: value = re.sub(r"\A\s*\s*", "", value, count=1, flags=re.DOTALL) match = re.search(r"^#{1,2}\s+(.+?)\s*$", value, flags=re.MULTILINE) if not match: return fallback, value.strip() title = re.sub(r"[*_`]", "", match.group(1)).strip() or fallback body = (value[: match.start()] + value[match.end() :]).strip() return title, body def _word_count(value: str) -> int: return len(re.findall(r"\b[\w'-]+\b", value, flags=re.UNICODE)) def _rewrite_publication_links(rendered: str) -> str: """Point source-relative references at their compact public equivalents.""" research_routes = { name: f"/research/{_page_slug(Path(name))}" for name in RESEARCH_PAGE_ORDER } notebook_routes = { name: f"/notebooks/view/{_page_slug(Path(name))}" for name in FOCUSED_NOTEBOOK_ORDER } def research_link(match: re.Match[str]) -> str: name = match.group("name") fragment = match.group("fragment") or "" return f'href="{research_routes.get(name, name)}{fragment}"' rendered = re.sub( r'href="(?:\.\./research/)?(?P[A-Za-z0-9_]+\.md)' r'(?P#[^"]+)?"', research_link, rendered, ) for name, route in notebook_routes.items(): rendered = rendered.replace( f'href="../notebooks/{name}"', f'href="{route}"' ) return ( rendered.replace('href="field_notebook"', 'href="/read/prologue"') .replace('href="../research/field_notebook"', 'href="/read/prologue"') .replace('href="sources.json"', 'href="/research/sources.json"') .replace('href="../research/sources.json"', 'href="/research/sources.json"') .replace( 'href="../notebooks/00_complete_rag_handbook.ipynb"', 'href="/notebooks/00_complete_rag_handbook.ipynb"', ) ) def _math_head() -> str: return r""" """ def _page_head( title: str, description: str, canonical_path: str, *, math: bool = False, python: bool = False, source_sha256: str = "", ) -> str: digest_meta = ( f' \n' if source_sha256 else "" ) python_css = f" \n" if python else "" math_runtime = _math_head() if math else "" canonical = PUBLIC_SITE_URL.rstrip("/") + canonical_path return f""" {digest_meta} {html.escape(title)} — The Evidence Path {python_css}{math_runtime}""" def _article_navigation( previous: Optional[Tuple[str, str]], following: Optional[Tuple[str, str]], ) -> str: previous_link = ( f'← {html.escape(previous[0])}' if previous else "" ) following_link = ( f'{html.escape(following[0])} →' if following else "" ) return f'
    {previous_link}{following_link}
    ' def _article_document( *, title: str, source: str, source_path: Path, route: str, current: str, position: int, total: int, previous: Optional[Tuple[str, str]], following: Optional[Tuple[str, str]], ) -> str: description = f"{title}, part of The Evidence Path." digest = sha256(source_path) body = _rewrite_publication_links(markdownish(source)) document = f""" {_page_head(title, description, route, math=True, source_sha256=digest)} Skip to article {_site_navigation(current)}

    {html.escape(title)}

    {position} of {total} · {_word_count(source):,} words

    {body}
    {_article_navigation(previous, following)} Typesetting mathematics…
    The Evidence Path · evidence cutoff 9 August 2026
    """ return "\n".join(line.rstrip() for line in document.splitlines()) + "\n" def _collection_index_document( *, title: str, intro: str, route: str, current: str, records: Sequence[Mapping[str, Any]], ) -> str: items = "".join( "
  • " f'' f'{html.escape(str(record["title"]))}' f'{html.escape(str(record["meta"]))}' "
    • " for record in records ) document = f""" {_page_head(title, intro, route)} {_site_navigation(current)}

    {html.escape(title)}

    {html.escape(intro)}

      {items}
    The Evidence Path · {len(records)} entries
    """ return "\n".join(line.rstrip() for line in document.splitlines()) + "\n" def _home_document(field_records: Sequence[Mapping[str, Any]]) -> str: entries = "".join( "
  • " f'' f'{html.escape(str(record["title"]))}' f'{html.escape(str(record["meta"]))}' "
    • " for record in field_records ) document = f""" {_page_head( "The Evidence Path", "A field notebook of retrieval-augmented generation, from first principles to the 2026 frontier.", "/", )} {_site_navigation("reader")}

    The Evidence Path

    A field notebook of retrieval-augmented generation—from first principles to the 2026 frontier.

    Reading order

      {entries}
    Research18 evidence and reference notes Notebooks8 focused executed notebooks Python{len(_public_python_artifacts())} implementation and tooling files PDFCompact reader's edition
    The Evidence Path · evidence cutoff 9 August 2026
    """ return "\n".join(line.rstrip() for line in document.splitlines()) + "\n" def _notebook_document( path: Path, position: int, total: int, previous: Optional[Tuple[str, str]], following: Optional[Tuple[str, str]], ) -> str: payload = _load_notebook(path) title = _notebook_title(payload, path.stem.replace("_", " ").title()) _, cover = _cover_parts(payload) rendered = [ render_cell( {**payload["cells"][0], "source": cover}, 1, path.name, raw_markdown=True, anchor="notebook-cover", ) ] rendered.extend( render_cell(cell, index, path.name) for index, cell in enumerate(payload.get("cells", [])[1:], start=2) ) stats = _notebook_stats({path: payload}) slug = _page_slug(path) description = f"Executed notebook: {title}." manifest = json.dumps( { "notebook": path.name, "sha256": sha256(path), "stats": stats, }, ensure_ascii=False, sort_keys=True, ) document = f""" {_page_head(title, description, f"/notebooks/view/{slug}", math=True, python=True, source_sha256=sha256(path))} Skip to notebook {_site_navigation("notebooks")}
    {rendered[0]}
    {position} of {total} · {stats['code_cells']} executed cells Download .ipynb
    {''.join(rendered[1:])} {_article_navigation(previous, following)}

    Typesetting mathematics…
    The Evidence Path · executed notebook
    """ return "\n".join(line.rstrip() for line in document.splitlines()) + "\n" def _records_for_markdown(paths: Sequence[Path], route_prefix: str) -> List[Dict[str, Any]]: records: List[Dict[str, Any]] = [] for path in paths: value = path.read_text(encoding="utf-8") title, body = _extract_document_title(value, path.stem.replace("_", " ").title()) records.append( { "path": path, "title": title, "body": body, "slug": _page_slug(path), "route": f"{route_prefix}/{_page_slug(path)}", "meta": f"{_word_count(body):,} words", } ) return records def _neighbor_links( records: Sequence[Mapping[str, Any]], index: int ) -> Tuple[Optional[Tuple[str, str]], Optional[Tuple[str, str]]]: previous = ( (str(records[index - 1]["title"]), str(records[index - 1]["route"])) if index > 0 else None ) following = ( (str(records[index + 1]["title"]), str(records[index + 1]["route"])) if index + 1 < len(records) else None ) return previous, following def _render_publication(notebook_path: Path, output_path: Path) -> Path: """Render one ordered publication as compact, source-bound pages.""" complete = _load_notebook(notebook_path) output_directory = output_path.parent output_directory.mkdir(parents=True, exist_ok=True) field_records = _records_for_markdown(FIELD_PAGE_PATHS, "/read") for record, slug in zip(field_records, FIELD_ROUTE_SLUGS): record["slug"] = slug record["route"] = f"/read/{slug}" read_directory = output_directory / "read" read_directory.mkdir(parents=True, exist_ok=True) for index, record in enumerate(field_records): previous, following = _neighbor_links(field_records, index) page = _article_document( title=str(record["title"]), source=str(record["body"]), source_path=record["path"], route=str(record["route"]), current="reader", position=index + 1, total=len(field_records), previous=previous, following=following, ) (read_directory / f"{record['slug']}.html").write_text( page, encoding="utf-8" ) research_paths = tuple(ROOT / "research" / name for name in RESEARCH_PAGE_ORDER) research_records = _records_for_markdown(research_paths, "/research") research_directory = output_directory / "research" research_directory.mkdir(parents=True, exist_ok=True) for index, record in enumerate(research_records): previous, following = _neighbor_links(research_records, index) page = _article_document( title=str(record["title"]), source=str(record["body"]), source_path=record["path"], route=str(record["route"]), current="research", position=index + 1, total=len(research_records), previous=previous, following=following, ) (research_directory / f"{record['slug']}.html").write_text( page, encoding="utf-8" ) (research_directory / "index.html").write_text( _collection_index_document( title="Research", intro="Evidence, methods, technical history, decisions, and reference material.", route="/research", current="research", records=research_records, ), encoding="utf-8", ) notebook_paths = tuple(NOTEBOOKS / name for name in FOCUSED_NOTEBOOK_ORDER) notebook_records: List[Dict[str, Any]] = [] for path in notebook_paths: payload = _load_notebook(path) stats = _notebook_stats({path: payload}) notebook_records.append( { "path": path, "title": _notebook_title(payload, path.stem.replace("_", " ").title()), "slug": _page_slug(path), "route": f"/notebooks/view/{_page_slug(path)}", "meta": f"{stats['code_cells']} executed cells", } ) notebook_directory = output_directory / "notebooks" notebook_directory.mkdir(parents=True, exist_ok=True) for index, record in enumerate(notebook_records): previous, following = _neighbor_links(notebook_records, index) page = _notebook_document( record["path"], index + 1, len(notebook_records), previous, following, ) (notebook_directory / f"{record['slug']}.html").write_text( page, encoding="utf-8" ) (notebook_directory / "index.html").write_text( _collection_index_document( title="Notebooks", intro="Focused, executed experiments with their saved outputs and raw .ipynb files.", route="/notebooks", current="notebooks", records=notebook_records, ), encoding="utf-8", ) source_pages = _write_source_pages(output_directory) output_path.write_text(_home_document(field_records), encoding="utf-8") site_manifest = { "schema_version": 2, "complete_notebook": { "path": notebook_path.relative_to(ROOT).as_posix(), "sha256": sha256(notebook_path), "cells": len(complete.get("cells", [])), }, "reader": { record["path"].relative_to(ROOT).as_posix(): str(record["route"]) for record in field_records }, "research": { record["path"].relative_to(ROOT).as_posix(): str(record["route"]) for record in research_records }, "notebooks": { record["path"].relative_to(ROOT).as_posix(): str(record["route"]) for record in notebook_records }, "python": source_pages, } (output_directory / "site_manifest.json").write_text( json.dumps(site_manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8", ) return output_path # Keep ``render`` as the stable public entry point used by tests and scripts. render = _render_publication def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--notebook", type=Path, default=DEFAULT_NOTEBOOK) parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT) arguments = parser.parse_args() try: path = render(arguments.notebook.resolve(), arguments.output.resolve()) except (OSError, RuntimeError, json.JSONDecodeError) as error: parser.error(str(error)) print(path.relative_to(ROOT) if path.is_relative_to(ROOT) else path) return 0 if __name__ == "__main__": raise SystemExit(main())