#!/usr/bin/env python3
"""Generate the exhaustive chronological source index from sources.json."""

import json
from collections import Counter, defaultdict
from pathlib import Path
from typing import Any, DefaultDict, Dict, List, Mapping, Sequence


ROOT = Path(__file__).resolve().parents[1]
REGISTRY = ROOT / "research" / "sources.json"
OUTPUT = ROOT / "research" / "chronological_index.md"


INTRO = """# Chronological index of RAG and its technical substrate

This is the complete chronological view of the primary-source registry as of
**2026-08-09**.  It complements the explanatory [chronology](chronology.md):
that chapter follows causal architectural transitions, while this index makes
all registered works—including parsing, chunking, sparse execution, ANN,
ranking, evaluation, privacy, security, multimodal retrieval, memory, and
serving—discoverable by first public date.

Dates are the earliest public date recorded in [`sources.json`](sources.json),
not necessarily the later proceedings year.  Status is explicit because an
influential preprint or industry report is not equivalent to a peer-reviewed
result.  Entries link to an original paper, official proceedings page, or
first-party program/report; the registry's title, date, venue, URL, status, and
topics are the canonical metadata.

## How to read the eras

1. **1971–2013: retrieval and indexing foundations.** Relevance feedback,
   probabilistic term specificity, the vector-space model, BM25, passage
   segmentation, relevance models, dynamic pruning, RRF, PQ/OPQ, and ANN
   establish the substrate later RAG systems inherit.
2. **2014–2019: differentiable memory and open-domain retrieve/read.** Memory
   Networks, DrQA, knowledge-grounded dialogue, graph QA, dense latent
   retrieval, kNN-LM, early BERT reranking, document expansion, DiskANN, and
   multi-hop benchmarks move external evidence into neural NLP.
3. **2020–2021: modern neural retrieval and retrieval-conditioned generation.**
   REALM, DPR, RAG, FiD, ColBERT, ANCE, KILT, RocketQA, learned sparse search,
   RETRO, BEIR, ScaNN, SPANN, and training/distillation work define the modern
   stack.
4. **2022–2023: generalization, instruction, long-form attribution, and
   retrieval-aware control.** Atlas, Contriever, E5/INSTRUCTOR/GTR, HyDE,
   FLARE, IRCoT, ALCE, Self-RAG, RAPTOR, and broader evaluation make retrieval
   more controllable and auditable.
5. **2024–2026: adaptive/agentic policies, graph and visual documents,
   reasoning-aware retrieval, memory, safety, and systems.** The frontier learns
   whether/when/how to retrieve and stop, while evaluation exposes citation,
   freshness, poisoning, privacy, multimodal, long-context, and production
   trade-offs.

Cross-paper scores are not comparable merely because methods appear in the same
year.  Corpus snapshots, qrels, retrieval depth, generators, prompts, models,
judges, and budgets differ.  Use this index to find evidence, then read the
mechanism and limitation analysis in the linked handbook chapter.
"""


def escape(value: str) -> str:
    return value.replace("|", "\\|").replace("\n", " ")


def year_of(source: Mapping[str, Any]) -> int:
    return int(str(source["first_public"])[:4])


def era_heading(year: int) -> str:
    if year <= 2013:
        return "Retrieval and indexing foundations (1971–2013)"
    if year <= 2019:
        return "Differentiable memory and open-domain retrieve/read (2014–2019)"
    if year <= 2021:
        return "Modern neural retrieval and RAG (2020–2021)"
    if year <= 2023:
        return "Generalization, attribution, and retrieval control (2022–2023)"
    return "Adaptive, multimodal, secure, and production RAG (2024–2026)"


def build(payload: Mapping[str, Any]) -> str:
    sources: Sequence[Mapping[str, Any]] = payload["sources"]
    grouped: DefaultDict[int, List[Mapping[str, Any]]] = defaultdict(list)
    for source in sources:
        grouped[year_of(source)].append(source)
    status_counts = Counter(str(source["status"]) for source in sources)
    topic_counts = Counter(topic for source in sources for topic in source["topics"])

    lines = [INTRO.rstrip(), "", "## Registry summary", ""]
    lines.append(
        f"The index contains **{len(sources)} works**: "
        + ", ".join(f"{status_counts[key]} {key}" for key in sorted(status_counts))
        + "."
    )
    lines.extend(["", "Most represented topic tags:", ""])
    for topic, count in topic_counts.most_common(30):
        lines.append(f"- `{topic}` — {count}")

    current_era = ""
    ordinal = 0
    for year in sorted(grouped):
        era = era_heading(year)
        if era != current_era:
            lines.extend(["", f"## {era}", ""])
            current_era = era
        lines.extend(
            [
                f"### {year}",
                "",
                "| # | First public | Work | Venue/status | Topics |",
                "|---:|---|---|---|---|",
            ]
        )
        works = sorted(
            grouped[year],
            key=lambda source: (
                str(source["first_public"]),
                str(source["title"]).casefold(),
                str(source["id"]),
            ),
        )
        for source in works:
            ordinal += 1
            title = escape(str(source["title"]))
            url = str(source["primary_url"])
            venue = escape(str(source["venue"]))
            status = escape(str(source["status"]))
            topics = ", ".join(f"`{escape(str(topic))}`" for topic in source["topics"])
            first_public = escape(str(source["first_public"]))
            lines.append(
                f"| {ordinal} | {first_public} | [{title}]({url}) | {venue}; {status} | {topics} |"
            )

    lines.extend(
        [
            "",
            "## Coverage and maintenance rules",
            "",
            "The index is broad by design, but it is not a claim that every publication "
            "ever using retrieval appears here. A work enters the registry when it is "
            "needed to support a historical, mechanism, empirical, evaluation, security, "
            "or systems claim in this repository. The [coverage matrix](coverage_matrix.md) "
            "shows which lifecycle surface each body of work supports.",
            "",
            "When adding a source:",
            "",
            "1. prefer final official proceedings, then accepted-paper/author manuscript, "
            "then an original preprint or first-party report;",
            "2. record earliest public date separately from venue year;",
            "3. use a unique stable ID and primary URL;",
            "4. label status without upgrading preprints or industry reports;",
            "5. attach specific topic tags and update the substantive chapter;",
            "6. regenerate this file and the complete handbook notebook; and",
            "7. run the full validator so dates, links, notebook execution, and coverage "
            "remain synchronized.",
            "",
        ]
    )
    return "\n".join(lines)


def main() -> int:
    payload: Dict[str, Any] = json.loads(REGISTRY.read_text(encoding="utf-8"))
    OUTPUT.write_text(build(payload), encoding="utf-8")
    print(OUTPUT.relative_to(ROOT), f"({len(payload['sources'])} works)")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())