Text

"""Text processing helpers with explicit, inspectable behavior."""

import re
from typing import Dict, Iterable, List, Mapping, Sequence, Set, Tuple

from .models import Chunk, Document


TOKEN_RE = re.compile(r"[a-z0-9]+(?:[-'][a-z0-9]+)?", re.IGNORECASE)
SENTENCE_RE = re.compile(r"(?<=[.!?])\s+")

STOPWORDS: Set[str] = {
    "a",
    "an",
    "and",
    "are",
    "as",
    "at",
    "be",
    "between",
    "by",
    "did",
    "do",
    "does",
    "for",
    "from",
    "how",
    "in",
    "into",
    "is",
    "it",
    "of",
    "on",
    "or",
    "original",
    "that",
    "the",
    "their",
    "this",
    "to",
    "was",
    "were",
    "what",
    "when",
    "which",
    "who",
    "with",
}

DEFAULT_SYNONYM_GROUPS: Tuple[Tuple[str, ...], ...] = (
    ("retrieve", "retrieval", "retriever", "search", "lookup", "fetch"),
    ("generate", "generation", "generator", "decode", "decoder"),
    ("document", "documents", "passage", "passages", "chunk", "chunks", "context"),
    ("graph", "network", "linked", "relation", "relations"),
    ("question", "query", "queries"),
    ("evaluate", "evaluation", "metric", "metrics", "benchmark"),
    ("adaptive", "dynamic", "conditional", "route", "routing"),
    ("combine", "fusion", "fuse", "aggregate", "aggregation"),
)


def tokenize(text: str) -> List[str]:
    """Lowercase alphanumeric tokenization used by every reference component."""

    return [match.group(0).lower() for match in TOKEN_RE.finditer(text)]


def content_terms(text: str) -> List[str]:
    """Tokens after removing a small, visible stop-word list."""

    return [token for token in tokenize(text) if token not in STOPWORDS]


def split_sentences(text: str) -> List[str]:
    """Split prose into non-empty sentences without an external NLP model."""

    return [part.strip() for part in SENTENCE_RE.split(text.strip()) if part.strip()]


def synonym_map(groups: Sequence[Sequence[str]] = DEFAULT_SYNONYM_GROUPS) -> Dict[str, str]:
    """Map synonyms to the first term of each declared group."""

    mapping: Dict[str, str] = {}
    for group in groups:
        if not group:
            continue
        canonical = group[0].lower()
        for term in group:
            mapping[term.lower()] = canonical
    return mapping


def semantic_terms(text: str, mapping: Mapping[str, str]) -> List[str]:
    """Canonicalized terms for the dependency-free semantic proxy."""

    return [mapping.get(token, token) for token in content_terms(text)]


def chunk_document(document: Document, chunk_size: int = 120, overlap: int = 20) -> List[Chunk]:
    """Split a document into token windows.

    Token offsets, source identity, and metadata are preserved so that a final
    citation can be traced back to the original document.  The function uses
    whitespace reconstruction for portability; production systems should keep
    exact character offsets as well.
    """

    if chunk_size <= 0:
        raise ValueError("chunk_size must be positive")
    if overlap < 0 or overlap >= chunk_size:
        raise ValueError("overlap must satisfy 0 <= overlap < chunk_size")
    matches = list(TOKEN_RE.finditer(document.text))
    if not matches:
        return []
    chunks: List[Chunk] = []
    step = chunk_size - overlap
    for start in range(0, len(matches), step):
        end = min(start + chunk_size, len(matches))
        chunk_id = f"{document.id}::c{len(chunks):03d}"
        char_start = matches[start].start()
        char_end = matches[end - 1].end()
        chunks.append(
            Chunk(
                id=chunk_id,
                document_id=document.id,
                text=document.text[char_start:char_end],
                start_token=start,
                end_token=end,
                title=document.title,
                source=document.source,
                metadata=document.metadata,
            )
        )
        if end == len(matches):
            break
    return chunks


def chunk_documents(
    documents: Iterable[Document], chunk_size: int = 120, overlap: int = 20
) -> List[Chunk]:
    """Chunk a document collection in stable input order."""

    chunks: List[Chunk] = []
    for document in documents:
        chunks.extend(chunk_document(document, chunk_size=chunk_size, overlap=overlap))
    return chunks


def jaccard(left: Iterable[str], right: Iterable[str]) -> float:
    """Set Jaccard similarity with well-defined empty-set behavior."""

    left_set, right_set = set(left), set(right)
    union = left_set | right_set
    if not union:
        return 1.0
    return len(left_set & right_set) / len(union)


def ordered_unique(values: Iterable[str]) -> Tuple[str, ...]:
    """Return unique values while retaining first occurrence order."""

    seen: Set[str] = set()
    output: List[str] = []
    for value in values:
        if value not in seen:
            output.append(value)
            seen.add(value)
    return tuple(output)