#!/usr/bin/env python3
"""
passages — the relevant PARTS of one note, with line numbers (vaultq library).

Fills the gap between `find`'s two settings. `find` ranks notes and shows one
grepped line (`snippet_for`); `read_file` returns the whole thing, and the
default cap is 10MB. For a long note — a folder note, a Knowledge article, a
`.chat` transcript — an agent asking "where in here?" had to choose between one
line and everything.

A passage is a scored span of a note with **file-absolute, 1-indexed, inclusive
line numbers**, so the natural next call is `read_file(path, start=, end=)` to
widen it. That composition is the design: this module does not replace reading a
note, it tells you which lines to read. It stays inside Alexandria's stance that
a read returns what it says it returns (mcp/tools.py MAX_TEXT, 2026-07-19) —
nothing here truncates a read, it only ranks offsets into one.

Borrowed from obsidian-semantic-mcp (MIT, aaronsb) — the proximity-cluster idea
in `proximity-index.ts` and the overlap dedup in `adaptive-index.ts`. What is
NOT borrowed is its scoring: passages are weighted with the idf `find` already
computes over the corpus, so a passage's rank and its note's rank agree about
which terms are rare. Its own indexes re-tokenized every document on every
scoring pass and scanned the whole vocabulary per query term; this runs over one
note that `find` has already selected.

Read-only; stdlib-only.
"""
import re

from ...tokens import est_tokens
from .. import sections as _sections

# The search tokenizer. Defined here rather than in `find` because passages must
# locate exactly the tokens that ranked the note — two tokenizers that agree
# today are two tokenizers that drift (vault_core/wikilink.py's docstring records
# what that cost the link graph). `find` re-exports these.
TOKEN = re.compile(r"[a-z0-9][a-z0-9\-']*")

MAX_GAP = 40          # words between two matches that still count as one cluster
WIDTH_TOKENS = 120    # ~1 short paragraph; the per-passage size budget
TF_CAP = 3            # occurrences of one term that can score inside a passage
W_PROX = 4.0          # weight of the tightness bonus
W_PHRASE = 6.0        # the whole query string, verbatim, inside the passage
W_COVER = 2.0         # per distinct query term present (× its idf)


def tokens(s):
    return TOKEN.findall((s or "").casefold())


def _stream(text):
    """The note as a word stream → ``(words, line_of, block_of)``, three parallel
    lists. Frontmatter is skipped (it is metadata `find` already filters on);
    every other block type is searchable, code included — a command or an error
    string in a fenced block is often the thing being looked for."""
    words, line_of, block_of = [], [], []
    for bi, blk in enumerate(_sections.blocks(text)):
        if blk["kind"] == "frontmatter":
            continue
        for offset, line in enumerate(blk["text"].splitlines()):
            for w in tokens(line):
                words.append(w)
                line_of.append(blk["line_start"] + offset)
                block_of.append(bi)
    return words, line_of, block_of


def _clusters(hits, block_of, max_gap):
    """Matched word positions → clusters of positions that sit close together.

    A cluster never spans two blocks: a match in the last bullet of a list and
    one in the paragraph after it are near each other by word distance and are
    not the same passage.
    """
    out, cur = [], []
    for pos in hits:
        if cur and (pos - cur[-1] > max_gap or block_of[pos] != block_of[cur[-1]]):
            out.append(cur)
            cur = []
        cur.append(pos)
    if cur:
        out.append(cur)
    return out


def _span(lines, line_of, cluster, budget_tokens, block_lines):
    """A cluster → the (start, end) 1-indexed line span to show for it.

    The budget BOUNDS the span, it does not merely stop it growing: a cluster's
    own matched lines can already exceed the budget when matches are dense (a
    term on every line of a long list), and a passage that reports a width
    budget must honour it in that case too, not only in the sparse one. So the
    span is built outward from the FIRST match — covering as much of the cluster
    as fits, then spending anything left on surrounding context — and never
    leaves the block.

    The one guaranteed inclusion is the first matched line itself, even if that
    single line is over budget. A passage that returned nothing would be worse
    than one that returns the line you asked about.
    """
    lo_blk, hi_blk = block_lines
    matched = sorted({line_of[pos] for pos in cluster})
    lo = hi = matched[0]
    used = est_tokens(lines[lo - 1])

    for ln in matched[1:]:                       # extend over the cluster…
        cost = est_tokens("\n".join(lines[hi:ln]))
        if used + cost > budget_tokens:
            break
        hi, used = ln, used + cost

    grew = True                                  # …then spend the remainder
    while grew and used < budget_tokens:
        grew = False
        if lo > lo_blk:
            cost = est_tokens(lines[lo - 2])
            if used + cost <= budget_tokens:
                lo, used, grew = lo - 1, used + cost, True
        if hi < hi_blk and used < budget_tokens:
            cost = est_tokens(lines[hi])
            if used + cost <= budget_tokens:
                hi, used, grew = hi + 1, used + cost, True
    return lo, hi


def _score(passage_words, qtokens, idf, cluster, query_norm, passage_text):
    """Coverage (× idf) + capped density + tightness + a verbatim-phrase bonus.

    Deliberately the same *shape* as `find`'s note score — distinct-term
    coverage dominates, repetition is capped so one spammy line can't win, and
    the reason is reconstructable from the parts.
    """
    present = set(passage_words) & set(qtokens)
    score = sum(W_COVER * idf.get(t, 1.0) for t in present)
    for t in present:
        score += min(passage_words.count(t), TF_CAP) * idf.get(t, 1.0)
    for a, b in zip(cluster, cluster[1:]):       # tightness: adjacent matches
        gap = b - a
        if gap:
            score += W_PROX / gap
    if len(qtokens) > 1 and query_norm and query_norm in " ".join(tokens(passage_text)):
        score += W_PHRASE
    return score


def passages(text, query, idf=None, limit=3, width_tokens=WIDTH_TOKENS,
             max_gap=MAX_GAP):
    """The best ``limit`` passages of one note for ``query``.

    ``idf`` is ``{token: weight}`` from the corpus (``find`` passes its own); a
    missing token weighs 1.0, so this is usable standalone. Returns a list of
    ``{text, line_start, line_end, section, score, terms}`` ordered best-first,
    with scores normalized to the top passage **of this note** — a relative rank
    within the note, exactly as `find` normalizes within a result set. Never
    returns overlapping spans.

    ``section`` is the heading path the passage sits under (``None`` above the
    first heading): a passage arrives knowing WHERE in the note it is, which is
    what makes it addressable by ``note_section_patch`` as well as readable by
    ``read_file``.
    """
    qtokens = list(dict.fromkeys(tokens(query)))
    if not qtokens or not str(text or "").strip():
        return []
    idf = idf or {}
    query_norm = " ".join(qtokens)
    lines = text.splitlines()
    words, line_of, block_of = _stream(text)
    if not words:
        return []

    wanted = set(qtokens)
    hits = [i for i, w in enumerate(words) if w in wanted]
    if not hits:
        return []

    # Block extents, so a passage can grow to fill its block and stop there.
    extent = {}
    for i, bi in enumerate(block_of):
        lo, hi = extent.get(bi, (line_of[i], line_of[i]))
        extent[bi] = (min(lo, line_of[i]), max(hi, line_of[i]))

    secs = _sections.sections(text)
    scored = []
    for cluster in _clusters(hits, block_of, max_gap):
        lo, hi = _span(lines, line_of, cluster, width_tokens,
                       extent[block_of[cluster[0]]])
        body = "\n".join(lines[lo - 1:hi])
        pwords = [w for i, w in enumerate(words) if lo <= line_of[i] <= hi]
        # Score the tightness of what the passage ACTUALLY contains: a cluster
        # the budget cut short must not be credited for matches left outside.
        inside = [p for p in cluster if lo <= line_of[p] <= hi]
        # Innermost enclosing section: sections nest, so the last one whose span
        # contains the passage's first line is the deepest.
        inner = [s["path"] for s in secs if s["line_start"] <= lo <= s["line_end"]]
        scored.append({
            "text": body.strip("\n"),
            "line_start": lo, "line_end": hi,
            "section": inner[-1] if inner else None,
            "score": _score(pwords, qtokens, idf, inside, query_norm, body),
            "terms": sorted(set(pwords) & wanted),
        })

    # Highest score wins an overlap — the borrowed dedup, on line spans.
    scored.sort(key=lambda p: (-p["score"], p["line_start"]))
    kept = []
    for p in scored:
        if any(p["line_start"] <= k["line_end"] and k["line_start"] <= p["line_end"]
               for k in kept):
            continue
        kept.append(p)
        if len(kept) >= limit:
            break
    top = kept[0]["score"] if kept else 1.0
    for p in kept:
        p["score"] = round(p["score"] / top, 3) if top else 0.0
    return kept
