"""The ingest core — one routing engine behind every path content takes in.

Alexandria had **two** ingestion doctrines. The rich one lives in the vault's
``AGENTS.md`` (§ *Ingesting and filing content*) and is executed by a chat agent
that calls ``vaultq_resolve`` as a dedup guard before creating any person note.
The thin one was a prompt in ``mcp/tools.py`` that showed the filing model a
directory listing — ``fs.tree(depth=2)[:150]``, alphabetical — and asked it to
"prefer appending over near-duplicate new notes".

On a real 1730-note brain those 150 slots covered ``Agent/``…``Knowledge/`` and
stopped: **0 of 378 ``People/`` notes were visible**, while ``Daily/`` spent 54
of them on dated capture notes. The model could not know a single person
existed, so it invented near-duplicates — or guessed a path that did exist and
took the whole manifest down with it (``apply_ingest`` was all-or-nothing).

The fix is not a bigger listing. It is that **routing context must come from
retrieval, not from a directory walk**: :func:`vault_core.query.route.route`
already returns ranked candidate homes *and* resolved entities, deterministically
and with no model spend. ``entities.resolved`` **is** the dedup guard the
doctrine asks for. This module makes every ingest path use it.

Full record and measurements: ``docs/INGESTION.md``.
"""
import json
import re
from datetime import datetime, timezone

from .tokens import est_tokens_json
from .vault_core.query import find as _find
from .vault_core.query import resolve as _resolve
from .vault_core.query import route as _route

# One model reply must carry the FULL markdown of every note it proposes, so the
# output ceiling — not the input allowance — is what actually bounds an ingest.
# Before this module the two were mismatched 12×: 200k chars in, 4096 tokens out
# (≈3 notes), and anything larger truncated the JSON mid-string and surfaced as
# "the model returned no usable JSON manifest — retry", which retrying never fixed.
MANIFEST_MAX_TOKENS = 16384

# Chars of source per pass. Sized so a pass's manifest — which restates the
# content as filed markdown, plus paths and reasons — fits MANIFEST_MAX_TOKENS
# with headroom for JSON escaping.
CHUNK_CHARS = 24_000

# Ranked homes and resolved entities handed to the model per pass.
ROUTE_CANDIDATES = 10

# A short tree still helps the model name a *new* destination in the right
# pillar. It is orientation only — `route` carries the routing signal now — so
# it no longer needs to be exhaustive, which is what made it misleading.
ORIENT_TREE_PATHS = 60

_HEADING = re.compile(r"^#{1,6} ", re.M)

# The staging queue. `Agent/inbox/` (not a root `Inbox/`) and the item shape below
# are adopted wholesale from the reference vault rather than reinvented —
# that design is further along than anything written here would have been, and
# converging costs Alexandria nothing because no rendered vault had the folder yet.
# Reasoning and the rejected alternatives: docs/INGESTION.md §7.
INBOX_REL = "Agent/inbox"
INBOX_FILES_REL = INBOX_REL + "/Files"      # capital F — see stage_item's note
_SAFE_LABEL = re.compile(r"[^\w \-—.,'()&+]+", re.U)


def stage_item(*, content, label=None, manifest=None, source=None,
               author="vault-mcp", stamp=None, route=None, extra=None):
    """Build one queue item — ``(vault_rel_path, markdown)``.

    One builder for every producer (`capture_inbox`, the bulk importer) so a
    drain never has to care which one wrote an item.

    The shape is the reference vault's: `status: pending` for the drain to glob,
    a `## Manifest` holding a best-effort routing plan, and the verbatim
    `## Raw content`. Verbatim matters — the drain re-validates the plan against
    the vault as it stands *then*, and it can only do that against the original.
    """
    stamp = stamp or datetime.now(timezone.utc)
    label = _SAFE_LABEL.sub(" ", str(label or "capture")).strip()[:80] or "capture"
    fm = ["---", "type: inbox-item", "status: pending",
          f"captured: {stamp.strftime('%Y-%m-%d')}",
          f"label: {label}", f"source: {source or 'unknown'}", f"author: {author}"]
    if route:
        fm.append(f"route_band: {route.get('band')}")
        fm.append(f"route_confidence: {route.get('confidence')}")
        cands = route.get("candidates") or []
        if cands:
            fm.append("route_candidates:")
            fm += [f"  - {c['rel']}" for c in cands[:5]]
        new = (route.get("entities") or {}).get("unresolved") or []
        if new:
            fm.append("new_entities:")
            fm += [f"  - {e}" for e in new[:10]]
    for k, v in (extra or {}).items():        # a producer's own keys (e.g. federation ids)
        fm.append(f"{k}: {v}")
    fm.append("---")
    body = "\n".join(fm) + "\n\n## Manifest\n\n" + (
        (manifest or "").strip()
        or "_No routing plan captured. Re-derive it on the drain._") + \
        "\n\n## Raw content\n\n" + content.rstrip() + "\n"
    rel = f"{INBOX_REL}/{stamp.strftime('%Y-%m-%d-%H%M')} {label}.md"
    return rel, body


class IngestIndex:
    """A corpus + entity index built once and reused across many items.

    ``route()`` builds both itself when called bare — sub-second, fine for one
    item. A bulk run must hoist them or it re-walks the whole vault per item
    (O(N × corpus)).

    The overlay is the part that matters for correctness, not just speed:
    a person filed at item 3 has to be visible to item 40, or one import
    produces the very duplicates this whole change exists to prevent.
    """

    def __init__(self, vault):
        self.vault = vault
        self.notes = _find.scan(vault)
        self.idx = _resolve.build_entity_index(vault)
        self._seen = {n["rel"] for n in self.notes}

    def note_created(self, rel, *, title=None, type_="", body=""):
        """Fold a just-written note into the index so later items can find it."""
        if rel in self._seen:
            return
        self._seen.add(rel)
        title = title or rel.rsplit("/", 1)[-1][:-3]
        self.notes.append({
            "rel": rel, "title": title, "aliases": [], "type": type_,
            "status": "", "tags": [], "area": _find.area_of(rel),
            "body": body, "date": None,
        })
        # Mirror build_entity_index's tables for the one new note. Only the
        # name-side matters: an ingest-created note has no frontmatter email or
        # phone yet, and those tables are keyed on values we do not have.
        norm = _resolve.norm_name(title)
        self.idx["names"].setdefault(norm, set()).add(rel)
        self.idx["toks"].append((rel, _resolve.name_tokens(title), norm))
        self.idx["meta"][rel] = {"title": title, "type": type_, "email": "",
                                 "phone": "", "aliases": []}

    def route(self, content, limit=ROUTE_CANDIDATES):
        return _route.route(self.vault, content, limit=limit,
                            notes=self.notes, idx=self.idx)


def chunks(content, size=CHUNK_CHARS):
    """Split on structure, never mid-word where a boundary exists. Headings
    first (a markdown document files best along its own sections), then blank
    lines, then a hard cut.

    Segmentation is **byte-exact**: ``"".join(chunks(t)) == t`` for any t that
    isn't all whitespace. Every segmenter below keeps its own separators, so
    nothing is dropped or fabricated — content silently mutated on the way in
    would be indistinguishable from a bad file once it is in the vault.
    """
    content = content or ""
    if not content.strip():
        return []
    if len(content) <= size:
        return [content]
    # Prefer heading boundaries: they are where a document's own topic changes.
    starts = [m.start() for m in _HEADING.finditer(content)]
    segs = _split_at(content, starts) if starts else _paragraphs(content)
    pieces, buf = [], ""
    for seg in segs:
        if buf and len(buf) + len(seg) > size:
            pieces.append(buf)
            buf = ""
        if len(seg) > size:                      # one oversized section
            if buf:
                pieces.append(buf)
                buf = ""
            pieces.extend(seg[i:i + size] for i in range(0, len(seg), size))
            continue
        buf += seg
    if buf:
        pieces.append(buf)
    return pieces


def _split_at(text, starts):
    """Slice at the given offsets — slices, so the join is the original."""
    bounds = [0] + [s for s in starts if s > 0] + [len(text)]
    return [text[a:b] for a, b in zip(bounds, bounds[1:]) if text[a:b]]


def _paragraphs(text):
    """Split after each blank-line run, keeping the separator on the piece
    before it (so re-joining is byte-exact)."""
    return [p for p in re.split(r"(?<=\n\n)", text) if p]


SYSTEM = (
    "You are the filing engine for a PEAK-organized Obsidian vault "
    "(Areas/, People/, Events/, Knowledge/, Daily/). Given raw content "
    "plus retrieval context, produce a routing manifest as STRICT JSON — no "
    "prose, no code fence:\n"
    '{"tldr": "<one line>", "items": [{"action": "create"|"append", '
    '"path": "<vault-relative .md path>", "content": "<full markdown to write>", '
    '"reason": "<why this destination>"}]}\n'
    "\n"
    "THE RETRIEVAL CONTEXT IS AUTHORITATIVE ABOUT WHAT ALREADY EXISTS:\n"
    "- `route.entities.resolved` — each {query, rel} is an EXISTING note for a "
    "person/org named in the content. Use `action: append` on that exact `rel`. "
    "NEVER create a second note for a resolved entity; a near-duplicate person "
    "note is the single worst outcome of a bad file.\n"
    "- `route.entities.unresolved` — genuinely new entities. These are the ones "
    "that may earn a new note.\n"
    "- `route.entities.ambiguous` — do not guess. Append the fact to the best "
    "topical candidate instead of picking one of the matches.\n"
    "- `route.candidates` — ranked existing homes, with a class. Prefer "
    "appending to one of these over creating anything new.\n"
    "- `vault_tree` is a SMALL orientation sample, not the vault. Never conclude "
    "a note is absent because it is not listed there — use route.\n"
    "\n"
    "Routing by kind (decompose the input, then route each piece):\n"
    "- a person the owner personally deals with -> People/ ; someone merely "
    "referenced -> a fact on the relevant Knowledge page\n"
    "- a neutral fact about a lookup-able subject -> Knowledge/ ; the owner's "
    "own view, thesis or strategy -> the area or a Concept note\n"
    "- substance about one of the owner's responsibilities -> the area/project "
    "folder note\n"
    "- something the owner must DO -> that area's '<Area> TODO.md'\n"
    "- a major area- or owner-level dated event -> the current-year Events note\n"
    "- a meeting/call with substance -> a dated meeting note linked from the area\n"
    "\n"
    "Rules: file into existing folders when they fit; new notes get frontmatter "
    "with author: vault-mcp and a type; never touch Agent/ except Agent/memory/ and Agent/inbox/; "
    "use short-form [[wikilinks]]. If the payload has related_notes (the "
    "neighborhood a graph thought activated), prefer filing into or beside those."
)


class IngestError(RuntimeError):
    """A failure the caller should surface verbatim — it names the fix."""


def plan(svc, content, instructions=None, *, index=None, related=None,
         tier="synthesize"):
    """content -> validated manifest items, routed against real retrieval.

    Chunks oversized content and merges the per-pass manifests, because one
    reply cannot carry more note text than MANIFEST_MAX_TOKENS however much
    input it was given.
    """
    from .models import ModelProviderError

    parts = chunks(content)
    if not parts:
        raise IngestError("nothing to file — content is empty")

    router = index if index is not None else IngestIndex(svc.vault)
    tree = svc.visible_filter(_fs_tree(svc))[:ORIENT_TREE_PATHS]

    merged, tldrs, usage = [], [], []
    for part in parts:
        r = router.route(part)
        payload = {
            "content": part,
            "instructions": instructions or "",
            "route": {"band": r["band"], "confidence": r["confidence"],
                      "candidates": r["candidates"], "entities": r["entities"]},
            "vault_tree": tree,
        }
        if related:
            payload["related_notes"] = related
        try:
            reply = svc.get_provider().chat(
                [{"role": "user", "content": json.dumps(payload, ensure_ascii=False)}],
                system=SYSTEM, max_tokens=MANIFEST_MAX_TOKENS,
                model=svc.resolve_tier_model("ingest", tier))
        except ModelProviderError as e:
            raise IngestError(f"model call failed: {e}")
        items, tldr = _parse(reply, len(part))
        tldrs.append(tldr)
        usage.append(reply.usage)
        _merge(merged, items)

    return {"items": merged,
            "tldr": tldrs[0] if len(tldrs) == 1 else
                    f"{len(merged)} item(s) from {len(parts)} passes",
            "passes": len(parts), "usage": _sum_usage(usage)}


def _sum_usage(per_pass):
    """One usage dict for the whole ingest, whatever the pass count. Chunking is
    an implementation detail — a caller reading usage.input_tokens must not have
    to learn about it, and must not get a list where it expected a mapping."""
    total = {}
    for u in per_pass:
        for k, v in (u or {}).items():
            if isinstance(v, (int, float)):
                total[k] = total.get(k, 0) + v
            else:
                total.setdefault(k, v)
    return total


def _fs_tree(svc):
    from .vault_core import fs
    return fs.tree(svc.vault, depth=2)


def _parse(reply, part_chars):
    """Reply -> (items, tldr). A truncated manifest is named as such: the old
    message told the caller to retry, which could never work — the reply had hit
    the token ceiling and a retry hits the same one."""
    text = reply.unfenced()
    try:
        manifest = json.loads(text)
        items = manifest["items"]
        if not isinstance(items, list) or not items:
            raise ValueError("no items")
    except (ValueError, KeyError, TypeError):
        if est_tokens_json(text) >= MANIFEST_MAX_TOKENS * 0.9:
            raise IngestError(
                f"the filing model hit its {MANIFEST_MAX_TOKENS:,}-token output "
                f"ceiling on a {part_chars:,}-char segment, so the manifest came "
                "back truncated. Retrying will hit the same ceiling — ingest this "
                "content in smaller pieces, or file it directly with note_save.")
        raise IngestError("the model returned no usable JSON manifest — retry, "
                          "or file by hand with note_save")
    for it in items:
        if it.get("action") not in ("create", "append"):
            raise IngestError(f"manifest item has bad action {it.get('action')!r}")
        if not str(it.get("path") or "").strip():
            raise IngestError("manifest item has no path")
    return items, manifest.get("tldr") or f"{len(items)} item(s)"


def _merge(merged, items):
    """Two passes proposing the same destination must not become two writes —
    the second becomes an append onto the first."""
    by_path = {it["path"]: it for it in merged}
    for it in items:
        prior = by_path.get(it["path"])
        if prior is None:
            merged.append(it)
            by_path[it["path"]] = it
            continue
        body = (it.get("content") or "").strip()
        if body:
            prior["content"] = (prior.get("content") or "").rstrip() + "\n\n" + body
