#!/usr/bin/env python3
"""
Diverse-source news fetcher (read-only)
=======================================
Pulls headlines from the vault's source registry
(``Agent/scripts/read-news/sources.json``) so the vault agent can synthesize a
world-news briefing in which every story is labeled corroborated vs
single-source. The `news` skill drives this; a person rarely runs it by hand.

The registry is Config — seeded by the news module, then the owner's to edit.
Each source is tagged on the diversity axes a briefing must cover (geography,
funding model, editorial lean, domain), because the defense against a narrowed
or manipulated picture of the world is structural — many independent newsrooms
fetched side by side — not editorial.

The script is deliberately dumb: it fetches every feed concurrently, normalizes
RSS 2.0 / Atom / RDF into one item shape, filters to a time window, and prints
a human summary or ``--json`` for the agent to synthesize. It does not rank,
dedupe, or editorialize — cross-source corroboration is judgment, so it lives
in the skill (the agent), not here. What the script *does* own is the coverage
report: any source that couldn't be fetched is listed explicitly, the JSON
carries a ``coverage`` block naming which diversity axes actually responded,
and every fetch run prints a one-line stderr coverage summary — because a
silently missing source is a silent narrowing of the picture (the repo rule:
a reader must report its coverage; a silent cap is a lie about completeness).

HOW MUCH TEXT COMES BACK
------------------------
Three fields, shortest to longest, so a caller can spend tokens deliberately:

    summary   the feed's blurb (RSS <description> / Atom <summary>), capped at
              --summary-chars (default 4000 — generous enough that it should
              never bite in practice; some feeds legitimately leave it empty).
    content   the publisher's own body text when the *feed* carries it
              (RSS <content:encoded> / Atom <content>). Only some feeds do, and
              it is only kept when it beats `summary`, so a feed that merely
              repeats its description doesn't pay for the duplicate.
    body      the article page's own text, fetched and extracted — only with
              --full-text or --article, never for free.

Body extraction is stdlib and two-pronged: schema.org `articleBody` out of
<script type="application/ld+json">, and a paragraph harvest off the stripped
HTML — whichever is longer wins, since publishers truncate whichever copy they
consider the machine-readable one. Some publishers answer 401/403 to any
non-browser client — that is a paywall, not a bug, and it is reported per item
as `body_error`, never silently blanked. Fetching article pages is opt-in
precisely because it is the expensive, impolite option: one HTTP request per
item against the publisher. --max-articles bounds it, and whatever the bound
skips is reported, never silently dropped.

Feeds are rolling windows (typically 24-72h of items): a --since beyond a
couple of days cannot reach further back than the feed itself does. Judge a
catch-up by what actually came back — the coverage report — never by the
window you asked for.

Optional config — read from Agent/config.json under the `news` key, all
optional (built-in defaults apply when absent):
    news.default_since     window when none is asked for   (default "24h")
    news.max_per_source    per-source item cap             (default 15)
    news.sources_file      path to the source registry
                           (default <vault>/Agent/scripts/read-news/sources.json)

Usage:
    python -m alexandria_vault.engine.news_read --vault .              # default window, human summary
    python -m alexandria_vault.engine.news_read --vault . --since 2d --json
    python -m alexandria_vault.engine.news_read --vault . --source bbc-world --source npr-news
    python -m alexandria_vault.engine.news_read --vault . --list-sources
    python -m alexandria_vault.engine.news_read --vault . --article "<url>" --json
    python -m alexandria_vault.engine.news_read --vault . --full-text --max-articles 20

Read-only against the world: fetches public feeds and pages, never posts,
subscribes, or signs in. Needs the network at fetch time; no accounts, no
stored credentials. Python 3.9+, standard library only.
"""
import argparse
import concurrent.futures
import gzip
import html
import json
import re
import ssl
import sys
import urllib.request
from datetime import datetime, timedelta, timezone
from email.utils import parsedate_to_datetime
from pathlib import Path
from xml.etree import ElementTree

from alexandria_vault.engine import consume_vault_arg, resolve_vault
from alexandria_vault.vault_core import load_config as _vc_load_config

# `--vault` must land in VAULT_DIR before the module-level config load below.
consume_vault_arg()

USER_AGENT = "Mozilla/5.0 (Macintosh) alexandria-news/1.0 (personal news briefing)"
FETCH_TIMEOUT = 20

# Article pages, unlike feeds, are served to browsers; several publishers 403 an
# honest bot string. This is the standard desktop UA — we read the public page a
# reader would get, nothing more (no login, no paywall circumvention).
ARTICLE_USER_AGENT = (
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
)
ARTICLE_TIMEOUT = 20
MAX_ARTICLE_BYTES = 4_000_000   # a news page is <1 MB; this only stops pathology

SUMMARY_CHARS = 4000            # above the longest blurb any observed feed sends
BODY_CHARS = 20000              # above the longest body any observed feed/page sends
MAX_ARTICLES = 40               # default ceiling on --full-text page fetches

# The diversity axes a registry entry is tagged on, and the coverage report reports on.
AXES = ("geography", "funding", "lean", "domain")

ATOM_NS = "{http://www.w3.org/2005/Atom}"
RDF_ITEM_NS = "{http://purl.org/rss/1.0/}"
DC_DATE = "{http://purl.org/dc/elements/1.1/}date"
MEDIA_DESC = "{http://search.yahoo.com/mrss/}description"
CONTENT_ENCODED = "{http://purl.org/rss/1.0/modules/content/}encoded"

TAG_RE = re.compile(r"<[^>]+>")
WS_RE = re.compile(r"\s+")

# Browser-free article extraction. Drop the furniture first, then read what's left.
BOILERPLATE_RE = re.compile(
    r"(?is)<(script|style|noscript|svg|form|nav|header|footer|aside|figure|iframe|template)\b.*?</\1>"
)
LD_JSON_RE = re.compile(
    r'(?is)<script[^>]+type=["\']application/ld\+json["\'][^>]*>(.*?)</script>'
)
PARAGRAPH_RE = re.compile(r"(?is)<p\b[^>]*>(.*?)</p>")
CHARSET_RE = re.compile(r"charset=([\w-]+)", re.I)
MIN_PARAGRAPH_CHARS = 40        # below this it's a caption, byline or nav crumb


# ── config (Core-reads-Config seam; personalization never hardcoded here) ─────
def _load_config():
    """Read optional `news` tuning from the vault's Config (vault_core: the one
    locator + loader). A missing file or key is fine — built-in defaults apply.
    No identity is read here; this only tunes defaults (window, per-source cap,
    registry path)."""
    cfg = _vc_load_config(resolve_vault())
    return cfg.get("news", {}) if isinstance(cfg, dict) else {}


NEWS = _load_config()
DEFAULT_SINCE = NEWS.get("default_since", "24h")
try:
    DEFAULT_MAX_PER_SOURCE = int(NEWS.get("max_per_source", 15))
except (TypeError, ValueError):
    DEFAULT_MAX_PER_SOURCE = 15
SOURCES_PATH = (Path(NEWS["sources_file"]).expanduser() if NEWS.get("sources_file")
                else resolve_vault() / "Agent" / "scripts" / "read-news" / "sources.json")


def load_sources():
    """The source registry, from the vault's Config. Absence is a hard, named
    error — the fetcher without its registry would silently cover nothing."""
    if not SOURCES_PATH.is_file():
        raise SystemExit(
            "source registry not found at %s\n"
            "The news module seeds this file (Agent/scripts/read-news/sources.json in the "
            "vault — Config bucket, yours to edit, never overwritten on update). Re-render "
            "the news module, or restore the file, then re-run." % SOURCES_PATH)
    try:
        data = json.loads(SOURCES_PATH.read_text(encoding="utf-8"))
    except ValueError as e:
        raise SystemExit("source registry at %s is not valid JSON: %s" % (SOURCES_PATH, e))
    sources = data.get("sources") if isinstance(data, dict) else None
    if not isinstance(sources, list) or not sources:
        raise SystemExit("source registry at %s has no non-empty 'sources' list" % SOURCES_PATH)
    for s in sources:
        if not isinstance(s, dict) or not s.get("id") or not s.get("url"):
            raise SystemExit(
                "source registry at %s: every source needs at least 'id' and 'url' "
                "(and should carry the diversity axes: %s)" % (SOURCES_PATH, ", ".join(AXES)))
    return sources


def parse_window(spec):
    m = re.fullmatch(r"(\d+)([hdw])", spec)
    if not m:
        sys.exit("bad --since value %r; use e.g. 24h, 2d, 1w — or 'all' for no filter" % spec)
    n, unit = int(m.group(1)), m.group(2)
    hours = n * {"h": 1, "d": 24, "w": 168}[unit]
    return timedelta(hours=hours)


def clean_text(s, limit=SUMMARY_CHARS):
    """Markup out, whitespace normalized, truncated only as a backstop.

    `limit=0` means no limit — the caller wants everything the publisher sent."""
    if not s:
        return ""
    s = html.unescape(TAG_RE.sub(" ", s))
    s = WS_RE.sub(" ", s).strip()
    if limit and len(s) > limit:
        return s[: limit - 1] + "…"
    return s


def parse_date(raw):
    """RFC 2822 (RSS) or ISO-8601 (Atom/RDF) → aware datetime, else None.
    A naive stamp is taken as UTC so window comparison never raises."""
    if not raw:
        return None
    raw = raw.strip()
    try:
        parsed = parsedate_to_datetime(raw)  # RFC 2822 (RSS)
    except (TypeError, ValueError):
        parsed = None
    if parsed is None:
        try:
            parsed = datetime.fromisoformat(raw.replace("Z", "+00:00"))  # ISO (Atom/RDF)
        except ValueError:
            return None
    return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)


def first_text(el, *tags):
    for tag in tags:
        child = el.find(tag)
        if child is not None and (child.text or "").strip():
            return child.text.strip()
    return ""


def extract_items(root):
    """Yield (title, link, date_raw, summary, content) from RSS 2.0, Atom, or RDF.

    `summary` is the short blurb, `content` the publisher's fuller body when the
    feed carries one — kept apart rather than coalesced so a caller can choose
    how much text to pay for, and so `content` staying empty is legible as "this
    feed ships no body" rather than looking like an extraction failure."""
    # RSS 2.0: <rss><channel><item>; RDF: items are siblings of <channel>
    for item in root.iter():
        if item.tag == "item" or item.tag == RDF_ITEM_NS + "item":
            yield (
                first_text(item, "title", RDF_ITEM_NS + "title"),
                first_text(item, "link", RDF_ITEM_NS + "link"),
                first_text(item, "pubDate", DC_DATE),
                first_text(item, "description", RDF_ITEM_NS + "description", MEDIA_DESC),
                first_text(item, CONTENT_ENCODED),
            )
        elif item.tag == ATOM_NS + "entry":
            link = ""
            for l in item.findall(ATOM_NS + "link"):
                if l.get("rel", "alternate") == "alternate":
                    link = l.get("href", "")
                    break
            yield (
                first_text(item, ATOM_NS + "title"),
                link,
                first_text(item, ATOM_NS + "published", ATOM_NS + "updated"),
                first_text(item, ATOM_NS + "summary"),
                first_text(item, ATOM_NS + "content"),
            )


def fetch_source(source, cutoff, max_items, summary_chars=SUMMARY_CHARS, body_chars=BODY_CHARS):
    req = urllib.request.Request(source["url"], headers={"User-Agent": USER_AGENT})
    ctx = ssl.create_default_context()
    with urllib.request.urlopen(req, timeout=FETCH_TIMEOUT, context=ctx) as resp:
        body = resp.read()
    root = ElementTree.fromstring(body)
    items = []
    for title, link, date_raw, summary, content in extract_items(root):
        if not title:
            continue
        when = parse_date(date_raw)
        if cutoff and when and when < cutoff:
            continue
        summary = clean_text(summary, summary_chars)
        content = clean_text(content, body_chars)
        items.append(
            {
                "title": clean_text(title, 200),
                "link": link,
                "published": when.isoformat() if when else None,
                "summary": summary,
                # Some feeds echo the description into content:encoded verbatim.
                # Only keep it when it actually adds text.
                "content": content if len(content) > len(summary) else "",
            }
        )
    # newest first; undated items sink to the end rather than being dropped
    items.sort(key=lambda i: i["published"] or "", reverse=True)
    return items[:max_items]


# ── Article body extraction ──────────────────────────────────────────────────
# Feeds are blurbs. When the briefing needs what an article actually *said*, this
# reads the public page — the same bytes a reader's browser gets. Opt-in only.


def _decode(raw, content_type):
    """Bytes → str, honouring the declared charset, never raising on bad bytes."""
    m = CHARSET_RE.search(content_type or "")
    for enc in ([m.group(1)] if m else []) + ["utf-8", "latin-1"]:
        try:
            return raw.decode(enc)
        except (LookupError, UnicodeDecodeError):
            continue
    return raw.decode("utf-8", "replace")


def _ld_json_bodies(doc):
    """schema.org articleBody values, wherever they're nested in the graph."""
    def walk(node):
        if isinstance(node, dict):
            body = node.get("articleBody")
            if isinstance(body, str):
                yield body
            for value in node.values():
                yield from walk(value)
        elif isinstance(node, list):
            for value in node:
                yield from walk(value)

    for match in LD_JSON_RE.finditer(doc):
        try:
            data = json.loads(match.group(1).strip())
        except (json.JSONDecodeError, ValueError):
            continue
        yield from walk(data)


def _paragraph_body(doc):
    """Furniture-stripped <p> harvest — the fallback when there's no ld+json."""
    stripped = BOILERPLATE_RE.sub(" ", doc)
    paragraphs = (clean_text(m.group(1), 0) for m in PARAGRAPH_RE.finditer(stripped))
    return " ".join(p for p in paragraphs if len(p) >= MIN_PARAGRAPH_CHARS)


def fetch_article(url, body_chars=BODY_CHARS):
    """Fetch one article page and return {body, body_source} or {body_error}.

    Never raises: a paywall, a timeout or a redesigned template costs one item's
    body, not the run. Both extraction methods are tried and the longer wins —
    publishers truncate whichever one they consider the machine-readable copy."""
    if not url:
        return {"body": "", "body_source": None, "body_error": "no link"}
    try:
        req = urllib.request.Request(
            url,
            headers={
                "User-Agent": ARTICLE_USER_AGENT,
                "Accept": "text/html,application/xhtml+xml",
                "Accept-Language": "en",
                "Accept-Encoding": "gzip",
            },
        )
        ctx = ssl.create_default_context()
        with urllib.request.urlopen(req, timeout=ARTICLE_TIMEOUT, context=ctx) as resp:
            raw = resp.read(MAX_ARTICLE_BYTES)
            if resp.headers.get("Content-Encoding") == "gzip":
                raw = gzip.decompress(raw)
            doc = _decode(raw, resp.headers.get("Content-Type"))
    except Exception as e:                      # network, TLS, HTTP, decompression
        return {"body": "", "body_source": None, "body_error": "%s: %s" % (type(e).__name__, e)}

    candidates = [("ld+json", clean_text(b, 0)) for b in _ld_json_bodies(doc)]
    candidates.append(("paragraphs", _paragraph_body(doc)))
    source, body = max(candidates, key=lambda c: len(c[1]))
    if not body:
        return {"body": "", "body_source": None, "body_error": "no article text found"}
    if body_chars and len(body) > body_chars:
        body = body[: body_chars - 1] + "…"
    return {"body": body, "body_source": source, "body_error": None}


def add_bodies(items, body_chars=BODY_CHARS, max_articles=MAX_ARTICLES, workers=8):
    """Attach `body` to every item in place, in the order given. Returns how many
    the cap left un-fetched — the caller must report it, because a silent cap
    reads as 'we looked at everything' when we didn't.

    Every item comes back with the body fields set, even the ones that were
    skipped or had no link, so a missing body always carries its own reason."""
    linked = [i for i in items if i.get("link")]
    wanted = linked[:max_articles] if max_articles else linked
    skipped = linked[len(wanted):]

    for item in items:
        item.update({"body": "", "body_source": None, "body_error": "no link"})
    for item in skipped:
        item["body_error"] = "skipped (--max-articles)"

    with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool:
        futures = {pool.submit(fetch_article, i["link"], body_chars): i for i in wanted}
        for fut in concurrent.futures.as_completed(futures):
            futures[fut].update(fut.result())
    return len(skipped)


# ── coverage reporting (stderr, so --json stdout stays parseable) ────────────
def warn(msg):
    print("news_read: %s" % msg, file=sys.stderr)


def report_coverage(sources, results, failures, window):
    """One stderr line stating what the run actually covered — items, sources,
    and how much of each diversity axis responded — plus the failed source ids,
    so a quietly narrowed picture can't pass for a full sweep."""
    total_items = sum(len(r["items"]) for r in results)
    axes = []
    for axis in AXES:
        registry = {s.get(axis) for s in sources if s.get(axis)}
        responded = {r.get(axis) for r in results if r["items"] and r.get(axis)}
        axes.append("%s %d/%d" % (axis, len(responded), len(registry)))
    line = "window %s → %d items from %d/%d sources · axes: %s" % (
        window, total_items, len(results), len(sources), ", ".join(axes))
    if failures:
        line += " · FAILED %d: %s" % (
            len(failures), ", ".join(sorted(str(f.get("id", "?")) for f in failures)))
    warn(line)


def main():
    ap = argparse.ArgumentParser(description="Fetch headlines from the vault's diverse source registry.")
    ap.add_argument("--since", default=DEFAULT_SINCE,
                    help="window: e.g. 24h, 2d, 1w; 'all' for no filter (default %s)" % DEFAULT_SINCE)
    ap.add_argument("--max-per-source", type=int, default=DEFAULT_MAX_PER_SOURCE,
                    help="cap items per source (default %d)" % DEFAULT_MAX_PER_SOURCE)
    ap.add_argument("--source", action="append", help="only these source ids (repeatable)")
    ap.add_argument("--json", action="store_true", help="machine-readable output")
    ap.add_argument("--list-sources", action="store_true", help="print the registry and exit")
    ap.add_argument("--summary-chars", type=int, default=SUMMARY_CHARS,
                    help="cap on the feed blurb (default %d; 0 = uncapped)" % SUMMARY_CHARS)
    ap.add_argument("--body-chars", type=int, default=BODY_CHARS,
                    help="cap on feed content + fetched article body (default %d; 0 = uncapped)" % BODY_CHARS)
    ap.add_argument("--full-text", action="store_true",
                    help="also fetch each article page and extract its body (one request per item)")
    ap.add_argument("--max-articles", type=int, default=MAX_ARTICLES,
                    help="ceiling on --full-text page fetches, newest first (default %d; 0 = no ceiling)" % MAX_ARTICLES)
    ap.add_argument("--article", action="append", metavar="URL",
                    help="skip the feeds: fetch and extract these article URLs (repeatable)")
    args = ap.parse_args()

    # --article is the "tell me more about X" path: the agent has already picked
    # its cluster and just wants the text behind a handful of links.
    if args.article:
        articles = [{"link": url} for url in args.article]
        add_bodies(articles, args.body_chars, max_articles=0)
        if args.json:
            print(json.dumps({"articles": articles}, indent=2))
        else:
            for a in articles:
                if a["body_error"]:
                    print("✗ %s\n   %s\n" % (a["link"], a["body_error"]))
                else:
                    print("── %s  [%s, %d chars]\n%s\n"
                          % (a["link"], a["body_source"], len(a["body"]), a["body"]))
        warn("articles %d → %d bodies, %d failed" % (
            len(articles),
            sum(1 for a in articles if a.get("body")),
            sum(1 for a in articles if a.get("body_error"))))
        return

    sources = load_sources()
    if args.list_sources:
        for s in sources:
            print("%-14s %-32s %s · %s · %s · %s" % (
                s["id"], s.get("name", s["id"]), s.get("geography", "?"),
                s.get("funding", "?"), s.get("lean", "?"), s.get("domain", "?")))
        return
    if args.source:
        unknown = set(args.source) - {s["id"] for s in sources}
        if unknown:
            sys.exit("unknown source id(s): %s" % ", ".join(sorted(unknown)))
        sources = [s for s in sources if s["id"] in args.source]

    now = datetime.now(timezone.utc)
    cutoff = None if args.since == "all" else now - parse_window(args.since)

    results, failures = [], []
    with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
        futures = {
            pool.submit(
                fetch_source, s, cutoff, args.max_per_source, args.summary_chars, args.body_chars
            ): s
            for s in sources
        }
        for fut in concurrent.futures.as_completed(futures):
            s = futures[fut]
            try:
                items = fut.result()
                results.append({**s, "items": items})
            except Exception as e:
                failures.append({**s, "error": "%s: %s" % (type(e).__name__, e)})

    results.sort(key=lambda r: r["id"])

    # Bodies are fetched across the whole run, newest first, so the cap spends its
    # budget on the freshest stories rather than on whichever source sorted first.
    full_text = None
    if args.full_text:
        every_item = [i for r in results for i in r["items"]]
        every_item.sort(key=lambda i: i["published"] or "", reverse=True)
        skipped = add_bodies(every_item, args.body_chars, args.max_articles)
        full_text = {
            "requested": len(every_item),
            "fetched": sum(1 for i in every_item if i.get("body")),
            "failed": sum(1 for i in every_item
                          if i.get("body_error") and i["body_error"] != "skipped (--max-articles)"),
            "skipped_by_cap": skipped,
        }

    def covered(axis):
        return sorted({r.get(axis) for r in results if r["items"] and r.get(axis)})

    if args.json:
        print(
            json.dumps(
                {
                    "fetched_at": now.isoformat(),
                    "window": args.since,
                    "sources_ok": len(results),
                    "sources_failed": len(failures),
                    "coverage": {axis: covered(axis) for axis in AXES},
                    "full_text": full_text,
                    "failures": failures,
                    "results": results,
                },
                indent=2,
            )
        )
        report_coverage(sources, results, failures, args.since)
        return

    total = sum(len(r["items"]) for r in results)
    print("news: %d items from %d/%d sources (window %s)\n"
          % (total, len(results), len(sources), args.since))
    for r in results:
        print("── %s  [%s · %s · %s · %s]" % (
            r.get("name", r["id"]), r.get("geography", "?"), r.get("funding", "?"),
            r.get("lean", "?"), r.get("domain", "?")))
        if not r["items"]:
            print("   (nothing in window)")
        for i in r["items"]:
            stamp = (i["published"] or "")[:16].replace("T", " ")
            text = len(i.get("body") or i.get("content") or i.get("summary") or "")
            print("   • %s  (%s, %d chars)" % (i["title"], stamp, text))
        print()
    if full_text:
        print("full text: %d bodies fetched, %d unavailable (paywall/block), "
              "%d skipped by --max-articles\n"
              % (full_text["fetched"], full_text["failed"], full_text["skipped_by_cap"]))
    if failures:
        print("⚠ FAILED SOURCES — the picture is narrower than the registry intends:")
        for f in failures:
            print("   ✗ %s (%s): %s" % (f.get("name", f.get("id", "?")),
                                        f.get("geography", "?"), f["error"]))
    report_coverage(sources, results, failures, args.since)


if __name__ == "__main__":
    main()
