#!/usr/bin/env python3
"""Omega UI search inventory. Python 3.10+, standard library only.

Credentials are read only from environment variables. `audit` makes no changes.
`verify-dns` and `submit` require --apply to write to Cloudflare or Google.
"""
import argparse
import csv
import ipaddress
import json
import os
import re
import socket
import sys
import urllib.error
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timezone
from pathlib import Path

DOMAINS = (
    "syncloudconnect.com", "syncloudconnect.store", "syncloudconnect.info",
    "omegaui.com", "syncloudconnect.org", "ucrash.cash",
)
GITHUB_USER = "abc3rd"
UA = "OmegaUI-SearchKit/1.0 (+https://omega-ui-workbench.syncloudconnect.info/search-setup)"
MAX_HOSTS = 300
MAX_SITEMAP_BYTES = 2_000_000


def api(url, token=None, method="GET", payload=None, timeout=15):
    headers = {"User-Agent": UA, "Accept": "application/json"}
    if token:
        headers["Authorization"] = "Bearer " + token
    data = json.dumps(payload).encode() if payload is not None else None
    if data is not None:
        headers["Content-Type"] = "application/json"
    req = urllib.request.Request(url, data=data, method=method, headers=headers)
    try:
        with urllib.request.urlopen(req, timeout=timeout) as r:
            body = r.read(4_000_000)
            return json.loads(body) if body else {}
    except urllib.error.HTTPError as e:
        detail = e.read(500).decode("utf-8", "replace")
        raise RuntimeError(f"HTTP {e.code} from {urllib.parse.urlsplit(url).netloc}: {detail[:180]}") from None


def cloudflare_zone(domain, token):
    data = api("https://api.cloudflare.com/client/v4/zones?" + urllib.parse.urlencode({"name": domain, "per_page": 50}), token)
    matches = [z for z in data.get("result", []) if z.get("name") == domain]
    return matches[0] if matches else None


def cloudflare_records(zone_id, token):
    found = []
    for page in range(1, 51):
        data = api(f"https://api.cloudflare.com/client/v4/zones/{zone_id}/dns_records?per_page=100&page={page}", token)
        found.extend(data.get("result", []))
        if page >= data.get("result_info", {}).get("total_pages", 1):
            break
    return found


def github_pages(token):
    headers_token = token or None
    sites = []
    for page in range(1, 6):
        repos = api(f"https://api.github.com/users/{GITHUB_USER}/repos?per_page=100&page={page}", headers_token)
        if not isinstance(repos, list):
            break
        for repo in repos:
            if repo.get("private") or not repo.get("has_pages"):
                continue
            name = repo.get("name", "")
            if not name:
                continue
            default = (f"https://{GITHUB_USER}.github.io/" if name.lower() == f"{GITHUB_USER}.github.io" else f"https://{GITHUB_USER}.github.io/{name}/")
            try:
                info = api(f"https://api.github.com/repos/{GITHUB_USER}/{urllib.parse.quote(name)}/pages", headers_token)
                candidate = info.get("html_url") or default
            except Exception:
                candidate = default
            sites.append({"url": candidate, "source": "GitHub Pages", "repo": repo.get("html_url", "")})
        if len(repos) < 100:
            break
    return sites


def safe_public_target(host):
    if not host or host.lower() in ("localhost", "localhost.localdomain"):
        return False
    try:
        literal = ipaddress.ip_address(host)
        return literal.is_global
    except ValueError:
        pass
    try:
        addrs = {info[4][0] for info in socket.getaddrinfo(host, 443, type=socket.SOCK_STREAM)}
        return all(ipaddress.ip_address(addr).is_global for addr in addrs)
    except OSError:
        # An HTTP proxy may resolve the public host even when local DNS is blocked.
        return True


def fetch_text(url, limit=MAX_SITEMAP_BYTES):
    req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept": "text/html,application/xml,text/xml,text/plain;q=0.8,*/*;q=0.5"})
    try:
        with urllib.request.urlopen(req, timeout=8) as response:
            final = response.geturl()
            host = urllib.parse.urlsplit(final).hostname or ""
            if urllib.parse.urlsplit(final).scheme != "https" or not safe_public_target(host):
                return 0, "", final, {}, "Unsafe or non-HTTPS redirect"
            data = response.read(limit + 1)
            if len(data) > limit:
                return response.status, "", final, dict(response.headers), "Response exceeds limit"
            return response.status, data.decode("utf-8", "replace"), final, dict(response.headers), ""
    except urllib.error.HTTPError as e:
        return e.code, "", url, dict(e.headers), "HTTP error"
    except Exception as e:
        return 0, "", url, {}, type(e).__name__


def sitemap_urls(xml_text, base_host):
    try:
        root = ET.fromstring(xml_text)
    except ET.ParseError:
        return [], []
    kind = root.tag.rsplit("}", 1)[-1]
    locs = [x.text.strip() for x in root.iter() if x.tag.rsplit("}", 1)[-1] == "loc" and x.text]
    safe = [u for u in locs if urllib.parse.urlsplit(u).scheme == "https" and urllib.parse.urlsplit(u).hostname == base_host]
    return (safe[:5000], []) if kind == "urlset" else ([], safe[:20] if kind == "sitemapindex" else [])


def inspect_site(entry):
    start = entry["url"].rstrip("/") + "/"
    host = urllib.parse.urlsplit(start).hostname or ""
    row = {"url": start, "source": entry["source"], "repo": entry.get("repo", ""), "status": 0,
           "final_url": "", "title": "", "canonical": "", "indexable_candidate": False,
           "sitemaps": [], "sitemap_page_count": 0, "issues": []}
    if not safe_public_target(host):
        row["issues"].append("Unsafe private destination")
        return row, []
    status, html, final, headers, error = fetch_text(start)
    row["status"], row["final_url"] = status, final
    if error:
        row["issues"].append(error)
    if status != 200:
        row["issues"].append(f"Homepage HTTP {status or 'unreachable'}")
    if status == 200 and "text/html" not in headers.get("Content-Type", "").lower():
        row["issues"].append("Homepage is not HTML")
    m = re.search(r"<title\b[^>]*>(.*?)</title>", html, re.I | re.S)
    row["title"] = re.sub(r"\s+", " ", re.sub(r"<[^>]+>", "", m.group(1))).strip()[:160] if m else ""
    if not row["title"]:
        row["issues"].append("Missing HTML title")
    m = re.search(r'<link\b(?=[^>]*rel=["\']canonical["\'])[^>]*href=["\']([^"\']+)', html, re.I)
    row["canonical"] = m.group(1) if m else ""
    if not row["canonical"]:
        row["issues"].append("No canonical link found")
    robots_meta = " ".join(re.findall(r'<meta\b(?=[^>]*name=["\']robots["\'])[^>]*>', html, re.I))
    noindex = "noindex" in (robots_meta + headers.get("X-Robots-Tag", "")).lower()
    if noindex:
        row["issues"].append("Noindex directive")
    _, robots, _, _, _ = fetch_text(f"https://{host}/robots.txt", 100_000)
    if re.search(r"(?im)^disallow:\s*/\s*$", robots):
        row["issues"].append("Robots.txt may block the whole site; inspect user-agent groups")
    candidates = re.findall(r"(?im)^sitemap:\s*(\S+)", robots)
    candidates += [f"https://{host}/sitemap.xml", f"https://{host}/sitemap_index.xml"]
    pages = []
    for sitemap in dict.fromkeys(candidates):
        if urllib.parse.urlsplit(sitemap).hostname != host:
            continue
        code, xml, _, _, _ = fetch_text(sitemap)
        if code != 200 or not ("<urlset" in xml or "<sitemapindex" in xml):
            continue
        row["sitemaps"].append(sitemap)
        found, nested = sitemap_urls(xml, host)
        pages.extend(found)
        for child in nested[:10]:
            c, body, _, _, _ = fetch_text(child)
            if c == 200:
                subpages, _ = sitemap_urls(body, host)
                pages.extend(subpages)
    pages = list(dict.fromkeys(pages))[:5000]
    row["sitemap_page_count"] = len(pages)
    if not row["sitemaps"]:
        row["issues"].append("No readable XML sitemap found")
    row["indexable_candidate"] = status == 200 and not noindex and "text/html" in headers.get("Content-Type", "").lower() and not any("block the whole" in x for x in row["issues"])
    return row, pages


def audit(args):
    cf_token = os.environ.get("CLOUDFLARE_API_TOKEN")
    candidates = [{"url": f"https://{domain}/", "source": "Owned domain"} for domain in DOMAINS]
    errors = []
    if args.max_hosts < 1 or args.max_hosts > 2000:
        raise RuntimeError("--max-hosts must be between 1 and 2000")
    if cf_token:
        for domain in DOMAINS:
            try:
                zone = cloudflare_zone(domain, cf_token)
                if not zone:
                    errors.append(f"Cloudflare zone not found: {domain}")
                    continue
                records = cloudflare_records(zone["id"], cf_token)
                names = {r.get("name", "").rstrip(".").lower() for r in records if r.get("type") in ("A", "AAAA", "CNAME")}
                for name in sorted(names):
                    if name == domain or not name.endswith("." + domain) or "*" in name or "_" in name:
                        continue
                    candidates.append({"url": f"https://{name}/", "source": f"Cloudflare DNS: {domain}"})
            except Exception as e:
                errors.append(f"Cloudflare {domain}: {e}")
    else:
        errors.append("Set CLOUDFLARE_API_TOKEN for a complete Cloudflare DNS inventory; this run checks six apex domains and public GitHub Pages.")
    try:
        candidates.extend(github_pages(os.environ.get("GITHUB_TOKEN")))
    except Exception as e:
        errors.append(f"GitHub Pages discovery: {e}")
    candidates = list({x["url"].rstrip("/") + "/": x for x in candidates}.values())
    if len(candidates) > args.max_hosts:
        errors.append(f"Capped crawl at {args.max_hosts} of {len(candidates)} candidates; rerun with --max-hosts to check more.")
    candidates = candidates[:args.max_hosts]
    checked = []
    with ThreadPoolExecutor(max_workers=8) as pool:
        futures = {pool.submit(inspect_site, candidate): candidate for candidate in candidates}
        for future in as_completed(futures):
            checked.append(future.result())
            row = checked[-1][0]
            print(f"[{len(checked)}/{len(candidates)}] {row['url']} — HTTP {row['status'] or 'unreachable'}", flush=True)
    checked.sort(key=lambda result: result[0]["url"])
    rows = [x[0] for x in checked]
    page_urls = sorted(set(u for _, urls in checked for u in urls))
    out = Path(args.out).expanduser().resolve()
    out.mkdir(parents=True, exist_ok=True)
    report = {"generated_at": datetime.now(timezone.utc).isoformat(), "domains": DOMAINS, "sites": rows, "errors": errors}
    (out / "report.json").write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
    with (out / "sites.csv").open("w", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(f, fieldnames=["url", "source", "repo", "status", "final_url", "title", "canonical", "indexable_candidate", "sitemaps", "sitemap_page_count", "issues"])
        writer.writeheader()
        for row in rows:
            writer.writerow({**row, "sitemaps": "; ".join(row["sitemaps"]), "issues": "; ".join(row["issues"])})
    (out / "sitemap-pages.txt").write_text("\n".join(page_urls) + ("\n" if page_urls else ""), encoding="utf-8")
    action_lines = ["# Omega UI search actions", "", "Technical checks identify *candidates* for indexing, not Google index status.", "", "## Verify these six Search Console Domain properties", ""]
    action_lines += [f"- `sc-domain:{d}` — verify with the DNS TXT token Google gives you" for d in DOMAINS]
    action_lines += ["", "## Review and submit sitemaps", ""]
    for row in rows:
        if row["indexable_candidate"] and row["sitemaps"]:
            action_lines.append(f"- {row['url']} → {', '.join(row['sitemaps'])}")
    action_lines += ["", "## Fix before expecting search visibility", ""]
    for row in rows:
        if row["issues"]:
            action_lines.append(f"- {row['url']}: {', '.join(row['issues'])}")
    action_lines += ["", "GitHub Pages project URLs require a verified URL-prefix property (or a verified custom domain) before sitemap submission. No script can guarantee Google indexing or ranking."]
    action_lines += ["", "## Inventory gaps", ""] + [f"- {x}" for x in errors]
    (out / "actions.md").write_text("\n".join(action_lines) + "\n", encoding="utf-8")
    print(f"Checked {len(rows)} sites; found {len(page_urls)} sitemap page URLs. Open {out / 'actions.md'} and {out / 'sites.csv'}.")


def verify_dns(args):
    token = os.environ.get("CLOUDFLARE_API_TOKEN")
    if not token:
        raise RuntimeError("Set CLOUDFLARE_API_TOKEN with Zone Read and DNS Read/Edit permissions for the six zones")
    values = json.loads(Path(args.tokens).read_text(encoding="utf-8"))
    if not isinstance(values, dict) or set(values) != set(DOMAINS):
        raise RuntimeError("Token JSON must map exactly the six listed domains to their Google TXT values")
    for domain in DOMAINS:
        value = values[domain]
        if not isinstance(value, str) or not re.fullmatch(r"google-site-verification=[A-Za-z0-9_-]{10,200}", value):
            raise RuntimeError(f"Invalid Google verification TXT value for {domain}")
        zone = cloudflare_zone(domain, token)
        if not zone:
            raise RuntimeError(f"Cloudflare zone not found: {domain}")
        records = cloudflare_records(zone["id"], token)
        exists = any(r.get("type") == "TXT" and r.get("name", "").rstrip(".") == domain and r.get("content", "").strip('"') == value for r in records)
        if exists:
            print(f"{domain}: Google TXT already present")
        elif args.apply:
            result = api(f"https://api.cloudflare.com/client/v4/zones/{zone['id']}/dns_records", token, "POST", {"type": "TXT", "name": domain, "content": value, "ttl": 1})
            if not result.get("success"):
                raise RuntimeError(f"Cloudflare rejected TXT for {domain}")
            print(f"{domain}: added Google TXT; now click Verify in Search Console")
        else:
            print(f"{domain}: would add Google TXT (--apply to write)")


def submit(args):
    token = os.environ.get("GOOGLE_SEARCH_CONSOLE_TOKEN")
    if not token:
        raise RuntimeError("Set GOOGLE_SEARCH_CONSOLE_TOKEN with the webmasters OAuth scope")
    report = json.loads(Path(args.report).read_text(encoding="utf-8"))
    entries = api("https://www.googleapis.com/webmasters/v3/sites", token).get("siteEntry", [])
    verified = {x.get("siteUrl") for x in entries if x.get("permissionLevel") not in (None, "siteUnverifiedUser")}
    work = []
    for row in report.get("sites", []):
        if not row.get("indexable_candidate"):
            continue
        host = urllib.parse.urlsplit(row.get("url", "")).hostname or ""
        root = next((d for d in DOMAINS if host == d or host.endswith("." + d)), None)
        if not root:
            continue
        property_id = "sc-domain:" + root
        if property_id not in verified:
            continue
        for sitemap in row.get("sitemaps", []):
            if urllib.parse.urlsplit(sitemap).hostname == host:
                work.append((property_id, sitemap))
    for property_id, sitemap in sorted(set(work)):
        if args.apply:
            url = "https://www.googleapis.com/webmasters/v3/sites/" + urllib.parse.quote(property_id, safe="") + "/sitemaps/" + urllib.parse.quote(sitemap, safe="")
            api(url, token, "PUT")
            print(f"Submitted {sitemap} to {property_id}")
        else:
            print(f"Would submit {sitemap} to {property_id}")
    print(f"{len(set(work))} sitemaps eligible; GitHub Pages outside owned domains require their own verified property.")


def main():
    parser = argparse.ArgumentParser(description="Audit six Omega UI domains and public GitHub Pages for Google search readiness")
    sub = parser.add_subparsers(dest="command", required=True)
    p = sub.add_parser("audit", help="Read-only inventory and crawl checks")
    p.add_argument("--out", default="seo-report", help="Report directory")
    p.add_argument("--max-hosts", type=int, default=MAX_HOSTS, help="Maximum HTTPS hosts to check (default 300)")
    p.set_defaults(func=audit)
    p = sub.add_parser("verify-dns", help="Plan or add the six Search Console DNS TXT records")
    p.add_argument("--tokens", required=True, help="Local JSON mapping each domain to its Google verification TXT value")
    p.add_argument("--apply", action="store_true", help="Actually write missing TXT records")
    p.set_defaults(func=verify_dns)
    p = sub.add_parser("submit", help="Plan or submit discovered sitemaps to verified Search Console Domain properties")
    p.add_argument("--report", default="seo-report/report.json")
    p.add_argument("--apply", action="store_true", help="Actually submit sitemaps")
    p.set_defaults(func=submit)
    args = parser.parse_args()
    try:
        args.func(args)
    except (RuntimeError, OSError, ValueError, json.JSONDecodeError) as e:
        print(f"Error: {e}", file=sys.stderr)
        return 1
    return 0


if __name__ == "__main__":
    raise SystemExit(main())