gen_indexes.py

#!/usr/bin/env python3
"""Generate the stats index files from challenge metadata.

Scans ``platforms/**/metadata.json`` and (re)writes:

    stats/index-overview.md       - global totals across every platform
    stats/index-<platform>.md     - per-platform breakdown (one per active platform)

Counting model
--------------
A *challenge* is a distinct ``(platform, id)`` pair. The same challenge solved
in several languages has one folder per language but counts as a single
challenge everywhere EXCEPT the "solved per language" breakdown, where each
language solution is counted once (so per-language counts can sum to more than
the total number of challenges).

Per the repository owner's definition, the overview metric "total catalogued
patterns" is the number of *distinct topics* (``metadata.topics``) seen across
all platforms and languages, deduplicated. The per-platform "solved per type"
breakdown also uses ``topics``.

Determinism
-----------
Output is fully sorted and carries no wall-clock timestamp, so re-running it
without any metadata change produces no diff. This keeps the pre-commit hook
idempotent. Run it manually with ``python scripts/gen_indexes.py`` or let the
hook (``core.hooksPath = scripts/hooks``) run it on every commit.
"""

from __future__ import annotations

import hashlib
import json
import re
from collections import Counter
from pathlib import Path

REPO = Path(__file__).resolve().parents[1]
PLATFORMS = REPO / "platforms"
STATS = REPO / "stats"
README = REPO / "README.md"
PATTERN_ALIASES_FILE = REPO / "scripts" / "pattern-aliases.json"

GEN_NOTE = "> Auto-generated by `scripts/gen_indexes.py`. Do not edit by hand."
STATS_BEGIN = "<!--STATS:BEGIN-->"
STATS_END = "<!--STATS:END-->"
DIFF_ORDER = {"Easy": 0, "Medium": 1, "Hard": 2}


# --------------------------------------------------------------------------- #
# Sort keys
# --------------------------------------------------------------------------- #
def id_key(cid: str):
    """Numeric ids sort numerically; slug ids sort alphabetically after them."""
    return (0, int(cid)) if cid.isdigit() else (1, cid)


def diff_key(name: str):
    return (DIFF_ORDER.get(name, 99), name)


def count_key(item):
    """Sort (name, count) pairs by count desc, then name asc."""
    name, count = item
    return (-count, name)


# --------------------------------------------------------------------------- #
# Pattern vocabulary (open registry — see scripts/pattern-aliases.json)
# --------------------------------------------------------------------------- #
def load_pattern_aliases() -> dict:
    """Load the synonym -> canonical map. Absent/invalid file is non-fatal:
    normalization then collapses to the mechanical rule only."""
    try:
        data = json.loads(PATTERN_ALIASES_FILE.read_text(encoding="utf-8"))
    except (FileNotFoundError, json.JSONDecodeError):
        return {}
    return data.get("aliases", {}) if isinstance(data, dict) else {}


PATTERN_ALIASES = load_pattern_aliases()


def normalize_pattern(name: str) -> str:
    """Mechanical normalization (trim, lowercase, whitespace -> '-') then the
    alias lookup, so the same technique always lands on one canonical label."""
    base = re.sub(r"\s+", "-", name.strip().lower())
    return PATTERN_ALIASES.get(base, base)


# --------------------------------------------------------------------------- #
# Data loading
# --------------------------------------------------------------------------- #
def load_challenges() -> dict:
    """Aggregate every metadata.json into a {(platform, id): info} dict."""
    challenges: dict[tuple[str, str], dict] = {}
    for meta in PLATFORMS.glob("*/*/*/metadata.json"):
        try:
            data = json.loads(meta.read_text(encoding="utf-8"))
        except json.JSONDecodeError as exc:
            raise SystemExit(f"Invalid JSON in {meta}: {exc}")

        platform = data.get("platform") or meta.parents[2].name
        cid = str(data.get("id") or meta.parent.name.split("-", 1)[0])
        key = (platform, cid)

        info = challenges.setdefault(
            key,
            {
                "platform": platform,
                "id": cid,
                "slug": "",
                "dateSolved": None,
                "title": "",
                "difficulty": "Unknown",
                "topics": set(),
                "patterns": set(),
                "url": "",
                "langs": {},  # language -> platformPath (posix, repo-relative)
            },
        )
        if data.get("slug"):
            info["slug"] = data["slug"]
        ds = data.get("dateSolved")
        if ds and (info["dateSolved"] is None or ds < info["dateSolved"]):
            info["dateSolved"] = ds
        if data.get("title"):
            info["title"] = data["title"]
        if data.get("difficulty"):
            info["difficulty"] = data["difficulty"]
        if data.get("url"):
            info["url"] = data["url"]
        for topic in data.get("topics", []):
            info["topics"].add(topic)
        for pattern in data.get("patterns", []):
            info["patterns"].add(normalize_pattern(pattern))

        lang = data.get("language") or data.get("languageExt") or "Unknown"
        ppath = data.get("platformPath") or meta.parent.relative_to(REPO).as_posix()
        info["langs"][lang] = ppath.replace("\\", "/")

    return challenges


def catalogued_platforms() -> int:
    """Number of platform directories catalogued under platforms/."""
    if not PLATFORMS.is_dir():
        return 0
    return sum(1 for p in PLATFORMS.iterdir() if p.is_dir())


# --------------------------------------------------------------------------- #
# Rendering helpers
# --------------------------------------------------------------------------- #
def render_count_table(header: str, counter: Counter, key=count_key) -> list[str]:
    lines = [f"| {header} | Solved |", "| --- | ---: |"]
    for name, count in sorted(counter.items(), key=key):
        lines.append(f"| {name} | {count} |")
    return lines


# --------------------------------------------------------------------------- #
# Overview
# --------------------------------------------------------------------------- #
def overview_metrics(challenges: dict) -> dict:
    """Headline overview numbers shared by the stats index and the README banner.

    A challenge is a distinct ``(platform, id)`` pair; per-language solutions of
    the same challenge each add one to the language counts.
    """
    by_platform: Counter = Counter(c["platform"] for c in challenges.values())
    languages: Counter = Counter()
    topics: set[str] = set()
    patterns: set[str] = set()
    for c in challenges.values():
        for lang in c["langs"]:
            languages[lang] += 1
        topics |= c["topics"]
        patterns |= c["patterns"]
    return {
        "by_platform": by_platform,
        "languages": languages,
        "topics": topics,
        "patterns": patterns,
        "total_challenges": len(challenges),
        "active_platforms": len(by_platform),
        "catalogued": catalogued_platforms(),
        "total_languages": len(languages),
        "total_topics": len(topics),
        "total_patterns": len(patterns),
    }


def build_overview(challenges: dict) -> str:
    m = overview_metrics(challenges)
    by_platform = m["by_platform"]
    languages = m["languages"]
    topics = m["topics"]
    total_challenges = m["total_challenges"]
    active_platforms = m["active_platforms"]
    catalogued = m["catalogued"]
    total_languages = m["total_languages"]
    total_topics = m["total_topics"]
    total_patterns = m["total_patterns"]
    patterns = m["patterns"]

    out: list[str] = [
        "# Index — Overview",
        "",
        GEN_NOTE,
        "",
        "| Metric | Value |",
        "| --- | ---: |",
        f"| Total challenges solved | {total_challenges} |",
        f"| Total source platforms | {active_platforms} (of {catalogued} catalogued) |",
        f"| Total languages used | {total_languages} |",
        f"| Total catalogued topics | {total_topics} |",
        f"| Total catalogued patterns | {total_patterns} |",
        "",
        "_Topics are the broad algorithmic domains; patterns are the finer reusable "
        "techniques. Both counted distinct across all platforms and languages._",
        "",
        "## Platforms",
        "",
    ]
    if by_platform:
        out += ["| Platform | Challenges solved |", "| --- | ---: |"]
        for platform, count in sorted(by_platform.items(), key=count_key):
            out.append(f"| [{platform}](index-{platform}.md) | {count} |")
    else:
        out.append("_No challenges solved yet._")
    out += ["", "## Languages used", ""]
    if languages:
        out += render_count_table("Language", languages)
    else:
        out.append("_None yet._")
    out += ["", "## Catalogued topics", ""]
    out.append(", ".join(sorted(topics)) if topics else "_None yet._")
    out += ["", "## Catalogued patterns", ""]
    out.append(
        f"{total_patterns} distinct patterns — full list in "
        "[index-patterns.md](index-patterns.md)." if patterns else "_None yet._"
    )
    out.append("")
    return "\n".join(out)


# --------------------------------------------------------------------------- #
# Per-platform
# --------------------------------------------------------------------------- #
def build_platform(platform: str, items: list[dict]) -> str:
    difficulty: Counter = Counter(c["difficulty"] for c in items)
    languages: Counter = Counter()
    topics: Counter = Counter()
    for c in items:
        for lang in c["langs"]:
            languages[lang] += 1
        for topic in c["topics"]:
            topics[topic] += 1

    out: list[str] = [
        f"# Index — {platform}",
        "",
        GEN_NOTE,
        "",
        f"**Total challenges solved:** {len(items)}",
        "",
        "## Solved per difficulty",
        "",
        *render_count_table("Difficulty", difficulty, key=lambda kv: diff_key(kv[0])),
        "",
        "## Solved per language",
        "",
        *render_count_table("Language", languages),
        "",
        "## Solved per type",
        "",
        *render_count_table("Type (topic)", topics),
        "",
        "## Challenges",
        "",
        "| ID | Title | Difficulty | Languages |",
        "| --- | --- | --- | --- |",
    ]
    for c in sorted(items, key=lambda x: id_key(x["id"])):
        langs = " · ".join(
            f"[{lang}](../{path})"
            for lang, path in sorted(c["langs"].items())
        )
        title = f"[{c['title']}]({c['url']})" if c["url"] else c["title"]
        cid = c["id"].zfill(4) if c["id"].isdigit() else c["id"]
        out.append(f"| {cid} | {title} | {c['difficulty']} | {langs} |")
    out.append("")
    return "\n".join(out)


# --------------------------------------------------------------------------- #
# Cross-cutting lens indices (by pattern / by topic)
# --------------------------------------------------------------------------- #
def build_lens_index(challenges: dict, attr: str, label: str) -> str:
    """Group challenges by a per-challenge label set (``attr`` is ``"topics"``
    or ``"patterns"``) and render one row per label listing its challenges.

    Rows are sorted by frequency (desc) then name, so reusable labels surface
    first and the long tail of one-offs sinks to the bottom — the markdown
    counterpart of the "sort by count" the html view will offer. Deterministic
    (no timestamp), so an unchanged catalogue produces no diff.
    """
    groups: dict[str, list[dict]] = {}
    for c in challenges.values():
        for value in c[attr]:
            groups.setdefault(value, []).append(c)

    out: list[str] = [
        f"# Index — {label}s",
        "",
        GEN_NOTE,
        "",
        f"**Distinct {label.lower()}s:** {len(groups)} across "
        f"{len(challenges)} challenges.",
        "",
        f"| {label} | Solved | Challenges |",
        "| --- | ---: | --- |",
    ]
    for name, items in sorted(
        groups.items(), key=lambda kv: count_key((kv[0], len(kv[1])))
    ):
        refs = []
        for c in sorted(items, key=lambda x: id_key(x["id"])):
            cid = c["id"].zfill(4) if c["id"].isdigit() else c["id"]
            refs.append(f"[{cid}]({c['url']})" if c["url"] else cid)
        out.append(f"| {name} | {len(items)} | {' · '.join(refs)} |")
    out.append("")
    return "\n".join(out)


# --------------------------------------------------------------------------- #
# README live-stats banner
# --------------------------------------------------------------------------- #
def readme_stats_lines(m: dict) -> list[str]:
    """Markdown injected between the README STATS markers. Deterministic (no
    timestamp), so an unchanged catalogue produces no diff."""
    return [
        "> Auto-generated by `scripts/gen_indexes.py` — do not edit between the "
        "STATS markers.",
        "",
        "| Metric | Value |",
        "| --- | ---: |",
        f"| Challenges solved | {m['total_challenges']} |",
        f"| Source platforms | {m['active_platforms']} active "
        f"(of {m['catalogued']} catalogued) |",
        f"| Languages used | {m['total_languages']} |",
        f"| Catalogued topics | {m['total_topics']} |",
        f"| Catalogued patterns | {m['total_patterns']} |",
    ]


def update_readme(m: dict) -> bool:
    """Rewrite the content between the README STATS markers in place. Returns
    True if the file changed. Non-fatal when the file or markers are absent so a
    repo without the banner still commits cleanly. Newlines are preserved as-is
    (read/write with ``newline=""``) to avoid rewriting the whole file's EOLs."""
    if not README.is_file():
        return False
    with README.open("r", encoding="utf-8", newline="") as fh:
        text = fh.read()
    marker = re.compile(
        re.escape(STATS_BEGIN) + ".*?" + re.escape(STATS_END), re.DOTALL
    )
    if not marker.search(text):
        print("gen_indexes: README STATS markers not found; skipped README banner.")
        return False
    nl = "\r\n" if "\r\n" in text else "\n"
    block = nl.join([STATS_BEGIN, *readme_stats_lines(m), STATS_END])
    new_text = marker.sub(lambda _: block, text, count=1)
    if new_text == text:
        return False
    with README.open("w", encoding="utf-8", newline="") as fh:
        fh.write(new_text)
    return True


# --------------------------------------------------------------------------- #
# HTML asset cache-busting
# --------------------------------------------------------------------------- #
HTML_FILES = [REPO / "index.html", *sorted((REPO / "src").glob("*.html"))]
ASSET_REF = re.compile(
    r'(?P<attr>src|href)="(?P<path>src/[^"?]+\.(?:js|css))(?:\?v=[^"]*)?"'
)


def _asset_hash(rel_path: str):
    f = REPO / rel_path
    if not f.is_file():
        return None
    return hashlib.sha1(f.read_bytes()).hexdigest()[:10]


def update_html_cache_busts() -> list[str]:
    """Append ``?v=<content-hash>`` to every local asset reference in the HTML
    files so browsers re-fetch a file only when its contents change — an
    unchanged file keeps its hash and stays cached. Deterministic; EOLs are
    preserved. Returns the names of the HTML files that changed."""
    changed: list[str] = []
    for html in HTML_FILES:
        if not html.is_file():
            continue
        with html.open("r", encoding="utf-8", newline="") as fh:
            text = fh.read()

        def repl(m):
            h = _asset_hash(m.group("path"))
            return m.group(0) if h is None else f'{m.group("attr")}="{m.group("path")}?v={h}"'

        new_text = ASSET_REF.sub(repl, text)
        if new_text != text:
            with html.open("w", encoding="utf-8", newline="") as fh:
                fh.write(new_text)
            changed.append(html.name)
    return changed


# --------------------------------------------------------------------------- #
# Main
# --------------------------------------------------------------------------- #
def assign_platform_indices(challenges: dict) -> None:
    """Mirror of build-manifest.mjs assignPlatformIndices: challenges whose id
    isn't a plain number (slug-based platforms) get a per-platform incremental
    index (1, 2, …) in solve order (dateSolved, then slug), so the stats markdown
    shows "#1" instead of the solution slug — matching the website."""
    by_platform: dict[str, list[dict]] = {}
    for c in challenges.values():
        if str(c["id"]).isdigit():
            continue
        by_platform.setdefault(c["platform"], []).append(c)
    for items in by_platform.values():
        items.sort(key=lambda c: (c.get("dateSolved") or "", c.get("slug") or ""))
        for i, c in enumerate(items, 1):
            c["id"] = str(i)


def main() -> None:
    challenges = load_challenges()
    assign_platform_indices(challenges)
    STATS.mkdir(parents=True, exist_ok=True)

    # Group challenges by platform.
    by_platform: dict[str, list[dict]] = {}
    for c in challenges.values():
        by_platform.setdefault(c["platform"], []).append(c)

    expected = {"index-overview.md"}
    (STATS / "index-overview.md").write_text(
        build_overview(challenges), encoding="utf-8"
    )
    for platform, items in by_platform.items():
        name = f"index-{platform}.md"
        expected.add(name)
        (STATS / name).write_text(build_platform(platform, items), encoding="utf-8")

    # Cross-cutting lenses: one row per pattern / topic with its challenges.
    for name, attr, label in (
        ("index-patterns.md", "patterns", "Pattern"),
        ("index-topics.md", "topics", "Topic"),
    ):
        expected.add(name)
        (STATS / name).write_text(
            build_lens_index(challenges, attr, label), encoding="utf-8"
        )

    # Drop stale index files for platforms that no longer have solutions.
    for path in STATS.glob("index-*.md"):
        if path.name not in expected:
            path.unlink()

    readme_changed = update_readme(overview_metrics(challenges))
    html_changed = update_html_cache_busts()

    extra = ""
    if readme_changed:
        extra += "; refreshed README banner"
    if html_changed:
        extra += f"; cache-busted {len(html_changed)} HTML file(s)"
    print(
        f"Generated {len(expected)} index file(s) for "
        f"{len(challenges)} challenge(s) across {len(by_platform)} platform(s){extra}."
    )


if __name__ == "__main__":
    main()