EverOS/scripts/check_file_sizes.py

"""Block oversized files entering the repository through a pull request.

Scope is the change under review, not the whole tree: paths added, modified,
renamed or copied since the merge base. Files already committed above the
ceiling are left alone, so the gate never fails a pull request for something
it did not introduce.

The same ceiling is enforced locally by the ``check-added-large-files``
pre-commit hook. Keep ``MAX_KB`` and ``--maxkb`` in ``.pre-commit-config.yaml``
in lockstep — a unit test pins them together, because a hook that only runs
locally is not a gate. Note the local hook is weaker by construction: it only
looks at files being *added*, so it cannot catch an existing file that grows.
That is the case this script covers on every pull request.

On a push to the base branch itself the merge base is ``HEAD``, the diff is
empty, and the check is a no-op. This is a pull-request gate.

``EXEMPT_PREFIXES`` carves out directories whose contents are machine-generated
and legitimately large. Keep it as short as possible: every entry is a place
the ceiling no longer protects. A unit test pins the list, so growing it shows
up as a deliberate change in review.
"""

from __future__ import annotations

import argparse
import os
import subprocess
from collections.abc import Iterable, Sequence
from dataclasses import dataclass
from pathlib import Path

# Kilobytes, matching the pre-commit hook's ``--maxkb`` semantics exactly:
# a file is oversized when ``size_bytes > MAX_KB * 1024``.
MAX_KB = 640

DEFAULT_BASE = "origin/main"

# Directories exempt from the ceiling, as posix path prefixes.
#
# tests/fixtures/search_seed/ holds embedded search corpora regenerated by
# tests/fixtures/_dump_search_seed.py. Two of them already sit near 1 MB and
# grew ~60% in a single release; a ceiling they have to clear would be raised
# on every refresh, which teaches contributors to edit the gate instead of
# their payload. Outside this directory the largest tracked file is ~300 KB.
EXEMPT_PREFIXES = ("tests/fixtures/search_seed/",)


class BaseRefError(RuntimeError):
    """The comparison base could not be resolved."""


@dataclass(frozen=True)
class Violation:
    path: str
    size_bytes: int


def _repo_root() -> Path:
    return Path(__file__).resolve().parent.parent


def is_exempt(path: str, prefixes: Sequence[str] = EXEMPT_PREFIXES) -> bool:
    """Whether ``path`` sits under an exempt directory prefix."""
    normalised = path.replace("\\", "/")
    return any(normalised.startswith(prefix) for prefix in prefixes)


def find_violations(
    paths: Iterable[str],
    *,
    root: Path,
    max_kb: int = MAX_KB,
    exempt_prefixes: Sequence[str] = EXEMPT_PREFIXES,
) -> list[Violation]:
    """Return oversized entries among ``paths``, resolved against ``root``.

    Entries under ``exempt_prefixes`` are skipped. So are entries that are not
    regular files: a diff can name a path that no longer exists in the working
    tree, and a symlink would be measured by its target rather than itself.
    """
    limit_bytes = max_kb * 1024
    violations: list[Violation] = []
    for path in paths:
        if is_exempt(path, exempt_prefixes):
            continue
        candidate = root / path
        if candidate.is_symlink() or not candidate.is_file():
            continue
        size_bytes = candidate.stat().st_size
        if size_bytes > limit_bytes:
            violations.append(Violation(path=path, size_bytes=size_bytes))
    return violations


def default_base_ref() -> str:
    """Resolve the base ref to diff against.

    ``GITHUB_BASE_REF`` is set by GitHub Actions on ``pull_request`` events
    and names the target branch; everywhere else fall back to the default
    branch's remote tracking ref.
    """
    github_base = os.environ.get("GITHUB_BASE_REF", "").strip()
    if github_base:
        return f"origin/{github_base}"
    return DEFAULT_BASE


def _git(root: Path, *args: str) -> str:
    result = subprocess.run(
        ["git", *args],
        cwd=root,
        capture_output=True,
        text=True,
    )
    if result.returncode != 0:
        raise BaseRefError(
            f"`git {' '.join(args)}` failed: {result.stderr.strip() or 'unknown error'}"
        )
    return result.stdout


def changed_paths(root: Path, base_ref: str) -> list[str]:
    """Paths this change adds or grows, relative to ``base_ref``.

    Two sources, unioned:

    * ``git diff`` from the merge base to the working tree (no second
      revision), covering committed and uncommitted edits to tracked files.
      ``--diff-filter=ACMR`` keeps additions, copies, modifications and
      renames; deletions are excluded.
    * untracked, non-ignored files. ``git diff`` cannot see a newly created
      file until it is staged, so a local pre-push run would otherwise pass a
      brand-new oversized file. In CI the checkout is clean and this source
      is empty.

    Raises :class:`BaseRefError` when ``base_ref`` cannot be resolved — a
    gate that silently passes on a shallow clone is worse than no gate.
    """
    merge_base = _git(root, "merge-base", base_ref, "HEAD").strip()
    if not merge_base:
        raise BaseRefError(f"no merge base between {base_ref} and HEAD")
    tracked = _git(
        root,
        "diff",
        "--name-only",
        "-z",
        "--diff-filter=ACMR",
        merge_base,
    )
    untracked = _git(root, "ls-files", "--others", "--exclude-standard", "-z")
    entries = {entry for entry in tracked.split("\0") if entry}
    entries.update(entry for entry in untracked.split("\0") if entry)
    return sorted(entries)


def main(argv: Sequence[str] | None = None) -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "--base",
        default=None,
        help=f"ref to diff against (default: $GITHUB_BASE_REF, else {DEFAULT_BASE})",
    )
    args = parser.parse_args(argv)

    root = _repo_root()
    base_ref = args.base or default_base_ref()
    try:
        paths = changed_paths(root, base_ref)
    except BaseRefError as exc:
        print(
            f"Repository file size check could not run: {exc}\n"
            "The gate diffs against the base branch, so it needs that ref and "
            "enough history to find a merge base. In CI, set "
            "`fetch-depth: 0` on actions/checkout; locally, run "
            "`git fetch origin main`."
        )
        return 1

    violations = find_violations(paths, root=root)
    if not violations:
        print(
            f"Repository file size check passed "
            f"({len(paths)} changed file(s) vs {base_ref}, ceiling {MAX_KB} KB)."
        )
        return 0

    print(
        f"Repository file size check failed: this change adds or grows files "
        f"above {MAX_KB} KB.\n"
        "Large payloads belong in release artifacts, external hosting, or "
        "another approved storage location, then linked from docs. If a "
        "generated corpus genuinely has to live in-tree, add its directory "
        "to EXEMPT_PREFIXES in this script rather than raising the ceiling "
        "for the whole repository — and say why in the commit.\n"
    )
    for violation in sorted(violations, key=lambda item: -item.size_bytes):
        size_kb = violation.size_bytes / 1024
        print(f"- {violation.path}: {size_kb:.1f} KB")
    return 1


if __name__ == "__main__":
    raise SystemExit(main())