* ci: enforce the file-size ceiling on pull requests `check-added-large-files` only ran in pre-commit, so the ceiling was absent from CI: an unhooked clone or `--no-verify` bypassed it entirely, and the hook is weaker than it looks even locally — it inspects only files being *added*, so an existing fixture that grows never trips it. Add `scripts/check_file_sizes.py`, wired into `make lint` and therefore the required `lint` check. It diffs against the base branch's merge base and measures additions, modifications and renames, leaving files already committed above the ceiling alone so no pull request fails for something it did not touch. An unresolvable base is a hard failure rather than a silent pass. The `lint` job now checks out with `fetch-depth: 0` to provide it. Lower the hook's `--maxkb` from 1024 to 640 to match, and pin the two limits equal in a unit test so they cannot drift back apart. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * ci: catch untracked oversized files in the size gate `git diff` cannot see a newly created file until it is staged, so a local `make check-file-sizes` passed a brand-new 700 KB file — verified against the real script, not reasoned about. CI was unaffected (its checkout has everything committed), but the docstring claimed the local run covered uncommitted work, which was only true for edits to already-tracked files. Union in `git ls-files --others --exclude-standard`, which respects .gitignore, and pin both the untracked and the ignored case in tests. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * ci: exempt the generated search-seed corpora from the size ceiling tests/fixtures/search_seed/ holds embedded corpora regenerated by _dump_search_seed.py. Two of them are already near 1 MB and grew ~60% in one release, so the next refresh would have hit the 640 KB ceiling and the cheapest fix would have been raising it for the whole repository — a gate that teaches people to edit the gate. Exempt that one directory by path prefix instead. Outside it the largest tracked file is ~300 KB, so 640 KB still binds where it matters, including the examples/ case that prompted this work. Three tests pin the carve-out: a sibling of the exempt directory is still caught, the list itself is asserted verbatim so growth shows up in review, and every prefix must name a directory that actually exists. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> --------- Co-authored-by: zhanghui <zhanghui@shanda.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
205 lines
7.3 KiB
Python
205 lines
7.3 KiB
Python
"""Block oversized files entering the repository through a pull request.
|
|
|
|
Scope is the change under review, not the whole tree: paths added, modified,
|
|
renamed or copied since the merge base. Files already committed above the
|
|
ceiling are left alone, so the gate never fails a pull request for something
|
|
it did not introduce.
|
|
|
|
The same ceiling is enforced locally by the ``check-added-large-files``
|
|
pre-commit hook. Keep ``MAX_KB`` and ``--maxkb`` in ``.pre-commit-config.yaml``
|
|
in lockstep — a unit test pins them together, because a hook that only runs
|
|
locally is not a gate. Note the local hook is weaker by construction: it only
|
|
looks at files being *added*, so it cannot catch an existing file that grows.
|
|
That is the case this script covers on every pull request.
|
|
|
|
On a push to the base branch itself the merge base is ``HEAD``, the diff is
|
|
empty, and the check is a no-op. This is a pull-request gate.
|
|
|
|
``EXEMPT_PREFIXES`` carves out directories whose contents are machine-generated
|
|
and legitimately large. Keep it as short as possible: every entry is a place
|
|
the ceiling no longer protects. A unit test pins the list, so growing it shows
|
|
up as a deliberate change in review.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import os
|
|
import subprocess
|
|
from collections.abc import Iterable, Sequence
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
|
|
# Kilobytes, matching the pre-commit hook's ``--maxkb`` semantics exactly:
|
|
# a file is oversized when ``size_bytes > MAX_KB * 1024``.
|
|
MAX_KB = 640
|
|
|
|
DEFAULT_BASE = "origin/main"
|
|
|
|
# Directories exempt from the ceiling, as posix path prefixes.
|
|
#
|
|
# tests/fixtures/search_seed/ holds embedded search corpora regenerated by
|
|
# tests/fixtures/_dump_search_seed.py. Two of them already sit near 1 MB and
|
|
# grew ~60% in a single release; a ceiling they have to clear would be raised
|
|
# on every refresh, which teaches contributors to edit the gate instead of
|
|
# their payload. Outside this directory the largest tracked file is ~300 KB.
|
|
EXEMPT_PREFIXES = ("tests/fixtures/search_seed/",)
|
|
|
|
|
|
class BaseRefError(RuntimeError):
|
|
"""The comparison base could not be resolved."""
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Violation:
|
|
path: str
|
|
size_bytes: int
|
|
|
|
|
|
def _repo_root() -> Path:
|
|
return Path(__file__).resolve().parent.parent
|
|
|
|
|
|
def is_exempt(path: str, prefixes: Sequence[str] = EXEMPT_PREFIXES) -> bool:
|
|
"""Whether ``path`` sits under an exempt directory prefix."""
|
|
normalised = path.replace("\\", "/")
|
|
return any(normalised.startswith(prefix) for prefix in prefixes)
|
|
|
|
|
|
def find_violations(
|
|
paths: Iterable[str],
|
|
*,
|
|
root: Path,
|
|
max_kb: int = MAX_KB,
|
|
exempt_prefixes: Sequence[str] = EXEMPT_PREFIXES,
|
|
) -> list[Violation]:
|
|
"""Return oversized entries among ``paths``, resolved against ``root``.
|
|
|
|
Entries under ``exempt_prefixes`` are skipped. So are entries that are not
|
|
regular files: a diff can name a path that no longer exists in the working
|
|
tree, and a symlink would be measured by its target rather than itself.
|
|
"""
|
|
limit_bytes = max_kb * 1024
|
|
violations: list[Violation] = []
|
|
for path in paths:
|
|
if is_exempt(path, exempt_prefixes):
|
|
continue
|
|
candidate = root / path
|
|
if candidate.is_symlink() or not candidate.is_file():
|
|
continue
|
|
size_bytes = candidate.stat().st_size
|
|
if size_bytes > limit_bytes:
|
|
violations.append(Violation(path=path, size_bytes=size_bytes))
|
|
return violations
|
|
|
|
|
|
def default_base_ref() -> str:
|
|
"""Resolve the base ref to diff against.
|
|
|
|
``GITHUB_BASE_REF`` is set by GitHub Actions on ``pull_request`` events
|
|
and names the target branch; everywhere else fall back to the default
|
|
branch's remote tracking ref.
|
|
"""
|
|
github_base = os.environ.get("GITHUB_BASE_REF", "").strip()
|
|
if github_base:
|
|
return f"origin/{github_base}"
|
|
return DEFAULT_BASE
|
|
|
|
|
|
def _git(root: Path, *args: str) -> str:
|
|
result = subprocess.run(
|
|
["git", *args],
|
|
cwd=root,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
if result.returncode != 0:
|
|
raise BaseRefError(
|
|
f"`git {' '.join(args)}` failed: {result.stderr.strip() or 'unknown error'}"
|
|
)
|
|
return result.stdout
|
|
|
|
|
|
def changed_paths(root: Path, base_ref: str) -> list[str]:
|
|
"""Paths this change adds or grows, relative to ``base_ref``.
|
|
|
|
Two sources, unioned:
|
|
|
|
* ``git diff`` from the merge base to the working tree (no second
|
|
revision), covering committed and uncommitted edits to tracked files.
|
|
``--diff-filter=ACMR`` keeps additions, copies, modifications and
|
|
renames; deletions are excluded.
|
|
* untracked, non-ignored files. ``git diff`` cannot see a newly created
|
|
file until it is staged, so a local pre-push run would otherwise pass a
|
|
brand-new oversized file. In CI the checkout is clean and this source
|
|
is empty.
|
|
|
|
Raises :class:`BaseRefError` when ``base_ref`` cannot be resolved — a
|
|
gate that silently passes on a shallow clone is worse than no gate.
|
|
"""
|
|
merge_base = _git(root, "merge-base", base_ref, "HEAD").strip()
|
|
if not merge_base:
|
|
raise BaseRefError(f"no merge base between {base_ref} and HEAD")
|
|
tracked = _git(
|
|
root,
|
|
"diff",
|
|
"--name-only",
|
|
"-z",
|
|
"--diff-filter=ACMR",
|
|
merge_base,
|
|
)
|
|
untracked = _git(root, "ls-files", "--others", "--exclude-standard", "-z")
|
|
entries = {entry for entry in tracked.split("\0") if entry}
|
|
entries.update(entry for entry in untracked.split("\0") if entry)
|
|
return sorted(entries)
|
|
|
|
|
|
def main(argv: Sequence[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument(
|
|
"--base",
|
|
default=None,
|
|
help=f"ref to diff against (default: $GITHUB_BASE_REF, else {DEFAULT_BASE})",
|
|
)
|
|
args = parser.parse_args(argv)
|
|
|
|
root = _repo_root()
|
|
base_ref = args.base or default_base_ref()
|
|
try:
|
|
paths = changed_paths(root, base_ref)
|
|
except BaseRefError as exc:
|
|
print(
|
|
f"Repository file size check could not run: {exc}\n"
|
|
"The gate diffs against the base branch, so it needs that ref and "
|
|
"enough history to find a merge base. In CI, set "
|
|
"`fetch-depth: 0` on actions/checkout; locally, run "
|
|
"`git fetch origin main`."
|
|
)
|
|
return 1
|
|
|
|
violations = find_violations(paths, root=root)
|
|
if not violations:
|
|
print(
|
|
f"Repository file size check passed "
|
|
f"({len(paths)} changed file(s) vs {base_ref}, ceiling {MAX_KB} KB)."
|
|
)
|
|
return 0
|
|
|
|
print(
|
|
f"Repository file size check failed: this change adds or grows files "
|
|
f"above {MAX_KB} KB.\n"
|
|
"Large payloads belong in release artifacts, external hosting, or "
|
|
"another approved storage location, then linked from docs. If a "
|
|
"generated corpus genuinely has to live in-tree, add its directory "
|
|
"to EXEMPT_PREFIXES in this script rather than raising the ceiling "
|
|
"for the whole repository — and say why in the commit.\n"
|
|
)
|
|
for violation in sorted(violations, key=lambda item: -item.size_bytes):
|
|
size_kb = violation.size_bytes / 1024
|
|
print(f"- {violation.path}: {size_kb:.1f} KB")
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|