fix(repair-encoding-cli): reconfigure stdio to UTF-8 like the other entry points

mempalace/_stdio.py states the rule in its own docstring -- "every console
entry point that touches stdio needs to fix this on Windows" -- and cli.py
and fact_checker.py both apply it. scripts/mempalace_repair_encoding.py,
added later, does not.

That matters more here than anywhere else: this tool exists for Windows users
whose palace carries legacy mojibake, and it prints a before/after preview for
every proposed change. Under the console codepage it was written for, the
preview cannot be encoded -- the lead bytes of the corruption it detects are
exactly the characters the codepage rejects -- so the run dies with
UnicodeEncodeError before repairing a single drawer.

Reproduced with stdout on cp936:

  UnicodeEncodeError: 'gbk' codec can't encode character '\xc3'
  in position 14: illegal multibyte sequence

U+00C3 is the lead character of the à family this tool repairs.

Uses replace on stdout/stderr, matching cli.py and fact_checker.py, because
the preview carries verbatim drawer text that may hold surrogate halves
round-tripped from filenames; strict would crash mid-preview.

Related to #1122 (same exception class on the main CLI's help output) but a
different entry point and a different fix -- that one can use ASCII-safe
static strings, this one prints arbitrary user content.
This commit is contained in:
Arnold Wender 2026-08-08 23:21:55 +02:00
parent c7a026bbd0
commit 0e60e51824
2 changed files with 108 additions and 0 deletions

View File

@ -134,7 +134,24 @@ def _print_change(
)
def _reconfigure_stdio_utf8_on_windows() -> None:
"""Decode stdio as UTF-8 on Windows for the encoding-repair CLI.
Thin wrapper around the shared helper in ``mempalace._stdio``, matching
``cli.py`` and ``fact_checker.py``. stdout/stderr override to ``replace``
because every proposed change prints a before/after preview of verbatim
drawer text -- under the legacy console codepage this tool is written for,
``strict`` raises on the mojibake lead bytes themselves and aborts the run
before a single drawer is repaired.
"""
from mempalace._stdio import reconfigure_stdio_utf8_on_windows
reconfigure_stdio_utf8_on_windows(stdout_errors="replace", stderr_errors="replace")
def main() -> int:
_reconfigure_stdio_utf8_on_windows()
parser = build_parser()
args = parser.parse_args()

View File

@ -0,0 +1,91 @@
"""Stdio tests for the encoding-repair CLI (scripts/).
The tool ships as a script, not a package module; load it directly, the
same way tests/test_backfill_authored_at.py does.
"""
import importlib.util
import io
import subprocess
import sys
import textwrap
from pathlib import Path
from unittest.mock import patch
_REPO_ROOT = Path(__file__).resolve().parents[1]
_SCRIPT = _REPO_ROOT / "scripts" / "mempalace_repair_encoding.py"
_spec = importlib.util.spec_from_file_location("mempalace_repair_encoding", _SCRIPT)
repair_cli = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(repair_cli)
class _ReconfigurableStringIO(io.StringIO):
def __init__(self, initial_value=""):
super().__init__(initial_value)
self.reconfigure_calls = []
def reconfigure(self, **kwargs):
self.reconfigure_calls.append(kwargs)
def test_reconfigures_stdio_to_utf8_on_windows():
"""This entry point must apply the same Windows stdio fix as the others.
``mempalace/_stdio.py`` states the rule: every console entry point that
touches stdio needs it. This one prints verbatim drawer text, so it needs
it more than most -- the characters it exists to repair are exactly the
ones the legacy console codepage cannot encode.
"""
stdin = _ReconfigurableStringIO()
stdout = _ReconfigurableStringIO()
stderr = _ReconfigurableStringIO()
with (
patch.object(sys, "platform", "win32"),
patch.object(sys, "stdin", stdin),
patch.object(sys, "stdout", stdout),
patch.object(sys, "stderr", stderr),
):
repair_cli._reconfigure_stdio_utf8_on_windows()
# Mirrors cli.py and fact_checker.py: stdout/stderr use ``replace`` because
# this tool prints verbatim drawer content that may carry surrogate halves,
# where ``strict`` would crash mid-preview and abandon the run.
assert stdin.reconfigure_calls == [{"encoding": "utf-8", "errors": "surrogateescape"}]
assert stdout.reconfigure_calls == [{"encoding": "utf-8", "errors": "replace"}]
assert stderr.reconfigure_calls == [{"encoding": "utf-8", "errors": "replace"}]
def test_change_preview_survives_a_legacy_console_codepage():
"""Printing a mojibake preview must not kill the run on a GBK console.
Reproduces the real failure rather than asserting a call happened: a
child process is given a legacy codepage for stdout, then asked to print
the same before/after preview the tool emits for every proposed change.
Without the reconfigure this raises UnicodeEncodeError on U+00C3 -- the
very character the repair targets -- and the whole repair run aborts.
"""
# sys.platform is forced to win32 because the shared helper is a no-op
# elsewhere; the same patch the cli/fact_checker stdio tests use. stdout is
# given a legacy codepage so the encode actually has to succeed.
program = textwrap.dedent(f"""
import importlib.util, sys
spec = importlib.util.spec_from_file_location("m", {str(_SCRIPT)!r})
m = importlib.util.module_from_spec(spec)
spec.loader.exec_module(m)
# Fake the platform only AFTER importing: mempalace.palace and friends
# branch on sys.platform at import time and would take the Windows
# path (msvcrt) on this host.
sys.platform = "win32"
m._reconfigure_stdio_utf8_on_windows()
m._print_change("drawer-1", "Coração", "Coração", preview_chars=80)
print("SURVIVED")
""")
result = subprocess.run(
[sys.executable, "-c", program],
capture_output=True,
text=True,
env={**dict(__import__("os").environ), "PYTHONIOENCODING": "cp936"},
cwd=str(_REPO_ROOT),
)
assert result.returncode == 0, f"tool crashed printing its own preview:\n{result.stderr}"
assert "SURVIVED" in result.stdout