fix(repair-encoding-cli): reconfigure stdio to UTF-8 like the other entry points
mempalace/_stdio.py states the rule in its own docstring -- "every console entry point that touches stdio needs to fix this on Windows" -- and cli.py and fact_checker.py both apply it. scripts/mempalace_repair_encoding.py, added later, does not. That matters more here than anywhere else: this tool exists for Windows users whose palace carries legacy mojibake, and it prints a before/after preview for every proposed change. Under the console codepage it was written for, the preview cannot be encoded -- the lead bytes of the corruption it detects are exactly the characters the codepage rejects -- so the run dies with UnicodeEncodeError before repairing a single drawer. Reproduced with stdout on cp936: UnicodeEncodeError: 'gbk' codec can't encode character '\xc3' in position 14: illegal multibyte sequence U+00C3 is the lead character of the à family this tool repairs. Uses replace on stdout/stderr, matching cli.py and fact_checker.py, because the preview carries verbatim drawer text that may hold surrogate halves round-tripped from filenames; strict would crash mid-preview. Related to #1122 (same exception class on the main CLI's help output) but a different entry point and a different fix -- that one can use ASCII-safe static strings, this one prints arbitrary user content.
This commit is contained in:
parent
c7a026bbd0
commit
0e60e51824
|
|
@ -134,7 +134,24 @@ def _print_change(
|
|||
)
|
||||
|
||||
|
||||
def _reconfigure_stdio_utf8_on_windows() -> None:
|
||||
"""Decode stdio as UTF-8 on Windows for the encoding-repair CLI.
|
||||
|
||||
Thin wrapper around the shared helper in ``mempalace._stdio``, matching
|
||||
``cli.py`` and ``fact_checker.py``. stdout/stderr override to ``replace``
|
||||
because every proposed change prints a before/after preview of verbatim
|
||||
drawer text -- under the legacy console codepage this tool is written for,
|
||||
``strict`` raises on the mojibake lead bytes themselves and aborts the run
|
||||
before a single drawer is repaired.
|
||||
"""
|
||||
from mempalace._stdio import reconfigure_stdio_utf8_on_windows
|
||||
|
||||
reconfigure_stdio_utf8_on_windows(stdout_errors="replace", stderr_errors="replace")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
_reconfigure_stdio_utf8_on_windows()
|
||||
|
||||
parser = build_parser()
|
||||
args = parser.parse_args()
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,91 @@
|
|||
"""Stdio tests for the encoding-repair CLI (scripts/).
|
||||
|
||||
The tool ships as a script, not a package module; load it directly, the
|
||||
same way tests/test_backfill_authored_at.py does.
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import io
|
||||
import subprocess
|
||||
import sys
|
||||
import textwrap
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
_SCRIPT = _REPO_ROOT / "scripts" / "mempalace_repair_encoding.py"
|
||||
_spec = importlib.util.spec_from_file_location("mempalace_repair_encoding", _SCRIPT)
|
||||
repair_cli = importlib.util.module_from_spec(_spec)
|
||||
_spec.loader.exec_module(repair_cli)
|
||||
|
||||
|
||||
class _ReconfigurableStringIO(io.StringIO):
|
||||
def __init__(self, initial_value=""):
|
||||
super().__init__(initial_value)
|
||||
self.reconfigure_calls = []
|
||||
|
||||
def reconfigure(self, **kwargs):
|
||||
self.reconfigure_calls.append(kwargs)
|
||||
|
||||
|
||||
def test_reconfigures_stdio_to_utf8_on_windows():
|
||||
"""This entry point must apply the same Windows stdio fix as the others.
|
||||
|
||||
``mempalace/_stdio.py`` states the rule: every console entry point that
|
||||
touches stdio needs it. This one prints verbatim drawer text, so it needs
|
||||
it more than most -- the characters it exists to repair are exactly the
|
||||
ones the legacy console codepage cannot encode.
|
||||
"""
|
||||
stdin = _ReconfigurableStringIO()
|
||||
stdout = _ReconfigurableStringIO()
|
||||
stderr = _ReconfigurableStringIO()
|
||||
with (
|
||||
patch.object(sys, "platform", "win32"),
|
||||
patch.object(sys, "stdin", stdin),
|
||||
patch.object(sys, "stdout", stdout),
|
||||
patch.object(sys, "stderr", stderr),
|
||||
):
|
||||
repair_cli._reconfigure_stdio_utf8_on_windows()
|
||||
|
||||
# Mirrors cli.py and fact_checker.py: stdout/stderr use ``replace`` because
|
||||
# this tool prints verbatim drawer content that may carry surrogate halves,
|
||||
# where ``strict`` would crash mid-preview and abandon the run.
|
||||
assert stdin.reconfigure_calls == [{"encoding": "utf-8", "errors": "surrogateescape"}]
|
||||
assert stdout.reconfigure_calls == [{"encoding": "utf-8", "errors": "replace"}]
|
||||
assert stderr.reconfigure_calls == [{"encoding": "utf-8", "errors": "replace"}]
|
||||
|
||||
|
||||
def test_change_preview_survives_a_legacy_console_codepage():
|
||||
"""Printing a mojibake preview must not kill the run on a GBK console.
|
||||
|
||||
Reproduces the real failure rather than asserting a call happened: a
|
||||
child process is given a legacy codepage for stdout, then asked to print
|
||||
the same before/after preview the tool emits for every proposed change.
|
||||
Without the reconfigure this raises UnicodeEncodeError on U+00C3 -- the
|
||||
very character the repair targets -- and the whole repair run aborts.
|
||||
"""
|
||||
# sys.platform is forced to win32 because the shared helper is a no-op
|
||||
# elsewhere; the same patch the cli/fact_checker stdio tests use. stdout is
|
||||
# given a legacy codepage so the encode actually has to succeed.
|
||||
program = textwrap.dedent(f"""
|
||||
import importlib.util, sys
|
||||
spec = importlib.util.spec_from_file_location("m", {str(_SCRIPT)!r})
|
||||
m = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(m)
|
||||
# Fake the platform only AFTER importing: mempalace.palace and friends
|
||||
# branch on sys.platform at import time and would take the Windows
|
||||
# path (msvcrt) on this host.
|
||||
sys.platform = "win32"
|
||||
m._reconfigure_stdio_utf8_on_windows()
|
||||
m._print_change("drawer-1", "Coração", "Coração", preview_chars=80)
|
||||
print("SURVIVED")
|
||||
""")
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", program],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env={**dict(__import__("os").environ), "PYTHONIOENCODING": "cp936"},
|
||||
cwd=str(_REPO_ROOT),
|
||||
)
|
||||
assert result.returncode == 0, f"tool crashed printing its own preview:\n{result.stderr}"
|
||||
assert "SURVIVED" in result.stdout
|
||||
Loading…
Reference in New Issue