diff --git a/memo b/memo index 0cc0ea5..45cc853 100755 --- a/memo +++ b/memo @@ -23,6 +23,14 @@ import sys from collections import deque +# The store is UTF-8 by construction, so the tool reads and prints UTF-8 +# no matter what the machine's locale says. Without this, one arrow in a +# memory makes wake crash on a latin-1 locale. +for _s in (sys.stdout, sys.stderr): + if hasattr(_s, "reconfigure"): # Python 3.7+ + _s.reconfigure(encoding="utf-8") + + def pretty(p): """A path as the user would type it: keep symlinks, fold $HOME to ~.""" p, home = os.path.abspath(p), os.path.expanduser("~") @@ -162,7 +170,7 @@ def overrides(d): p = os.path.join(d, "config") if not os.path.exists(p): return out - for n, line in enumerate(open(p), 1): + for n, line in enumerate(open(p, encoding="utf-8"), 1): line = line.split("#")[0].strip() if "=" not in line: continue @@ -191,7 +199,7 @@ def write_config(d, over): for k, (default, what) in KNOBS.items(): out.append("%-2s%-12s = %-6d # %s" % ("" if k in over else "# ", k, over.get(k, default), what)) - with open(os.path.join(d, "config"), "w") as f: + with open(os.path.join(d, "config"), "w", encoding="utf-8") as f: f.write("\n".join(out) + "\n") @@ -274,7 +282,11 @@ def tree_get(d, lo, hi): rec = f.read(TREE_REC) except FileNotFoundError: # not built yet; any other failure is real return None - return rec.decode().rstrip() or None + try: + return rec.decode().rstrip() or None + except UnicodeDecodeError: + die("The summary of #%d-%d is corrupt. Run: %s forget %d-%d" + % (lo, hi - 1, ME, lo, hi - 1)) def pad(text, rec): @@ -745,7 +757,11 @@ def cmd_import(d, args): For bootstrapping an identity from older records. Used once.""" if len(args) != 1: die("usage: %s import # lines of 'YYYY-MM-DD '" % ME) - src = open(args[0]).readlines() + try: + src = open(args[0], encoding="utf-8").readlines() + except UnicodeDecodeError: + die("%s is not UTF-8 text. Convert it, then import again." + % pretty(args[0])) last = log_get(d, log_len(d) - 1)[1] if log_len(d) else "0000-00-00" out = [] for i, line in enumerate(src, 1): diff --git a/test.py b/test.py index 10e3d9b..06b2632 100755 --- a/test.py +++ b/test.py @@ -537,6 +537,36 @@ check(r.returncode == 1 and "forget 0-15" in r.stderr, "a blank half summary must point at forget: " + r.stdout + r.stderr) shutil.rmtree(d4) +# the store is UTF-8 whatever the locale says: without pinning the streams, +# one arrow in a memory made wake crash forever on a latin-1 machine +d5 = tempfile.mkdtemp(prefix="optmem-utf8-") +run("note", "an arrow \u2192 survives any locale", store=d5) +r_ = subprocess.run(memo + ["wake"], capture_output=True, + env=dict(os.environ, MEMORY_DIR=d5, + PYTHONIOENCODING="latin-1")) +check(r_.returncode == 0 and "\u2192".encode() in r_.stdout, + "wake must print UTF-8 on a non-UTF-8 locale: " + + repr(r_.stdout + r_.stderr)) + +# an import file that is not UTF-8 is refused -- neither a traceback nor, +# worse, silently mis-decoded into mojibake and stored forever +with open(os.path.join(d5, "latin1.txt"), "wb") as f: + f.write(b"2027-01-01 caf\xe9 in latin-1\n") +r = run("import", os.path.join(d5, "latin1.txt"), store=d5) +check(r.returncode == 1 and "not UTF-8" in r.stderr, + "a non-UTF-8 import file must be refused: " + r.stdout + r.stderr) + +# a summary record holding invalid UTF-8 is the blank-record dead end in +# another coat: it must name forget, not print a traceback +run("note", "utf8 probe second memory", store=d5) +run("nap", "0-1", "both utf8 probes", store=d5) +with open(os.path.join(d5, "TREE", "2"), "r+b") as f: + f.write(b"\xff\xfe corrupt bytes") +r = run("zoom", "0-3", store=d5) +check(r.returncode == 1 and "forget 0-1" in r.stderr, + "a corrupt summary must point at forget: " + r.stdout + r.stderr) +shutil.rmtree(d5) + # re-running init on a lived-in store must not touch one byte of it: the # whole setup is idempotent, so it is safe on every provision. `init` only # ever makedirs(exist_ok), opens LOG.txt for APPEND, and writes `config`