the store is UTF-8, so the tool is too: no locale can crash a wake

memo

@@ -23,6 +23,14 @@ import sys
 from collections import deque
 
 
+# The store is UTF-8 by construction, so the tool reads and prints UTF-8
+# no matter what the machine's locale says. Without this, one arrow in a
+# memory makes wake crash on a latin-1 locale.
+for _s in (sys.stdout, sys.stderr):
+    if hasattr(_s, "reconfigure"):  # Python 3.7+
+        _s.reconfigure(encoding="utf-8")
+
+
 def pretty(p):
     """A path as the user would type it: keep symlinks, fold $HOME to ~."""
     p, home = os.path.abspath(p), os.path.expanduser("~")
@@ -162,7 +170,7 @@ def overrides(d):
     p = os.path.join(d, "config")
     if not os.path.exists(p):
         return out
-    for n, line in enumerate(open(p), 1):
+    for n, line in enumerate(open(p, encoding="utf-8"), 1):
         line = line.split("#")[0].strip()
         if "=" not in line:
             continue
@@ -191,7 +199,7 @@ def write_config(d, over):
     for k, (default, what) in KNOBS.items():
         out.append("%-2s%-12s = %-6d # %s"
                    % ("" if k in over else "# ", k, over.get(k, default), what))
-    with open(os.path.join(d, "config"), "w") as f:
+    with open(os.path.join(d, "config"), "w", encoding="utf-8") as f:
         f.write("\n".join(out) + "\n")
 
 
@@ -274,7 +282,11 @@ def tree_get(d, lo, hi):
             rec = f.read(TREE_REC)
     except FileNotFoundError:  # not built yet; any other failure is real
         return None
+    try:
         return rec.decode().rstrip() or None
+    except UnicodeDecodeError:
+        die("The summary of #%d-%d is corrupt. Run: %s forget %d-%d"
+            % (lo, hi - 1, ME, lo, hi - 1))
 
 
 def pad(text, rec):
@@ -745,7 +757,11 @@ def cmd_import(d, args):
     For bootstrapping an identity from older records. Used once."""
     if len(args) != 1:
         die("usage: %s import <file>   # lines of 'YYYY-MM-DD <text>'" % ME)
-    src = open(args[0]).readlines()
+    try:
+        src = open(args[0], encoding="utf-8").readlines()
+    except UnicodeDecodeError:
+        die("%s is not UTF-8 text. Convert it, then import again."
+            % pretty(args[0]))
     last = log_get(d, log_len(d) - 1)[1] if log_len(d) else "0000-00-00"
     out = []
     for i, line in enumerate(src, 1):

test.py

@@ -537,6 +537,36 @@ check(r.returncode == 1 and "forget 0-15" in r.stderr,
       "a blank half summary must point at forget: " + r.stdout + r.stderr)
 shutil.rmtree(d4)
 
+# the store is UTF-8 whatever the locale says: without pinning the streams,
+# one arrow in a memory made wake crash forever on a latin-1 machine
+d5 = tempfile.mkdtemp(prefix="optmem-utf8-")
+run("note", "an arrow \u2192 survives any locale", store=d5)
+r_ = subprocess.run(memo + ["wake"], capture_output=True,
+                    env=dict(os.environ, MEMORY_DIR=d5,
+                             PYTHONIOENCODING="latin-1"))
+check(r_.returncode == 0 and "\u2192".encode() in r_.stdout,
+      "wake must print UTF-8 on a non-UTF-8 locale: "
+      + repr(r_.stdout + r_.stderr))
+
+# an import file that is not UTF-8 is refused -- neither a traceback nor,
+# worse, silently mis-decoded into mojibake and stored forever
+with open(os.path.join(d5, "latin1.txt"), "wb") as f:
+    f.write(b"2027-01-01 caf\xe9 in latin-1\n")
+r = run("import", os.path.join(d5, "latin1.txt"), store=d5)
+check(r.returncode == 1 and "not UTF-8" in r.stderr,
+      "a non-UTF-8 import file must be refused: " + r.stdout + r.stderr)
+
+# a summary record holding invalid UTF-8 is the blank-record dead end in
+# another coat: it must name forget, not print a traceback
+run("note", "utf8 probe second memory", store=d5)
+run("nap", "0-1", "both utf8 probes", store=d5)
+with open(os.path.join(d5, "TREE", "2"), "r+b") as f:
+    f.write(b"\xff\xfe corrupt bytes")
+r = run("zoom", "0-3", store=d5)
+check(r.returncode == 1 and "forget 0-1" in r.stderr,
+      "a corrupt summary must point at forget: " + r.stdout + r.stderr)
+shutil.rmtree(d5)
+
 # re-running init on a lived-in store must not touch one byte of it: the
 # whole setup is idempotent, so it is safe on every provision. `init` only
 # ever makedirs(exist_ok), opens LOG.txt for APPEND, and writes `config`