audit fixes: UTF-8 record corruption, byte-vs-char limits

- log_slice decoded the whole buffer then sliced at 320-CHARACTER offsets;
  one multi-byte char (memory #42: 'Gestão') shifted every later record and
  crashed recall on the live store. Records are bytes: slice, then decode.
- limits counted characters while records are bytes, so 150 'ã's passed
  check() and died inside pad() mid-append. check/import/paginate now count
  UTF-8 bytes; config rejects an ENTRY_CHARS the records cannot fit.
- tests were ASCII-only (why 239k assertions missed it): UTF-8 cases added.

memo

@@ -76,6 +76,9 @@ def config(d):
             PART_CHARS = int(v)
         elif k == "PART_LINES":
             PART_LINES = int(v)
+    if ENTRY_CHARS > min(TREE_REC - 8, LOG_REC - 40):
+        die("config: ENTRY_CHARS=%d cannot fit the %d/%d-byte records."
+            % (ENTRY_CHARS, LOG_REC, TREE_REC))
 
 
 def log_path(d):
@@ -108,13 +111,15 @@ def log_get(d, i):
 
 
 def log_slice(d, lo, hi):
-    """Memories [lo,hi) in one read."""
+    """Memories [lo,hi) in one read. Records are sliced as BYTES and decoded
+    one by one -- slicing decoded text would shift every boundary after the
+    first multi-byte character."""
     with open(log_path(d), "rb") as f:
         f.seek(lo * LOG_REC)
-        buf = f.read((hi - lo) * LOG_REC).decode()
+        buf = f.read((hi - lo) * LOG_REC)
     out = []
     for i in range(hi - lo):
-        line = buf[i * LOG_REC:(i + 1) * LOG_REC].rstrip()
+        line = buf[i * LOG_REC:(i + 1) * LOG_REC].decode().rstrip()
         head, _, rest = line.partition(" ")
         date, _, text = rest.partition(" ")
         out.append((int(head[1:]), date, text))
@@ -206,9 +211,11 @@ def check(text):
     if "\n" in text or "\r" in text:
         die("REJECTED: %d lines. A memory is exactly one line." %
             (text.count("\n") + 1))
-    if len(text) > ENTRY_CHARS:
+    n = len(text.encode())
-        die("REJECTED: %d chars, %d over the %d limit. Compress it further."
+    if n > ENTRY_CHARS:
-            % (len(text), len(text) - ENTRY_CHARS, ENTRY_CHARS))
+        die("REJECTED: %d bytes, %d over the %d limit (accents and symbols "
+            "cost more than one). Compress it further."
+            % (n, n - ENTRY_CHARS, ENTRY_CHARS))
     return text
 
 
@@ -270,11 +277,12 @@ def paginate(lines):
     """Split the document into parts that survive any harness's output cap."""
     parts, cur, size = [], [], 0
     for line in lines:
-        if cur and (len(cur) >= PART_LINES or size + len(line) > PART_CHARS):
+        n = len(line.encode())
+        if cur and (len(cur) >= PART_LINES or size + n > PART_CHARS):
             parts.append(cur)
             cur, size = [], 0
         cur.append(line)
-        size += len(line) + 1
+        size += n + 1
     if cur:
         parts.append(cur)
     return parts
@@ -417,8 +425,8 @@ def cmd_import(d, args):
             die("line %d: date %s is older than the previous memory (%s). "
                 "Memories must be in order." % (i, date, last))
         text = text.strip()
-        if not text or len(text) > ENTRY_CHARS:
+        if not text or len(text.encode()) > ENTRY_CHARS:
-            die("line %d: %d chars (limit %d)." % (i, len(text), ENTRY_CHARS))
+            die("line %d: %d bytes (limit %d)." % (i, len(text.encode()), ENTRY_CHARS))
         out.append("#%d %s %s" % (T + len(out), date, text))
         last = date
     log_append(d, out)

test.py

@@ -155,7 +155,6 @@ check("REJECTED" in run("sleep", "0-2", "attempted overwrite").stderr,
 r = run("recall", "memory number 7,")
 check(r.returncode == 0 and "#7 " in r.stdout, "recall missed a memory")
 
-# a wrong summary can be dropped, with everything built on top of it
 def treesize():
     t = os.path.join(d, "TREE")
     return sum(os.path.getsize(os.path.join(t, f)) for f in os.listdir(t))
@@ -180,6 +179,28 @@ check(treesize() == before, "tree did not return to its original size")
 check(run("forget", "17-33").returncode == 1, "forgetting a non-block should fail")
 check(run("forget", "999998-1000000").returncode == 1, "forgetting a missing block should fail")
 
+# UTF-8: multi-byte characters must not shift record boundaries or dodge limits
+run("note", "reunião com João em São Paulo: ação aprovada, coração tranquilo")
+run("note", "a plain ascii memory right after the accented one")
+r = run("recall", "coração")
+check("João" in r.stdout, "recall lost the accented memory: " + r.stdout + r.stderr)
+r = run("recall", "plain ascii memory right after")
+check("#%d " % (N + 2) in r.stdout, "record after a multi-byte one reads shifted")
+r = run("note", "ã" * 150)
+check(r.returncode == 1 and "300 bytes" in r.stderr,
+      "multi-byte note dodged the byte limit: " + r.stderr)
+
+# a wrong summary can be dropped, with everything built on top of it
+
+# note landed -> its blocks are pending; settle before the final wake check
+while True:
+    r = run("sleep")
+    if "You woke up" in r.stdout:
+        break
+    bid = [l for l in r.stdout.splitlines() if l.strip().startswith("memo sleep ")][0].split()[2]
+    run("sleep", bid, "settled")
+check(run("wake").returncode == 0, "wake refuses at the very end")
+
 shutil.rmtree(d)
 print("\n%d passed, %d failed" % (ok, fail))
 sys.exit(1 if fail else 0)