diff --git a/memo b/memo index b4e24dd..daedcd7 100755 --- a/memo +++ b/memo @@ -76,6 +76,9 @@ def config(d): PART_CHARS = int(v) elif k == "PART_LINES": PART_LINES = int(v) + if ENTRY_CHARS > min(TREE_REC - 8, LOG_REC - 40): + die("config: ENTRY_CHARS=%d cannot fit the %d/%d-byte records." + % (ENTRY_CHARS, LOG_REC, TREE_REC)) def log_path(d): @@ -108,13 +111,15 @@ def log_get(d, i): def log_slice(d, lo, hi): - """Memories [lo,hi) in one read.""" + """Memories [lo,hi) in one read. Records are sliced as BYTES and decoded + one by one -- slicing decoded text would shift every boundary after the + first multi-byte character.""" with open(log_path(d), "rb") as f: f.seek(lo * LOG_REC) - buf = f.read((hi - lo) * LOG_REC).decode() + buf = f.read((hi - lo) * LOG_REC) out = [] for i in range(hi - lo): - line = buf[i * LOG_REC:(i + 1) * LOG_REC].rstrip() + line = buf[i * LOG_REC:(i + 1) * LOG_REC].decode().rstrip() head, _, rest = line.partition(" ") date, _, text = rest.partition(" ") out.append((int(head[1:]), date, text)) @@ -206,9 +211,11 @@ def check(text): if "\n" in text or "\r" in text: die("REJECTED: %d lines. A memory is exactly one line." % (text.count("\n") + 1)) - if len(text) > ENTRY_CHARS: - die("REJECTED: %d chars, %d over the %d limit. Compress it further." - % (len(text), len(text) - ENTRY_CHARS, ENTRY_CHARS)) + n = len(text.encode()) + if n > ENTRY_CHARS: + die("REJECTED: %d bytes, %d over the %d limit (accents and symbols " + "cost more than one). Compress it further." + % (n, n - ENTRY_CHARS, ENTRY_CHARS)) return text @@ -270,11 +277,12 @@ def paginate(lines): """Split the document into parts that survive any harness's output cap.""" parts, cur, size = [], [], 0 for line in lines: - if cur and (len(cur) >= PART_LINES or size + len(line) > PART_CHARS): + n = len(line.encode()) + if cur and (len(cur) >= PART_LINES or size + n > PART_CHARS): parts.append(cur) cur, size = [], 0 cur.append(line) - size += len(line) + 1 + size += n + 1 if cur: parts.append(cur) return parts @@ -417,8 +425,8 @@ def cmd_import(d, args): die("line %d: date %s is older than the previous memory (%s). " "Memories must be in order." % (i, date, last)) text = text.strip() - if not text or len(text) > ENTRY_CHARS: - die("line %d: %d chars (limit %d)." % (i, len(text), ENTRY_CHARS)) + if not text or len(text.encode()) > ENTRY_CHARS: + die("line %d: %d bytes (limit %d)." % (i, len(text.encode()), ENTRY_CHARS)) out.append("#%d %s %s" % (T + len(out), date, text)) last = date log_append(d, out) diff --git a/test.py b/test.py index 1c9dcbb..96ee839 100755 --- a/test.py +++ b/test.py @@ -155,7 +155,6 @@ check("REJECTED" in run("sleep", "0-2", "attempted overwrite").stderr, r = run("recall", "memory number 7,") check(r.returncode == 0 and "#7 " in r.stdout, "recall missed a memory") -# a wrong summary can be dropped, with everything built on top of it def treesize(): t = os.path.join(d, "TREE") return sum(os.path.getsize(os.path.join(t, f)) for f in os.listdir(t)) @@ -180,6 +179,28 @@ check(treesize() == before, "tree did not return to its original size") check(run("forget", "17-33").returncode == 1, "forgetting a non-block should fail") check(run("forget", "999998-1000000").returncode == 1, "forgetting a missing block should fail") +# UTF-8: multi-byte characters must not shift record boundaries or dodge limits +run("note", "reunião com João em São Paulo: ação aprovada, coração tranquilo") +run("note", "a plain ascii memory right after the accented one") +r = run("recall", "coração") +check("João" in r.stdout, "recall lost the accented memory: " + r.stdout + r.stderr) +r = run("recall", "plain ascii memory right after") +check("#%d " % (N + 2) in r.stdout, "record after a multi-byte one reads shifted") +r = run("note", "ã" * 150) +check(r.returncode == 1 and "300 bytes" in r.stderr, + "multi-byte note dodged the byte limit: " + r.stderr) + +# a wrong summary can be dropped, with everything built on top of it + +# note landed -> its blocks are pending; settle before the final wake check +while True: + r = run("sleep") + if "You woke up" in r.stdout: + break + bid = [l for l in r.stdout.splitlines() if l.strip().startswith("memo sleep ")][0].split()[2] + run("sleep", bid, "settled") +check(run("wake").returncode == 0, "wake refuses at the very end") + shutil.rmtree(d) print("\n%d passed, %d failed" % (ok, fail)) sys.exit(1 if fail else 0)