diff --git a/README.md b/README.md index 8ffcf1c..2929f77 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,8 @@ export MEMORY_DIR="$HOME/memory" # required; there is no default ```sh memo wake # who you are. run this first, every session, - # then `memo wake 2`, `3`... until it says so. + # then the command each part names, until + # one of them prints `awake.` memo note "..." # record a memory. one line, <= 280 chars. memo sleep # compress. keep going until it says you woke up. memo recall # search the raw log for detail a summary lost. @@ -86,11 +87,11 @@ keeps a block whole when its size is small relative to its age, so **detail is proportional to recency**, and it spends exactly `WAKE_LINES` lines doing it: ``` -10,000 memories, WAKE_LINES = 320: +10,000 memories, WAKE_LINES = 256: block size: 1 2 4 8 16 32 64 128 256 - how many: 70 35 35 35 36 35 35 35 4 - └ the last 70, verbatim ───────────▶ the first 1,000, 256:1 + how many: 54 27 27 27 28 27 27 27 12 + └ the last 54, verbatim ───────────▶ the first 3,000, 256:1 ``` The oldest memories are recalled as a vague shape, the newest word for word, @@ -126,21 +127,29 @@ decisions; drop wording. ``` bad worked on the memory system today and made good progress on the design -good OptMem design settled: LOG.txt append-only truth, TREE.txt binary merge - tree of 280-char summaries, wake renders a fixed 320-line document +good OptMem design settled: LOG.txt append-only truth, TREE binary merge + tree of 280-char summaries, wake renders a fixed 256-line document ``` ## Output is delivered in parts Every harness truncates an over-long command, and each one drops a different -piece: Codex cuts at 10 KiB or 256 lines, Claude Code at 30,000 characters, -pi at 50 KB. A 320-line memory is ~79 KB, so a single-shot `memo wake` gets -mangled everywhere — and silently. +piece: -So `memo wake` pages the document into parts that fit the strictest of them +``` + Claude Code 30,000 chars drops the MIDDLE + pi 50 KB / 2000 lines drops the HEAD + Codex 10,000 tokens (configurable per call) +``` + +A 256-line memory is ~64 KB, so a single-shot `memo wake` is mangled +everywhere, and silently. + +So `memo wake` pages the document into parts that fit all of them (`PART_CHARS`, `PART_LINES`), and each part ends by naming the exact command -for the next one. Nothing is ever dropped, and no harness is special-cased: if -yours is more generous, raise the two settings and get fewer parts. +for the next one, including the `T` it was rendered at — so a memory written +mid-wake cannot shift a boundary and drop a line. Nothing is special-cased per +harness: if yours is more generous, raise the two settings for fewer parts. ## Files @@ -152,9 +161,9 @@ $MEMORY_DIR/ TREE/8 by position ... config ENTRY_CHARS=280 longest a memory may be - WAKE_LINES=320 how many lines `memo wake` prints (~24k tokens) - PART_CHARS=8000 how much of it fits in one command's output - PART_LINES=200 ...and in how many lines + WAKE_LINES=256 how many lines `memo wake` prints (~16k tokens) + PART_CHARS=20000 how much of it fits in one command's output + PART_LINES=500 ...and in how many lines ``` **Records are fixed width**: 320 bytes in `LOG.txt`, 288 in the `TREE` files. diff --git a/memo b/memo index daedcd7..bc45c61 100755 --- a/memo +++ b/memo @@ -1,14 +1,14 @@ #!/usr/bin/env python3 """OptMem: a permanent, append-only memory for AI agents. - memo wake [part] print who you are + memo wake [part [T]] print who you are memo note "..." record a memory memo sleep [id "..."] compress memo recall search the raw log memo forget drop a wrong summary so it is compressed again memo import bulk-append historical memories (bootstrap only) -Everything lives in $MEMORY_DIR as two append-only text files. See README.md. +Everything lives in $MEMORY_DIR. See README.md. """ import datetime @@ -18,18 +18,18 @@ import re import sys sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) -from blocks import complete, cover # noqa: E402 +from blocks import cover # noqa: E402 ENTRY_CHARS = 280 -WAKE_LINES = 320 +WAKE_LINES = 256 RAW_MAX = 16 # blocks up to this many memories compress from the raw log -# Every agent harness silently truncates a command that prints too much -- -# Codex at 10 KiB or 256 lines, Claude Code at 30k chars, pi at 50 KB -- and -# each drops a different part. So the memory is handed over in parts that fit -# the strictest of them. These are transport limits, not memory limits. -PART_CHARS = 8000 -PART_LINES = 200 +# Every harness truncates a command that prints too much, and each drops a +# different piece: Claude Code cuts the middle at 30,000 chars, pi cuts the +# head at 50 KB, Codex budgets 10,000 tokens. So the memory is handed over in +# parts that fit all of them. These are transport limits, not memory limits. +PART_CHARS = 20000 +PART_LINES = 500 # Records are FIXED WIDTH, so a memory or a block is found by seeking to its @@ -45,8 +45,7 @@ TREE_REC = 288 def store(): d = os.environ.get("MEMORY_DIR") if not d: - die("MEMORY_DIR is not set. It must name this machine's memory " - "directory, e.g. export MEMORY_DIR=~/memory") + die("MEMORY_DIR is not set. Example: export MEMORY_DIR=~/memory") d = os.path.expanduser(d) os.makedirs(os.path.join(d, "TREE"), exist_ok=True) p = os.path.join(d, "LOG.txt") @@ -77,7 +76,7 @@ def config(d): elif k == "PART_LINES": PART_LINES = int(v) if ENTRY_CHARS > min(TREE_REC - 8, LOG_REC - 40): - die("config: ENTRY_CHARS=%d cannot fit the %d/%d-byte records." + die("config: ENTRY_CHARS=%d does not fit the %d/%d-byte records." % (ENTRY_CHARS, LOG_REC, TREE_REC)) @@ -100,14 +99,30 @@ def log_len(d): return count(log_path(d), LOG_REC) +def repair(path, rec): + """Drop a partial trailing record left by a crash. It was never + acknowledged. Without this the next append lands at a wrong offset and + every later record is misaligned. Callers hold the lock.""" + try: + n = os.path.getsize(path) + except OSError: + return + if n % rec: + with open(path, "r+b") as f: + f.truncate(n - n % rec) + + +def parse(line): + head, _, rest = line.partition(" ") + date, _, text = rest.partition(" ") + return int(head[1:]), date, text + + def log_get(d, i): """(id, date, text) of memory i, in one seek.""" with open(log_path(d), "rb") as f: f.seek(i * LOG_REC) - line = f.read(LOG_REC).decode().rstrip() - head, _, rest = line.partition(" ") - date, _, text = rest.partition(" ") - return int(head[1:]), date, text + return parse(f.read(LOG_REC).decode().rstrip()) def log_slice(d, lo, hi): @@ -117,28 +132,26 @@ def log_slice(d, lo, hi): with open(log_path(d), "rb") as f: f.seek(lo * LOG_REC) buf = f.read((hi - lo) * LOG_REC) - out = [] - for i in range(hi - lo): - line = buf[i * LOG_REC:(i + 1) * LOG_REC].decode().rstrip() - head, _, rest = line.partition(" ") - date, _, text = rest.partition(" ") - out.append((int(head[1:]), date, text)) - return out + return [parse(buf[i * LOG_REC:(i + 1) * LOG_REC].decode().rstrip()) + for i in range(hi - lo)] def tree_get(d, lo, hi): """The summary of block [lo,hi), in one seek. None if not built yet.""" size = hi - lo - with open(tree_path(d, size), "rb") as f: - f.seek((lo // size) * TREE_REC) - rec = f.read(TREE_REC) + try: + with open(tree_path(d, size), "rb") as f: + f.seek((lo // size) * TREE_REC) + rec = f.read(TREE_REC) + except OSError: + return None return rec.decode().rstrip() or None def pad(text, rec): b = text.encode() if len(b) > rec - 1: - die("REJECTED: %d bytes, over the %d-byte record." % (len(b), rec - 1)) + die("Too long: %d bytes. The record holds %d." % (len(b), rec - 1)) return b + b" " * (rec - 1 - len(b)) + b"\n" @@ -148,15 +161,20 @@ def locked(d): return lock -def log_append(d, entries): - """Append memories. The only way LOG.txt ever changes.""" +def log_append(d, items): + """Append memories, items = [(date, text)]. The only way LOG.txt ever + changes. Ids are assigned INSIDE the lock: two sessions noting at the same + moment must not be handed the same id. Returns the first id used.""" lock = locked(d) try: + repair(log_path(d), LOG_REC) + base = log_len(d) with open(log_path(d), "ab") as f: - for e in entries: - f.write(pad(e, LOG_REC)) + for k, (date, text) in enumerate(items): + f.write(pad("#%d %s %s" % (base + k, date, text), LOG_REC)) f.flush() os.fsync(f.fileno()) + return base finally: lock.close() @@ -168,6 +186,7 @@ def tree_put(d, lo, hi, text): lock = locked(d) try: p = tree_path(d, size) + repair(p, TREE_REC) if count(p, TREE_REC) != lo // size: return False with open(p, "ab") as f: @@ -207,15 +226,13 @@ def die(msg): def check(text): text = text.strip() if not text: - die("REJECTED: empty.") + die("Empty.") if "\n" in text or "\r" in text: - die("REJECTED: %d lines. A memory is exactly one line." % - (text.count("\n") + 1)) + die("%d lines. A memory is one line." % (text.count("\n") + 1)) n = len(text.encode()) if n > ENTRY_CHARS: - die("REJECTED: %d bytes, %d over the %d limit (accents and symbols " - "cost more than one). Compress it further." - % (n, n - ENTRY_CHARS, ENTRY_CHARS)) + die("Too long: %d bytes, limit %d. Accented characters cost 2 bytes. " + "Compress it further." % (n, ENTRY_CHARS)) return text @@ -247,20 +264,16 @@ def pending_count(d, T): def nap_prompt(d, lo, hi, left): if hi - lo <= RAW_MAX: body = "\n".join(" #%d %s %s" % e for e in log_slice(d, lo, hi)) - what = "these %d memories" % (hi - lo) else: mid = (lo + hi) // 2 - body = "\n".join(" " + tree_get(d, a, b) for a, b in - ((lo, mid), (mid, hi))) - what = "these two summaries" - return ( - "You are dreaming. Compress {what} into ONE line of at most {n} " - "characters.\nKeep every name, number, date and decision. Drop wording, " - "not facts.\nInvent nothing. Write it as a memory, not as a description " - "of memories.\n\n{body}\n\nThen run exactly:\n" - " memo sleep {lo}-{hi} \"\"\n\n" - "{left} nap(s) left after this one." - ).format(what=what, n=ENTRY_CHARS, body=body, lo=lo, hi=hi, left=left) + body = "\n".join(" " + (tree_get(d, a, b) or "?") for a, b in + ((lo, mid), (mid, hi))) + return ("Compress into one line, at most %d characters.\n" + "Keep every name, number, date and decision.\n" + "Invent nothing. State the facts; do not describe them.\n\n" + "%s\n\n" + "memo sleep %d-%d \"\"\n" + "%d left after this." % (ENTRY_CHARS, body, lo, hi, left)) def next_nap(d, T): @@ -277,65 +290,66 @@ def paginate(lines): """Split the document into parts that survive any harness's output cap.""" parts, cur, size = [], [], 0 for line in lines: - n = len(line.encode()) + n = len(line.encode()) + 1 if cur and (len(cur) >= PART_LINES or size + n > PART_CHARS): parts.append(cur) cur, size = [], 0 cur.append(line) - size += n + 1 + size += n if cur: parts.append(cur) return parts def cmd_wake(d, args): - T = log_len(d) - nap = next_nap(d, T) - if nap: - print("YOU CANNOT WAKE UP YET: %d compression(s) are pending, and a " - "memory\nwith work left in it is not yet the truth. Sleep first " - "-- it is quick.\n" % pending_count(d, T)) - print(nap) + now = log_len(d) + k, T = 1, now + if args: + if len(args) > 2 or not all(a.isdigit() for a in args): + die("usage: memo wake [part [T]]") + k = int(args[0]) + if len(args) == 2: + T = int(args[1]) + if T > now: + die("T=%d, but the memory holds %d entries." % (T, now)) + # A part is rendered as of T, so a note landing between two parts cannot + # shift a boundary and drop a line. + n = pending_count(d, T) + if n: + print("Cannot wake: %d compression(s) pending.\n" % n) + print(next_nap(d, T)) sys.exit(1) if not T: - print("You have no memories yet. This is your first moment.\n" - "Record what matters with: memo note \"...\"") + print("No memories yet. Record one with: memo note \"...\"") return lines = [] for lo, hi in cover(T, WAKE_LINES): if hi - lo == 1: lines.append("#%d %s %s" % log_get(d, lo)) else: - lines.append("#%d-%d %s" % (lo, hi - 1, tree_get(d, lo, hi))) + s = tree_get(d, lo, hi) + if s is None: + die("Summary %d-%d is missing. Run: memo sleep" % (lo, hi - 1)) + lines.append("#%d-%d %s" % (lo, hi - 1, s)) parts = paginate(lines) - k = 1 - if args: - if len(args) != 1 or not args[0].isdigit(): - die("usage: memo wake [part]") - k = int(args[0]) - if not 1 <= k <= len(parts): - die("There is no part %d. Your memory has %d." % (k, len(parts))) + if not 1 <= k <= len(parts): + die("No part %d. The memory has %d." % (k, len(parts))) if len(parts) > 1: - print("--- your memory, part %d of %d (oldest first) ---" % (k, len(parts))) + print("memory, part %d of %d, oldest first" % (k, len(parts))) print("\n".join(parts[k - 1])) if k < len(parts): - print("--- END OF PART %d. You are still half awake: you have NOT seen " - "your most recent\n memories yet, and acting now would be " - "acting blind. Run `memo wake %d`\n RIGHT NOW, and keep going " - "until a part tells you that was the last one. ---" % (k, k + 1)) + print("next: memo wake %d %d" % (k + 1, T)) elif len(parts) > 1: - print("--- END OF PART %d, the last one. That is your whole memory. " - "You are awake. ---" % k) + print("awake.") def cmd_note(d, args): if len(args) != 1: die("usage: memo note \"\"" % ENTRY_CHARS) text = check(args[0]) - T = log_len(d) - log_append(d, ["#%d %s %s" % (T, datetime.date.today().isoformat(), text)]) - print("ok, memory #%d." % T) - nap = next_nap(d, T + 1) + i = log_append(d, [(datetime.date.today().isoformat(), text)]) + print("saved as #%d." % i) + nap = next_nap(d, i + 1) if nap: print("\n" + nap) @@ -347,23 +361,21 @@ def cmd_sleep(d, args): die("usage: memo sleep - \"\"") m = re.fullmatch(r"(\d+)-(\d+)", args[0]) if not m: - die("REJECTED: '%s' is not a block id. Copy it from the prompt." - % args[0]) + die("'%s' is not a block id. Copy it from the prompt." % args[0]) lo, hi = int(m.group(1)), int(m.group(2)) todo = pending(d, T, limit=1) if not todo: - die("REJECTED: nothing is pending. You are already awake.") + die("Nothing pending.") if (lo, hi) != todo[0]: - die("REJECTED: %d-%d is not the block to compress. Blocks are built " - "in order,\nand yours is %d-%d. Run `memo sleep` to see it." - % (lo, hi, todo[0][0], todo[0][1])) + die("Wrong block: %d-%d. Blocks are built in order; the next is " + "%d-%d." % (lo, hi, todo[0][0], todo[0][1])) if not tree_put(d, lo, hi, check(args[1])): - print("Already dreamt; another session got there first. Skipping.") + print("Another session already wrote %d-%d." % (lo, hi)) else: - print("ok, %d-%d remembered." % (lo, hi)) + print("%d-%d saved." % (lo, hi)) nap = next_nap(d, T) if not nap: - print("You woke up. Nothing left to compress.") + print("Nothing left to compress. You are awake.") return print("\n" + nap) @@ -376,17 +388,16 @@ def cmd_forget(d, args): die("usage: memo forget -") m = re.fullmatch(r"(\d+)-(\d+)", args[0]) if not m: - die("REJECTED: '%s' is not a block id." % args[0]) + die("'%s' is not a block id." % args[0]) lo, hi = int(m.group(1)), int(m.group(2)) size = hi - lo if size < 2 or size & (size - 1) or lo % size: - die("REJECTED: %d-%d is not a block. A block covers an aligned " - "power-of-two range." % (lo, hi)) + die("%d-%d is not a block. A block covers an aligned power-of-two " + "range." % (lo, hi)) gone = tree_drop(d, lo, hi) if not gone: - die("There is no summary at %d-%d to forget." % (lo, hi)) - print("forgot %d summaries (%d-%d and everything built from it). They will " - "be compressed again on your next sleep." + die("No summary at %d-%d." % (lo, hi)) + print("Forgot %d summaries, from %d-%d up. Run: memo sleep" % (len(gone), gone[0][0], gone[0][1])) @@ -399,11 +410,22 @@ def cmd_recall(d, args): die("bad regex: %s" % e) hits = [e for e in log_slice(d, 0, log_len(d)) if pat.search(e[2])] if not hits: - print("Nothing in your memory matches that.") + print("No match.") return - for e in hits: - print("#%d %s %s" % e) - print("\n%d memories matched." % len(hits)) + # Newest first is what a search is usually for, and the output has to fit + # the same cap `wake` respects. + out, size = [], 0 + for e in reversed(hits): + line = "#%d %s %s" % e + size += len(line.encode()) + 1 + if size > PART_CHARS: + break + out.append(line) + print("\n".join(reversed(out))) + if len(out) < len(hits): + print("newest %d of %d matches. Narrow the regex." % (len(out), len(hits))) + else: + print("%d matches." % len(hits)) def cmd_import(d, args): @@ -411,8 +433,7 @@ def cmd_import(d, args): For bootstrapping an identity from older records. Used once.""" if len(args) != 1: die("usage: memo import # lines of 'YYYY-MM-DD '") - T = log_len(d) - last = log_get(d, T - 1)[1] if T else "0000-00-00" + last = log_get(d, log_len(d) - 1)[1] if log_len(d) else "0000-00-00" out = [] for i, line in enumerate(open(args[0]), 1): line = line.rstrip("\n") @@ -422,19 +443,19 @@ def cmd_import(d, args): if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", date): die("line %d: expected 'YYYY-MM-DD ', got: %s" % (i, line)) if date < last: - die("line %d: date %s is older than the previous memory (%s). " - "Memories must be in order." % (i, date, last)) + die("line %d: date %s precedes the previous memory (%s)." + % (i, date, last)) text = text.strip() if not text or len(text.encode()) > ENTRY_CHARS: - die("line %d: %d bytes (limit %d)." % (i, len(text.encode()), ENTRY_CHARS)) - out.append("#%d %s %s" % (T + len(out), date, text)) + die("line %d: %d bytes, limit %d." % (i, len(text.encode()), ENTRY_CHARS)) + out.append((date, text)) last = date - log_append(d, out) - print("imported %d memories (#%d..#%d)." % (len(out), T, T + len(out) - 1)) + base = log_append(d, out) + print("imported %d memories, #%d to #%d." % (len(out), base, base + len(out) - 1)) n = pending_count(d, log_len(d)) if n: - print("%d compressions are now pending. Run `memo sleep` until it " - "says you woke up." % n) + print("%d compressions pending. Run `memo sleep` until it says you " + "are awake." % n) COMMANDS = {"wake": cmd_wake, "note": cmd_note, "sleep": cmd_sleep, diff --git a/test.py b/test.py index 96ee839..2cf353d 100755 --- a/test.py +++ b/test.py @@ -16,7 +16,12 @@ sys.path.insert(0, HERE) from blocks import complete, cover # noqa: E402 N = 2000 -WAKE_LINES = 320 +WAKE_LINES = 256 +PART_CHARS = 20000 +# Verified caps of the harnesses in the wild: Claude Code cuts a command's +# output at 30,000 chars (middle), pi at 50 KB / 2000 lines (head), Codex +# budgets 10,000 tokens. A part must fit the strictest of each kind. +CAP_CHARS, CAP_LINES = 30000, 2000 ok, fail = 0, 0 @@ -76,12 +81,12 @@ def run(*args): r = run("note", "x" * 281) -check(r.returncode == 1 and "REJECTED" in r.stderr, "over-long note accepted") +check(r.returncode == 1 and "Too long" in r.stderr, "over-long note accepted") r = run("note", "two\nlines") -check(r.returncode == 1 and "REJECTED" in r.stderr, "multi-line note accepted") +check(r.returncode == 1 and "one line" in r.stderr, "multi-line note accepted") r = run("note", " ") check(r.returncode == 1, "empty note accepted") -check("no memories yet" in run("wake").stdout, "empty wake should say so") +check("No memories yet" in run("wake").stdout, "empty wake should say so") with open(os.path.join(d, "seed.txt"), "w") as f: day = datetime.date(2020, 1, 1) @@ -92,13 +97,13 @@ r = run("import", os.path.join(d, "seed.txt")) check("imported %d" % N in r.stdout, "import failed: " + r.stdout + r.stderr) r = run("wake") -check(r.returncode == 1 and "CANNOT WAKE" in r.stdout, +check(r.returncode == 1 and "Cannot wake" in r.stdout, "wake must refuse while work is pending") # sleep loop, with a fake compressor naps = 0 r = run("sleep") -while "You woke up" not in r.stdout: +while "You are awake" not in r.stdout: line = [l for l in r.stdout.splitlines() if l.strip().startswith("memo sleep ")] check(bool(line), "no command offered:\n" + r.stdout + r.stderr) if not line: @@ -108,25 +113,24 @@ while "You woke up" not in r.stdout: if l.startswith(" #") or (l.startswith(" ") and l.strip() and not l.strip().startswith("memo"))] r = run("sleep", bid, (" ".join(body)[:280]).strip() or "empty") - check("REJECTED" not in r.stderr, "sleep rejected a valid nap: " + r.stderr) + check(r.returncode == 0, "sleep rejected a valid nap: " + r.stderr) naps += 1 check(naps == len(complete(N)), "did %d naps, expected %d" % (naps, len(complete(N)))) r = run("wake") check(r.returncode == 0, "wake still refuses after a full sleep") -# the document survives pagination, and every part fits the strictest harness -# output cap in the wild (Codex: 10 KiB or 256 lines) +# the document survives pagination, and every part fits every harness's cap parts, k = [], 1 while True: r = run("wake", str(k)) if r.returncode != 0: break - body = [l for l in r.stdout.splitlines() - if not l.startswith("---") and not l.startswith(" ")] - check(len(r.stdout) < 10240, "part %d is %d bytes, over Codex's 10 KiB cap" - % (k, len(r.stdout))) - check(len(r.stdout.splitlines()) < 256, "part %d is over Codex's 256-line cap" % k) + body = [l for l in r.stdout.splitlines() if l.startswith("#")] + check(len(r.stdout) < CAP_CHARS, "part %d is %d chars, over the %d cap" + % (k, len(r.stdout), CAP_CHARS)) + check(len(r.stdout.splitlines()) < CAP_LINES, "part %d is over %d lines" + % (k, CAP_LINES)) parts.append(body) k += 1 check(len(parts) > 1, "a %d-line memory should need more than one part" % WAKE_LINES) @@ -135,7 +139,7 @@ check(len(lines) == WAKE_LINES, "woke with %d lines, want %d" % (len(lines), WAK check(lines[-1].startswith("#%d " % (N - 1)), "newest memory not last / not raw") check(lines[0].startswith("#0-"), "oldest line should be a summary block") check("memo wake 2" in run("wake").stdout, "part 1 must name the next command") -check("last one" in run("wake", str(len(parts))).stdout, "last part must say it is last") +check("awake." in run("wake", str(len(parts))).stdout, "last part must say it is last") check(run("wake", str(len(parts) + 1)).returncode == 1, "a nonexistent part should fail") # append-only: nothing was ever rewritten @@ -148,7 +152,7 @@ for f in os.listdir(os.path.join(d, "TREE")): "TREE/%s is not a whole number of records" % f) # a block already written cannot be rewritten -check("REJECTED" in run("sleep", "0-2", "attempted overwrite").stderr, +check(run("sleep", "0-2", "attempted overwrite").returncode == 1, "rewriting a settled block was allowed") # recall reaches memories the summaries lost @@ -168,10 +172,10 @@ check(run("wake").returncode == 1, "wake should refuse after a forget") n = 0 while True: r = run("sleep") - if "You woke up" in r.stdout: + if "You are awake" in r.stdout: break bid = [l for l in r.stdout.splitlines() if l.strip().startswith("memo sleep ")][0].split()[2] - check("REJECTED" not in run("sleep", bid, "rebuilt after forget").stderr, "rebuild rejected") + check(run("sleep", bid, "rebuilt after forget").returncode == 0, "rebuild rejected") n += 1 check(n > 0, "forget created no work") check(run("wake").returncode == 0, "wake still refuses after rebuilding") @@ -190,17 +194,63 @@ r = run("note", "ã" * 150) check(r.returncode == 1 and "300 bytes" in r.stderr, "multi-byte note dodged the byte limit: " + r.stderr) -# a wrong summary can be dropped, with everything built on top of it - # note landed -> its blocks are pending; settle before the final wake check while True: r = run("sleep") - if "You woke up" in r.stdout: + if "You are awake" in r.stdout: break bid = [l for l in r.stdout.splitlines() if l.strip().startswith("memo sleep ")][0].split()[2] run("sleep", bid, "settled") check(run("wake").returncode == 0, "wake refuses at the very end") +# a part is rendered as of T, so a note landing mid-wake cannot shift a +# boundary and silently drop a line +T0 = os.path.getsize(os.path.join(d, "LOG.txt")) // 320 +before = run("wake", "1", str(T0)) +check(before.returncode == 0, "as-of-T wake failed: " + before.stdout + before.stderr) +run("note", "a note that lands between two wake calls") +check(run("wake", "1", str(T0)).stdout == before.stdout, + "a note between parts changed an already-rendered part") +check(run("wake", "1", str(T0 + 99)).returncode == 1, "wake accepted a future T") + +# recall must not hand back more than a harness will carry +r = run("recall", "memory number") +check(len(r.stdout) < CAP_CHARS, "recall returned %d chars" % len(r.stdout)) +check("Narrow the regex" in r.stdout, "recall did not say it had been capped") + +# ---- concurrency and crash recovery ---------------------------------- + +d2 = tempfile.mkdtemp(prefix="optmem-race-") +env2 = dict(os.environ, MEMORY_DIR=d2) +P = 16 +procs = [subprocess.Popen(memo + ["note", "parallel note %d" % i], env=env2, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + for i in range(P)] +for p in procs: + p.wait() +with open(os.path.join(d2, "LOG.txt"), "rb") as f: + recs = [f.read(320) for _ in range(P)] +ids = [r.decode().split()[0] for r in recs if r.strip()] +check(len(ids) == P, "%d of %d parallel notes survived" % (len(ids), P)) +check(len(set(ids)) == P, "parallel notes collided on an id: %s" % sorted(ids)) +check(sorted(ids) == sorted("#%d" % i for i in range(P)), + "parallel note ids are not 0..%d: %s" % (P - 1, sorted(ids))) + +# a crash mid-append leaves a partial record; the next append must drop it, +# or every later record is misaligned forever +with open(os.path.join(d2, "LOG.txt"), "ab") as f: + f.write(b"#99 2026-01-01 a half-written record killed by a power cut") +r = subprocess.run(memo + ["note", "the memory right after a torn write"], + env=env2, capture_output=True, text=True) +check(r.returncode == 0, "note failed after a torn write: " + r.stderr) +sz = os.path.getsize(os.path.join(d2, "LOG.txt")) +check(sz % 320 == 0, "LOG.txt left misaligned after a torn write: %d" % sz) +check("saved as #%d" % P in r.stdout, "torn record was counted as a memory") +r = subprocess.run(memo + ["recall", "right after a torn write"], env=env2, + capture_output=True, text=True) +check("#%d " % P in r.stdout, "the memory after a torn write reads wrong") + +shutil.rmtree(d2) shutil.rmtree(d) print("\n%d passed, %d failed" % (ok, fail)) sys.exit(1 if fail else 0)