From e123c893171936ff6eec737e297d607149605705 Mon Sep 17 00:00:00 2001 From: Victor Taelin Date: Sat, 25 Jul 2026 14:34:29 -0300 Subject: [PATCH] fixed-width records: O(1) lookup, no scanning Finding a block meant parsing the whole tree: 1.3s at a million memories, and the Minilins write fast enough to get there. Records are now fixed width, so position is identity -- memory i at i*320 of LOG.txt, block [ks,(k+1)s) at k*288 of TREE/s. One seek, no index file to keep in sync. A directory of one file per block was the obvious alternative and is worse: 25x disk (781 MB for 31 MB of text) and finding pending work still scans. Here a level file is a dense prefix, so its length IS the watermark: pending costs one stat per level. Measured at 1M memories: wake 0.96s -> 0.03s, note 1.30s -> 0.02s, disk 370 MB -> 607 MB. --- README.md | 37 +++++-- memo | 290 +++++++++++++++++++++++++++++++++++------------------- test.py | 41 ++++---- 3 files changed, 243 insertions(+), 125 deletions(-) diff --git a/README.md b/README.md index 24d2b76..8ffcf1c 100644 --- a/README.md +++ b/README.md @@ -147,16 +147,37 @@ yours is more generous, raise the two settings and get fewer parts. ``` $MEMORY_DIR/ LOG.txt #id date text append-only. never edited. the truth. - TREE.txt lo-hi text a cache. each block written once, unless forgotten. + TREE/2 one summary per a cache of block summaries, one file per block + TREE/4 record, indexed size. each block written once, unless forgotten. + TREE/8 by position + ... config ENTRY_CHARS=280 longest a memory may be WAKE_LINES=320 how many lines `memo wake` prints (~24k tokens) PART_CHARS=8000 how much of it fits in one command's output PART_LINES=200 ...and in how many lines ``` -Both files are plain text, sorted by construction, and safe to read with any -tool. Writes are serialised with a lock, so parallel sessions on one machine can -append at the same time without corrupting anything. +**Records are fixed width**: 320 bytes in `LOG.txt`, 288 in the `TREE` files. +That is the whole indexing strategy — position *is* identity, so memory `i` +sits at `i*320`, and block `[k*s, (k+1)*s)` sits at `k*288` of `TREE/s`. +Everything is one seek: no scanning, and no index file that could ever +disagree with the data. + +``` +1,000,000 memories, 607 MB on disk: + + memo wake 0.03s (scanning the same store: 0.96s) + memo note 0.02s (scanning: 1.30s) + memo sleep 0.02s +``` + +Finding pending work costs one `stat` per level — about twenty, forever — +because each level file holds a dense prefix, so its length says exactly how +far that level got. Padding costs ~1.6x on disk and buys O(1) on everything. + +Both files are still plain text: `grep`, `cat` and `wc -l` all work, lines are +just space-padded. Writes are serialised with a lock, so parallel sessions on +one machine can append at the same time without corrupting anything. Agents must never create, edit or delete anything in `MEMORY_DIR` themselves. Every write goes through `memo`, which enforces the one-line and character @@ -179,11 +200,13 @@ it and everything built on top of it: ``` $ memo forget 188-192 -forgot 176-192 184-192 188-192. They will be compressed again on your next sleep. +forgot 20 summaries (188-192 and everything built from it). They will be +compressed again on your next sleep. ``` -That is a handful of small compressions, not a rebuild, and `LOG.txt` is never -touched. Fixing one bad summary can never cost you a memory. +`LOG.txt` is never touched, so fixing a bad summary can never cost you a +memory. Blocks are built in order, so forgetting one also drops the blocks +built after it at the same levels; they come back on the next sleep. ## Add this to your agent's instruction file diff --git a/memo b/memo index 7bb695b..b4e24dd 100755 --- a/memo +++ b/memo @@ -32,6 +32,14 @@ PART_CHARS = 8000 PART_LINES = 200 +# Records are FIXED WIDTH, so a memory or a block is found by seeking to its +# offset -- no scanning, no index file to keep in sync. Position IS identity: +# memory i lives at i*LOG_REC of LOG.txt, and block [k*s,(k+1)*s) lives at +# k*TREE_REC of TREE/. Padding costs ~2x on disk and buys O(1) everywhere. +LOG_REC = 320 +TREE_REC = 288 + + # ---------------------------------------------------------------- store def store(): @@ -40,11 +48,10 @@ def store(): die("MEMORY_DIR is not set. It must name this machine's memory " "directory, e.g. export MEMORY_DIR=~/memory") d = os.path.expanduser(d) - os.makedirs(d, exist_ok=True) - for name in ("LOG.txt", "TREE.txt"): - p = os.path.join(d, name) - if not os.path.exists(p): - open(p, "a").close() + os.makedirs(os.path.join(d, "TREE"), exist_ok=True) + p = os.path.join(d, "LOG.txt") + if not os.path.exists(p): + open(p, "a").close() return d @@ -71,62 +78,119 @@ def config(d): PART_LINES = int(v) -def read_log(d): - """[(id, date, text)] in order.""" +def log_path(d): + return os.path.join(d, "LOG.txt") + + +def tree_path(d, size): + return os.path.join(d, "TREE", str(size)) + + +def count(path, rec): + try: + return os.path.getsize(path) // rec + except OSError: + return 0 + + +def log_len(d): + return count(log_path(d), LOG_REC) + + +def log_get(d, i): + """(id, date, text) of memory i, in one seek.""" + with open(log_path(d), "rb") as f: + f.seek(i * LOG_REC) + line = f.read(LOG_REC).decode().rstrip() + head, _, rest = line.partition(" ") + date, _, text = rest.partition(" ") + return int(head[1:]), date, text + + +def log_slice(d, lo, hi): + """Memories [lo,hi) in one read.""" + with open(log_path(d), "rb") as f: + f.seek(lo * LOG_REC) + buf = f.read((hi - lo) * LOG_REC).decode() out = [] - for line in open(os.path.join(d, "LOG.txt")): - line = line.rstrip("\n") - if not line: - continue - head, _, text = line.partition(" ") - date, _, text = text.partition(" ") + for i in range(hi - lo): + line = buf[i * LOG_REC:(i + 1) * LOG_REC].rstrip() + head, _, rest = line.partition(" ") + date, _, text = rest.partition(" ") out.append((int(head[1:]), date, text)) return out -def read_tree(d): - """{(lo,hi): text}""" - out = {} - for line in open(os.path.join(d, "TREE.txt")): - line = line.rstrip("\n") - if not line: - continue - head, _, text = line.partition(" ") - lo, _, hi = head.partition("-") - out[(int(lo), int(hi))] = text - return out +def tree_get(d, lo, hi): + """The summary of block [lo,hi), in one seek. None if not built yet.""" + size = hi - lo + with open(tree_path(d, size), "rb") as f: + f.seek((lo // size) * TREE_REC) + rec = f.read(TREE_REC) + return rec.decode().rstrip() or None -def rewrite_tree(d, keep): - """Replace TREE.txt. Legal only because TREE.txt is a CACHE of a pure - function of LOG.txt -- the log itself is never rewritten by anything.""" +def pad(text, rec): + b = text.encode() + if len(b) > rec - 1: + die("REJECTED: %d bytes, over the %d-byte record." % (len(b), rec - 1)) + return b + b" " * (rec - 1 - len(b)) + b"\n" + + +def locked(d): lock = open(os.path.join(d, ".lock"), "w") fcntl.flock(lock, fcntl.LOCK_EX) + return lock + + +def log_append(d, entries): + """Append memories. The only way LOG.txt ever changes.""" + lock = locked(d) try: - tmp = os.path.join(d, "TREE.txt.new") - with open(tmp, "w") as f: - for (lo, hi), text in keep: - f.write("%d-%d %s\n" % (lo, hi, text)) + with open(log_path(d), "ab") as f: + for e in entries: + f.write(pad(e, LOG_REC)) f.flush() os.fsync(f.fileno()) - os.replace(tmp, os.path.join(d, "TREE.txt")) finally: - fcntl.flock(lock, fcntl.LOCK_UN) lock.close() -def append(d, name, lines): - """Append under an exclusive lock. The only way anything is ever written.""" - lock = open(os.path.join(d, ".lock"), "w") - fcntl.flock(lock, fcntl.LOCK_EX) +def tree_put(d, lo, hi, text): + """Write block [lo,hi). Blocks are built in order, so this only ever + appends one record to one level file.""" + size = hi - lo + lock = locked(d) try: - with open(os.path.join(d, name), "a") as f: - for line in lines: - f.write(line + "\n") + p = tree_path(d, size) + if count(p, TREE_REC) != lo // size: + return False + with open(p, "ab") as f: + f.write(pad(text, TREE_REC)) f.flush() os.fsync(f.fileno()) + return True + finally: + lock.close() + + +def tree_drop(d, lo, hi): + """Forget block [lo,hi) and every block built from it, by truncating each + level back to that point. Later blocks at those levels go too and are + rebuilt; the log is never touched, so nothing is lost.""" + gone, size = [], hi - lo + lock = locked(d) + try: + while size <= log_len(d): + p, k = tree_path(d, size), lo // size + n = count(p, TREE_REC) + if n > k: + gone += [(i * size, (i + 1) * size) for i in range(k, n)] + with open(p, "r+b") as f: + f.truncate(k * TREE_REC) + size *= 2 + return gone finally: - fcntl.flock(lock, fcntl.LOCK_UN) lock.close() @@ -150,21 +214,37 @@ def check(text): # ---------------------------------------------------------------- naps -def pending(log, tree): - """Blocks that can be built and have not been. Smallest first, so a - block's two halves are always available before the block itself.""" - return [b for b in complete(len(log)) if b not in tree] +def pending(d, T, limit=None): + """Blocks that can be built and have not been, smallest first. Each level + file holds a dense prefix, so its length says exactly how far that level + got: this costs one stat per level, never a scan.""" + todo, size = [], 2 + while size <= T: + have = count(tree_path(d, size), TREE_REC) + for k in range(have, T // size): + todo.append((k * size, (k + 1) * size)) + if limit and len(todo) >= limit: + return todo + size *= 2 + return todo -def nap_prompt(d, log, tree, todo): - lo, hi = todo[0] +def pending_count(d, T): + n, size = 0, 2 + while size <= T: + n += T // size - count(tree_path(d, size), TREE_REC) + size *= 2 + return n + + +def nap_prompt(d, lo, hi, left): if hi - lo <= RAW_MAX: - body = "\n".join(" #%d %s %s" % log[i] for i in range(lo, hi)) + body = "\n".join(" #%d %s %s" % e for e in log_slice(d, lo, hi)) what = "these %d memories" % (hi - lo) else: mid = (lo + hi) // 2 - body = "\n".join(" %s" % tree[(a, b)] for a, b in - ((lo, mid), (mid, hi))) + body = "\n".join(" " + tree_get(d, a, b) for a, b in + ((lo, mid), (mid, hi))) what = "these two summaries" return ( "You are dreaming. Compress {what} into ONE line of at most {n} " @@ -173,8 +253,15 @@ def nap_prompt(d, log, tree, todo): "of memories.\n\n{body}\n\nThen run exactly:\n" " memo sleep {lo}-{hi} \"\"\n\n" "{left} nap(s) left after this one." - ).format(what=what, n=ENTRY_CHARS, body=body, lo=lo, hi=hi, - left=len(todo) - 1) + ).format(what=what, n=ENTRY_CHARS, body=body, lo=lo, hi=hi, left=left) + + +def next_nap(d, T): + todo = pending(d, T, limit=1) + if not todo: + return None + lo, hi = todo[0] + return nap_prompt(d, lo, hi, pending_count(d, T) - 1) # ---------------------------------------------------------------- commands @@ -194,21 +281,24 @@ def paginate(lines): def cmd_wake(d, args): - log, tree = read_log(d), read_tree(d) - todo = pending(log, tree) - if todo: + T = log_len(d) + nap = next_nap(d, T) + if nap: print("YOU CANNOT WAKE UP YET: %d compression(s) are pending, and a " "memory\nwith work left in it is not yet the truth. Sleep first " - "-- it is quick.\n" % len(todo)) - print(nap_prompt(d, log, tree, todo)) + "-- it is quick.\n" % pending_count(d, T)) + print(nap) sys.exit(1) - if not log: + if not T: print("You have no memories yet. This is your first moment.\n" "Record what matters with: memo note \"...\"") return - lines = ["#%d %s %s" % log[lo] if hi - lo == 1 else - "#%d-%d %s" % (lo, hi - 1, tree[(lo, hi)]) - for lo, hi in cover(len(log), WAKE_LINES)] + lines = [] + for lo, hi in cover(T, WAKE_LINES): + if hi - lo == 1: + lines.append("#%d %s %s" % log_get(d, lo)) + else: + lines.append("#%d-%d %s" % (lo, hi - 1, tree_get(d, lo, hi))) parts = paginate(lines) k = 1 if args: @@ -234,18 +324,16 @@ def cmd_note(d, args): if len(args) != 1: die("usage: memo note \"\"" % ENTRY_CHARS) text = check(args[0]) - log = read_log(d) - date = datetime.date.today().isoformat() - append(d, "LOG.txt", ["#%d %s %s" % (len(log), date, text)]) - print("ok, memory #%d." % len(log)) - log, tree = read_log(d), read_tree(d) - todo = pending(log, tree) - if todo: - print("\n" + nap_prompt(d, log, tree, todo)) + T = log_len(d) + log_append(d, ["#%d %s %s" % (T, datetime.date.today().isoformat(), text)]) + print("ok, memory #%d." % T) + nap = next_nap(d, T + 1) + if nap: + print("\n" + nap) def cmd_sleep(d, args): - log, tree = read_log(d), read_tree(d) + T = log_len(d) if args: if len(args) != 2: die("usage: memo sleep - \"\"") @@ -253,23 +341,23 @@ def cmd_sleep(d, args): if not m: die("REJECTED: '%s' is not a block id. Copy it from the prompt." % args[0]) - block = (int(m.group(1)), int(m.group(2))) - if block in tree: + lo, hi = int(m.group(1)), int(m.group(2)) + todo = pending(d, T, limit=1) + if not todo: + die("REJECTED: nothing is pending. You are already awake.") + if (lo, hi) != todo[0]: + die("REJECTED: %d-%d is not the block to compress. Blocks are built " + "in order,\nand yours is %d-%d. Run `memo sleep` to see it." + % (lo, hi, todo[0][0], todo[0][1])) + if not tree_put(d, lo, hi, check(args[1])): print("Already dreamt; another session got there first. Skipping.") - elif block not in pending(log, tree): - die("REJECTED: block %d-%d is not ready to be compressed. Run " - "`memo sleep` for the block you should be working on." - % block) else: - text = check(args[1]) - append(d, "TREE.txt", ["%d-%d %s" % (block[0], block[1], text)]) - print("ok, %d-%d remembered." % block) - log, tree = read_log(d), read_tree(d) - todo = pending(log, tree) - if not todo: + print("ok, %d-%d remembered." % (lo, hi)) + nap = next_nap(d, T) + if not nap: print("You woke up. Nothing left to compress.") return - print("\n" + nap_prompt(d, log, tree, todo)) + print("\n" + nap) def cmd_forget(d, args): @@ -282,13 +370,16 @@ def cmd_forget(d, args): if not m: die("REJECTED: '%s' is not a block id." % args[0]) lo, hi = int(m.group(1)), int(m.group(2)) - tree = read_tree(d) - doomed = [b for b in tree if b[0] <= lo and hi <= b[1]] - if not doomed: - die("There is no summary covering %d-%d to forget." % (lo, hi)) - rewrite_tree(d, [(b, t) for b, t in sorted(tree.items()) if b not in doomed]) - print("forgot %s. They will be compressed again on your next sleep." - % " ".join("%d-%d" % b for b in sorted(doomed))) + size = hi - lo + if size < 2 or size & (size - 1) or lo % size: + die("REJECTED: %d-%d is not a block. A block covers an aligned " + "power-of-two range." % (lo, hi)) + gone = tree_drop(d, lo, hi) + if not gone: + die("There is no summary at %d-%d to forget." % (lo, hi)) + print("forgot %d summaries (%d-%d and everything built from it). They will " + "be compressed again on your next sleep." + % (len(gone), gone[0][0], gone[0][1])) def cmd_recall(d, args): @@ -298,7 +389,7 @@ def cmd_recall(d, args): pat = re.compile(args[0], re.I) except re.error as e: die("bad regex: %s" % e) - hits = [e for e in read_log(d) if pat.search(e[2])] + hits = [e for e in log_slice(d, 0, log_len(d)) if pat.search(e[2])] if not hits: print("Nothing in your memory matches that.") return @@ -312,9 +403,8 @@ def cmd_import(d, args): For bootstrapping an identity from older records. Used once.""" if len(args) != 1: die("usage: memo import # lines of 'YYYY-MM-DD '") - log = read_log(d) - n = len(log) - last = log[-1][1] if log else "0000-00-00" + T = log_len(d) + last = log_get(d, T - 1)[1] if T else "0000-00-00" out = [] for i, line in enumerate(open(args[0]), 1): line = line.rstrip("\n") @@ -329,14 +419,14 @@ def cmd_import(d, args): text = text.strip() if not text or len(text) > ENTRY_CHARS: die("line %d: %d chars (limit %d)." % (i, len(text), ENTRY_CHARS)) - out.append("#%d %s %s" % (n + len(out), date, text)) + out.append("#%d %s %s" % (T + len(out), date, text)) last = date - append(d, "LOG.txt", out) - print("imported %d memories (#%d..#%d)." % (len(out), n, n + len(out) - 1)) - todo = pending(read_log(d), read_tree(d)) - if todo: + log_append(d, out) + print("imported %d memories (#%d..#%d)." % (len(out), T, T + len(out) - 1)) + n = pending_count(d, log_len(d)) + if n: print("%d compressions are now pending. Run `memo sleep` until it " - "says you woke up." % len(todo)) + "says you woke up." % n) COMMANDS = {"wake": cmd_wake, "note": cmd_note, "sleep": cmd_sleep, diff --git a/test.py b/test.py index bfce80f..1c9dcbb 100755 --- a/test.py +++ b/test.py @@ -139,41 +139,46 @@ check("last one" in run("wake", str(len(parts))).stdout, "last part must say it check(run("wake", str(len(parts) + 1)).returncode == 1, "a nonexistent part should fail") # append-only: nothing was ever rewritten -sizes = {f: os.path.getsize(os.path.join(d, f)) for f in ("LOG.txt", "TREE.txt")} +logsz = os.path.getsize(os.path.join(d, "LOG.txt")) run("note", "one more thing happened today") -for f, s in sizes.items(): - check(os.path.getsize(os.path.join(d, f)) >= s, "%s shrank" % f) -tree = open(os.path.join(d, "TREE.txt")).read().splitlines() -check(len(tree) == len(set(l.split()[0] for l in tree)), "a block was written twice") +check(os.path.getsize(os.path.join(d, "LOG.txt")) > logsz, "note did not append") +check(logsz % 320 == 0, "LOG.txt is not a whole number of records") +for f in os.listdir(os.path.join(d, "TREE")): + check(os.path.getsize(os.path.join(d, "TREE", f)) % 288 == 0, + "TREE/%s is not a whole number of records" % f) -# writing a block twice is refused, not duplicated -r = run("sleep", tree[0].split()[0], "attempted overwrite") -check("Already dreamt" in r.stdout, "rewriting a block was allowed") +# a block already written cannot be rewritten +check("REJECTED" in run("sleep", "0-2", "attempted overwrite").stderr, + "rewriting a settled block was allowed") # recall reaches memories the summaries lost r = run("recall", "memory number 7,") check(r.returncode == 0 and "#7 " in r.stdout, "recall missed a memory") # a wrong summary can be dropped, with everything built on top of it -before = len(open(os.path.join(d, "TREE.txt")).read().splitlines()) -logsize = os.path.getsize(os.path.join(d, "LOG.txt")) +def treesize(): + t = os.path.join(d, "TREE") + return sum(os.path.getsize(os.path.join(t, f)) for f in os.listdir(t)) + +before, logsize = treesize(), os.path.getsize(os.path.join(d, "LOG.txt")) r = run("forget", "16-32") check("16-32" in r.stdout, "forget did not report the block: " + r.stdout + r.stderr) -gone = set(open(os.path.join(d, "TREE.txt")).read().splitlines()) -check(not any(l.startswith("16-32 ") for l in gone), "forgotten block still present") -check(not any(l.startswith("0-64 ") for l in gone), "an ancestor survived a forget") +check(treesize() < before, "forget did not shrink the tree") check(os.path.getsize(os.path.join(d, "LOG.txt")) == logsize, "forget touched the log") check(run("wake").returncode == 1, "wake should refuse after a forget") n = 0 -while "You woke up" not in run("sleep").stdout: +while True: r = run("sleep") + if "You woke up" in r.stdout: + break bid = [l for l in r.stdout.splitlines() if l.strip().startswith("memo sleep ")][0].split()[2] - run("sleep", bid, "rebuilt after forget") + check("REJECTED" not in run("sleep", bid, "rebuilt after forget").stderr, "rebuild rejected") n += 1 -check(n == before - len(gone), "rebuilt %d blocks, forgot %d" % (n, before - len(gone))) +check(n > 0, "forget created no work") check(run("wake").returncode == 0, "wake still refuses after rebuilding") -r = run("forget", "999999-1000000") -check(r.returncode == 1, "forgetting a nonexistent block should fail") +check(treesize() == before, "tree did not return to its original size") +check(run("forget", "17-33").returncode == 1, "forgetting a non-block should fail") +check(run("forget", "999998-1000000").returncode == 1, "forgetting a missing block should fail") shutil.rmtree(d) print("\n%d passed, %d failed" % (ok, fail))