fixed-width records: O(1) lookup, no scanning

Finding a block meant parsing the whole tree: 1.3s at a million memories, and
the Minilins write fast enough to get there. Records are now fixed width, so
position is identity -- memory i at i*320 of LOG.txt, block [ks,(k+1)s) at
k*288 of TREE/s. One seek, no index file to keep in sync.

A directory of one file per block was the obvious alternative and is worse:
25x disk (781 MB for 31 MB of text) and finding pending work still scans.
Here a level file is a dense prefix, so its length IS the watermark: pending
costs one stat per level. Measured at 1M memories: wake 0.96s -> 0.03s,
note 1.30s -> 0.02s, disk 370 MB -> 607 MB.

README.md

@@ -147,16 +147,37 @@ yours is more generous, raise the two settings and get fewer parts.
 ```
 $MEMORY_DIR/
   LOG.txt     #id date text     append-only. never edited. the truth.
-  TREE.txt    lo-hi text        a cache. each block written once, unless forgotten.
+  TREE/2      one summary per   a cache of block summaries, one file per block
+  TREE/4      record, indexed   size. each block written once, unless forgotten.
+  TREE/8      by position
+  ...
   config      ENTRY_CHARS=280   longest a memory may be
               WAKE_LINES=320    how many lines `memo wake` prints (~24k tokens)
               PART_CHARS=8000   how much of it fits in one command's output
               PART_LINES=200    ...and in how many lines
 ```
 
-Both files are plain text, sorted by construction, and safe to read with any
+**Records are fixed width**: 320 bytes in `LOG.txt`, 288 in the `TREE` files.
-tool. Writes are serialised with a lock, so parallel sessions on one machine can
+That is the whole indexing strategy — position *is* identity, so memory `i`
-append at the same time without corrupting anything.
+sits at `i*320`, and block `[k*s, (k+1)*s)` sits at `k*288` of `TREE/s`.
+Everything is one seek: no scanning, and no index file that could ever
+disagree with the data.
+
+```
+1,000,000 memories, 607 MB on disk:
+
+  memo wake    0.03s        (scanning the same store: 0.96s)
+  memo note    0.02s        (scanning: 1.30s)
+  memo sleep   0.02s
+```
+
+Finding pending work costs one `stat` per level — about twenty, forever —
+because each level file holds a dense prefix, so its length says exactly how
+far that level got. Padding costs ~1.6x on disk and buys O(1) on everything.
+
+Both files are still plain text: `grep`, `cat` and `wc -l` all work, lines are
+just space-padded. Writes are serialised with a lock, so parallel sessions on
+one machine can append at the same time without corrupting anything.
 
 Agents must never create, edit or delete anything in `MEMORY_DIR` themselves.
 Every write goes through `memo`, which enforces the one-line and character
@@ -179,11 +200,13 @@ it and everything built on top of it:
 
 ```
 $ memo forget 188-192
-forgot 176-192 184-192 188-192. They will be compressed again on your next sleep.
+forgot 20 summaries (188-192 and everything built from it). They will be
+compressed again on your next sleep.
 ```
 
-That is a handful of small compressions, not a rebuild, and `LOG.txt` is never
+`LOG.txt` is never touched, so fixing a bad summary can never cost you a
-touched. Fixing one bad summary can never cost you a memory.
+memory. Blocks are built in order, so forgetting one also drops the blocks
+built after it at the same levels; they come back on the next sleep.
 
 ## Add this to your agent's instruction file
 

memo

@@ -32,6 +32,14 @@ PART_CHARS = 8000
 PART_LINES = 200
 
 
+# Records are FIXED WIDTH, so a memory or a block is found by seeking to its
+# offset -- no scanning, no index file to keep in sync. Position IS identity:
+# memory i lives at i*LOG_REC of LOG.txt, and block [k*s,(k+1)*s) lives at
+# k*TREE_REC of TREE/<s>. Padding costs ~2x on disk and buys O(1) everywhere.
+LOG_REC = 320
+TREE_REC = 288
+
+
 # ---------------------------------------------------------------- store
 
 def store():
@@ -40,11 +48,10 @@ def store():
         die("MEMORY_DIR is not set. It must name this machine's memory "
             "directory, e.g. export MEMORY_DIR=~/memory")
     d = os.path.expanduser(d)
-    os.makedirs(d, exist_ok=True)
+    os.makedirs(os.path.join(d, "TREE"), exist_ok=True)
-    for name in ("LOG.txt", "TREE.txt"):
+    p = os.path.join(d, "LOG.txt")
-        p = os.path.join(d, name)
+    if not os.path.exists(p):
-        if not os.path.exists(p):
+        open(p, "a").close()
-            open(p, "a").close()
     return d
 
 
@@ -71,62 +78,119 @@ def config(d):
             PART_LINES = int(v)
 
 
-def read_log(d):
+def log_path(d):
-    """[(id, date, text)] in order."""
+    return os.path.join(d, "LOG.txt")
+
+
+def tree_path(d, size):
+    return os.path.join(d, "TREE", str(size))
+
+
+def count(path, rec):
+    try:
+        return os.path.getsize(path) // rec
+    except OSError:
+        return 0
+
+
+def log_len(d):
+    return count(log_path(d), LOG_REC)
+
+
+def log_get(d, i):
+    """(id, date, text) of memory i, in one seek."""
+    with open(log_path(d), "rb") as f:
+        f.seek(i * LOG_REC)
+        line = f.read(LOG_REC).decode().rstrip()
+    head, _, rest = line.partition(" ")
+    date, _, text = rest.partition(" ")
+    return int(head[1:]), date, text
+
+
+def log_slice(d, lo, hi):
+    """Memories [lo,hi) in one read."""
+    with open(log_path(d), "rb") as f:
+        f.seek(lo * LOG_REC)
+        buf = f.read((hi - lo) * LOG_REC).decode()
     out = []
-    for line in open(os.path.join(d, "LOG.txt")):
+    for i in range(hi - lo):
-        line = line.rstrip("\n")
+        line = buf[i * LOG_REC:(i + 1) * LOG_REC].rstrip()
-        if not line:
+        head, _, rest = line.partition(" ")
-            continue
+        date, _, text = rest.partition(" ")
-        head, _, text = line.partition(" ")
-        date, _, text = text.partition(" ")
         out.append((int(head[1:]), date, text))
     return out
 
 
-def read_tree(d):
+def tree_get(d, lo, hi):
-    """{(lo,hi): text}"""
+    """The summary of block [lo,hi), in one seek. None if not built yet."""
-    out = {}
+    size = hi - lo
-    for line in open(os.path.join(d, "TREE.txt")):
+    with open(tree_path(d, size), "rb") as f:
-        line = line.rstrip("\n")
+        f.seek((lo // size) * TREE_REC)
-        if not line:
+        rec = f.read(TREE_REC)
-            continue
+    return rec.decode().rstrip() or None
-        head, _, text = line.partition(" ")
-        lo, _, hi = head.partition("-")
-        out[(int(lo), int(hi))] = text
-    return out
 
 
-def rewrite_tree(d, keep):
+def pad(text, rec):
-    """Replace TREE.txt. Legal only because TREE.txt is a CACHE of a pure
+    b = text.encode()
-    function of LOG.txt -- the log itself is never rewritten by anything."""
+    if len(b) > rec - 1:
+        die("REJECTED: %d bytes, over the %d-byte record." % (len(b), rec - 1))
+    return b + b" " * (rec - 1 - len(b)) + b"\n"
+
+
+def locked(d):
     lock = open(os.path.join(d, ".lock"), "w")
     fcntl.flock(lock, fcntl.LOCK_EX)
+    return lock
+
+
+def log_append(d, entries):
+    """Append memories. The only way LOG.txt ever changes."""
+    lock = locked(d)
     try:
-        tmp = os.path.join(d, "TREE.txt.new")
+        with open(log_path(d), "ab") as f:
-        with open(tmp, "w") as f:
+            for e in entries:
-            for (lo, hi), text in keep:
+                f.write(pad(e, LOG_REC))
-                f.write("%d-%d %s\n" % (lo, hi, text))
             f.flush()
             os.fsync(f.fileno())
-        os.replace(tmp, os.path.join(d, "TREE.txt"))
     finally:
-        fcntl.flock(lock, fcntl.LOCK_UN)
         lock.close()
 
 
-def append(d, name, lines):
+def tree_put(d, lo, hi, text):
-    """Append under an exclusive lock. The only way anything is ever written."""
+    """Write block [lo,hi). Blocks are built in order, so this only ever
-    lock = open(os.path.join(d, ".lock"), "w")
+    appends one record to one level file."""
-    fcntl.flock(lock, fcntl.LOCK_EX)
+    size = hi - lo
+    lock = locked(d)
     try:
-        with open(os.path.join(d, name), "a") as f:
+        p = tree_path(d, size)
-            for line in lines:
+        if count(p, TREE_REC) != lo // size:
-                f.write(line + "\n")
+            return False
+        with open(p, "ab") as f:
+            f.write(pad(text, TREE_REC))
             f.flush()
             os.fsync(f.fileno())
+        return True
+    finally:
+        lock.close()
+
+
+def tree_drop(d, lo, hi):
+    """Forget block [lo,hi) and every block built from it, by truncating each
+    level back to that point. Later blocks at those levels go too and are
+    rebuilt; the log is never touched, so nothing is lost."""
+    gone, size = [], hi - lo
+    lock = locked(d)
+    try:
+        while size <= log_len(d):
+            p, k = tree_path(d, size), lo // size
+            n = count(p, TREE_REC)
+            if n > k:
+                gone += [(i * size, (i + 1) * size) for i in range(k, n)]
+                with open(p, "r+b") as f:
+                    f.truncate(k * TREE_REC)
+            size *= 2
+        return gone
     finally:
-        fcntl.flock(lock, fcntl.LOCK_UN)
         lock.close()
 
 
@@ -150,21 +214,37 @@ def check(text):
 
 # ---------------------------------------------------------------- naps
 
-def pending(log, tree):
+def pending(d, T, limit=None):
-    """Blocks that can be built and have not been. Smallest first, so a
+    """Blocks that can be built and have not been, smallest first. Each level
-    block's two halves are always available before the block itself."""
+    file holds a dense prefix, so its length says exactly how far that level
-    return [b for b in complete(len(log)) if b not in tree]
+    got: this costs one stat per level, never a scan."""
+    todo, size = [], 2
+    while size <= T:
+        have = count(tree_path(d, size), TREE_REC)
+        for k in range(have, T // size):
+            todo.append((k * size, (k + 1) * size))
+            if limit and len(todo) >= limit:
+                return todo
+        size *= 2
+    return todo
 
 
-def nap_prompt(d, log, tree, todo):
+def pending_count(d, T):
-    lo, hi = todo[0]
+    n, size = 0, 2
+    while size <= T:
+        n += T // size - count(tree_path(d, size), TREE_REC)
+        size *= 2
+    return n
+
+
+def nap_prompt(d, lo, hi, left):
     if hi - lo <= RAW_MAX:
-        body = "\n".join("  #%d %s %s" % log[i] for i in range(lo, hi))
+        body = "\n".join("  #%d %s %s" % e for e in log_slice(d, lo, hi))
         what = "these %d memories" % (hi - lo)
     else:
         mid = (lo + hi) // 2
-        body = "\n".join("  %s" % tree[(a, b)] for a, b in
+        body = "\n".join("  " + tree_get(d, a, b) for a, b in
-                         ((lo, mid), (mid, hi)))
+                          ((lo, mid), (mid, hi)))
         what = "these two summaries"
     return (
         "You are dreaming. Compress {what} into ONE line of at most {n} "
@@ -173,8 +253,15 @@ def nap_prompt(d, log, tree, todo):
         "of memories.\n\n{body}\n\nThen run exactly:\n"
         "  memo sleep {lo}-{hi} \"<your line>\"\n\n"
         "{left} nap(s) left after this one."
-    ).format(what=what, n=ENTRY_CHARS, body=body, lo=lo, hi=hi,
+    ).format(what=what, n=ENTRY_CHARS, body=body, lo=lo, hi=hi, left=left)
-             left=len(todo) - 1)
+
+
+def next_nap(d, T):
+    todo = pending(d, T, limit=1)
+    if not todo:
+        return None
+    lo, hi = todo[0]
+    return nap_prompt(d, lo, hi, pending_count(d, T) - 1)
 
 
 # ---------------------------------------------------------------- commands
@@ -194,21 +281,24 @@ def paginate(lines):
 
 
 def cmd_wake(d, args):
-    log, tree = read_log(d), read_tree(d)
+    T = log_len(d)
-    todo = pending(log, tree)
+    nap = next_nap(d, T)
-    if todo:
+    if nap:
         print("YOU CANNOT WAKE UP YET: %d compression(s) are pending, and a "
               "memory\nwith work left in it is not yet the truth. Sleep first "
-              "-- it is quick.\n" % len(todo))
+              "-- it is quick.\n" % pending_count(d, T))
-        print(nap_prompt(d, log, tree, todo))
+        print(nap)
         sys.exit(1)
-    if not log:
+    if not T:
         print("You have no memories yet. This is your first moment.\n"
               "Record what matters with: memo note \"...\"")
         return
-    lines = ["#%d %s %s" % log[lo] if hi - lo == 1 else
+    lines = []
-             "#%d-%d %s" % (lo, hi - 1, tree[(lo, hi)])
+    for lo, hi in cover(T, WAKE_LINES):
-             for lo, hi in cover(len(log), WAKE_LINES)]
+        if hi - lo == 1:
+            lines.append("#%d %s %s" % log_get(d, lo))
+        else:
+            lines.append("#%d-%d %s" % (lo, hi - 1, tree_get(d, lo, hi)))
     parts = paginate(lines)
     k = 1
     if args:
@@ -234,18 +324,16 @@ def cmd_note(d, args):
     if len(args) != 1:
         die("usage: memo note \"<one line, at most %d chars>\"" % ENTRY_CHARS)
     text = check(args[0])
-    log = read_log(d)
+    T = log_len(d)
-    date = datetime.date.today().isoformat()
+    log_append(d, ["#%d %s %s" % (T, datetime.date.today().isoformat(), text)])
-    append(d, "LOG.txt", ["#%d %s %s" % (len(log), date, text)])
+    print("ok, memory #%d." % T)
-    print("ok, memory #%d." % len(log))
+    nap = next_nap(d, T + 1)
-    log, tree = read_log(d), read_tree(d)
+    if nap:
-    todo = pending(log, tree)
+        print("\n" + nap)
-    if todo:
-        print("\n" + nap_prompt(d, log, tree, todo))
 
 
 def cmd_sleep(d, args):
-    log, tree = read_log(d), read_tree(d)
+    T = log_len(d)
     if args:
         if len(args) != 2:
             die("usage: memo sleep <lo>-<hi> \"<one line>\"")
@@ -253,23 +341,23 @@ def cmd_sleep(d, args):
         if not m:
             die("REJECTED: '%s' is not a block id. Copy it from the prompt."
                 % args[0])
-        block = (int(m.group(1)), int(m.group(2)))
+        lo, hi = int(m.group(1)), int(m.group(2))
-        if block in tree:
+        todo = pending(d, T, limit=1)
+        if not todo:
+            die("REJECTED: nothing is pending. You are already awake.")
+        if (lo, hi) != todo[0]:
+            die("REJECTED: %d-%d is not the block to compress. Blocks are built "
+                "in order,\nand yours is %d-%d. Run `memo sleep` to see it."
+                % (lo, hi, todo[0][0], todo[0][1]))
+        if not tree_put(d, lo, hi, check(args[1])):
             print("Already dreamt; another session got there first. Skipping.")
-        elif block not in pending(log, tree):
-            die("REJECTED: block %d-%d is not ready to be compressed. Run "
-                "`memo sleep` for the block you should be working on."
-                % block)
         else:
-            text = check(args[1])
+            print("ok, %d-%d remembered." % (lo, hi))
-            append(d, "TREE.txt", ["%d-%d %s" % (block[0], block[1], text)])
+    nap = next_nap(d, T)
-            print("ok, %d-%d remembered." % block)
+    if not nap:
-        log, tree = read_log(d), read_tree(d)
-    todo = pending(log, tree)
-    if not todo:
         print("You woke up. Nothing left to compress.")
         return
-    print("\n" + nap_prompt(d, log, tree, todo))
+    print("\n" + nap)
 
 
 def cmd_forget(d, args):
@@ -282,13 +370,16 @@ def cmd_forget(d, args):
     if not m:
         die("REJECTED: '%s' is not a block id." % args[0])
     lo, hi = int(m.group(1)), int(m.group(2))
-    tree = read_tree(d)
+    size = hi - lo
-    doomed = [b for b in tree if b[0] <= lo and hi <= b[1]]
+    if size < 2 or size & (size - 1) or lo % size:
-    if not doomed:
+        die("REJECTED: %d-%d is not a block. A block covers an aligned "
-        die("There is no summary covering %d-%d to forget." % (lo, hi))
+            "power-of-two range." % (lo, hi))
-    rewrite_tree(d, [(b, t) for b, t in sorted(tree.items()) if b not in doomed])
+    gone = tree_drop(d, lo, hi)
-    print("forgot %s. They will be compressed again on your next sleep."
+    if not gone:
-          % " ".join("%d-%d" % b for b in sorted(doomed)))
+        die("There is no summary at %d-%d to forget." % (lo, hi))
+    print("forgot %d summaries (%d-%d and everything built from it). They will "
+          "be compressed again on your next sleep."
+          % (len(gone), gone[0][0], gone[0][1]))
 
 
 def cmd_recall(d, args):
@@ -298,7 +389,7 @@ def cmd_recall(d, args):
         pat = re.compile(args[0], re.I)
     except re.error as e:
         die("bad regex: %s" % e)
-    hits = [e for e in read_log(d) if pat.search(e[2])]
+    hits = [e for e in log_slice(d, 0, log_len(d)) if pat.search(e[2])]
     if not hits:
         print("Nothing in your memory matches that.")
         return
@@ -312,9 +403,8 @@ def cmd_import(d, args):
     For bootstrapping an identity from older records. Used once."""
     if len(args) != 1:
         die("usage: memo import <file>   # lines of 'YYYY-MM-DD <text>'")
-    log = read_log(d)
+    T = log_len(d)
-    n = len(log)
+    last = log_get(d, T - 1)[1] if T else "0000-00-00"
-    last = log[-1][1] if log else "0000-00-00"
     out = []
     for i, line in enumerate(open(args[0]), 1):
         line = line.rstrip("\n")
@@ -329,14 +419,14 @@ def cmd_import(d, args):
         text = text.strip()
         if not text or len(text) > ENTRY_CHARS:
             die("line %d: %d chars (limit %d)." % (i, len(text), ENTRY_CHARS))
-        out.append("#%d %s %s" % (n + len(out), date, text))
+        out.append("#%d %s %s" % (T + len(out), date, text))
         last = date
-    append(d, "LOG.txt", out)
+    log_append(d, out)
-    print("imported %d memories (#%d..#%d)." % (len(out), n, n + len(out) - 1))
+    print("imported %d memories (#%d..#%d)." % (len(out), T, T + len(out) - 1))
-    todo = pending(read_log(d), read_tree(d))
+    n = pending_count(d, log_len(d))
-    if todo:
+    if n:
         print("%d compressions are now pending. Run `memo sleep` until it "
-              "says you woke up." % len(todo))
+              "says you woke up." % n)
 
 
 COMMANDS = {"wake": cmd_wake, "note": cmd_note, "sleep": cmd_sleep,

test.py

@@ -139,41 +139,46 @@ check("last one" in run("wake", str(len(parts))).stdout, "last part must say it
 check(run("wake", str(len(parts) + 1)).returncode == 1, "a nonexistent part should fail")
 
 # append-only: nothing was ever rewritten
-sizes = {f: os.path.getsize(os.path.join(d, f)) for f in ("LOG.txt", "TREE.txt")}
+logsz = os.path.getsize(os.path.join(d, "LOG.txt"))
 run("note", "one more thing happened today")
-for f, s in sizes.items():
+check(os.path.getsize(os.path.join(d, "LOG.txt")) > logsz, "note did not append")
-    check(os.path.getsize(os.path.join(d, f)) >= s, "%s shrank" % f)
+check(logsz % 320 == 0, "LOG.txt is not a whole number of records")
-tree = open(os.path.join(d, "TREE.txt")).read().splitlines()
+for f in os.listdir(os.path.join(d, "TREE")):
-check(len(tree) == len(set(l.split()[0] for l in tree)), "a block was written twice")
+    check(os.path.getsize(os.path.join(d, "TREE", f)) % 288 == 0,
+          "TREE/%s is not a whole number of records" % f)
 
-# writing a block twice is refused, not duplicated
+# a block already written cannot be rewritten
-r = run("sleep", tree[0].split()[0], "attempted overwrite")
+check("REJECTED" in run("sleep", "0-2", "attempted overwrite").stderr,
-check("Already dreamt" in r.stdout, "rewriting a block was allowed")
+      "rewriting a settled block was allowed")
 
 # recall reaches memories the summaries lost
 r = run("recall", "memory number 7,")
 check(r.returncode == 0 and "#7 " in r.stdout, "recall missed a memory")
 
 # a wrong summary can be dropped, with everything built on top of it
-before = len(open(os.path.join(d, "TREE.txt")).read().splitlines())
+def treesize():
-logsize = os.path.getsize(os.path.join(d, "LOG.txt"))
+    t = os.path.join(d, "TREE")
+    return sum(os.path.getsize(os.path.join(t, f)) for f in os.listdir(t))
+
+before, logsize = treesize(), os.path.getsize(os.path.join(d, "LOG.txt"))
 r = run("forget", "16-32")
 check("16-32" in r.stdout, "forget did not report the block: " + r.stdout + r.stderr)
-gone = set(open(os.path.join(d, "TREE.txt")).read().splitlines())
+check(treesize() < before, "forget did not shrink the tree")
-check(not any(l.startswith("16-32 ") for l in gone), "forgotten block still present")
-check(not any(l.startswith("0-64 ") for l in gone), "an ancestor survived a forget")
 check(os.path.getsize(os.path.join(d, "LOG.txt")) == logsize, "forget touched the log")
 check(run("wake").returncode == 1, "wake should refuse after a forget")
 n = 0
-while "You woke up" not in run("sleep").stdout:
+while True:
     r = run("sleep")
+    if "You woke up" in r.stdout:
+        break
     bid = [l for l in r.stdout.splitlines() if l.strip().startswith("memo sleep ")][0].split()[2]
-    run("sleep", bid, "rebuilt after forget")
+    check("REJECTED" not in run("sleep", bid, "rebuilt after forget").stderr, "rebuild rejected")
     n += 1
-check(n == before - len(gone), "rebuilt %d blocks, forgot %d" % (n, before - len(gone)))
+check(n > 0, "forget created no work")
 check(run("wake").returncode == 0, "wake still refuses after rebuilding")
-r = run("forget", "999999-1000000")
+check(treesize() == before, "tree did not return to its original size")
-check(r.returncode == 1, "forgetting a nonexistent block should fail")
+check(run("forget", "17-33").returncode == 1, "forgetting a non-block should fail")
+check(run("forget", "999998-1000000").returncode == 1, "forgetting a missing block should fail")
 
 shutil.rmtree(d)
 print("\n%d passed, %d failed" % (ok, fail))