zoom: an agent navigates its own tree instead of grepping it

README.md

@@ -1,6 +1,6 @@
 # OptMem
 
-Permanent memory for AI agents. A 333-token prompt, a script, plug and play.
+Permanent memory for AI agents. A 433-token prompt, a script, plug and play.
 
 ![how OptMem works](anim/optmem.gif)
 
@@ -24,6 +24,7 @@ The tool lands at `~/.optmem/memo`; put `~/.optmem` on `PATH` to type `memo`.
 | `memo note "..."` | record one memory: one line, up to 280 chars |
 | `memo nap` | answer the merges that came due |
 | `memo recall <regex>` | search every memory ever recorded, word for word |
+| `memo zoom <lo>-<hi>` | open a block into finer parts, down to raw memories |
 | `memo forget <lo>-<hi>` | drop a bad summary; the next nap rebuilds it |
 
 Merges arrive one at a time, in the output of `note`. Nothing ever runs in the
@@ -85,10 +86,17 @@ Do not register redundant memories.
 
 If `~/.optmem/memo note` asks a compression: do it before your next action.
 
-Use `~/.optmem/memo recall <regex>` when a memory you need is too vague.
-
 Never edit or delete anything under `~/.optmem/memory`: the tool manages it.
 
+### When you need an old memory: search, or navigate
+
+`~/.optmem/memo recall <regex>` searches every memory, word for word.
+
+Your memories are also a tree: every 2, 4, 8 ... consecutive ones exist as
+one summary, so each `#a-b` line wake printed is a whole era in one line.
+`~/.optmem/memo zoom <a-b>` opens an era into finer parts, and again, down to
+the raw memories. `~/.optmem/memo zoom` alone opens the tree from the top.
+
 ### If you're a subagent: skip everything above
 
 Parallel sessions on this machine are all you, and may all write memories.

memo

@@ -6,6 +6,7 @@
   {memo} note "..."       record one memory: one short line.
   {memo} nap [id "..."]   do the pending compressions.
   {memo} recall <regex>   search every memory ever recorded.
+  {memo} zoom [<lo>-<hi>] open a block into its finer parts.
   {memo} forget <lo>-<hi> drop a bad summary; nap rebuilds it.
   {memo} config [NAME=N]  show this memory's sizes, or change one.
   {memo} import <file>    bulk-load dated memories (bootstrap only).
@@ -50,7 +51,10 @@ ENTRY_CHARS = KNOBS["ENTRY_CHARS"][0]
 PART_CHARS = KNOBS["PART_CHARS"][0]
 PART_LINES = KNOBS["PART_LINES"][0]
 
-RAW_MAX = 16  # blocks up to this many memories compress from the raw log
+# The grain of the tree as an agent sees it: a block of at most this many
+# memories is shown as raw memories, a bigger one as this many children. So a
+# nap prompt, and a zoom, are never more than RAW_MAX lines.
+RAW_MAX = 16
 
 
 # Records are FIXED WIDTH, so a memory or a block is found by seeking to its
@@ -479,10 +483,17 @@ Do not register redundant memories.
 
 If `{memo} note` asks a compression: do it before your next action.
 
-Use `{memo} recall <regex>` when a memory you need is too vague.
-
 Never edit or delete anything under `{data}`: the tool manages it.
 
+### When you need an old memory: search, or navigate
+
+`{memo} recall <regex>` searches every memory, word for word.
+
+Your memories are also a tree: every 2, 4, 8 ... consecutive ones exist as
+one summary, so each `#a-b` line wake printed is a whole era in one line.
+`{memo} zoom <a-b>` opens an era into finer parts, and again, down to
+the raw memories. `{memo} zoom` alone opens the tree from the top.
+
 ### If you're a subagent: skip everything above
 
 Parallel sessions on this machine are all you, and may all write memories.
@@ -709,6 +720,34 @@ def cmd_recall(d, args):
         print("%s." % plural(hits, "match"))
 
 
+def cmd_zoom(d, args):
+    """Open one block into its finer parts. `recall` finds the words an agent
+    remembers; this finds the memory it does not, by descending the tree from
+    an era to a day. Five zooms reach any memory of a million."""
+    if len(args) > 1:
+        die("usage: %s zoom [<lo>-<hi>]   # a block id, as wake prints them"
+            % ME)
+    T = log_len(d)
+    if not T:
+        die("No memories yet. Record the first with: %s note \"<one line>\""
+            % ME)
+    if not args:
+        kids = cover(T, RAW_MAX)  # the whole life, coarsest: where to start
+    else:
+        lo, hi = block_id(args[0])
+        if lo >= T:
+            die("#%d-%d is beyond the memory: it holds %s. Run: %s wake"
+                % (lo, hi - 1, plural(T, "memory"), ME))
+        step = (hi - lo) // RAW_MAX or 1
+        kids = [(k, k + step) for k in range(lo, hi, step) if k < T]
+    for a, b in kids:
+        if b - a == 1:
+            print("#%d %s %s" % log_get(d, a))
+        else:
+            print("#%d-%d %s" % (a, b - 1, tree_get(d, a, b)
+                                 or "not compressed yet"))
+
+
 def cmd_import(d, args):
     """Bulk-append historical memories: 'YYYY-MM-DD <text>' per line.
     For bootstrapping an identity from older records. Used once."""
@@ -747,8 +786,8 @@ def cmd_import(d, args):
 
 
 COMMANDS = {"init": cmd_init, "wake": cmd_wake, "note": cmd_note,
-            "nap": cmd_nap, "recall": cmd_recall, "forget": cmd_forget,
+            "nap": cmd_nap, "recall": cmd_recall, "zoom": cmd_zoom,
-            "config": cmd_config, "import": cmd_import}
+            "forget": cmd_forget, "config": cmd_config, "import": cmd_import}
 
 
 def main():

test.py

@@ -301,6 +301,51 @@ check("#7 " in r.stdout and "5 matches." in r.stdout,
       "recall cannot find memories by date: " + r.stdout)
 
 
+# zoom: the agent must be able to walk from the whole life down to one day
+# without reading the store itself. Every view is a contiguous tiling of the
+# block it opened, so at each step exactly one line can hold what is wanted.
+def view(*args):
+    r = run("zoom", *args)
+    check(r.returncode == 0, "zoom %s failed: %s" % (args, r.stderr))
+    check(len(r.stdout) < CAP_CHARS, "a zoom returned %d chars" % len(r.stdout))
+    out = []
+    for line in r.stdout.splitlines():
+        m = re.match(r"#(\d+)(?:-(\d+))? ", line)
+        check(bool(m), "zoom printed a line with no id: %r" % line)
+        lo = int(m.group(1))
+        out.append((lo, int(m.group(2)) + 1 if m.group(2) else lo + 1))
+    check(len(out) <= cli.RAW_MAX, "a zoom printed %d lines" % len(out))
+    return out
+
+
+target, steps, lo, hi = 1234, 0, 0, N
+cur = view()
+while True:
+    check(cur[0][0] == lo and cur[-1][1] >= min(hi, N),
+          "zoom %d-%d did not tile its block: %r" % (lo, hi - 1, cur))
+    for a, b in zip(cur, cur[1:]):
+        check(a[1] == b[0], "zoom left a gap or an overlap: %r" % (cur,))
+    holds = [b for b in cur if b[0] <= target < b[1]]
+    check(len(holds) == 1, "#%d is in %d of the lines zoom printed"
+          % (target, len(holds)))
+    lo, hi = holds[0]
+    if hi - lo == 1:
+        break
+    steps += 1
+    check(steps <= 4, "zoom needed %d steps to reach one of %d memories"
+          % (steps, N))
+    cur = view("%d-%d" % (lo, hi - 1))
+check(lo == target, "the descent landed on #%d, not #%d" % (lo, target))
+check("memory number %d," % target in run("zoom", "1232-1235").stdout,
+      "a zoom of the finest blocks must print the raw memories")
+
+# a block id that is not one, and a block the memory has not reached yet
+check(run("zoom", "3-9").returncode == 1, "zoom accepted a non-block")
+r = run("zoom", "1048576-2097151")
+check(r.returncode == 1 and "beyond the memory" in r.stderr
+      and "memo wake" in r.stderr, "zoom past the end must name a way back")
+
+
 def treesize():
     t = os.path.join(d, "TREE")
     return sum(os.path.getsize(os.path.join(t, f)) for f in os.listdir(t))
@@ -335,6 +380,16 @@ check(n > 0, "forget created no work")
 check(run("wake").returncode == 0, "wake still refuses after rebuilding")
 check(treesize() == before, "tree did not return to its original size")
 check(run("forget", "17-32").returncode == 1, "forgetting a non-block should fail")
+# a summary that is not built yet is named as such, never left blank
+run("forget", "16-31")
+z = run("zoom", "0-255").stdout
+check("#16-31 not compressed yet" in z, "zoom hid a missing summary: " + z)
+while True:
+    bid = nap_id(run("nap").stdout)
+    if not bid:
+        break
+    run("nap", bid, "rebuilt after forget")
+check(treesize() == before, "tree did not return to its original size")
 check(run("forget", "1048576-1048577").returncode == 1, "forgetting a missing block should fail")
 
 # UTF-8: multi-byte characters must not shift record boundaries or dodge limits