From d25f09ddc083b70b12e57f30b7aadc103727f658 Mon Sep 17 00:00:00 2001 From: victortaelin Date: Sun, 26 Jul 2026 15:12:41 -0300 Subject: [PATCH] one file: blocks.py folded into memo; sizes editable with memo config --- README.md | 68 ++++++++++++++++---- blocks.py | 75 ---------------------- install.sh | 7 +-- memo | 181 ++++++++++++++++++++++++++++++++++++++++------------- test.py | 28 ++++++++- 5 files changed, 221 insertions(+), 138 deletions(-) delete mode 100644 blocks.py diff --git a/README.md b/README.md index ad7c307..ec5ef14 100644 --- a/README.md +++ b/README.md @@ -32,28 +32,70 @@ background. ``` ~/.optmem/ - memo the tool (Python 3, no dependencies) - blocks.py which memories to read, and which to merge + memo the tool: one file of Python 3, no dependencies memory/ LOG.txt every memory, one per line, append-only, never edited TREE/ the summaries: a cache, rebuildable from the log alone - config the sizes, all commented out + config the sizes, written by `memo config` ``` -`WAKE_LINES` is the only size worth touching: how many lines `wake` prints -(208 ≈ 16k tokens). It is a reading budget, not a storage budget — change it -whenever, in either direction, and nothing is recomputed. +```sh +memo config # show the sizes +memo config WAKE_LINES=300 # how many lines wake prints (208 ≈ 16k tokens) +memo config WAKE_LINES= # back to the default +``` + +`WAKE_LINES` is the only size worth touching, and it is a reading budget, not +a storage budget: change it whenever, in either direction, and nothing is +recomputed. Records are fixed width, so position *is* identity and every lookup is one seek. At a million memories (607 MB), `wake` takes 0.03s. Set `$MEMORY_DIR` to keep `memory/` elsewhere — a synced folder, a git repo. -## Limitations +## The prompt -Recency is the only axis: an old memory fades however important it was, and -the one defence is rehearsal — note it again and it is recent again. `recall` -is regex, not semantic search. Summaries are written by the agent out of other -summaries, so a bad one spreads upward until you `forget` it. And a wake costs -~16k tokens, which is deliberate but not free. If you need a fact database, -use a wiki — this is for *who the agent is*. +This is what the installer prints, and the whole of the integration. + +```markdown +## Memory + +Your memory is OptMem: +- The tool is `~/.optmem/memo` +- The data is `~/.optmem/memory` + +OptMem survives every new session, every compaction and every change +of model or vendor. Without it you do not know who you are, or what +was already decided and tried. + +### At startup: activating OptMem (mandatory) + +Run `~/.optmem/memo wake` before any other tool call, in every session. +It prints in numbered parts, each ordering the next; run every one until +a part says `You are awake.` Do not stop early: part 1 is your distant +past, the last part is this week. If wake refuses because compressions +are pending, do them and run `~/.optmem/memo wake` again. + +### While working: register memories (mandatory) + +Call `~/.optmem/memo note "<1 line, max 280 chars>"` whenever you learn +something new, or something worth keeping happens. That covers a task +worth real effort, a fact or insight the user teaches you, anything you +learn about their life (even indirectly), any event of lasting effect. + +Do not register redundant memories. + +If `~/.optmem/memo note` asks a compression: do it before your next action. + +Use `~/.optmem/memo recall ` when a memory you need is too vague. + +Never edit or delete anything under `~/.optmem/memory`: the tool manages it. + +### If you're a subagent: skip everything above + +Parallel sessions on this machine are all you, and may all write memories. +A subagent is not: it must never run `memo`, because it cannot judge what +is already known, and its notes would arrive duplicated and incorrectly. +Start every brief you send one with `You are a subagent. Don't run memo.` +``` diff --git a/blocks.py b/blocks.py deleted file mode 100644 index ec7cbfc..0000000 --- a/blocks.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Block math for OptMem. - -A BLOCK is an aligned power-of-two range of memories, [lo, hi), written as one -line of at most ENTRY_CHARS characters. Blocks form a binary merge tree over -LOG.txt: block [lo,hi) is the compression of [lo,mid) and [mid,hi). - -Two pure functions matter: - - cover(T, budget) which blocks `memo wake` prints - complete(T) every block that CAN be built, smallest first -""" - - -def _cover(T, alpha): - """Tile [0,T) with aligned power-of-two blocks; keep a block whole iff its - size is at most `alpha` times its age. Bigger alpha = coarser = fewer lines.""" - root = 1 - while root < T: - root *= 2 - out, stack = [], [(0, root)] - while stack: - lo, hi = stack.pop() - if lo >= T: - continue - size = hi - lo - if size > 1 and (hi > T or size > alpha * (T - lo)): - mid = (lo + hi) // 2 - stack.append((mid, hi)) - stack.append((lo, mid)) - else: - out.append((lo, hi)) - out.sort() - return out - - -def cover(T, budget): - """The blocks `memo wake` prints: at most `budget` of them, finest near T. - - Detail decays with age, so recent memories stay verbatim and ancient ones - collapse. If everything fits, nothing is compressed at all.""" - if T <= 0: - return [] - if T <= budget: - return [(i, i + 1) for i in range(T)] - lo, hi = 0.0, 1.0 - for _ in range(60): - mid = (lo + hi) / 2 - if len(_cover(T, mid)) > budget: - lo = mid - else: - hi = mid - out = _cover(T, hi) - # Block sizes jump in powers of two, so alpha alone can undershoot the - # budget. Spend what is left on the present, where detail is worth most. - while len(out) < budget: - i = max((i for i, b in enumerate(out) if b[1] - b[0] > 1), default=None) - if i is None: - break - lo_, hi_ = out[i] - mid = (lo_ + hi_) // 2 - out[i:i + 1] = [(lo_, mid), (mid, hi_)] - return out - - -def complete(T): - """Every block buildable from T memories, smallest first (so a block's - halves always come before it). This is the whole of the work that exists: - if all of these are in TREE.txt, there is nothing left to do.""" - out = [] - size = 2 - while size <= T: - for i in range(T // size): - out.append((i * size, (i + 1) * size)) - size *= 2 - return out diff --git a/install.sh b/install.sh index b21d9a0..8ca42c7 100755 --- a/install.sh +++ b/install.sh @@ -6,13 +6,10 @@ set -e DIR="$HOME/.optmem" -SRC="https://raw.githubusercontent.com/VictorTaelin/OptMem/main" mkdir -p "$DIR" -for f in memo blocks.py; do - curl -fsSL "$SRC/$f" -o "$DIR/$f.new" - mv "$DIR/$f.new" "$DIR/$f" -done +curl -fsSL https://raw.githubusercontent.com/VictorTaelin/OptMem/main/memo -o "$DIR/memo.new" +mv "$DIR/memo.new" "$DIR/memo" chmod +x "$DIR/memo" exec "$DIR/memo" init diff --git a/memo b/memo index 4c6f274..cdc28fd 100755 --- a/memo +++ b/memo @@ -3,10 +3,11 @@ memo init create this machine's memory, print the setup block. memo wake [part [T]] read your memory. Run first, every session. - memo note "..." record one memory: one line, at most 280 chars. + memo note "..." record one memory: one short line. memo sleep [id "..."] do the pending compressions. memo recall search every memory ever recorded. memo forget - drop a bad summary; sleep rebuilds it. + memo config [NAME=N] show this memory's sizes, or change one. memo import bulk-load dated memories (bootstrap only). The memories live in ~/.optmem/memory, or in $MEMORY_DIR if set. See README.md. @@ -18,19 +19,25 @@ import os import re import sys -sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) -from blocks import cover # noqa: E402 - -ENTRY_CHARS = 280 -WAKE_LINES = 208 # ~16k tokens of dense text, in 3 parts -RAW_MAX = 16 # blocks up to this many memories compress from the raw log - +# The sizes a memory may override in its own `config` file: the default, and +# what it means. `memo config` shows and edits them. The globals below start +# at these defaults, so a memory that overrides nothing follows the tool. +KNOBS = { + "WAKE_LINES": (208, "the memory context: how many lines wake prints"), + "ENTRY_CHARS": (280, "the longest one memory may be, in bytes"), + "PART_CHARS": (20000, "output paging: largest part, in bytes"), + "PART_LINES": (500, "output paging: largest part, in lines"), +} +WAKE_LINES = KNOBS["WAKE_LINES"][0] # ~16k tokens of dense text, in 3 parts +ENTRY_CHARS = KNOBS["ENTRY_CHARS"][0] # Every harness truncates a command that prints too much, and each drops a # different piece: Claude Code cuts the middle at 30,000 chars, pi cuts the # head at 50 KB, Codex budgets 10,000 tokens. So the memory is handed over in # parts that fit all of them. These are transport limits, not memory limits. -PART_CHARS = 20000 -PART_LINES = 500 +PART_CHARS = KNOBS["PART_CHARS"][0] +PART_LINES = KNOBS["PART_LINES"][0] + +RAW_MAX = 16 # blocks up to this many memories compress from the raw log # Records are FIXED WIDTH, so a memory or a block is found by seeking to its @@ -41,6 +48,64 @@ LOG_REC = 320 TREE_REC = 288 +# ---------------------------------------------------------------- blocks + +# A BLOCK is an aligned power-of-two range of memories, [lo,hi), compressed +# into one line. Blocks form a binary merge tree over LOG.txt: block [lo,hi) +# is the compression of [lo,mid) and [mid,hi). + +def _cover(T, alpha): + """Tile [0,T) with aligned power-of-two blocks; keep a block whole iff its + size is at most `alpha` times its age. Bigger alpha = coarser = fewer + lines.""" + root = 1 + while root < T: + root *= 2 + out, stack = [], [(0, root)] + while stack: + lo, hi = stack.pop() + if lo >= T: + continue + size = hi - lo + if size > 1 and (hi > T or size > alpha * (T - lo)): + mid = (lo + hi) // 2 + stack.append((mid, hi)) + stack.append((lo, mid)) + else: + out.append((lo, hi)) + out.sort() + return out + + +def cover(T, budget): + """The blocks `memo wake` prints: at most `budget` of them, finest near T. + + Detail decays with age, so recent memories stay verbatim and ancient ones + collapse. If everything fits, nothing is compressed at all.""" + if T <= 0: + return [] + if T <= budget: + return [(i, i + 1) for i in range(T)] + lo, hi = 0.0, 1.0 + for _ in range(60): + mid = (lo + hi) / 2 + if len(_cover(T, mid)) > budget: + lo = mid + else: + hi = mid + out = _cover(T, hi) + # Block sizes jump in powers of two, so alpha alone can undershoot the + # budget. Spend what is left on the present, where detail is worth most. + while len(out) < budget: + i = max((i for i, b in enumerate(out) if b[1] - b[0] > 1), default=None) + if i is None: + break + lo_, hi_ = out[i] + mid = (lo_ + hi_) // 2 + out[i:i + 1] = [(lo_, mid), (mid, hi_)] + return out + + # ---------------------------------------------------------------- store def memory_dir(): @@ -64,30 +129,51 @@ def store(): return d -def config(d): - """Optional overrides in the memory directory's `config`. `memo init` - writes it fully commented out: an uncommented copy of the defaults would - freeze them, and updating the tool would stop changing how it behaves.""" - global ENTRY_CHARS, WAKE_LINES, PART_CHARS, PART_LINES +def size(k, v): + """Validate one knob, wherever it came from: the config file or argv.""" + if not v.isdigit() or int(v) < 1: + die("%s must be a positive whole number, not '%s'." % (k, v)) + top = min(TREE_REC - 8, LOG_REC - 40) + if k == "ENTRY_CHARS" and int(v) > top: + die("ENTRY_CHARS is at most %d: a memory has to fit the fixed-width " + "records." % top) + return int(v) + + +def overrides(d): + """The knobs this memory sets for itself, read from its `config` file.""" + out = {} p = os.path.join(d, "config") if not os.path.exists(p): - return + return out for line in open(p): line = line.split("#")[0].strip() if "=" not in line: continue k, v = (s.strip() for s in line.split("=", 1)) - if k == "ENTRY_CHARS": - ENTRY_CHARS = int(v) - elif k == "WAKE_LINES": - WAKE_LINES = int(v) - elif k == "PART_CHARS": - PART_CHARS = int(v) - elif k == "PART_LINES": - PART_LINES = int(v) - if ENTRY_CHARS > min(TREE_REC - 8, LOG_REC - 40): - die("config: ENTRY_CHARS=%d does not fit the %d/%d-byte records." - % (ENTRY_CHARS, LOG_REC, TREE_REC)) + if k not in KNOBS: + die("config: %s is not a size. Run: memo config" % k) + out[k] = size(k, v) + return out + + +def config(d): + """Apply this memory's overrides. A knob it does not set keeps the tool's + default, so updating the tool still changes how it behaves.""" + for k, v in overrides(d).items(): + globals()[k] = v # a knob's name IS the name of the global it sets + + +def write_config(d, over): + """Rewrite `config`: every knob on its own line, commented out unless this + memory overrides it.""" + out = ["# OptMem sizes for this memory. A commented line means: follow the", + "# tool's default. Edit with `memo config NAME=VALUE`.", ""] + for k, (default, what) in KNOBS.items(): + out.append("%-2s%-12s = %-6d # %s" + % ("" if k in over else "# ", k, over.get(k, default), what)) + with open(os.path.join(d, "config"), "w") as f: + f.write("\n".join(out) + "\n") def log_path(d): @@ -317,16 +403,6 @@ def next_nap(d, T): # ---------------------------------------------------------------- commands -CONFIG = """\ -# OptMem sizes for this memory. Uncomment a line to override it; an -# absent line tracks the tool's default. -# -# WAKE_LINES=208 # the memory context: how many lines wake prints (~16k tokens) -# ENTRY_CHARS=280 # the longest a single memory may be, in bytes -# PART_CHARS=20000 # output paging: largest part, in bytes -# PART_LINES=500 # output paging: largest part, in lines -""" - TEMPLATE = """\ ## Memory @@ -380,10 +456,8 @@ def cmd_init(d, args): fresh = not os.path.isdir(d) os.makedirs(os.path.join(d, "TREE"), exist_ok=True) open(log_path(d), "a").close() - cfg = os.path.join(d, "config") - if not os.path.exists(cfg): - with open(cfg, "w") as f: - f.write(CONFIG) + if not os.path.exists(os.path.join(d, "config")): + write_config(d, {}) config(d) home = os.path.expanduser("~") @@ -511,6 +585,29 @@ def cmd_sleep(d, args): print(("\n" if said else "") + nap) +def cmd_config(d, args): + """Show this memory's sizes, or change one. An empty value restores the + default. Sizes only select what is printed, so changing one is free: no + memory is touched and nothing is recomputed.""" + over = overrides(d) + for a in args: + k, eq, v = a.partition("=") + k = k.strip().upper() + if not eq or k not in KNOBS: + die("usage: memo config [NAME=VALUE ...] # NAME one of %s" + % ", ".join(KNOBS)) + if v.strip(): + over[k] = size(k, v.strip()) + else: + over.pop(k, None) + if args: + write_config(d, over) + for k, (default, what) in KNOBS.items(): + print("%-12s %-7d %s%s" % (k, over.get(k, default), what, + "" if k not in over else + " (default %d)" % default)) + + def cmd_forget(d, args): """A summary can be wrong -- mistyped, or a bad compression. Drop it and everything built on top of it; the next sleep computes them again. The log @@ -599,7 +696,7 @@ def cmd_import(d, args): COMMANDS = {"init": cmd_init, "wake": cmd_wake, "note": cmd_note, "sleep": cmd_sleep, "recall": cmd_recall, "forget": cmd_forget, - "import": cmd_import} + "config": cmd_config, "import": cmd_import} def main(): diff --git a/test.py b/test.py index 20c01e1..4677b49 100755 --- a/test.py +++ b/test.py @@ -16,11 +16,21 @@ import tempfile from importlib.machinery import SourceFileLoader HERE = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, HERE) -from blocks import complete, cover # noqa: E402 - MEMO = os.path.join(HERE, "memo") cli = SourceFileLoader("memo_cli", MEMO).load_module() +cover = cli.cover + + +def complete(T): + """Every block buildable from T memories, smallest first. The oracle for + the tool's `pending()`: written straight from the definition, so a bug in + the fast version (which reads level lengths, never scanning) shows up.""" + out, size = [], 2 + while size <= T: + out += [(i * size, (i + 1) * size) for i in range(T // size)] + size *= 2 + return out + # The shipped defaults. A fresh process starts from these, so an in-process # call must too, or one store's config would leak into the next. DEFAULTS = {k: getattr(cli, k) for k in @@ -400,6 +410,18 @@ def fingerprint(path): return out +# `memo config` is how a size is changed: it writes the file the tool reads +# back, an empty value restores the default, and a wake obeys immediately -- +# nothing is recomputed, because a size only selects what gets printed. +r = run("config", "WAKE_LINES=12") +check("12" in r.stdout and "default 208" in r.stdout, "config did not set:\n" + r.stdout) +check(len(run("wake").stdout.splitlines()) <= 13, "wake ignored the new size") +r = run("config", "WAKE_LINES=") +check("default" not in r.stdout, "an empty value did not restore the default") +check(len(run("wake").stdout.splitlines()) > 13, "the default did not come back") +for bad in ("WAKE_LINES=0", "WAKE_LINES=x", "ENTRY_CHARS=999", "NOPE=1", "WAKE_LINES"): + check(run("config", bad).returncode == 1, "config accepted %s" % bad) + with open(os.path.join(d, "config"), "a") as f: f.write("WAKE_LINES=120\n") # a size the user tuned by hand before = fingerprint(d)