a6abc264c5
WAKE_LINES=256 was converted from Taelin's 16k-token budget at 4 bytes per token. Dense technical memories run 3.36, so the real document was 19,015 tokens on the live store -- 19% over the number he set, and it needed a fourth tool call to deliver a 17-line stub. 256 lines 63,922 bytes 19,015 tokens 4 parts 208 lines 54,011 bytes 16,393 tokens 3 parts <- shipped Nothing is recomputed: the budget only selects which existing lines print. test.py now reads the budget from memo instead of keeping its own copy.
523 lines
17 KiB
Python
Executable file
523 lines
17 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""OptMem: a permanent, append-only memory for AI agents.
|
|
|
|
memo wake [part [T]] read your memory. Run first, every session.
|
|
memo note "..." record one memory: one line, at most 280 chars.
|
|
memo sleep [id "..."] do the pending compressions.
|
|
memo recall <regex> search every memory ever recorded.
|
|
memo forget <lo>-<hi> drop a bad summary; sleep rebuilds it.
|
|
memo import <file> bulk-load dated memories (bootstrap only).
|
|
|
|
Everything lives in $MEMORY_DIR. See README.md.
|
|
"""
|
|
|
|
import datetime
|
|
import fcntl
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
|
|
from blocks import cover # noqa: E402
|
|
|
|
ENTRY_CHARS = 280
|
|
WAKE_LINES = 208 # ~16k tokens of dense text, in 3 parts
|
|
RAW_MAX = 16 # blocks up to this many memories compress from the raw log
|
|
|
|
# Every harness truncates a command that prints too much, and each drops a
|
|
# different piece: Claude Code cuts the middle at 30,000 chars, pi cuts the
|
|
# head at 50 KB, Codex budgets 10,000 tokens. So the memory is handed over in
|
|
# parts that fit all of them. These are transport limits, not memory limits.
|
|
PART_CHARS = 20000
|
|
PART_LINES = 500
|
|
|
|
|
|
# Records are FIXED WIDTH, so a memory or a block is found by seeking to its
|
|
# offset -- no scanning, no index file to keep in sync. Position IS identity:
|
|
# memory i lives at i*LOG_REC of LOG.txt, and block [k*s,(k+1)*s) lives at
|
|
# k*TREE_REC of TREE/<s>. Padding costs ~2x on disk and buys O(1) everywhere.
|
|
LOG_REC = 320
|
|
TREE_REC = 288
|
|
|
|
|
|
# ---------------------------------------------------------------- store
|
|
|
|
def store():
|
|
d = os.environ.get("MEMORY_DIR")
|
|
if not d:
|
|
die("MEMORY_DIR is not set. Example: export MEMORY_DIR=~/memory")
|
|
d = os.path.expanduser(d)
|
|
# The directory is never created here. A typo in MEMORY_DIR would then
|
|
# open an empty store, and the agent would wake with no past and start
|
|
# writing a second identity. Making the directory IS creating the
|
|
# identity, and that is a deliberate act: mkdir.
|
|
if not os.path.isdir(d):
|
|
die("MEMORY_DIR=%s does not exist.\nIf this is a new identity, run: "
|
|
"mkdir -p %s" % (d, d))
|
|
os.makedirs(os.path.join(d, "TREE"), exist_ok=True)
|
|
p = os.path.join(d, "LOG.txt")
|
|
if not os.path.exists(p):
|
|
open(p, "a").close()
|
|
return d
|
|
|
|
|
|
def config(d):
|
|
"""Optional overrides in $MEMORY_DIR/config. The file is never written:
|
|
a store that kept its own copy of the defaults would freeze them, and
|
|
updating the tool would stop changing how it behaves."""
|
|
global ENTRY_CHARS, WAKE_LINES, PART_CHARS, PART_LINES
|
|
p = os.path.join(d, "config")
|
|
if not os.path.exists(p):
|
|
return
|
|
for line in open(p):
|
|
line = line.split("#")[0].strip()
|
|
if "=" not in line:
|
|
continue
|
|
k, v = (s.strip() for s in line.split("=", 1))
|
|
if k == "ENTRY_CHARS":
|
|
ENTRY_CHARS = int(v)
|
|
elif k == "WAKE_LINES":
|
|
WAKE_LINES = int(v)
|
|
elif k == "PART_CHARS":
|
|
PART_CHARS = int(v)
|
|
elif k == "PART_LINES":
|
|
PART_LINES = int(v)
|
|
if ENTRY_CHARS > min(TREE_REC - 8, LOG_REC - 40):
|
|
die("config: ENTRY_CHARS=%d does not fit the %d/%d-byte records."
|
|
% (ENTRY_CHARS, LOG_REC, TREE_REC))
|
|
|
|
|
|
def log_path(d):
|
|
return os.path.join(d, "LOG.txt")
|
|
|
|
|
|
def tree_path(d, size):
|
|
return os.path.join(d, "TREE", str(size))
|
|
|
|
|
|
def count(path, rec):
|
|
try:
|
|
return os.path.getsize(path) // rec
|
|
except OSError:
|
|
return 0
|
|
|
|
|
|
def log_len(d):
|
|
return count(log_path(d), LOG_REC)
|
|
|
|
|
|
def repair(path, rec):
|
|
"""Drop a partial trailing record left by a crash. It was never
|
|
acknowledged. Without this the next append lands at a wrong offset and
|
|
every later record is misaligned. Callers hold the lock."""
|
|
try:
|
|
n = os.path.getsize(path)
|
|
except OSError:
|
|
return
|
|
if n % rec:
|
|
with open(path, "r+b") as f:
|
|
f.truncate(n - n % rec)
|
|
|
|
|
|
def parse(line):
|
|
head, _, rest = line.partition(" ")
|
|
date, _, text = rest.partition(" ")
|
|
return int(head[1:]), date, text
|
|
|
|
|
|
def log_get(d, i):
|
|
"""(id, date, text) of memory i, in one seek."""
|
|
with open(log_path(d), "rb") as f:
|
|
f.seek(i * LOG_REC)
|
|
return parse(f.read(LOG_REC).decode().rstrip())
|
|
|
|
|
|
def log_slice(d, lo, hi):
|
|
"""Memories [lo,hi) in one read. Records are sliced as BYTES and decoded
|
|
one by one -- slicing decoded text would shift every boundary after the
|
|
first multi-byte character."""
|
|
with open(log_path(d), "rb") as f:
|
|
f.seek(lo * LOG_REC)
|
|
buf = f.read((hi - lo) * LOG_REC)
|
|
return [parse(buf[i * LOG_REC:(i + 1) * LOG_REC].decode().rstrip())
|
|
for i in range(hi - lo)]
|
|
|
|
|
|
def tree_get(d, lo, hi):
|
|
"""The summary of block [lo,hi), in one seek. None if not built yet."""
|
|
size = hi - lo
|
|
try:
|
|
with open(tree_path(d, size), "rb") as f:
|
|
f.seek((lo // size) * TREE_REC)
|
|
rec = f.read(TREE_REC)
|
|
except OSError:
|
|
return None
|
|
return rec.decode().rstrip() or None
|
|
|
|
|
|
def pad(text, rec):
|
|
b = text.encode()
|
|
if len(b) > rec - 1:
|
|
die("Too long: %d bytes. The record holds %d." % (len(b), rec - 1))
|
|
return b + b" " * (rec - 1 - len(b)) + b"\n"
|
|
|
|
|
|
def locked(d):
|
|
lock = open(os.path.join(d, ".lock"), "w")
|
|
fcntl.flock(lock, fcntl.LOCK_EX)
|
|
return lock
|
|
|
|
|
|
def log_append(d, items):
|
|
"""Append memories, items = [(date, text)]. The only way LOG.txt ever
|
|
changes. Ids are assigned INSIDE the lock: two sessions noting at the same
|
|
moment must not be handed the same id. Returns the first id used."""
|
|
lock = locked(d)
|
|
try:
|
|
repair(log_path(d), LOG_REC)
|
|
base = log_len(d)
|
|
with open(log_path(d), "ab") as f:
|
|
for k, (date, text) in enumerate(items):
|
|
f.write(pad("#%d %s %s" % (base + k, date, text), LOG_REC))
|
|
f.flush()
|
|
os.fsync(f.fileno())
|
|
return base
|
|
finally:
|
|
lock.close()
|
|
|
|
|
|
def tree_put(d, lo, hi, text):
|
|
"""Write block [lo,hi). Blocks are built in order, so this only ever
|
|
appends one record to one level file."""
|
|
size = hi - lo
|
|
lock = locked(d)
|
|
try:
|
|
p = tree_path(d, size)
|
|
repair(p, TREE_REC)
|
|
if count(p, TREE_REC) != lo // size:
|
|
return False
|
|
with open(p, "ab") as f:
|
|
f.write(pad(text, TREE_REC))
|
|
f.flush()
|
|
os.fsync(f.fileno())
|
|
return True
|
|
finally:
|
|
lock.close()
|
|
|
|
|
|
def tree_drop(d, lo, hi):
|
|
"""Forget block [lo,hi) and every block built from it, by truncating each
|
|
level back to that point. Later blocks at those levels go too and are
|
|
rebuilt; the log is never touched, so nothing is lost."""
|
|
gone, size = [], hi - lo
|
|
lock = locked(d)
|
|
try:
|
|
while size <= log_len(d):
|
|
p, k = tree_path(d, size), lo // size
|
|
n = count(p, TREE_REC)
|
|
if n > k:
|
|
gone += [(i * size, (i + 1) * size) for i in range(k, n)]
|
|
with open(p, "r+b") as f:
|
|
f.truncate(k * TREE_REC)
|
|
size *= 2
|
|
return gone
|
|
finally:
|
|
lock.close()
|
|
|
|
|
|
def die(msg):
|
|
print(msg, file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
|
|
def plural(n, word):
|
|
if n == 1:
|
|
return "1 " + word
|
|
if word.endswith("y"):
|
|
word = word[:-1] + "ie"
|
|
elif word.endswith(("s", "h", "x")):
|
|
word += "e"
|
|
return "%d %ss" % (n, word)
|
|
|
|
|
|
def check(text):
|
|
text = text.strip()
|
|
if not text:
|
|
die("Empty. A memory is one line of text.")
|
|
if "\n" in text or "\r" in text:
|
|
die("%d lines. A memory is one line: merge them, or note them "
|
|
"separately." % (text.count("\n") + 1))
|
|
n = len(text.encode())
|
|
if n > ENTRY_CHARS:
|
|
die("Too long: %d bytes, limit %d. Accented characters cost 2 bytes. "
|
|
"Compress it further." % (n, ENTRY_CHARS))
|
|
return text
|
|
|
|
|
|
# ---------------------------------------------------------------- naps
|
|
|
|
def pending(d, T, limit=None):
|
|
"""Blocks that can be built and have not been, smallest first. Each level
|
|
file holds a dense prefix, so its length says exactly how far that level
|
|
got: this costs one stat per level, never a scan."""
|
|
todo, size = [], 2
|
|
while size <= T:
|
|
have = count(tree_path(d, size), TREE_REC)
|
|
for k in range(have, T // size):
|
|
todo.append((k * size, (k + 1) * size))
|
|
if limit and len(todo) >= limit:
|
|
return todo
|
|
size *= 2
|
|
return todo
|
|
|
|
|
|
def pending_count(d, T):
|
|
"""How many blocks pending() would list, without listing them. A level can
|
|
hold MORE blocks than T needs -- T is a snapshot, and memories keep
|
|
arriving while an agent reads -- so each level is clamped at zero."""
|
|
n, size = 0, 2
|
|
while size <= T:
|
|
n += max(0, T // size - count(tree_path(d, size), TREE_REC))
|
|
size *= 2
|
|
return n
|
|
|
|
|
|
def nap_prompt(d, lo, hi, left):
|
|
if hi - lo <= RAW_MAX:
|
|
body = "\n".join(" #%d %s %s" % e for e in log_slice(d, lo, hi))
|
|
else:
|
|
mid, halves = (lo + hi) // 2, []
|
|
for a, b in ((lo, mid), (mid, hi)):
|
|
s = tree_get(d, a, b)
|
|
if s is None:
|
|
die("Summary %d-%d is missing. Run: memo sleep" % (a, b - 1))
|
|
halves.append(" #%d-%d %s" % (a, b - 1, s))
|
|
body = "\n".join(halves)
|
|
tail = "" if not left else "\n%s after this one." % (
|
|
"1 compression remains" if left == 1 else
|
|
"%d compressions remain" % left)
|
|
return ("Compress memories #%d-%d into one line of at most %d characters.\n"
|
|
"Keep every name, number, date, decision and outcome.\n"
|
|
"Drop wording, not facts. Invent nothing.\n\n"
|
|
"%s\n%s\n"
|
|
"Run: memo sleep %d-%d \"<your line>\""
|
|
% (lo, hi - 1, ENTRY_CHARS, body, tail, lo, hi - 1))
|
|
|
|
|
|
def next_nap(d, T):
|
|
todo = pending(d, T, limit=1)
|
|
if not todo:
|
|
return None
|
|
lo, hi = todo[0]
|
|
return nap_prompt(d, lo, hi, pending_count(d, T) - 1)
|
|
|
|
|
|
# ---------------------------------------------------------------- commands
|
|
|
|
def paginate(lines):
|
|
"""Split the document into parts that survive any harness's output cap."""
|
|
parts, cur, size = [], [], 0
|
|
for line in lines:
|
|
n = len(line.encode()) + 1
|
|
if cur and (len(cur) >= PART_LINES or size + n > PART_CHARS):
|
|
parts.append(cur)
|
|
cur, size = [], 0
|
|
cur.append(line)
|
|
size += n
|
|
if cur:
|
|
parts.append(cur)
|
|
return parts
|
|
|
|
|
|
def cmd_wake(d, args):
|
|
now = log_len(d)
|
|
k, T = 1, now
|
|
if args:
|
|
if len(args) > 2 or not all(a.isdigit() for a in args):
|
|
die("usage: memo wake [part [T]]")
|
|
k = int(args[0])
|
|
if len(args) == 2:
|
|
T = int(args[1])
|
|
if T > now:
|
|
die("T=%d, but the memory holds %s. Run: memo wake"
|
|
% (T, plural(now, "entry")))
|
|
# A part is rendered as of T, so a note landing between two parts cannot
|
|
# shift a boundary and drop a line.
|
|
n = pending_count(d, T)
|
|
if n:
|
|
print("Cannot wake: %s pending. Do %s, then run memo wake again.\n"
|
|
% (plural(n, "compression"), "it" if n == 1 else "them"))
|
|
print(next_nap(d, T))
|
|
sys.exit(1)
|
|
if not T:
|
|
print("No memories yet. Record the first with: memo note \"<one line>\"")
|
|
print("You are awake.")
|
|
return
|
|
lines = []
|
|
for lo, hi in cover(T, WAKE_LINES):
|
|
if hi - lo == 1:
|
|
lines.append("#%d %s %s" % log_get(d, lo))
|
|
else:
|
|
s = tree_get(d, lo, hi)
|
|
if s is None:
|
|
die("Summary %d-%d is missing. Run: memo sleep" % (lo, hi - 1))
|
|
lines.append("#%d-%d %s" % (lo, hi - 1, s))
|
|
parts = paginate(lines)
|
|
if not 1 <= k <= len(parts):
|
|
die("No part %d: the memory has %s. Run: memo wake"
|
|
% (k, plural(len(parts), "part")))
|
|
if len(parts) > 1:
|
|
print("Your memory, part %d of %d, oldest first." % (k, len(parts)))
|
|
print("\n".join(parts[k - 1]))
|
|
if k < len(parts):
|
|
print("Run: memo wake %d %d" % (k + 1, T))
|
|
else:
|
|
# always, even for a one-part memory: the contract an agent is given
|
|
# is "run parts until one says awake", so it must always arrive
|
|
print("You are awake.")
|
|
|
|
|
|
def cmd_note(d, args):
|
|
if len(args) != 1:
|
|
die("usage: memo note \"<one line, at most %d chars>\"" % ENTRY_CHARS)
|
|
text = check(args[0])
|
|
i = log_append(d, [(datetime.date.today().isoformat(), text)])
|
|
print("Saved as #%d." % i)
|
|
nap = next_nap(d, i + 1)
|
|
if nap:
|
|
print("\n" + nap)
|
|
|
|
|
|
def cmd_sleep(d, args):
|
|
T, said = log_len(d), False
|
|
if args:
|
|
said = True
|
|
if len(args) != 2:
|
|
die("usage: memo sleep <lo>-<hi> \"<one line>\"")
|
|
m = re.fullmatch(r"(\d+)-(\d+)", args[0])
|
|
if not m:
|
|
die("'%s' is not a block id. Copy it from the prompt." % args[0])
|
|
lo, hi = int(m.group(1)), int(m.group(2)) + 1
|
|
todo = pending(d, T, limit=1)
|
|
if not todo:
|
|
print("Nothing left to compress.")
|
|
return
|
|
if (lo, hi) != todo[0]:
|
|
die("Wrong block: %s. Blocks are built in order; the next is "
|
|
"%d-%d. Run: memo sleep"
|
|
% (args[0], todo[0][0], todo[0][1] - 1))
|
|
if not tree_put(d, lo, hi, check(args[1])):
|
|
print("Another session already wrote %d-%d." % (lo, hi - 1))
|
|
else:
|
|
print("%d-%d saved." % (lo, hi - 1))
|
|
nap = next_nap(d, T)
|
|
if not nap:
|
|
print("Nothing left to compress.")
|
|
return
|
|
print(("\n" if said else "") + nap)
|
|
|
|
|
|
def cmd_forget(d, args):
|
|
"""A summary can be wrong -- mistyped, or a bad compression. Drop it and
|
|
everything built on top of it; the next sleep computes them again. The log
|
|
is untouched, so nothing is ever actually lost."""
|
|
if len(args) != 1:
|
|
die("usage: memo forget <lo>-<hi>")
|
|
m = re.fullmatch(r"(\d+)-(\d+)", args[0])
|
|
if not m:
|
|
die("'%s' is not a block id." % args[0])
|
|
lo, hi = int(m.group(1)), int(m.group(2)) + 1
|
|
size = hi - lo
|
|
if size < 2 or size & (size - 1) or lo % size:
|
|
die("%s is not a block. Copy the id printed by wake, like 16-31."
|
|
% args[0])
|
|
gone = tree_drop(d, lo, hi)
|
|
if not gone:
|
|
die("No summary at %s." % args[0])
|
|
print("Forgot %s, from %d-%d up. Run: memo sleep"
|
|
% (plural(len(gone), "summary"), gone[0][0], gone[0][1] - 1))
|
|
|
|
|
|
def cmd_recall(d, args):
|
|
if len(args) != 1:
|
|
die("usage: memo recall <regex>")
|
|
try:
|
|
pat = re.compile(args[0], re.I)
|
|
except re.error as e:
|
|
die("bad regex: %s" % e)
|
|
hits = [e for e in log_slice(d, 0, log_len(d)) if pat.search(e[2])]
|
|
if not hits:
|
|
print("No match.")
|
|
return
|
|
# Newest first is what a search is usually for, and the output has to fit
|
|
# the same cap `wake` respects.
|
|
out, size = [], 0
|
|
for e in reversed(hits):
|
|
line = "#%d %s %s" % e
|
|
size += len(line.encode()) + 1
|
|
if size > PART_CHARS:
|
|
break
|
|
out.append(line)
|
|
print("\n".join(reversed(out)))
|
|
if len(out) < len(hits):
|
|
print("Newest %d of %s. Narrow the regex."
|
|
% (len(out), plural(len(hits), "match")))
|
|
else:
|
|
print("%s." % plural(len(hits), "match"))
|
|
|
|
|
|
def cmd_import(d, args):
|
|
"""Bulk-append historical memories: 'YYYY-MM-DD <text>' per line.
|
|
For bootstrapping an identity from older records. Used once."""
|
|
if len(args) != 1:
|
|
die("usage: memo import <file> # lines of 'YYYY-MM-DD <text>'")
|
|
try:
|
|
src = open(args[0]).readlines()
|
|
except OSError as e:
|
|
die("Cannot read %s: %s" % (args[0], e.strerror))
|
|
last = log_get(d, log_len(d) - 1)[1] if log_len(d) else "0000-00-00"
|
|
out = []
|
|
for i, line in enumerate(src, 1):
|
|
line = line.rstrip("\n")
|
|
if not line.strip():
|
|
continue
|
|
date, _, text = line.partition(" ")
|
|
if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", date):
|
|
die("line %d: expected 'YYYY-MM-DD <text>', got: %s" % (i, line))
|
|
if date < last:
|
|
die("line %d: date %s precedes the previous memory (%s)."
|
|
% (i, date, last))
|
|
text = text.strip()
|
|
if not text or len(text.encode()) > ENTRY_CHARS:
|
|
die("line %d: %d bytes, limit %d." % (i, len(text.encode()), ENTRY_CHARS))
|
|
out.append((date, text))
|
|
last = date
|
|
if not out:
|
|
die("%s has no memories." % args[0])
|
|
base = log_append(d, out)
|
|
print("Imported %s, #%d to #%d."
|
|
% (plural(len(out), "memory"), base, base + len(out) - 1))
|
|
n = pending_count(d, log_len(d))
|
|
if n:
|
|
print("%s pending. Run: memo sleep" % plural(n, "compression"))
|
|
|
|
|
|
COMMANDS = {"wake": cmd_wake, "note": cmd_note, "sleep": cmd_sleep,
|
|
"recall": cmd_recall, "forget": cmd_forget, "import": cmd_import}
|
|
|
|
|
|
def main():
|
|
if len(sys.argv) < 2:
|
|
print(__doc__.strip())
|
|
sys.exit(0)
|
|
if sys.argv[1] not in COMMANDS:
|
|
print("No such command: %s\n" % sys.argv[1], file=sys.stderr)
|
|
print(__doc__.strip(), file=sys.stderr)
|
|
sys.exit(1)
|
|
d = store()
|
|
config(d)
|
|
COMMANDS[sys.argv[1]](d, sys.argv[2:])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|