a6abc264c5
WAKE_LINES=256 was converted from Taelin's 16k-token budget at 4 bytes per token. Dense technical memories run 3.36, so the real document was 19,015 tokens on the live store -- 19% over the number he set, and it needed a fourth tool call to deliver a 17-line stub. 256 lines 63,922 bytes 19,015 tokens 4 parts 208 lines 54,011 bytes 16,393 tokens 3 parts <- shipped Nothing is recomputed: the budget only selects which existing lines print. test.py now reads the budget from memo instead of keeping its own copy.
363 lines
15 KiB
Python
Executable file
363 lines
15 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""OptMem invariants, checked against a synthetic life of 5000 memories.
|
|
|
|
Uses a fake compressor (join + truncate) so the run is deterministic and free.
|
|
"""
|
|
|
|
import contextlib
|
|
import datetime
|
|
import io
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from importlib.machinery import SourceFileLoader
|
|
|
|
HERE = os.path.dirname(os.path.realpath(__file__))
|
|
sys.path.insert(0, HERE)
|
|
from blocks import complete, cover # noqa: E402
|
|
|
|
MEMO = os.path.join(HERE, "memo")
|
|
cli = SourceFileLoader("memo_cli", MEMO).load_module()
|
|
# The shipped defaults. A fresh process starts from these, so an in-process
|
|
# call must too, or one store's config would leak into the next.
|
|
DEFAULTS = {k: getattr(cli, k) for k in
|
|
("ENTRY_CHARS", "WAKE_LINES", "PART_CHARS", "PART_LINES")}
|
|
|
|
N = 2000
|
|
WAKE_LINES = cli.WAKE_LINES # the shipped budget, not a second copy of it
|
|
PART_CHARS = cli.PART_CHARS
|
|
# Verified caps of the harnesses in the wild: Claude Code cuts a command's
|
|
# output at 30,000 chars (middle), pi at 50 KB / 2000 lines (head), Codex
|
|
# budgets 10,000 tokens. A part must fit the strictest of each kind.
|
|
CAP_CHARS, CAP_LINES = 30000, 2000
|
|
ok, fail = 0, 0
|
|
|
|
|
|
def check(cond, msg):
|
|
global ok, fail
|
|
if cond:
|
|
ok += 1
|
|
else:
|
|
fail += 1
|
|
print("FAIL: " + msg)
|
|
|
|
|
|
# ---- pure block math -------------------------------------------------
|
|
|
|
for T in list(range(1, 400)) + [1000, 4096, 10000, 65536, 100003]:
|
|
c = cover(T, WAKE_LINES)
|
|
check(len(c) <= WAKE_LINES, "T=%d: %d lines > budget" % (T, len(c)))
|
|
check(c[0][0] == 0 and c[-1][1] == T, "T=%d: does not span [0,T)" % T)
|
|
for a, b in zip(c, c[1:]):
|
|
check(a[1] == b[0], "T=%d: gap or overlap at %s %s" % (T, a, b))
|
|
for lo, hi in c:
|
|
s = hi - lo
|
|
check(s & (s - 1) == 0 and lo % s == 0,
|
|
"T=%d: [%d,%d) is not an aligned power-of-two block" % (T, lo, hi))
|
|
for a, b in zip(c, c[1:]):
|
|
check(b[1] - b[0] <= a[1] - a[0],
|
|
"T=%d: detail does not increase toward the present" % T)
|
|
|
|
check(cover(300, 320) == [(i, i + 1) for i in range(300)],
|
|
"under budget, memory should be verbatim")
|
|
|
|
# every block a cover ever needs must be buildable. cover() costs a 60-step
|
|
# binary search, so this walks every tree shape up to 300 and then samples:
|
|
# the property is structural, not a function of the exact T.
|
|
seen = set()
|
|
for T in list(range(1, 300)) + [512, 700, 1000, 1023, 1024, 2000, 2999]:
|
|
seen.update(b for b in cover(T, WAKE_LINES) if b[1] - b[0] > 1)
|
|
buildable = set(complete(3000))
|
|
check(seen <= buildable, "a cover wants a block that complete() never yields")
|
|
|
|
# work never spikes: naps created by one new memory
|
|
worst, prev = 0, 0
|
|
for T in range(1, N):
|
|
cur = len(complete(T))
|
|
worst = max(worst, cur - prev)
|
|
prev = cur
|
|
check(worst <= 16, "a single memory created %d naps" % worst)
|
|
|
|
# ---- the real CLI ----------------------------------------------------
|
|
|
|
d = tempfile.mkdtemp(prefix="optmem-test-")
|
|
memo = [sys.executable, MEMO]
|
|
|
|
|
|
class Result:
|
|
def __init__(self, returncode, stdout, stderr):
|
|
self.returncode, self.stdout, self.stderr = returncode, stdout, stderr
|
|
|
|
|
|
def run(*args, store=None):
|
|
"""One `memo` command, in-process. Spawning an interpreter per call cost
|
|
~40ms x ~2000 naps; the cross-process behaviour that genuinely needs real
|
|
processes (the lock) is tested with real processes below."""
|
|
os.environ["MEMORY_DIR"] = store or d
|
|
for k, v in DEFAULTS.items():
|
|
setattr(cli, k, v)
|
|
out, err, code = io.StringIO(), io.StringIO(), 0
|
|
try:
|
|
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
|
|
sd = cli.store()
|
|
cli.config(sd)
|
|
cli.COMMANDS[args[0]](sd, list(args[1:]))
|
|
except SystemExit as e:
|
|
code = e.code if isinstance(e.code, int) else 0
|
|
return Result(code, out.getvalue(), err.getvalue())
|
|
|
|
|
|
def nap_id(out):
|
|
"""The block id from the command a nap prompt offers."""
|
|
m = re.search(r"memo sleep (\d+)-(\d+)", out)
|
|
return "%s-%s" % m.groups() if m else None
|
|
|
|
|
|
def offered(out):
|
|
"""The line offering a command. Every command handed to an agent must be
|
|
an order, not a label: `Run: memo ...`, never `next: memo ...`."""
|
|
return [l for l in out.splitlines() if "memo sleep " in l or "memo wake " in l]
|
|
|
|
|
|
# the real entry point still has to work: shebang, argv parsing, exit code
|
|
smoke = subprocess.run(memo + ["wake"], env=dict(os.environ, MEMORY_DIR=d),
|
|
capture_output=True, text=True)
|
|
check(smoke.returncode == 0 and "No memories yet" in smoke.stdout,
|
|
"the memo CLI does not run: " + smoke.stdout + smoke.stderr)
|
|
|
|
# a typo in MEMORY_DIR must not silently open a second, empty identity
|
|
ghost = subprocess.run(memo + ["wake"], capture_output=True, text=True,
|
|
env=dict(os.environ, MEMORY_DIR=d + "-typo"))
|
|
check(ghost.returncode == 1 and "does not exist" in ghost.stderr,
|
|
"a missing MEMORY_DIR was created instead of reported")
|
|
check(not os.path.exists(d + "-typo"), "a missing MEMORY_DIR was created")
|
|
noenv = subprocess.run(memo + ["wake"], capture_output=True, text=True,
|
|
env={k: v for k, v in os.environ.items()
|
|
if k != "MEMORY_DIR"})
|
|
check(noenv.returncode == 1 and "MEMORY_DIR is not set" in noenv.stderr,
|
|
"an unset MEMORY_DIR must fail loudly")
|
|
|
|
|
|
r = run("note", "x" * 281)
|
|
check(r.returncode == 1 and "Too long" in r.stderr, "over-long note accepted")
|
|
r = run("note", "two\nlines")
|
|
check(r.returncode == 1 and "one line" in r.stderr, "multi-line note accepted")
|
|
r = run("note", " ")
|
|
check(r.returncode == 1, "empty note accepted")
|
|
r = run("wake")
|
|
check("No memories yet" in r.stdout, "empty wake should say so")
|
|
check(r.stdout.rstrip().endswith("You are awake."),
|
|
"an empty wake must still end with `You are awake.`")
|
|
|
|
with open(os.path.join(d, "seed.txt"), "w") as f:
|
|
day = datetime.date(2020, 1, 1)
|
|
for i in range(N):
|
|
f.write("%s memory number %d, a thing that happened\n"
|
|
% ((day + datetime.timedelta(days=i // 5)).isoformat(), i))
|
|
r = run("import", os.path.join(d, "seed.txt"))
|
|
check("Imported %d" % N in r.stdout, "import failed: " + r.stdout + r.stderr)
|
|
check(not os.path.exists(os.path.join(d, "config")),
|
|
"a store wrote its own config file: the defaults are now frozen in it")
|
|
|
|
r = run("wake")
|
|
check(r.returncode == 1 and "Cannot wake" in r.stdout,
|
|
"wake must refuse while work is pending")
|
|
check("run memo wake again" in r.stdout,
|
|
"the refusal must order the agent back to wake")
|
|
|
|
# sleep loop, with a fake compressor
|
|
naps = 0
|
|
r = run("sleep")
|
|
check("Compress memories #" in r.stdout, "nap prompt must name its object")
|
|
while "Nothing left to compress" not in r.stdout:
|
|
line = offered(r.stdout)
|
|
check(bool(line), "no command offered:\n" + r.stdout + r.stderr)
|
|
if not line:
|
|
break
|
|
check(line[0].startswith("Run: "), "a command was offered as a label, not "
|
|
"an order: %r" % line[0])
|
|
bid = nap_id(r.stdout)
|
|
body = [l.strip() for l in r.stdout.splitlines() if l.startswith(" #")]
|
|
r = run("sleep", bid, (" ".join(body)[:280]).strip() or "empty")
|
|
check(r.returncode == 0, "sleep rejected a valid nap: " + r.stderr)
|
|
naps += 1
|
|
check("You are awake" not in r.stdout,
|
|
"sleep must never claim the agent is awake; only wake may")
|
|
check(naps == len(complete(N)), "did %d naps, expected %d" % (naps, len(complete(N))))
|
|
|
|
r = run("wake")
|
|
check(r.returncode == 0, "wake still refuses after a full sleep")
|
|
|
|
# the document survives pagination, and every part fits every harness's cap
|
|
parts, k = [], 1
|
|
while True:
|
|
r = run("wake", str(k))
|
|
if r.returncode != 0:
|
|
break
|
|
body = [l for l in r.stdout.splitlines() if l.startswith("#")]
|
|
check(len(r.stdout) < CAP_CHARS, "part %d is %d chars, over the %d cap"
|
|
% (k, len(r.stdout), CAP_CHARS))
|
|
check(len(r.stdout.splitlines()) < CAP_LINES, "part %d is over %d lines"
|
|
% (k, CAP_LINES))
|
|
parts.append(body)
|
|
k += 1
|
|
check(len(parts) > 1, "a %d-line memory should need more than one part" % WAKE_LINES)
|
|
lines = [l for p in parts for l in p]
|
|
check(len(lines) == WAKE_LINES, "woke with %d lines, want %d" % (len(lines), WAKE_LINES))
|
|
check(lines[-1].startswith("#%d " % (N - 1)), "newest memory not last / not raw")
|
|
check(lines[0].startswith("#0-"), "oldest line should be a summary block")
|
|
check("Run: memo wake 2" in run("wake").stdout,
|
|
"part 1 must ORDER the next command, not label it")
|
|
check("You are awake." in run("wake", str(len(parts))).stdout,
|
|
"last part must say it is last")
|
|
check(run("wake", str(len(parts) + 1)).returncode == 1, "a nonexistent part should fail")
|
|
|
|
# append-only: nothing was ever rewritten
|
|
logsz = os.path.getsize(os.path.join(d, "LOG.txt"))
|
|
run("note", "one more thing happened today")
|
|
check(os.path.getsize(os.path.join(d, "LOG.txt")) > logsz, "note did not append")
|
|
check(logsz % 320 == 0, "LOG.txt is not a whole number of records")
|
|
for f in os.listdir(os.path.join(d, "TREE")):
|
|
check(os.path.getsize(os.path.join(d, "TREE", f)) % 288 == 0,
|
|
"TREE/%s is not a whole number of records" % f)
|
|
|
|
# a sleep when nothing is pending writes nothing and says so
|
|
r = run("sleep", "0-1", "attempted overwrite")
|
|
check(r.returncode == 0 and "Nothing left to compress" in r.stdout,
|
|
"sleep with nothing pending must say so and write nothing")
|
|
|
|
# recall reaches memories the summaries lost
|
|
r = run("recall", "memory number 7,")
|
|
check(r.returncode == 0 and "#7 " in r.stdout, "recall missed a memory")
|
|
check("1 match." in r.stdout, "a single match is not `1 matches`: " + r.stdout)
|
|
|
|
def treesize():
|
|
t = os.path.join(d, "TREE")
|
|
return sum(os.path.getsize(os.path.join(t, f)) for f in os.listdir(t))
|
|
|
|
before, logsize = treesize(), os.path.getsize(os.path.join(d, "LOG.txt"))
|
|
r = run("forget", "16-31")
|
|
check("16-31" in r.stdout, "forget did not report the block: " + r.stdout + r.stderr)
|
|
check(treesize() < before, "forget did not shrink the tree")
|
|
check(os.path.getsize(os.path.join(d, "LOG.txt")) == logsize, "forget touched the log")
|
|
check(run("wake").returncode == 1, "wake should refuse after a forget")
|
|
# a block already settled cannot be rewritten: only the first pending block
|
|
# is ever accepted
|
|
r = run("sleep", "0-1", "attempted overwrite")
|
|
check(r.returncode == 1 and "Wrong block" in r.stderr,
|
|
"rewriting a settled block was allowed")
|
|
n = 0
|
|
while True:
|
|
r = run("sleep")
|
|
if "Nothing left to compress" in r.stdout:
|
|
break
|
|
bid = nap_id(r.stdout)
|
|
check(run("sleep", bid, "rebuilt after forget").returncode == 0, "rebuild rejected")
|
|
n += 1
|
|
check(n > 0, "forget created no work")
|
|
check(run("wake").returncode == 0, "wake still refuses after rebuilding")
|
|
check(treesize() == before, "tree did not return to its original size")
|
|
check(run("forget", "17-32").returncode == 1, "forgetting a non-block should fail")
|
|
check(run("forget", "1048576-1048577").returncode == 1, "forgetting a missing block should fail")
|
|
|
|
# UTF-8: multi-byte characters must not shift record boundaries or dodge limits
|
|
run("note", "reunião com João em São Paulo: ação aprovada, coração tranquilo")
|
|
run("note", "a plain ascii memory right after the accented one")
|
|
r = run("recall", "coração")
|
|
check("João" in r.stdout, "recall lost the accented memory: " + r.stdout + r.stderr)
|
|
r = run("recall", "plain ascii memory right after")
|
|
check("#%d " % (N + 2) in r.stdout, "record after a multi-byte one reads shifted")
|
|
r = run("note", "ã" * 150)
|
|
check(r.returncode == 1 and "300 bytes" in r.stderr,
|
|
"multi-byte note dodged the byte limit: " + r.stderr)
|
|
|
|
# note landed -> its blocks are pending; settle before the final wake check
|
|
while True:
|
|
r = run("sleep")
|
|
if "Nothing left to compress" in r.stdout:
|
|
break
|
|
bid = nap_id(r.stdout)
|
|
run("sleep", bid, "settled")
|
|
check(run("wake").returncode == 0, "wake refuses at the very end")
|
|
|
|
# a part is rendered as of T, so a note landing mid-wake cannot shift a
|
|
# boundary and silently drop a line
|
|
T0 = os.path.getsize(os.path.join(d, "LOG.txt")) // 320
|
|
before = run("wake", "1", str(T0))
|
|
check(before.returncode == 0, "as-of-T wake failed: " + before.stdout + before.stderr)
|
|
run("note", "a note that lands between two wake calls")
|
|
check(run("wake", "1", str(T0)).stdout == before.stdout,
|
|
"a note between parts changed an already-rendered part")
|
|
check(run("wake", "1", str(T0 + 99)).returncode == 1, "wake accepted a future T")
|
|
|
|
# ...and the agent pays that note's compressions on the spot, as it is told
|
|
# to. The tree then holds MORE blocks than the snapshot needs: a level must
|
|
# never count as negative work, or the rest of the wake is refused with an
|
|
# impossible number.
|
|
while True:
|
|
r = run("sleep")
|
|
if "Nothing left to compress" in r.stdout:
|
|
break
|
|
run("sleep", nap_id(r.stdout), "settled mid-wake")
|
|
r = run("wake", "1", str(T0))
|
|
check(r.returncode == 0 and r.stdout == before.stdout,
|
|
"a compression paid mid-wake broke the rest of the wake:\n"
|
|
+ r.stdout + r.stderr)
|
|
for T in list(range(1, 40)) + [T0 - 1, T0, T0 + 1]:
|
|
check(cli.pending_count(d, T) == len(cli.pending(d, T)),
|
|
"pending_count disagrees with pending at T=%d" % T)
|
|
|
|
# recall must not hand back more than a harness will carry
|
|
r = run("recall", "memory number")
|
|
check(len(r.stdout) < CAP_CHARS, "recall returned %d chars" % len(r.stdout))
|
|
check("Narrow the regex" in r.stdout, "recall did not say it had been capped")
|
|
|
|
# ---- concurrency and crash recovery ----------------------------------
|
|
|
|
d2 = tempfile.mkdtemp(prefix="optmem-race-")
|
|
env2 = dict(os.environ, MEMORY_DIR=d2)
|
|
P = 16 # real processes: this is the cross-process lock under test
|
|
procs = [subprocess.Popen(memo + ["note", "parallel note %d" % i], env=env2,
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
for i in range(P)]
|
|
for p in procs:
|
|
p.wait()
|
|
with open(os.path.join(d2, "LOG.txt"), "rb") as f:
|
|
recs = [f.read(320) for _ in range(P)]
|
|
ids = [r.decode().split()[0] for r in recs if r.strip()]
|
|
check(len(ids) == P, "%d of %d parallel notes survived" % (len(ids), P))
|
|
check(len(set(ids)) == P, "parallel notes collided on an id: %s" % sorted(ids))
|
|
check(sorted(ids) == sorted("#%d" % i for i in range(P)),
|
|
"parallel note ids are not 0..%d: %s" % (P - 1, sorted(ids)))
|
|
|
|
# a crash mid-append leaves a partial record; the next append must drop it,
|
|
# or every later record is misaligned forever
|
|
with open(os.path.join(d2, "LOG.txt"), "ab") as f:
|
|
f.write(b"#99 2026-01-01 a half-written record killed by a power cut")
|
|
r = run("note", "the memory right after a torn write", store=d2)
|
|
check(r.returncode == 0, "note failed after a torn write: " + r.stderr)
|
|
sz = os.path.getsize(os.path.join(d2, "LOG.txt"))
|
|
check(sz % 320 == 0, "LOG.txt left misaligned after a torn write: %d" % sz)
|
|
check("Saved as #%d" % P in r.stdout, "torn record was counted as a memory")
|
|
r = run("recall", "right after a torn write", store=d2)
|
|
check("#%d " % P in r.stdout, "the memory after a torn write reads wrong")
|
|
|
|
# a memory small enough to fit one part must still end with the terminator
|
|
# the agent was told to wait for
|
|
while True:
|
|
r = run("sleep", store=d2)
|
|
if "Nothing left to compress" in r.stdout:
|
|
break
|
|
bid = nap_id(r.stdout)
|
|
run("sleep", bid, "settled", store=d2)
|
|
r = run("wake", store=d2)
|
|
check(r.stdout.rstrip().endswith("You are awake."),
|
|
"a one-part wake never says `You are awake.`:\n" + r.stdout)
|
|
|
|
shutil.rmtree(d2)
|
|
shutil.rmtree(d)
|
|
print("\n%d passed, %d failed" % (ok, fail))
|
|
sys.exit(1 if fail else 0)
|