af17f94c2f
'next: memo wake 2 246' is an annotation, not an instruction. ASD-STE100: an instruction is an imperative, verb first. Same defect in the nap prompt, where the command sat on a bare line with nothing telling the agent to run it. The wake terminator is 'You are awake.', the same words memo sleep already used, instead of a bare 'awake.' -- one term, one meaning. test.py now asserts that every offered command starts with 'Run: '.
319 lines
13 KiB
Python
Executable file
319 lines
13 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""OptMem invariants, checked against a synthetic life of 5000 memories.
|
|
|
|
Uses a fake compressor (join + truncate) so the run is deterministic and free.
|
|
"""
|
|
|
|
import contextlib
|
|
import datetime
|
|
import io
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from importlib.machinery import SourceFileLoader
|
|
|
|
HERE = os.path.dirname(os.path.realpath(__file__))
|
|
sys.path.insert(0, HERE)
|
|
from blocks import complete, cover # noqa: E402
|
|
|
|
MEMO = os.path.join(HERE, "memo")
|
|
cli = SourceFileLoader("memo_cli", MEMO).load_module()
|
|
# The shipped defaults. A fresh process starts from these, so an in-process
|
|
# call must too, or one store's config would leak into the next.
|
|
DEFAULTS = {k: getattr(cli, k) for k in
|
|
("ENTRY_CHARS", "WAKE_LINES", "PART_CHARS", "PART_LINES")}
|
|
|
|
N = 2000
|
|
WAKE_LINES = 256
|
|
PART_CHARS = 20000
|
|
# Verified caps of the harnesses in the wild: Claude Code cuts a command's
|
|
# output at 30,000 chars (middle), pi at 50 KB / 2000 lines (head), Codex
|
|
# budgets 10,000 tokens. A part must fit the strictest of each kind.
|
|
CAP_CHARS, CAP_LINES = 30000, 2000
|
|
ok, fail = 0, 0
|
|
|
|
|
|
def check(cond, msg):
|
|
global ok, fail
|
|
if cond:
|
|
ok += 1
|
|
else:
|
|
fail += 1
|
|
print("FAIL: " + msg)
|
|
|
|
|
|
# ---- pure block math -------------------------------------------------
|
|
|
|
for T in list(range(1, 400)) + [1000, 4096, 10000, 65536, 100003]:
|
|
c = cover(T, WAKE_LINES)
|
|
check(len(c) <= WAKE_LINES, "T=%d: %d lines > budget" % (T, len(c)))
|
|
check(c[0][0] == 0 and c[-1][1] == T, "T=%d: does not span [0,T)" % T)
|
|
for a, b in zip(c, c[1:]):
|
|
check(a[1] == b[0], "T=%d: gap or overlap at %s %s" % (T, a, b))
|
|
for lo, hi in c:
|
|
s = hi - lo
|
|
check(s & (s - 1) == 0 and lo % s == 0,
|
|
"T=%d: [%d,%d) is not an aligned power-of-two block" % (T, lo, hi))
|
|
for a, b in zip(c, c[1:]):
|
|
check(b[1] - b[0] <= a[1] - a[0],
|
|
"T=%d: detail does not increase toward the present" % T)
|
|
|
|
check(cover(300, 320) == [(i, i + 1) for i in range(300)],
|
|
"under budget, memory should be verbatim")
|
|
|
|
# every block a cover ever needs must be buildable. cover() costs a 60-step
|
|
# binary search, so this walks every tree shape up to 300 and then samples:
|
|
# the property is structural, not a function of the exact T.
|
|
seen = set()
|
|
for T in list(range(1, 300)) + [512, 700, 1000, 1023, 1024, 2000, 2999]:
|
|
seen.update(b for b in cover(T, WAKE_LINES) if b[1] - b[0] > 1)
|
|
buildable = set(complete(3000))
|
|
check(seen <= buildable, "a cover wants a block that complete() never yields")
|
|
|
|
# work never spikes: naps created by one new memory
|
|
worst, prev = 0, 0
|
|
for T in range(1, N):
|
|
cur = len(complete(T))
|
|
worst = max(worst, cur - prev)
|
|
prev = cur
|
|
check(worst <= 16, "a single memory created %d naps" % worst)
|
|
|
|
# ---- the real CLI ----------------------------------------------------
|
|
|
|
d = tempfile.mkdtemp(prefix="optmem-test-")
|
|
memo = [sys.executable, MEMO]
|
|
|
|
|
|
class Result:
|
|
def __init__(self, returncode, stdout, stderr):
|
|
self.returncode, self.stdout, self.stderr = returncode, stdout, stderr
|
|
|
|
|
|
def run(*args, store=None):
|
|
"""One `memo` command, in-process. Spawning an interpreter per call cost
|
|
~40ms x ~2000 naps; the cross-process behaviour that genuinely needs real
|
|
processes (the lock) is tested with real processes below."""
|
|
os.environ["MEMORY_DIR"] = store or d
|
|
for k, v in DEFAULTS.items():
|
|
setattr(cli, k, v)
|
|
out, err, code = io.StringIO(), io.StringIO(), 0
|
|
try:
|
|
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
|
|
sd = cli.store()
|
|
cli.config(sd)
|
|
cli.COMMANDS[args[0]](sd, list(args[1:]))
|
|
except SystemExit as e:
|
|
code = e.code if isinstance(e.code, int) else 0
|
|
return Result(code, out.getvalue(), err.getvalue())
|
|
|
|
|
|
def nap_id(out):
|
|
"""The block id from the command a nap prompt offers."""
|
|
m = re.search(r"memo sleep (\d+)-(\d+)", out)
|
|
return "%s-%s" % m.groups() if m else None
|
|
|
|
|
|
def offered(out):
|
|
"""The line offering a command. Every command handed to an agent must be
|
|
an order, not a label: `Run: memo ...`, never `next: memo ...`."""
|
|
return [l for l in out.splitlines() if "memo sleep " in l or "memo wake " in l]
|
|
|
|
|
|
# the real entry point still has to work: shebang, argv parsing, exit code
|
|
smoke = subprocess.run(memo + ["wake"], env=dict(os.environ, MEMORY_DIR=d),
|
|
capture_output=True, text=True)
|
|
check(smoke.returncode == 0 and "No memories yet" in smoke.stdout,
|
|
"the memo CLI does not run: " + smoke.stdout + smoke.stderr)
|
|
|
|
|
|
r = run("note", "x" * 281)
|
|
check(r.returncode == 1 and "Too long" in r.stderr, "over-long note accepted")
|
|
r = run("note", "two\nlines")
|
|
check(r.returncode == 1 and "one line" in r.stderr, "multi-line note accepted")
|
|
r = run("note", " ")
|
|
check(r.returncode == 1, "empty note accepted")
|
|
check("No memories yet" in run("wake").stdout, "empty wake should say so")
|
|
|
|
with open(os.path.join(d, "seed.txt"), "w") as f:
|
|
day = datetime.date(2020, 1, 1)
|
|
for i in range(N):
|
|
f.write("%s memory number %d, a thing that happened\n"
|
|
% ((day + datetime.timedelta(days=i // 5)).isoformat(), i))
|
|
r = run("import", os.path.join(d, "seed.txt"))
|
|
check("imported %d" % N in r.stdout, "import failed: " + r.stdout + r.stderr)
|
|
|
|
r = run("wake")
|
|
check(r.returncode == 1 and "Cannot wake" in r.stdout,
|
|
"wake must refuse while work is pending")
|
|
|
|
# sleep loop, with a fake compressor
|
|
naps = 0
|
|
r = run("sleep")
|
|
while "You are awake" not in r.stdout:
|
|
line = offered(r.stdout)
|
|
check(bool(line), "no command offered:\n" + r.stdout + r.stderr)
|
|
if not line:
|
|
break
|
|
check(line[0].startswith("Run: "), "a command was offered as a label, not "
|
|
"an order: %r" % line[0])
|
|
bid = nap_id(r.stdout)
|
|
body = [l.strip() for l in r.stdout.splitlines()
|
|
if l.startswith(" #") or (l.startswith(" ") and l.strip()
|
|
and not l.strip().startswith("memo"))]
|
|
r = run("sleep", bid, (" ".join(body)[:280]).strip() or "empty")
|
|
check(r.returncode == 0, "sleep rejected a valid nap: " + r.stderr)
|
|
naps += 1
|
|
check(naps == len(complete(N)), "did %d naps, expected %d" % (naps, len(complete(N))))
|
|
|
|
r = run("wake")
|
|
check(r.returncode == 0, "wake still refuses after a full sleep")
|
|
|
|
# the document survives pagination, and every part fits every harness's cap
|
|
parts, k = [], 1
|
|
while True:
|
|
r = run("wake", str(k))
|
|
if r.returncode != 0:
|
|
break
|
|
body = [l for l in r.stdout.splitlines() if l.startswith("#")]
|
|
check(len(r.stdout) < CAP_CHARS, "part %d is %d chars, over the %d cap"
|
|
% (k, len(r.stdout), CAP_CHARS))
|
|
check(len(r.stdout.splitlines()) < CAP_LINES, "part %d is over %d lines"
|
|
% (k, CAP_LINES))
|
|
parts.append(body)
|
|
k += 1
|
|
check(len(parts) > 1, "a %d-line memory should need more than one part" % WAKE_LINES)
|
|
lines = [l for p in parts for l in p]
|
|
check(len(lines) == WAKE_LINES, "woke with %d lines, want %d" % (len(lines), WAKE_LINES))
|
|
check(lines[-1].startswith("#%d " % (N - 1)), "newest memory not last / not raw")
|
|
check(lines[0].startswith("#0-"), "oldest line should be a summary block")
|
|
check("Run: memo wake 2" in run("wake").stdout,
|
|
"part 1 must ORDER the next command, not label it")
|
|
check("You are awake." in run("wake", str(len(parts))).stdout,
|
|
"last part must say it is last")
|
|
check(run("wake", str(len(parts) + 1)).returncode == 1, "a nonexistent part should fail")
|
|
|
|
# append-only: nothing was ever rewritten
|
|
logsz = os.path.getsize(os.path.join(d, "LOG.txt"))
|
|
run("note", "one more thing happened today")
|
|
check(os.path.getsize(os.path.join(d, "LOG.txt")) > logsz, "note did not append")
|
|
check(logsz % 320 == 0, "LOG.txt is not a whole number of records")
|
|
for f in os.listdir(os.path.join(d, "TREE")):
|
|
check(os.path.getsize(os.path.join(d, "TREE", f)) % 288 == 0,
|
|
"TREE/%s is not a whole number of records" % f)
|
|
|
|
# a block already written cannot be rewritten
|
|
check(run("sleep", "0-2", "attempted overwrite").returncode == 1,
|
|
"rewriting a settled block was allowed")
|
|
|
|
# recall reaches memories the summaries lost
|
|
r = run("recall", "memory number 7,")
|
|
check(r.returncode == 0 and "#7 " in r.stdout, "recall missed a memory")
|
|
|
|
def treesize():
|
|
t = os.path.join(d, "TREE")
|
|
return sum(os.path.getsize(os.path.join(t, f)) for f in os.listdir(t))
|
|
|
|
before, logsize = treesize(), os.path.getsize(os.path.join(d, "LOG.txt"))
|
|
r = run("forget", "16-32")
|
|
check("16-32" in r.stdout, "forget did not report the block: " + r.stdout + r.stderr)
|
|
check(treesize() < before, "forget did not shrink the tree")
|
|
check(os.path.getsize(os.path.join(d, "LOG.txt")) == logsize, "forget touched the log")
|
|
check(run("wake").returncode == 1, "wake should refuse after a forget")
|
|
n = 0
|
|
while True:
|
|
r = run("sleep")
|
|
if "You are awake" in r.stdout:
|
|
break
|
|
bid = nap_id(r.stdout)
|
|
check(run("sleep", bid, "rebuilt after forget").returncode == 0, "rebuild rejected")
|
|
n += 1
|
|
check(n > 0, "forget created no work")
|
|
check(run("wake").returncode == 0, "wake still refuses after rebuilding")
|
|
check(treesize() == before, "tree did not return to its original size")
|
|
check(run("forget", "17-33").returncode == 1, "forgetting a non-block should fail")
|
|
check(run("forget", "999998-1000000").returncode == 1, "forgetting a missing block should fail")
|
|
|
|
# UTF-8: multi-byte characters must not shift record boundaries or dodge limits
|
|
run("note", "reunião com João em São Paulo: ação aprovada, coração tranquilo")
|
|
run("note", "a plain ascii memory right after the accented one")
|
|
r = run("recall", "coração")
|
|
check("João" in r.stdout, "recall lost the accented memory: " + r.stdout + r.stderr)
|
|
r = run("recall", "plain ascii memory right after")
|
|
check("#%d " % (N + 2) in r.stdout, "record after a multi-byte one reads shifted")
|
|
r = run("note", "ã" * 150)
|
|
check(r.returncode == 1 and "300 bytes" in r.stderr,
|
|
"multi-byte note dodged the byte limit: " + r.stderr)
|
|
|
|
# note landed -> its blocks are pending; settle before the final wake check
|
|
while True:
|
|
r = run("sleep")
|
|
if "You are awake" in r.stdout:
|
|
break
|
|
bid = nap_id(r.stdout)
|
|
run("sleep", bid, "settled")
|
|
check(run("wake").returncode == 0, "wake refuses at the very end")
|
|
|
|
# a part is rendered as of T, so a note landing mid-wake cannot shift a
|
|
# boundary and silently drop a line
|
|
T0 = os.path.getsize(os.path.join(d, "LOG.txt")) // 320
|
|
before = run("wake", "1", str(T0))
|
|
check(before.returncode == 0, "as-of-T wake failed: " + before.stdout + before.stderr)
|
|
run("note", "a note that lands between two wake calls")
|
|
check(run("wake", "1", str(T0)).stdout == before.stdout,
|
|
"a note between parts changed an already-rendered part")
|
|
check(run("wake", "1", str(T0 + 99)).returncode == 1, "wake accepted a future T")
|
|
|
|
# recall must not hand back more than a harness will carry
|
|
r = run("recall", "memory number")
|
|
check(len(r.stdout) < CAP_CHARS, "recall returned %d chars" % len(r.stdout))
|
|
check("Narrow the regex" in r.stdout, "recall did not say it had been capped")
|
|
|
|
# ---- concurrency and crash recovery ----------------------------------
|
|
|
|
d2 = tempfile.mkdtemp(prefix="optmem-race-")
|
|
env2 = dict(os.environ, MEMORY_DIR=d2)
|
|
P = 16 # real processes: this is the cross-process lock under test
|
|
procs = [subprocess.Popen(memo + ["note", "parallel note %d" % i], env=env2,
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
for i in range(P)]
|
|
for p in procs:
|
|
p.wait()
|
|
with open(os.path.join(d2, "LOG.txt"), "rb") as f:
|
|
recs = [f.read(320) for _ in range(P)]
|
|
ids = [r.decode().split()[0] for r in recs if r.strip()]
|
|
check(len(ids) == P, "%d of %d parallel notes survived" % (len(ids), P))
|
|
check(len(set(ids)) == P, "parallel notes collided on an id: %s" % sorted(ids))
|
|
check(sorted(ids) == sorted("#%d" % i for i in range(P)),
|
|
"parallel note ids are not 0..%d: %s" % (P - 1, sorted(ids)))
|
|
|
|
# a crash mid-append leaves a partial record; the next append must drop it,
|
|
# or every later record is misaligned forever
|
|
with open(os.path.join(d2, "LOG.txt"), "ab") as f:
|
|
f.write(b"#99 2026-01-01 a half-written record killed by a power cut")
|
|
r = run("note", "the memory right after a torn write", store=d2)
|
|
check(r.returncode == 0, "note failed after a torn write: " + r.stderr)
|
|
sz = os.path.getsize(os.path.join(d2, "LOG.txt"))
|
|
check(sz % 320 == 0, "LOG.txt left misaligned after a torn write: %d" % sz)
|
|
check("saved as #%d" % P in r.stdout, "torn record was counted as a memory")
|
|
r = run("recall", "right after a torn write", store=d2)
|
|
check("#%d " % P in r.stdout, "the memory after a torn write reads wrong")
|
|
|
|
# a memory small enough to fit one part must still end with the terminator
|
|
# the agent was told to wait for
|
|
while True:
|
|
r = run("sleep", store=d2)
|
|
if "You are awake" in r.stdout:
|
|
break
|
|
bid = nap_id(r.stdout)
|
|
run("sleep", bid, "settled", store=d2)
|
|
r = run("wake", store=d2)
|
|
check(r.stdout.rstrip().endswith("You are awake."),
|
|
"a one-part wake never says `You are awake.`:\n" + r.stdout)
|
|
|
|
shutil.rmtree(d2)
|
|
shutil.rmtree(d)
|
|
print("\n%d passed, %d failed" % (ok, fail))
|
|
sys.exit(1 if fail else 0)
|