audit: a paid compression no longer breaks a snapshot wake

Found by battle-testing the real flow: wake part 1, note something, pay the
compression the note creates, then continue the wake. pending_count subtracted
per level without clamping, so a level that had moved PAST the snapshot
counted as negative work:

  Cannot wake: -1 compressions pending. Do them, then run memo wake again.
  None

The rest of the memory was unreadable for the rest of the session. Clamped at
zero, with a test that pending_count always equals len(pending).

Also: MEMORY_DIR is never created (a typo opened a second, empty identity
instead of failing); the config file is never written (a store froze the
defaults at creation, so updating the tool stopped changing behaviour);
missing import file reports instead of raising; unknown command says so;
a nap prompt with nothing left after it drops the '0 compressions remain'
line; a missing half dies instead of compressing a '?'; plurals.

README.md

@@ -27,10 +27,12 @@ OptMem fixes the two halves separately:
 git clone https://github.com/VictorTaelin/OptMem ~/OptMem
 export PATH="$HOME/OptMem:$PATH"
 export MEMORY_DIR="$HOME/memory"     # required; there is no default
+mkdir -p "$MEMORY_DIR"               # this is what creates the identity
 ```
 
 `MEMORY_DIR` is the only machine-specific fact in the system. One machine, one
-`MEMORY_DIR`, one identity.
+`MEMORY_DIR`, one identity. `memo` never creates that directory itself: if it
+did, one typo would open a second, empty identity instead of an error.
 
 ## Use
 
@@ -168,7 +170,10 @@ $MEMORY_DIR/
   TREE/4      record, indexed   size. each block written once, unless forgotten.
   TREE/8      by position
   ...
-  config      ENTRY_CHARS=280   longest a memory may be
+  config      optional. absent on a normal store; the defaults below live in
+              `memo` and are the only home for them.
+
+              ENTRY_CHARS=280   longest a memory may be
               WAKE_LINES=256    how many lines `memo wake` prints (~16k tokens)
               PART_CHARS=20000  how much of it fits in one command's output
               PART_LINES=500    ...and in how many lines

memo

@@ -47,6 +47,13 @@ def store():
     if not d:
         die("MEMORY_DIR is not set. Example: export MEMORY_DIR=~/memory")
     d = os.path.expanduser(d)
+    # The directory is never created here. A typo in MEMORY_DIR would then
+    # open an empty store, and the agent would wake with no past and start
+    # writing a second identity. Making the directory IS creating the
+    # identity, and that is a deliberate act: mkdir.
+    if not os.path.isdir(d):
+        die("MEMORY_DIR=%s does not exist.\nIf this is a new identity, run: "
+            "mkdir -p %s" % (d, d))
     os.makedirs(os.path.join(d, "TREE"), exist_ok=True)
     p = os.path.join(d, "LOG.txt")
     if not os.path.exists(p):
@@ -55,12 +62,12 @@ def store():
 
 
 def config(d):
+    """Optional overrides in $MEMORY_DIR/config. The file is never written:
+    a store that kept its own copy of the defaults would freeze them, and
+    updating the tool would stop changing how it behaves."""
     global ENTRY_CHARS, WAKE_LINES, PART_CHARS, PART_LINES
     p = os.path.join(d, "config")
     if not os.path.exists(p):
-        with open(p, "w") as f:
-            f.write("ENTRY_CHARS=%d\nWAKE_LINES=%d\nPART_CHARS=%d\nPART_LINES=%d\n"
-                    % (ENTRY_CHARS, WAKE_LINES, PART_CHARS, PART_LINES))
         return
     for line in open(p):
         line = line.split("#")[0].strip()
@@ -223,10 +230,20 @@ def die(msg):
     sys.exit(1)
 
 
+def plural(n, word):
+    if n == 1:
+        return "1 " + word
+    if word.endswith("y"):
+        word = word[:-1] + "ie"
+    elif word.endswith(("s", "h", "x")):
+        word += "e"
+    return "%d %ss" % (n, word)
+
+
 def check(text):
     text = text.strip()
     if not text:
-        die("Empty.")
+        die("Empty. A memory is one line of text.")
     if "\n" in text or "\r" in text:
         die("%d lines. A memory is one line: merge them, or note them "
             "separately." % (text.count("\n") + 1))
@@ -255,9 +272,12 @@ def pending(d, T, limit=None):
 
 
 def pending_count(d, T):
+    """How many blocks pending() would list, without listing them. A level can
+    hold MORE blocks than T needs -- T is a snapshot, and memories keep
+    arriving while an agent reads -- so each level is clamped at zero."""
     n, size = 0, 2
     while size <= T:
-        n += T // size - count(tree_path(d, size), TREE_REC)
+        n += max(0, T // size - count(tree_path(d, size), TREE_REC))
         size *= 2
     return n
 
@@ -266,16 +286,20 @@ def nap_prompt(d, lo, hi, left):
     if hi - lo <= RAW_MAX:
         body = "\n".join("  #%d %s %s" % e for e in log_slice(d, lo, hi))
     else:
-        mid = (lo + hi) // 2
+        mid, halves = (lo + hi) // 2, []
-        body = "\n".join("  #%d-%d %s" % (a, b - 1, tree_get(d, a, b) or "?")
+        for a, b in ((lo, mid), (mid, hi)):
-                         for a, b in ((lo, mid), (mid, hi)))
+            s = tree_get(d, a, b)
-    tail = ("1 compression remains" if left == 1 else
+            if s is None:
-            "%d compressions remain" % left)
+                die("Summary %d-%d is missing. Run: memo sleep" % (a, b - 1))
+            halves.append("  #%d-%d %s" % (a, b - 1, s))
+        body = "\n".join(halves)
+    tail = "" if not left else "\n%s after this one." % (
+        "1 compression remains" if left == 1 else
+        "%d compressions remain" % left)
     return ("Compress memories #%d-%d into one line of at most %d characters.\n"
             "Keep every name, number, date, decision and outcome.\n"
             "Drop wording, not facts. Invent nothing.\n\n"
-            "%s\n\n"
+            "%s\n%s\n"
-            "%s after this one.\n"
             "Run: memo sleep %d-%d \"<your line>\""
             % (lo, hi - 1, ENTRY_CHARS, body, tail, lo, hi - 1))
 
@@ -315,15 +339,14 @@ def cmd_wake(d, args):
         if len(args) == 2:
             T = int(args[1])
             if T > now:
-                die("T=%d, but the memory holds %d entries. Run: memo wake"
+                die("T=%d, but the memory holds %s. Run: memo wake"
-                    % (T, now))
+                    % (T, plural(now, "entry")))
     # A part is rendered as of T, so a note landing between two parts cannot
     # shift a boundary and drop a line.
     n = pending_count(d, T)
     if n:
         print("Cannot wake: %s pending. Do %s, then run memo wake again.\n"
-              % ("1 compression" if n == 1 else "%d compressions" % n,
+              % (plural(n, "compression"), "it" if n == 1 else "them"))
-                 "it" if n == 1 else "them"))
         print(next_nap(d, T))
         sys.exit(1)
     if not T:
@@ -341,8 +364,8 @@ def cmd_wake(d, args):
             lines.append("#%d-%d %s" % (lo, hi - 1, s))
     parts = paginate(lines)
     if not 1 <= k <= len(parts):
-        die("No part %d: the memory has %d parts. Run: memo wake"
+        die("No part %d: the memory has %s. Run: memo wake"
-            % (k, len(parts)))
+            % (k, plural(len(parts), "part")))
     if len(parts) > 1:
         print("Your memory, part %d of %d, oldest first." % (k, len(parts)))
     print("\n".join(parts[k - 1]))
@@ -410,8 +433,8 @@ def cmd_forget(d, args):
     gone = tree_drop(d, lo, hi)
     if not gone:
         die("No summary at %s." % args[0])
-    print("Forgot %d summaries, from %d-%d up. Run: memo sleep"
+    print("Forgot %s, from %d-%d up. Run: memo sleep"
-          % (len(gone), gone[0][0], gone[0][1] - 1))
+          % (plural(len(gone), "summary"), gone[0][0], gone[0][1] - 1))
 
 
 def cmd_recall(d, args):
@@ -436,9 +459,10 @@ def cmd_recall(d, args):
         out.append(line)
     print("\n".join(reversed(out)))
     if len(out) < len(hits):
-        print("Newest %d of %d matches. Narrow the regex." % (len(out), len(hits)))
+        print("Newest %d of %s. Narrow the regex."
+              % (len(out), plural(len(hits), "match")))
     else:
-        print("%d matches." % len(hits))
+        print("%s." % plural(len(hits), "match"))
 
 
 def cmd_import(d, args):
@@ -446,9 +470,13 @@ def cmd_import(d, args):
     For bootstrapping an identity from older records. Used once."""
     if len(args) != 1:
         die("usage: memo import <file>   # lines of 'YYYY-MM-DD <text>'")
+    try:
+        src = open(args[0]).readlines()
+    except OSError as e:
+        die("Cannot read %s: %s" % (args[0], e.strerror))
     last = log_get(d, log_len(d) - 1)[1] if log_len(d) else "0000-00-00"
     out = []
-    for i, line in enumerate(open(args[0]), 1):
+    for i, line in enumerate(src, 1):
         line = line.rstrip("\n")
         if not line.strip():
             continue
@@ -463,11 +491,14 @@ def cmd_import(d, args):
             die("line %d: %d bytes, limit %d." % (i, len(text.encode()), ENTRY_CHARS))
         out.append((date, text))
         last = date
+    if not out:
+        die("%s has no memories." % args[0])
     base = log_append(d, out)
-    print("imported %d memories, #%d to #%d." % (len(out), base, base + len(out) - 1))
+    print("Imported %s, #%d to #%d."
+          % (plural(len(out), "memory"), base, base + len(out) - 1))
     n = pending_count(d, log_len(d))
     if n:
-        print("%d compressions pending. Run: memo sleep" % n)
+        print("%s pending. Run: memo sleep" % plural(n, "compression"))
 
 
 COMMANDS = {"wake": cmd_wake, "note": cmd_note, "sleep": cmd_sleep,
@@ -475,9 +506,13 @@ COMMANDS = {"wake": cmd_wake, "note": cmd_note, "sleep": cmd_sleep,
 
 
 def main():
-    if len(sys.argv) < 2 or sys.argv[1] not in COMMANDS:
+    if len(sys.argv) < 2:
         print(__doc__.strip())
-        sys.exit(0 if len(sys.argv) < 2 else 1)
+        sys.exit(0)
+    if sys.argv[1] not in COMMANDS:
+        print("No such command: %s\n" % sys.argv[1], file=sys.stderr)
+        print(__doc__.strip(), file=sys.stderr)
+        sys.exit(1)
     d = store()
     config(d)
     COMMANDS[sys.argv[1]](d, sys.argv[2:])

test.py

@@ -128,6 +128,18 @@ smoke = subprocess.run(memo + ["wake"], env=dict(os.environ, MEMORY_DIR=d),
 check(smoke.returncode == 0 and "No memories yet" in smoke.stdout,
       "the memo CLI does not run: " + smoke.stdout + smoke.stderr)
 
+# a typo in MEMORY_DIR must not silently open a second, empty identity
+ghost = subprocess.run(memo + ["wake"], capture_output=True, text=True,
+                       env=dict(os.environ, MEMORY_DIR=d + "-typo"))
+check(ghost.returncode == 1 and "does not exist" in ghost.stderr,
+      "a missing MEMORY_DIR was created instead of reported")
+check(not os.path.exists(d + "-typo"), "a missing MEMORY_DIR was created")
+noenv = subprocess.run(memo + ["wake"], capture_output=True, text=True,
+                       env={k: v for k, v in os.environ.items()
+                            if k != "MEMORY_DIR"})
+check(noenv.returncode == 1 and "MEMORY_DIR is not set" in noenv.stderr,
+      "an unset MEMORY_DIR must fail loudly")
+
 
 r = run("note", "x" * 281)
 check(r.returncode == 1 and "Too long" in r.stderr, "over-long note accepted")
@@ -146,7 +158,9 @@ with open(os.path.join(d, "seed.txt"), "w") as f:
         f.write("%s memory number %d, a thing that happened\n"
                 % ((day + datetime.timedelta(days=i // 5)).isoformat(), i))
 r = run("import", os.path.join(d, "seed.txt"))
-check("imported %d" % N in r.stdout, "import failed: " + r.stdout + r.stderr)
+check("Imported %d" % N in r.stdout, "import failed: " + r.stdout + r.stderr)
+check(not os.path.exists(os.path.join(d, "config")),
+      "a store wrote its own config file: the defaults are now frozen in it")
 
 r = run("wake")
 check(r.returncode == 1 and "Cannot wake" in r.stdout,
@@ -218,6 +232,7 @@ check(r.returncode == 0 and "Nothing left to compress" in r.stdout,
 # recall reaches memories the summaries lost
 r = run("recall", "memory number 7,")
 check(r.returncode == 0 and "#7 " in r.stdout, "recall missed a memory")
+check("1 match." in r.stdout, "a single match is not `1 matches`: " + r.stdout)
 
 def treesize():
     t = os.path.join(d, "TREE")
@@ -278,6 +293,23 @@ check(run("wake", "1", str(T0)).stdout == before.stdout,
       "a note between parts changed an already-rendered part")
 check(run("wake", "1", str(T0 + 99)).returncode == 1, "wake accepted a future T")
 
+# ...and the agent pays that note's compressions on the spot, as it is told
+# to. The tree then holds MORE blocks than the snapshot needs: a level must
+# never count as negative work, or the rest of the wake is refused with an
+# impossible number.
+while True:
+    r = run("sleep")
+    if "Nothing left to compress" in r.stdout:
+        break
+    run("sleep", nap_id(r.stdout), "settled mid-wake")
+r = run("wake", "1", str(T0))
+check(r.returncode == 0 and r.stdout == before.stdout,
+      "a compression paid mid-wake broke the rest of the wake:\n"
+      + r.stdout + r.stderr)
+for T in list(range(1, 40)) + [T0 - 1, T0, T0 + 1]:
+    check(cli.pending_count(d, T) == len(cli.pending(d, T)),
+          "pending_count disagrees with pending at T=%d" % T)
+
 # recall must not hand back more than a harness will carry
 r = run("recall", "memory number")
 check(len(r.stdout) < CAP_CHARS, "recall returned %d chars" % len(r.stdout))