test(manifold): merge probe-API + spine-invariant e2e specs

manifold/tests/e2e/helpers.ts

@@ -0,0 +1,101 @@
+/**
+ * Playwright helpers for the Manifold app.
+ *
+ * Ported from `playground/tests/e2e/helpers.ts` and adapted to Manifold's
+ * probe surface. Differences from the playground probe:
+ *   - Manifold's probe has NO `__init()` — it is installed by `App.tsx` only
+ *     AFTER the engine (and its WASM) are live, and `__ready` is a live getter
+ *     over the spine state. So "ready" == `window.__nisps.__ready === true`.
+ *   - The probe is gated behind `?debug=1` (see `src/debug/probe.ts`), not a
+ *     route. We always navigate to `/?debug=1`.
+ *   - `routedOutputs()` (plural) is the post-output-pipeline vector.
+ */
+import type { Page } from '@playwright/test';
+import type { DebugProbe } from '../../src/debug/probe';
+
+declare global {
+  interface Window {
+    __nisps?: DebugProbe;
+  }
+}
+
+const READY_TIMEOUT = 20_000;
+
+/**
+ * Navigate to the Manifold app with the `?debug=1` probe installed and
+ * localStorage cleared (fresh default weights, not a prior test's state), then
+ * wait until `window.__nisps` reports the WASM engine is ready.
+ *
+ * `extraQuery` is appended after `debug=1` (leading `&` optional).
+ */
+export async function loadProbe(page: Page, extraQuery = ''): Promise<void> {
+  // Clear persisted ML/settings state before the SPA boots so initial
+  // inference uses default weights.
+  await page.addInitScript(() => {
+    try {
+      localStorage.clear();
+      sessionStorage.clear();
+    } catch {
+      /* private mode etc — ignore */
+    }
+  });
+
+  let q = extraQuery.trim();
+  if (q && !q.startsWith('&')) q = '&' + q;
+  await page.goto(`/?debug=1${q}`);
+
+  await page.waitForFunction(
+    () => {
+      const n = window.__nisps;
+      return !!n && n.__ready === true && n.getOutputs().length > 0;
+    },
+    undefined,
+    { timeout: READY_TIMEOUT },
+  );
+}
+
+/** Read the live post-ML output vector as a JSON-safe number[]. */
+export async function getOutputs(page: Page): Promise<number[]> {
+  return page.evaluate(() => Array.from(window.__nisps!.getOutputs()));
+}
+
+/** Read the live routed (post output-pipeline) vector as a JSON-safe number[]. */
+export async function getRouted(page: Page): Promise<number[]> {
+  return page.evaluate(() => Array.from(window.__nisps!.routedOutputs()));
+}
+
+/** Push the same raw XY input `n` times so the input/output EMA smoothing
+ *  settles, then return the converged post-ML outputs. */
+export async function settleInputs(page: Page, x: number, y: number, n = 40): Promise<number[]> {
+  return page.evaluate(
+    ([px, py, count]) => {
+      const probe = window.__nisps!;
+      for (let i = 0; i < count; i++) probe.setInputs(px, py);
+      return Array.from(probe.getOutputs());
+    },
+    [x, y, n] as const,
+  );
+}
+
+/**
+ * How many values differ by more than `eps` between two snapshots. Length
+ * mismatch counts as the absolute size difference.
+ */
+export function countChanged(a: number[], b: number[], eps = 1e-3): number {
+  if (a.length !== b.length) return Math.abs(a.length - b.length);
+  let n = 0;
+  for (let i = 0; i < a.length; ++i) {
+    if (Math.abs(a[i]! - b[i]!) > eps) ++n;
+  }
+  return n;
+}
+
+/** True iff every value is within [lo, hi] (with a tiny float tolerance). */
+export function allWithin(xs: number[], lo = 0, hi = 1, tol = 1e-6): boolean {
+  for (const v of xs) {
+    if (!(v >= lo - tol && v <= hi + tol)) return false;
+  }
+  return true;
+}
+
+export type Probe = DebugProbe;

manifold/tests/e2e/probe-api.spec.ts

@@ -0,0 +1,178 @@
+/**
+ * Debug-probe API contract — `window.__nisps` (gated behind `?debug=1`).
+ *
+ * Ported from `playground/tests/e2e/ml-engine.spec.ts`. This is the ENGINE
+ * contract, not playground UI, so it survives the playground's retirement:
+ * every probe accessor must return the documented shape and never throw.
+ *
+ * Adaptations vs. the playground original:
+ *   - Manifold's WASM net is `MLP<32,10,14,18,126>` (playground was `<2,...>`),
+ *     so `getWeights()` has 3148 elements, not 2848 (derivation below).
+ *   - No `probe.__init()` / no `mlStore.iml` poke-through: Manifold's probe
+ *     exposes `addExample()` and `routedOutputs()` directly, so the training
+ *     tests drive the real public surface instead of an escape hatch.
+ *   - The playground's `test.skip(!probeReady)` guard is gone — on Manifold the
+ *     probe is only installed once WASM is live, so a not-ready probe is a
+ *     genuine failure, not a pending-stream skip.
+ */
+import { test, expect } from '@playwright/test';
+import { loadProbe, getOutputs, countChanged, allWithin } from './helpers';
+
+// Fixed by the WASM build (`nisps/wasm/bindings.cpp`: MLP<32,10,14,18,126>).
+const N_OUTPUTS = 126;
+// weight_count = 32*10 + 10*14 + 14*18 + 18*126  (weights)
+//              + 10 + 14 + 18 + 126              (biases)
+//              = 320 + 140 + 252 + 2268 + 168 = 3148
+const WEIGHT_COUNT = 3148;
+// DefaultMLP::kNumLayers (4) * 4 stats per layer.
+const LAYER_STATS = 16;
+
+const EXAMPLE_LOW = { input: [0.1, 0.9], output: new Array(N_OUTPUTS).fill(0.1) };
+const EXAMPLE_HIGH = { input: [0.9, 0.1], output: new Array(N_OUTPUTS).fill(0.9) };
+
+test.beforeEach(async ({ page }) => {
+  await loadProbe(page);
+});
+
+test.describe('ML engine — debug probe contract', () => {
+  test('probe is installed and reports ready', async ({ page }) => {
+    const kind = await page.evaluate(() => typeof window.__nisps);
+    expect(kind).toBe('object');
+    const ready = await page.evaluate(() => window.__nisps!.__ready);
+    expect(ready).toBe(true);
+  });
+
+  test('initial outputs are bounded in [0, 1]', async ({ page }) => {
+    const outs = await getOutputs(page);
+    expect(outs).toHaveLength(N_OUTPUTS);
+    expect(allWithin(outs, 0, 1)).toBe(true);
+  });
+
+  test('initial state is 0 examples and no loss', async ({ page }) => {
+    const count = await page.evaluate(() => window.__nisps!.getExampleCount());
+    expect(count).toBe(0);
+    const loss = await page.evaluate(() => window.__nisps!.getLoss());
+    expect(loss).toBeNull();
+  });
+
+  test('randomise changes outputs', async ({ page }) => {
+    await page.evaluate(() => window.__nisps!.setInputs(0.3, 0.7));
+    const before = await getOutputs(page);
+    await page.evaluate(() => window.__nisps!.randomise());
+    await page.evaluate(() => window.__nisps!.setInputs(0.3, 0.7));
+    const after = await getOutputs(page);
+    expect(countChanged(before, after, 1e-3)).toBeGreaterThan(0);
+  });
+
+  test('setInputs runs inference and yields bounded outputs', async ({ page }) => {
+    await page.evaluate(() => window.__nisps!.setInputs(0.25, 0.75));
+    const outs = await getOutputs(page);
+    expect(outs).toHaveLength(N_OUTPUTS);
+    expect(allWithin(outs, 0, 1)).toBe(true);
+  });
+
+  test('thumbsUp returns a finite FeedbackAction and keeps the count sane', async ({ page }) => {
+    await page.evaluate(() => window.__nisps!.setInputs(0.4, 0.6));
+    const action = await page.evaluate(() => window.__nisps!.thumbsUp());
+    expect(typeof action).toBe('number');
+    expect(Number.isFinite(action)).toBe(true);
+    const count = await page.evaluate(() => window.__nisps!.getExampleCount());
+    expect(Number.isInteger(count)).toBe(true);
+    expect(count).toBeGreaterThanOrEqual(0);
+  });
+
+  test('thumbsDown moves weights and changes outputs', async ({ page }) => {
+    await page.evaluate(() => window.__nisps!.setInputs(0.3, 0.7));
+    const before = await getOutputs(page);
+    await page.evaluate(() => window.__nisps!.thumbsDown());
+    await page.evaluate(() => window.__nisps!.setInputs(0.3, 0.7));
+    const after = await getOutputs(page);
+    expect(countChanged(before, after, 1e-4)).toBeGreaterThan(0);
+  });
+
+  test('addExample reports success and bumps the example count', async ({ page }) => {
+    const ok = await page.evaluate(
+      ([ex]) => window.__nisps!.addExample(ex.input, ex.output),
+      [EXAMPLE_LOW],
+    );
+    expect(typeof ok).toBe('boolean');
+    expect(ok).toBe(true);
+    const count = await page.evaluate(() => window.__nisps!.getExampleCount());
+    expect(count).toBe(1);
+  });
+
+  test('train() with two contrasting examples does not increase loss', async ({ page }) => {
+    await page.evaluate(
+      ([low, high]) => {
+        window.__nisps!.addExample(low.input, low.output);
+        window.__nisps!.addExample(high.input, high.output);
+      },
+      [EXAMPLE_LOW, EXAMPLE_HIGH],
+    );
+
+    const loss1 = await page.evaluate(() => window.__nisps!.train());
+    expect(typeof loss1).toBe('number');
+    expect(Number.isFinite(loss1)).toBe(true);
+    expect(loss1).toBeGreaterThanOrEqual(0);
+
+    const loss2 = await page.evaluate(() => window.__nisps!.train());
+    expect(loss2).toBeLessThanOrEqual(loss1 + 1e-6);
+  });
+
+  test('async training resolves to a finite non-negative loss', async ({ page }) => {
+    await page.evaluate(
+      ([low, high]) => {
+        window.__nisps!.addExample(low.input, low.output);
+        window.__nisps!.addExample(high.input, high.output);
+      },
+      [EXAMPLE_LOW, EXAMPLE_HIGH],
+    );
+    const loss = await page.evaluate(() => window.__nisps!.trainAsync());
+    expect(typeof loss).toBe('number');
+    expect(Number.isFinite(loss)).toBe(true);
+    expect(loss).toBeGreaterThanOrEqual(0);
+  });
+
+  test('clearExamples resets the dataset count to 0', async ({ page }) => {
+    await page.evaluate(
+      ([ex]) => window.__nisps!.addExample(ex.input, ex.output),
+      [EXAMPLE_LOW],
+    );
+    expect(await page.evaluate(() => window.__nisps!.getExampleCount())).toBe(1);
+    await page.evaluate(() => window.__nisps!.clearExamples());
+    expect(await page.evaluate(() => window.__nisps!.getExampleCount())).toBe(0);
+  });
+
+  test('evalLoss returns a non-negative number or null', async ({ page }) => {
+    const v = await page.evaluate(() => window.__nisps!.evalLoss());
+    if (v !== null) {
+      expect(Number.isFinite(v)).toBe(true);
+      expect(v).toBeGreaterThanOrEqual(0);
+    }
+  });
+
+  test('inferBatch returns N * outputSize bounded floats', async ({ page }) => {
+    const points: ReadonlyArray<readonly [number, number]> = [
+      [0.0, 0.0],
+      [0.5, 0.5],
+      [1.0, 1.0],
+    ];
+    const flat = await page.evaluate(
+      (pts) => Array.from(window.__nisps!.inferBatch(pts as [number, number][])),
+      points,
+    );
+    expect(flat).toHaveLength(points.length * N_OUTPUTS);
+    expect(allWithin(flat, 0, 1)).toBe(true);
+  });
+
+  test('getLayerStats returns 4 floats per layer, all finite', async ({ page }) => {
+    const stats = await page.evaluate(() => Array.from(window.__nisps!.getLayerStats()));
+    expect(stats).toHaveLength(LAYER_STATS);
+    for (const v of stats) expect(Number.isFinite(v)).toBe(true);
+  });
+
+  test('getWeights returns the full weight vector', async ({ page }) => {
+    const len = await page.evaluate(() => window.__nisps!.getWeights().length);
+    expect(len).toBe(WEIGHT_COUNT);
+  });
+});

manifold/tests/e2e/spine.spec.ts

@@ -0,0 +1,104 @@
+/**
+ * Spine invariant.
+ *
+ * The engine spine (`src/engine/spine.ts`) is: setInputs → processed → ml →
+ * routed, derived EAGERLY + SYNCHRONOUSLY off React's render cycle. This spec
+ * asserts the two properties that make the spine the trustworthy core of the
+ * app:
+ *
+ *   1. Pushing inputs through the spine yields BOUNDED, CONSISTENT routed
+ *      outputs — distinct inputs map to distinct outputs, the same input
+ *      (once the input/output EMA smoothing settles) converges to a stable
+ *      vector, and both the post-ML and routed vectors stay in [0, 1].
+ *   2. The probe (hence the WASM engine and the reactive spine) stays ALIVE
+ *      across output-mode switches — flipping the convertible Console between
+ *      Stages must never tear down or re-init the net.
+ *
+ * Distilled from `playground/tests/e2e/modes.spec.ts` ("cycling through all
+ * modes leaves probe alive"). The playground drove mode changes by reloading
+ * with a `nisps-mode-store` localStorage key; Manifold has no such store, so we
+ * drive the real dock mode selector instead — a genuine in-tab Stage switch,
+ * no reload, no probe teardown.
+ */
+import { test, expect } from '@playwright/test';
+import { loadProbe, getOutputs, getRouted, settleInputs, countChanged, allWithin } from './helpers';
+
+test.beforeEach(async ({ page }) => {
+  await loadProbe(page);
+});
+
+test.describe('spine — bounded, consistent routed outputs', () => {
+  test('setInputs yields bounded post-ML and routed vectors', async ({ page }) => {
+    await page.evaluate(() => window.__nisps!.setInputs(0.25, 0.75));
+    const outs = await getOutputs(page);
+    const routed = await getRouted(page);
+
+    expect(outs.length).toBeGreaterThan(0);
+    expect(routed.length).toBe(outs.length);
+    expect(allWithin(outs, 0, 1)).toBe(true);
+    expect(allWithin(routed, 0, 1)).toBe(true);
+  });
+
+  test('distinct inputs produce distinct outputs; the same input converges', async ({ page }) => {
+    // Settle at A, then read A again — smoothing has converged, so the two
+    // reads must be (near-)identical: the mapping is a stable function of the
+    // input once the pipeline state has caught up.
+    const a1 = await settleInputs(page, 0.3, 0.7);
+    const a2 = await settleInputs(page, 0.3, 0.7, 5);
+    expect(a1.length).toBe(a2.length);
+    expect(countChanged(a1, a2, 1e-3)).toBe(0);
+    expect(allWithin(a1, 0, 1)).toBe(true);
+
+    // A different input must move the mapping (spine actually propagates).
+    const b = await settleInputs(page, 0.8, 0.2);
+    expect(countChanged(a1, b, 1e-3)).toBeGreaterThan(0);
+    expect(allWithin(b, 0, 1)).toBe(true);
+  });
+
+  test('routed output tracks the input across a sweep, staying bounded', async ({ page }) => {
+    const points: Array<[number, number]> = [
+      [0.1, 0.1],
+      [0.5, 0.5],
+      [0.9, 0.9],
+    ];
+    for (const [x, y] of points) {
+      await settleInputs(page, x, y);
+      const routed = await getRouted(page);
+      expect(routed.length).toBeGreaterThan(0);
+      expect(allWithin(routed, 0, 1)).toBe(true);
+    }
+  });
+});
+
+test.describe('spine — probe stays alive across mode switches', () => {
+  /**
+   * Switch the dock's output-mode selector via the real UI. The "M" button
+   * (title `Mode: <label>`) opens the popover; each mode is a button named by
+   * its label. We assert the selector's title updates (deterministic wait, no
+   * sleep) so we know the switch landed before probing.
+   */
+  async function switchMode(page: import('@playwright/test').Page, label: string): Promise<void> {
+    await page.getByTitle(/^Mode:/).click();
+    await page.getByRole('button', { name: label, exact: true }).click();
+    await expect(page.getByTitle(`Mode: ${label}`)).toBeVisible();
+  }
+
+  test('cycling output modes keeps the probe ready and the spine live', async ({ page }) => {
+    // Network-free modes only (OSC / CV need a bridge; Editor is a serial stub).
+    const modes = ['Built-in Synth', 'MIDI', 'Particle System'];
+
+    for (const label of modes) {
+      await switchMode(page, label);
+
+      // The probe (and thus the WASM engine + spine) must survive the switch.
+      const ready = await page.evaluate(() => window.__nisps?.__ready === true);
+      expect(ready, `probe not ready after switching to ${label}`).toBe(true);
+
+      // And the spine must still propagate a fresh input to bounded outputs.
+      await page.evaluate(() => window.__nisps!.setInputs(0.35, 0.65));
+      const outs = await getOutputs(page);
+      expect(outs.length, `no outputs after ${label}`).toBeGreaterThan(0);
+      expect(allWithin(outs, 0, 1), `unbounded outputs after ${label}`).toBe(true);
+    }
+  });
+});