diff --git a/manifold/tests/e2e/helpers.ts b/manifold/tests/e2e/helpers.ts new file mode 100644 index 0000000..1227415 --- /dev/null +++ b/manifold/tests/e2e/helpers.ts @@ -0,0 +1,101 @@ +/** + * Playwright helpers for the Manifold app. + * + * Ported from `playground/tests/e2e/helpers.ts` and adapted to Manifold's + * probe surface. Differences from the playground probe: + * - Manifold's probe has NO `__init()` — it is installed by `App.tsx` only + * AFTER the engine (and its WASM) are live, and `__ready` is a live getter + * over the spine state. So "ready" == `window.__nisps.__ready === true`. + * - The probe is gated behind `?debug=1` (see `src/debug/probe.ts`), not a + * route. We always navigate to `/?debug=1`. + * - `routedOutputs()` (plural) is the post-output-pipeline vector. + */ +import type { Page } from '@playwright/test'; +import type { DebugProbe } from '../../src/debug/probe'; + +declare global { + interface Window { + __nisps?: DebugProbe; + } +} + +const READY_TIMEOUT = 20_000; + +/** + * Navigate to the Manifold app with the `?debug=1` probe installed and + * localStorage cleared (fresh default weights, not a prior test's state), then + * wait until `window.__nisps` reports the WASM engine is ready. + * + * `extraQuery` is appended after `debug=1` (leading `&` optional). + */ +export async function loadProbe(page: Page, extraQuery = ''): Promise { + // Clear persisted ML/settings state before the SPA boots so initial + // inference uses default weights. + await page.addInitScript(() => { + try { + localStorage.clear(); + sessionStorage.clear(); + } catch { + /* private mode etc — ignore */ + } + }); + + let q = extraQuery.trim(); + if (q && !q.startsWith('&')) q = '&' + q; + await page.goto(`/?debug=1${q}`); + + await page.waitForFunction( + () => { + const n = window.__nisps; + return !!n && n.__ready === true && n.getOutputs().length > 0; + }, + undefined, + { timeout: READY_TIMEOUT }, + ); +} + +/** Read the live post-ML output vector as a JSON-safe number[]. */ +export async function getOutputs(page: Page): Promise { + return page.evaluate(() => Array.from(window.__nisps!.getOutputs())); +} + +/** Read the live routed (post output-pipeline) vector as a JSON-safe number[]. */ +export async function getRouted(page: Page): Promise { + return page.evaluate(() => Array.from(window.__nisps!.routedOutputs())); +} + +/** Push the same raw XY input `n` times so the input/output EMA smoothing + * settles, then return the converged post-ML outputs. */ +export async function settleInputs(page: Page, x: number, y: number, n = 40): Promise { + return page.evaluate( + ([px, py, count]) => { + const probe = window.__nisps!; + for (let i = 0; i < count; i++) probe.setInputs(px, py); + return Array.from(probe.getOutputs()); + }, + [x, y, n] as const, + ); +} + +/** + * How many values differ by more than `eps` between two snapshots. Length + * mismatch counts as the absolute size difference. + */ +export function countChanged(a: number[], b: number[], eps = 1e-3): number { + if (a.length !== b.length) return Math.abs(a.length - b.length); + let n = 0; + for (let i = 0; i < a.length; ++i) { + if (Math.abs(a[i]! - b[i]!) > eps) ++n; + } + return n; +} + +/** True iff every value is within [lo, hi] (with a tiny float tolerance). */ +export function allWithin(xs: number[], lo = 0, hi = 1, tol = 1e-6): boolean { + for (const v of xs) { + if (!(v >= lo - tol && v <= hi + tol)) return false; + } + return true; +} + +export type Probe = DebugProbe; diff --git a/manifold/tests/e2e/probe-api.spec.ts b/manifold/tests/e2e/probe-api.spec.ts new file mode 100644 index 0000000..07a2d60 --- /dev/null +++ b/manifold/tests/e2e/probe-api.spec.ts @@ -0,0 +1,178 @@ +/** + * Debug-probe API contract — `window.__nisps` (gated behind `?debug=1`). + * + * Ported from `playground/tests/e2e/ml-engine.spec.ts`. This is the ENGINE + * contract, not playground UI, so it survives the playground's retirement: + * every probe accessor must return the documented shape and never throw. + * + * Adaptations vs. the playground original: + * - Manifold's WASM net is `MLP<32,10,14,18,126>` (playground was `<2,...>`), + * so `getWeights()` has 3148 elements, not 2848 (derivation below). + * - No `probe.__init()` / no `mlStore.iml` poke-through: Manifold's probe + * exposes `addExample()` and `routedOutputs()` directly, so the training + * tests drive the real public surface instead of an escape hatch. + * - The playground's `test.skip(!probeReady)` guard is gone — on Manifold the + * probe is only installed once WASM is live, so a not-ready probe is a + * genuine failure, not a pending-stream skip. + */ +import { test, expect } from '@playwright/test'; +import { loadProbe, getOutputs, countChanged, allWithin } from './helpers'; + +// Fixed by the WASM build (`nisps/wasm/bindings.cpp`: MLP<32,10,14,18,126>). +const N_OUTPUTS = 126; +// weight_count = 32*10 + 10*14 + 14*18 + 18*126 (weights) +// + 10 + 14 + 18 + 126 (biases) +// = 320 + 140 + 252 + 2268 + 168 = 3148 +const WEIGHT_COUNT = 3148; +// DefaultMLP::kNumLayers (4) * 4 stats per layer. +const LAYER_STATS = 16; + +const EXAMPLE_LOW = { input: [0.1, 0.9], output: new Array(N_OUTPUTS).fill(0.1) }; +const EXAMPLE_HIGH = { input: [0.9, 0.1], output: new Array(N_OUTPUTS).fill(0.9) }; + +test.beforeEach(async ({ page }) => { + await loadProbe(page); +}); + +test.describe('ML engine — debug probe contract', () => { + test('probe is installed and reports ready', async ({ page }) => { + const kind = await page.evaluate(() => typeof window.__nisps); + expect(kind).toBe('object'); + const ready = await page.evaluate(() => window.__nisps!.__ready); + expect(ready).toBe(true); + }); + + test('initial outputs are bounded in [0, 1]', async ({ page }) => { + const outs = await getOutputs(page); + expect(outs).toHaveLength(N_OUTPUTS); + expect(allWithin(outs, 0, 1)).toBe(true); + }); + + test('initial state is 0 examples and no loss', async ({ page }) => { + const count = await page.evaluate(() => window.__nisps!.getExampleCount()); + expect(count).toBe(0); + const loss = await page.evaluate(() => window.__nisps!.getLoss()); + expect(loss).toBeNull(); + }); + + test('randomise changes outputs', async ({ page }) => { + await page.evaluate(() => window.__nisps!.setInputs(0.3, 0.7)); + const before = await getOutputs(page); + await page.evaluate(() => window.__nisps!.randomise()); + await page.evaluate(() => window.__nisps!.setInputs(0.3, 0.7)); + const after = await getOutputs(page); + expect(countChanged(before, after, 1e-3)).toBeGreaterThan(0); + }); + + test('setInputs runs inference and yields bounded outputs', async ({ page }) => { + await page.evaluate(() => window.__nisps!.setInputs(0.25, 0.75)); + const outs = await getOutputs(page); + expect(outs).toHaveLength(N_OUTPUTS); + expect(allWithin(outs, 0, 1)).toBe(true); + }); + + test('thumbsUp returns a finite FeedbackAction and keeps the count sane', async ({ page }) => { + await page.evaluate(() => window.__nisps!.setInputs(0.4, 0.6)); + const action = await page.evaluate(() => window.__nisps!.thumbsUp()); + expect(typeof action).toBe('number'); + expect(Number.isFinite(action)).toBe(true); + const count = await page.evaluate(() => window.__nisps!.getExampleCount()); + expect(Number.isInteger(count)).toBe(true); + expect(count).toBeGreaterThanOrEqual(0); + }); + + test('thumbsDown moves weights and changes outputs', async ({ page }) => { + await page.evaluate(() => window.__nisps!.setInputs(0.3, 0.7)); + const before = await getOutputs(page); + await page.evaluate(() => window.__nisps!.thumbsDown()); + await page.evaluate(() => window.__nisps!.setInputs(0.3, 0.7)); + const after = await getOutputs(page); + expect(countChanged(before, after, 1e-4)).toBeGreaterThan(0); + }); + + test('addExample reports success and bumps the example count', async ({ page }) => { + const ok = await page.evaluate( + ([ex]) => window.__nisps!.addExample(ex.input, ex.output), + [EXAMPLE_LOW], + ); + expect(typeof ok).toBe('boolean'); + expect(ok).toBe(true); + const count = await page.evaluate(() => window.__nisps!.getExampleCount()); + expect(count).toBe(1); + }); + + test('train() with two contrasting examples does not increase loss', async ({ page }) => { + await page.evaluate( + ([low, high]) => { + window.__nisps!.addExample(low.input, low.output); + window.__nisps!.addExample(high.input, high.output); + }, + [EXAMPLE_LOW, EXAMPLE_HIGH], + ); + + const loss1 = await page.evaluate(() => window.__nisps!.train()); + expect(typeof loss1).toBe('number'); + expect(Number.isFinite(loss1)).toBe(true); + expect(loss1).toBeGreaterThanOrEqual(0); + + const loss2 = await page.evaluate(() => window.__nisps!.train()); + expect(loss2).toBeLessThanOrEqual(loss1 + 1e-6); + }); + + test('async training resolves to a finite non-negative loss', async ({ page }) => { + await page.evaluate( + ([low, high]) => { + window.__nisps!.addExample(low.input, low.output); + window.__nisps!.addExample(high.input, high.output); + }, + [EXAMPLE_LOW, EXAMPLE_HIGH], + ); + const loss = await page.evaluate(() => window.__nisps!.trainAsync()); + expect(typeof loss).toBe('number'); + expect(Number.isFinite(loss)).toBe(true); + expect(loss).toBeGreaterThanOrEqual(0); + }); + + test('clearExamples resets the dataset count to 0', async ({ page }) => { + await page.evaluate( + ([ex]) => window.__nisps!.addExample(ex.input, ex.output), + [EXAMPLE_LOW], + ); + expect(await page.evaluate(() => window.__nisps!.getExampleCount())).toBe(1); + await page.evaluate(() => window.__nisps!.clearExamples()); + expect(await page.evaluate(() => window.__nisps!.getExampleCount())).toBe(0); + }); + + test('evalLoss returns a non-negative number or null', async ({ page }) => { + const v = await page.evaluate(() => window.__nisps!.evalLoss()); + if (v !== null) { + expect(Number.isFinite(v)).toBe(true); + expect(v).toBeGreaterThanOrEqual(0); + } + }); + + test('inferBatch returns N * outputSize bounded floats', async ({ page }) => { + const points: ReadonlyArray = [ + [0.0, 0.0], + [0.5, 0.5], + [1.0, 1.0], + ]; + const flat = await page.evaluate( + (pts) => Array.from(window.__nisps!.inferBatch(pts as [number, number][])), + points, + ); + expect(flat).toHaveLength(points.length * N_OUTPUTS); + expect(allWithin(flat, 0, 1)).toBe(true); + }); + + test('getLayerStats returns 4 floats per layer, all finite', async ({ page }) => { + const stats = await page.evaluate(() => Array.from(window.__nisps!.getLayerStats())); + expect(stats).toHaveLength(LAYER_STATS); + for (const v of stats) expect(Number.isFinite(v)).toBe(true); + }); + + test('getWeights returns the full weight vector', async ({ page }) => { + const len = await page.evaluate(() => window.__nisps!.getWeights().length); + expect(len).toBe(WEIGHT_COUNT); + }); +}); diff --git a/manifold/tests/e2e/spine.spec.ts b/manifold/tests/e2e/spine.spec.ts new file mode 100644 index 0000000..5b08108 --- /dev/null +++ b/manifold/tests/e2e/spine.spec.ts @@ -0,0 +1,104 @@ +/** + * Spine invariant. + * + * The engine spine (`src/engine/spine.ts`) is: setInputs → processed → ml → + * routed, derived EAGERLY + SYNCHRONOUSLY off React's render cycle. This spec + * asserts the two properties that make the spine the trustworthy core of the + * app: + * + * 1. Pushing inputs through the spine yields BOUNDED, CONSISTENT routed + * outputs — distinct inputs map to distinct outputs, the same input + * (once the input/output EMA smoothing settles) converges to a stable + * vector, and both the post-ML and routed vectors stay in [0, 1]. + * 2. The probe (hence the WASM engine and the reactive spine) stays ALIVE + * across output-mode switches — flipping the convertible Console between + * Stages must never tear down or re-init the net. + * + * Distilled from `playground/tests/e2e/modes.spec.ts` ("cycling through all + * modes leaves probe alive"). The playground drove mode changes by reloading + * with a `nisps-mode-store` localStorage key; Manifold has no such store, so we + * drive the real dock mode selector instead — a genuine in-tab Stage switch, + * no reload, no probe teardown. + */ +import { test, expect } from '@playwright/test'; +import { loadProbe, getOutputs, getRouted, settleInputs, countChanged, allWithin } from './helpers'; + +test.beforeEach(async ({ page }) => { + await loadProbe(page); +}); + +test.describe('spine — bounded, consistent routed outputs', () => { + test('setInputs yields bounded post-ML and routed vectors', async ({ page }) => { + await page.evaluate(() => window.__nisps!.setInputs(0.25, 0.75)); + const outs = await getOutputs(page); + const routed = await getRouted(page); + + expect(outs.length).toBeGreaterThan(0); + expect(routed.length).toBe(outs.length); + expect(allWithin(outs, 0, 1)).toBe(true); + expect(allWithin(routed, 0, 1)).toBe(true); + }); + + test('distinct inputs produce distinct outputs; the same input converges', async ({ page }) => { + // Settle at A, then read A again — smoothing has converged, so the two + // reads must be (near-)identical: the mapping is a stable function of the + // input once the pipeline state has caught up. + const a1 = await settleInputs(page, 0.3, 0.7); + const a2 = await settleInputs(page, 0.3, 0.7, 5); + expect(a1.length).toBe(a2.length); + expect(countChanged(a1, a2, 1e-3)).toBe(0); + expect(allWithin(a1, 0, 1)).toBe(true); + + // A different input must move the mapping (spine actually propagates). + const b = await settleInputs(page, 0.8, 0.2); + expect(countChanged(a1, b, 1e-3)).toBeGreaterThan(0); + expect(allWithin(b, 0, 1)).toBe(true); + }); + + test('routed output tracks the input across a sweep, staying bounded', async ({ page }) => { + const points: Array<[number, number]> = [ + [0.1, 0.1], + [0.5, 0.5], + [0.9, 0.9], + ]; + for (const [x, y] of points) { + await settleInputs(page, x, y); + const routed = await getRouted(page); + expect(routed.length).toBeGreaterThan(0); + expect(allWithin(routed, 0, 1)).toBe(true); + } + }); +}); + +test.describe('spine — probe stays alive across mode switches', () => { + /** + * Switch the dock's output-mode selector via the real UI. The "M" button + * (title `Mode: