diff --git a/.factory/library/user-testing.md b/.factory/library/user-testing.md index 895b631..c0ef92c 100644 --- a/.factory/library/user-testing.md +++ b/.factory/library/user-testing.md @@ -72,3 +72,31 @@ These assertions require physical hardware and cannot be validated in headless b - Max concurrent: 5 validators (5 × 500MB = 2.5GB, well within budget) **Playwright tests**: Run serially (single browser, single server) + +--- + +## Flow Validator Guidance: Browser + +**Surface**: Browser-based web application served by Vite dev server + +**Isolation rules**: +- Each validator gets its own agent-browser session +- All validators share the same Vite dev server at http://localhost:5174 +- localStorage is per-browser-profile — since each agent-browser session is independent, there's no state leakage between validators +- The debug probe (`window.__nisps`) is the primary testing interface; use JavaScript evaluation to call probe methods +- The app state is ephemeral per page load — clearing localStorage or reloading resets state + +**Shared state concerns**: +- The Vite dev server is shared — if one validator causes a crash, it affects all validators +- No database or server-side state to worry about +- WASM binary is loaded fresh per page load + +**Resource constraints**: +- Max 5 concurrent agent-browser sessions +- Each session: ~300MB RAM + browser process +- Keep screenshots minimal to avoid disk pressure + +**Navigation patterns**: +- Load app with `?debug=1` to enable debug probe: `http://localhost:5174/?debug=1` +- Load without query params for production behavior: `http://localhost:5174` +- Use `localStorage.clear()` before tests requiring fresh state diff --git a/.factory/validation/skeleton-wasm/user-testing/flows/group-1.json b/.factory/validation/skeleton-wasm/user-testing/flows/group-1.json new file mode 100644 index 0000000..fbf5501 --- /dev/null +++ b/.factory/validation/skeleton-wasm/user-testing/flows/group-1.json @@ -0,0 +1,124 @@ +{ + "groupId": "group-1", + "testedAt": "2026-04-07T00:00:00Z", + "isolation": { + "appUrl": "http://localhost:5174/?debug=1", + "session": "9776ea6c39e9__g1", + "localStorageCleared": true + }, + "toolsUsed": ["agent-browser"], + "assertions": [ + { + "id": "VAL-PROJ-001", + "title": "Vite dev server starts with COOP/COEP headers", + "status": "pass", + "steps": [ + { + "action": "Navigate to http://localhost:5174/?debug=1", + "expected": "Page loads without errors", + "observed": "Page loaded successfully with title 'MEMLNaut NISPS'" + }, + { + "action": "Check console errors", + "expected": "No JavaScript errors", + "observed": "No console errors" + }, + { + "action": "Check SharedArrayBuffer availability via typeof SharedArrayBuffer !== 'undefined'", + "expected": "true (SharedArrayBuffer available)", + "observed": "true" + }, + { + "action": "Check window.crossOriginIsolated", + "expected": "true (cross-origin isolation active)", + "observed": "true" + }, + { + "action": "Fetch '/' and check response headers", + "expected": "COOP: same-origin, COEP: require-corp", + "observed": "COOP: same-origin, COEP: require-corp" + } + ], + "evidence": { + "screenshots": ["skeleton-wasm/group-1/VAL-PROJ-001-app-loaded.png"], + "consoleErrors": "none", + "network": "GET / -> 200, headers: {coop: 'same-origin', coep: 'require-corp'}" + }, + "issues": null + }, + { + "id": "VAL-PROJ-002", + "title": "SolidJS app renders without errors", + "status": "pass", + "steps": [ + { + "action": "Clear localStorage and set nisps-help-seen, reload with ?debug=1", + "expected": "Page loads fresh without help modal", + "observed": "Page loaded cleanly, no help modal" + }, + { + "action": "Wait for page to fully load and WASM to initialize", + "expected": "window.__nisps becomes available", + "observed": "window.__nisps available after ~3s" + }, + { + "action": "Check DOM for heading and status", + "expected": "Heading 'MEMLNaut NISPS' and status 'Ready — 126 outputs loaded'", + "observed": "heading 'MEMLNaut NISPS' [ref=e1], paragraph 'Ready — 126 outputs loaded'" + }, + { + "action": "Check console errors", + "expected": "No JavaScript errors", + "observed": "No console errors" + }, + { + "action": "Take screenshot", + "expected": "App renders with immersive layout visible", + "observed": "Screenshot shows app rendered correctly with heading and status text" + } + ], + "evidence": { + "screenshots": ["skeleton-wasm/group-1/VAL-PROJ-002-app-rendered.png"], + "consoleErrors": "none" + }, + "issues": null + }, + { + "id": "VAL-PROJ-003", + "title": "WASM ML engine loads", + "status": "pass", + "steps": [ + { + "action": "Wait for WASM initialization (window.__nisps !== undefined)", + "expected": "Debug probe available", + "observed": "window.__nisps defined and available" + }, + { + "action": "Call window.__nisps.getOutputs() and check length, bounds", + "expected": "Float32Array length 126, all values in [0,1]", + "observed": "length=126, min=2.066e-13, max=1, allBounded=true, sample=[0.300, 0.835, 0.688, 8.96e-8, 0.9999]" + }, + { + "action": "Check console errors", + "expected": "No JavaScript errors", + "observed": "No console errors" + } + ], + "evidence": { + "screenshots": ["skeleton-wasm/group-1/VAL-PROJ-003-wasm-loaded.png"], + "consoleErrors": "none", + "probeOutput": { + "length": 126, + "min": "2.066e-13", + "max": 1, + "allBounded": true, + "sample": [0.30002, 0.83461, 0.68781, 8.96e-8, 0.99994] + } + }, + "issues": null + } + ], + "frictions": [], + "blockers": [], + "summary": "Tested 3 assertions: all 3 passed. VAL-PROJ-001: Vite dev server correctly serves COOP/COEP headers with SharedArrayBuffer available and crossOriginIsolated=true. VAL-PROJ-002: SolidJS app renders without errors showing 'MEMLNaut NISPS' heading and 'Ready — 126 outputs loaded' status. VAL-PROJ-003: WASM ML engine loads successfully, debug probe returns 126 bounded outputs in [0,1]." +} diff --git a/.factory/validation/skeleton-wasm/user-testing/flows/group-2.json b/.factory/validation/skeleton-wasm/user-testing/flows/group-2.json new file mode 100644 index 0000000..a4491a4 --- /dev/null +++ b/.factory/validation/skeleton-wasm/user-testing/flows/group-2.json @@ -0,0 +1,114 @@ +{ + "groupId": "group-2", + "testedAt": "2026-04-07T00:00:00Z", + "isolation": { + "appUrl": "http://localhost:5174", + "session": "9776ea6c39e9__g2", + "localStorageClearedBeforeTests": true + }, + "toolsUsed": ["agent-browser"], + "assertions": [ + { + "id": "VAL-PROJ-004", + "title": "Signal bus creates topics and emits/receives", + "status": "pass", + "steps": [ + { + "action": "Navigate to http://localhost:5174/?debug=1, clear localStorage, reload", + "expected": "Page loads without errors, __nispsBus available", + "observed": "Page loaded successfully, window.__nispsBus is an object with createTopic, getTopic, match methods" + }, + { + "action": "Create topics test.a and test.b, subscribe to each, emit values 42 and 99", + "expected": "Subscribers receive emitted values", + "observed": "topicResults = [{sub:'a',data:42},{sub:'b',data:99},{sub:'a',data:100},{sub:'b',data:200}] — subscribers received all emitted values" + }, + { + "action": "Subscribe to wildcard pattern 'test.*' via bus.match(), then emit 100 and 200 to test.a and test.b", + "expected": "Wildcard subscriber captures emits after match() subscription", + "observed": "wildcardResults = [100, 200, 1, 1] — captured emits from test.a, test.b, and test.same matching test.*" + }, + { + "action": "Subscribe to test.same, emit 1 twice", + "expected": "sameDataCount is 2 (equals:false ensures every emit triggers subscriber)", + "observed": "sameDataCount = 2 — both emits triggered the subscriber even with identical data" + } + ], + "evidence": { + "screenshots": [ + "skeleton-wasm/group-2/VAL-PROJ-004-initial-load.png", + "skeleton-wasm/group-2/VAL-PROJ-004-after-test.png" + ], + "consoleErrors": "none", + "javascriptOutput": "{\"topicResults\":[{\"sub\":\"a\",\"data\":42},{\"sub\":\"b\",\"data\":99},{\"sub\":\"a\",\"data\":100},{\"sub\":\"b\",\"data\":200}],\"wildcardResults\":[100,200,1,1],\"sameDataCount\":2}" + }, + "issues": null + }, + { + "id": "VAL-PROJ-005", + "title": "Debug probe exposed with ?debug=1", + "status": "pass", + "steps": [ + { + "action": "Navigate to http://localhost:5174/?debug=1 (already loaded from VAL-PROJ-004)", + "expected": "window.__nisps is an object", + "observed": "window.__nisps exists and is an object (not null)" + }, + { + "action": "Check typeof for all 15 required methods", + "expected": "All methods are functions", + "observed": "All 15 methods present: getOutputs, getLoss, getWeights, getExampleCount, setInputs, thumbsUp, thumbsDown, train, trainAsync, randomise, clearExamples, saveState, evalLoss, inferBatch, getLayerStats — allPresent: true" + } + ], + "evidence": { + "screenshots": [ + "skeleton-wasm/group-2/VAL-PROJ-005-debug-probe-present.png" + ], + "consoleErrors": "none", + "javascriptOutput": "{\"exists\":true,\"methods\":[{\"name\":\"getOutputs\",\"exists\":true},{\"name\":\"getLoss\",\"exists\":true},{\"name\":\"getWeights\",\"exists\":true},{\"name\":\"getExampleCount\",\"exists\":true},{\"name\":\"setInputs\",\"exists\":true},{\"name\":\"thumbsUp\",\"exists\":true},{\"name\":\"thumbsDown\",\"exists\":true},{\"name\":\"train\",\"exists\":true},{\"name\":\"trainAsync\",\"exists\":true},{\"name\":\"randomise\",\"exists\":true},{\"name\":\"clearExamples\",\"exists\":true},{\"name\":\"saveState\",\"exists\":true},{\"name\":\"evalLoss\",\"exists\":true},{\"name\":\"inferBatch\",\"exists\":true},{\"name\":\"getLayerStats\",\"exists\":true}],\"allPresent\":true}" + }, + "issues": null + }, + { + "id": "VAL-PROJ-006", + "title": "Debug probe NOT exposed without ?debug=1", + "status": "pass", + "steps": [ + { + "action": "Navigate to http://localhost:5174/ (without ?debug=1)", + "expected": "Page loads successfully", + "observed": "Page loaded without errors" + }, + { + "action": "Check typeof window.__nisps", + "expected": "typeof window.__nisps === 'undefined'", + "observed": "nispsType: 'undefined', isUndefined: true" + } + ], + "evidence": { + "screenshots": [ + "skeleton-wasm/group-2/VAL-PROJ-006-no-debug-probe.png" + ], + "consoleErrors": "none", + "javascriptOutput": "{\"nispsType\":\"undefined\",\"isUndefined\":true}" + }, + "issues": null + } + ], + "frictions": [ + { + "description": "The validation contract task description referenced bus.topic() API but the actual implementation uses bus.createTopic()/getTopic(). The match() method takes (pattern, callback) directly rather than returning a subscribable object. Required reading the source code to discover the correct API.", + "resolved": true, + "resolution": "Read src/bus/signal-bus.ts to identify the actual API: createTopic(name), getTopic(name), match(pattern, fn)", + "affectedAssertions": ["VAL-PROJ-004"] + }, + { + "description": "Arrow function callbacks in eval() caused 'ms.fn is not a function' error on first attempt after page reload with cleared localStorage. Switching to function declarations resolved it, suggesting a possible serialization issue with arrow functions in the browser eval context.", + "resolved": true, + "resolution": "Used function declarations instead of arrow functions in eval() JavaScript code", + "affectedAssertions": ["VAL-PROJ-004"] + } + ], + "blockers": [], + "summary": "Tested 3 assertions: all 3 passed. VAL-PROJ-004 (signal bus): topics emit/receive correctly, wildcard matching works, equals:false ensures duplicate data triggers subscribers. VAL-PROJ-005 (debug probe with ?debug=1): all 15 required methods present. VAL-PROJ-006 (no probe without ?debug=1): window.__nisps is undefined." +} diff --git a/.factory/validation/skeleton-wasm/user-testing/synthesis.json b/.factory/validation/skeleton-wasm/user-testing/synthesis.json new file mode 100644 index 0000000..8b45339 --- /dev/null +++ b/.factory/validation/skeleton-wasm/user-testing/synthesis.json @@ -0,0 +1,29 @@ +{ + "milestone": "skeleton-wasm", + "round": 1, + "status": "pass", + "assertionsSummary": { + "total": 6, + "passed": 6, + "failed": 0, + "blocked": 0 + }, + "passedAssertions": [ + "VAL-PROJ-001", + "VAL-PROJ-002", + "VAL-PROJ-003", + "VAL-PROJ-004", + "VAL-PROJ-005", + "VAL-PROJ-006" + ], + "failedAssertions": [], + "blockedAssertions": [], + "appliedUpdates": [ + { + "target": "user-testing.md", + "description": "Added 'Flow Validator Guidance: Browser' section with isolation rules, shared state concerns, resource constraints, and navigation patterns for agent-browser flow validators.", + "source": "setup" + } + ], + "previousRound": null +}