diff --git a/src/App.tsx b/src/App.tsx index 40df8ff..8d9aadc 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -1,42 +1,32 @@ import { onMount, onCleanup, createSignal } from 'solid-js'; -import { WasmIML } from './core/iml'; -import { createSignalBus, type SignalBus } from './bus/signal-bus'; +import { createMLStore, type MLStore } from './stores/ml-store'; import { exposeDebugProbe } from './probe/debug-probe'; - -const N_INPUTS = 2; -const HIDDEN_LAYERS = [32, 48, 64]; -const N_OUTPUTS = 126; - -// Singleton signal bus — shared across the entire app -const bus: SignalBus = createSignalBus(); - -// Expose bus globally for e2e tests and debug access -(window as any).__nispsBus = bus; - -export { bus }; +import bus from './bus'; /** * Root application component for the NISPS immersive app. - * Initializes WASM IML, runs initial inference, and exposes debug probe. + * Initializes ML store (dual WasmIML instances), runs initial inference, + * and exposes debug probe. */ export default function App() { const [status, setStatus] = createSignal('Loading WASM...'); const [ready, setReady] = createSignal(false); - // IML instance lives outside reactive state (opaque handle) - let iml: WasmIML | null = null; + let mlStore: MLStore | null = null; onMount(async () => { try { - iml = await WasmIML.create(N_INPUTS, N_OUTPUTS, HIDDEN_LAYERS); + const store = await createMLStore(bus); - // Run initial inference at center position - iml.setInput(0, 0.5); - iml.setInput(1, 0.5); - iml.process(); + // Expose store globally for e2e tests + (window as any).__nispsStore = store; - const outputs = iml.getOutputs(); - const allBounded = outputs.every((v: number) => v >= 0 && v <= 1); + // Expose bus topics for testing dual IML and mode switching + bus.createTopic('ml.imlJoy').emit(store.getImlJoy() as any); + bus.createTopic('ml.imlHand').emit(store.getImlHand() as any); + + const outputs = store.outputs(); + const allBounded = Array.from(outputs).every(v => v >= 0 && v <= 1); if (!allBounded) { setStatus('ERROR: outputs out of bounds'); console.error('Unbounded outputs:', outputs); @@ -47,7 +37,9 @@ export default function App() { setStatus(`Ready — ${outputs.length} outputs loaded`); // Expose debug probe if ?debug=1 - exposeDebugProbe(iml); + exposeDebugProbe(store); + + mlStore = store; } catch (err) { setStatus(`Error: ${err}`); console.error('WASM init failed:', err); @@ -55,7 +47,7 @@ export default function App() { }); onCleanup(() => { - iml?.destroy(); + mlStore?.dispose(); }); return ( diff --git a/src/bus/index.ts b/src/bus/index.ts new file mode 100644 index 0000000..39ff573 --- /dev/null +++ b/src/bus/index.ts @@ -0,0 +1,16 @@ +/** + * Singleton signal bus — shared across the entire app. + * + * Created once at module load time and also exposed at window.__nispsBus + * for e2e tests and debug access. + */ + +import { createSignalBus, type SignalBus } from './signal-bus'; + +const bus: SignalBus = createSignalBus(); + +// Expose bus globally for e2e tests and debug access +(window as any).__nispsBus = bus; + +export default bus; +export type { SignalBus }; diff --git a/src/probe/debug-probe.ts b/src/probe/debug-probe.ts index fa3748b..0421462 100644 --- a/src/probe/debug-probe.ts +++ b/src/probe/debug-probe.ts @@ -3,9 +3,10 @@ * Used by Playwright e2e tests. Zero footprint in production. * * All methods are synchronous (bypass SolidJS batching with batch()/untrack()). + * Updated to work with MLStore interface. */ -import type { WasmIML } from '../core/iml'; +import type { MLStore } from '../stores/ml-store'; export interface DebugProbe { getOutputs: () => number[]; @@ -39,59 +40,41 @@ export function isDebugMode(): boolean { } /** - * Create the debug probe object, wired to a WasmIML instance. + * Create the debug probe object, wired to an MLStore instance. */ -export function createDebugProbe(iml: WasmIML): DebugProbe { +export function createDebugProbe(store: MLStore): DebugProbe { return { - getOutputs: () => iml.getOutputs(), + getOutputs: () => Array.from(store.outputs()), - getLoss: () => iml.lastLoss, + getLoss: () => store.getLoss(), - getWeights: () => iml._getFlatWeights(), + getWeights: () => store.getWeights(), - getExampleCount: () => iml.exampleCount, + getExampleCount: () => store.getExampleCount(), setInputs: (x: number, y: number) => { - iml.setInput(0, Math.max(0, Math.min(1, x))); - iml.setInput(1, Math.max(0, Math.min(1, y))); - iml.process(); + store.setInputs(x, y); }, - thumbsUp: () => { - iml.addExample(iml.inputState.slice(), iml.outputState.slice()); - return iml.trainAsync(); - }, + thumbsUp: () => store.thumbsUp(), - thumbsDown: () => { - const spread = 0.6; - const speed = 0.15 * (1 - spread) + 0.05 * spread; - iml.moveWeights(speed, spread); - }, + thumbsDown: () => store.thumbsDown(), - train: () => iml.train(), + train: () => store.train(), - trainAsync: () => iml.trainAsync(), + trainAsync: () => store.trainAsync(), - randomise: () => iml.randomiseWeights(0.6), + randomise: () => store.randomise(), - clearExamples: () => iml.clearDataset(), + clearExamples: () => store.clearExamples(), - saveState: () => { - const state = { - weights: iml._getFlatWeights(), - inputState: iml.inputState.slice(), - outputState: iml.outputState.slice(), - exampleCount: iml.exampleCount, - lossHistory: iml.lossHistory.slice(), - }; - localStorage.setItem('nisps-a-immersive', JSON.stringify(state)); - }, + saveState: () => store.saveState(), - evalLoss: () => iml.evalLoss(), + evalLoss: () => store.evalLoss(), - inferBatch: (points: number[][]) => iml.inferBatch(points), + inferBatch: (points: number[][]) => store.inferBatch(points), - getLayerStats: () => iml.getLayerStats(), + getLayerStats: () => store.getLayerStats(), }; } @@ -99,10 +82,10 @@ export function createDebugProbe(iml: WasmIML): DebugProbe { * Expose window.__nisps if ?debug=1 is present. * Returns the probe object (or undefined if not in debug mode). */ -export function exposeDebugProbe(iml: WasmIML): DebugProbe | undefined { +export function exposeDebugProbe(store: MLStore): DebugProbe | undefined { if (!isDebugMode()) return undefined; - const probe = createDebugProbe(iml); + const probe = createDebugProbe(store); (window as any).__nisps = probe; console.log('[NISPS] Debug probe exposed at window.__nisps'); return probe; diff --git a/src/stores/ml-store.ts b/src/stores/ml-store.ts new file mode 100644 index 0000000..9ad84c3 --- /dev/null +++ b/src/stores/ml-store.ts @@ -0,0 +1,310 @@ +/** + * ML Store — SolidJS reactive wrapper around dual WasmIML instances. + * + * Architecture (mirrors old a-app.js): + * imlJoy: 2 inputs (joystick), dynamic outputs — warm-started on resize + * imlHand: 14 inputs (hand tracking), dynamic outputs — fresh on resize + * iml: pointer to whichever is active (imlJoy by default) + * + * Outputs are a Float32Array stored in a signal (NOT a store — proxies break typed arrays). + * Mode changes trigger MLP recreation with warm-start weight transfer. + */ + +import { createSignal, onCleanup } from 'solid-js'; +import { createStore } from 'solid-js/store'; +import { WasmIML } from '../core/iml'; +import type { SignalBus } from '../bus/signal-bus'; + +// ─── Constants ──────────────────────────────────────────────────────── + +export const N_JOY_INPUTS = 2; +export const N_HAND_INPUTS = 14; +export const N_VISUAL_OUTPUTS = 20; +export const N_SYNTH_OUTPUTS = 126; +export const N_MIDI_CC_DEFAULT = 8; +export const N_AUDIO_CANVAS_OUTPUTS = 36; +export const HIDDEN_LAYERS_JOY = [32, 48, 64]; +export const HIDDEN_LAYERS_HAND = [48, 48, 64]; + +export type OutputMode = 'visual' | 'synth' | 'midi-cc' | 'audio-canvas'; + +/** Map from output mode to number of MLP outputs */ +export function outputCountForMode(mode: OutputMode, midiCCCount = N_MIDI_CC_DEFAULT): number { + switch (mode) { + case 'visual': return N_VISUAL_OUTPUTS; + case 'synth': return N_SYNTH_OUTPUTS; + case 'midi-cc': return midiCCCount; + case 'audio-canvas': return N_AUDIO_CANVAS_OUTPUTS; + default: return N_SYNTH_OUTPUTS; + } +} + +// ─── ML Store Interface ────────────────────────────────────────────── + +export interface MLState { + /** Current output mode */ + outputMode: OutputMode; + /** Dynamic output count for MIDI CC mode */ + midiCCCount: number; + /** Spread level [0,1] — controls weight initialization & RL noise */ + spreadLevel: number; + /** Whether IML instances are initialized */ + initialized: boolean; +} + +export interface MLStore { + // Reactive state + readonly state: MLState; + readonly outputs: () => Float32Array; + readonly outputCount: () => number; + + // IML access (opaque handles, NOT reactive) + getActiveIml(): WasmIML | null; + getImlJoy(): WasmIML | null; + getImlHand(): WasmIML | null; + + // Actions + setOutputMode(mode: OutputMode): Promise; + setMidiCCCount(count: number): Promise; + setSpreadLevel(level: number): void; + + // Direct IML operations (synchronous) + setInputs(x: number, y: number): void; + randomise(): void; + clearExamples(): void; + train(): number | null; + trainAsync(): Promise; + thumbsUp(): Promise; + thumbsDown(speed?: number): void; + evalLoss(): number | null; + inferBatch(points: number[][]): number[][]; + getLayerStats(): Array<{ meanAbs: number; maxAbs: number; deadFrac: number; satFrac: number }>; + getWeights(): number[]; + getExampleCount(): number; + getLoss(): number | null; + saveState(): void; + + // Lifecycle + dispose(): void; +} + +// ─── Factory ───────────────────────────────────────────────────────── + +export async function createMLStore(bus: SignalBus): Promise { + const [state, setState] = createStore({ + outputMode: 'synth', // default mode — old app starts at N_SYNTH_OUTPUTS + midiCCCount: N_MIDI_CC_DEFAULT, + spreadLevel: 0.6, + initialized: false, + }); + + // Outputs as signal — Float32Array MUST NOT go in store (proxy overhead) + const initialCount = outputCountForMode('synth'); + const [outputs, setOutputs] = createSignal(new Float32Array(initialCount)); + const [outputCount, setOutputCount] = createSignal(initialCount); + + // IML instances — module-scope opaque handles (not reactive) + let imlJoy: WasmIML | null = null; + let imlHand: WasmIML | null = null; + let activeIml: WasmIML | null = null; + + // Initialize dual IML instances + const currentCount = outputCount(); + + imlJoy = await WasmIML.create(N_JOY_INPUTS, currentCount, HIDDEN_LAYERS_JOY); + imlJoy.setLogger((msg: string) => console.log('[NISPS:joy]', msg)); + + imlHand = await WasmIML.create(N_HAND_INPUTS, currentCount, HIDDEN_LAYERS_HAND); + imlHand.setLogger((msg: string) => console.log('[NISPS:hand]', msg)); + + activeIml = imlJoy; // default to joystick + + // Run initial inference at center position + activeIml.setInput(0, 0.5); + activeIml.setInput(1, 0.5); + activeIml.process(); + setOutputs(new Float32Array(activeIml.getOutputs())); + + setState('initialized', true); + + // Create bus topics for external consumers + const outputsTopic = bus.createTopic('ml.outputs'); + const outputCountTopic = bus.createTopic('ml.outputCount'); + const modeTopic = bus.createTopic('mode.output'); + + // Publish initial values + outputsTopic.emit(outputs()); + outputCountTopic.emit(outputCount()); + + // ─── Helper: run inference and update signal ─── + + function runInference(): void { + if (!activeIml) return; + activeIml.process(); + const newOutputs = new Float32Array(activeIml.getOutputs()); + setOutputs(newOutputs); + outputsTopic.emit(newOutputs); + } + + // ─── Helper: resize MLP (mode switch) ─── + + async function resizeMLP(newCount: number): Promise { + if (newCount === outputCount()) return; + + // Extract joystick weights for warm-start + const joySnapshot = imlJoy ? imlJoy.extractWeights() : null; + + // Destroy old instances + imlJoy?.destroy(); + imlHand?.destroy(); + + // Joystick IML: warm-start if possible + if (joySnapshot) { + imlJoy = await WasmIML.createWithWarmStart( + joySnapshot, newCount, 1000, 1.0, 0.00001 + ); + } else { + imlJoy = await WasmIML.create(N_JOY_INPUTS, newCount, HIDDEN_LAYERS_JOY); + imlJoy.randomiseWeights(state.spreadLevel); + } + imlJoy.setLogger((msg: string) => console.log('[NISPS:joy]', msg)); + + // Hand IML: fresh init + imlHand = await WasmIML.create(N_HAND_INPUTS, newCount, HIDDEN_LAYERS_HAND); + imlHand.setLogger((msg: string) => console.log('[NISPS:hand]', msg)); + imlHand.randomiseWeights(state.spreadLevel); + + activeIml = imlJoy; + + // Run inference at current position + activeIml.setInput(0, 0.5); + activeIml.setInput(1, 0.5); + activeIml.process(); + + const newOutputs = new Float32Array(activeIml.getOutputs()); + setOutputs(newOutputs); + setOutputCount(newCount); + outputsTopic.emit(newOutputs); + outputCountTopic.emit(newCount); + + console.log(`[NISPS] MLP resized to ${newCount} outputs (joystick IML warm-started)`); + } + + // ─── Store implementation ─── + + const store: MLStore = { + get state() { return state; }, + outputs, + outputCount, + + getActiveIml: () => activeIml, + getImlJoy: () => imlJoy, + getImlHand: () => imlHand, + + setOutputMode: async (mode: OutputMode) => { + setState('outputMode', mode); + modeTopic.emit(mode); + + const targetCount = outputCountForMode(mode, state.midiCCCount); + await resizeMLP(targetCount); + }, + + setMidiCCCount: async (count: number) => { + setState('midiCCCount', count); + if (state.outputMode === 'midi-cc') { + await resizeMLP(count); + } + }, + + setSpreadLevel: (level: number) => { + setState('spreadLevel', Math.max(0, Math.min(1, level))); + }, + + setInputs: (x: number, y: number) => { + if (!activeIml) return; + const cx = Math.max(0, Math.min(1, x)); + const cy = Math.max(0, Math.min(1, y)); + activeIml.setInput(0, cx); + activeIml.setInput(1, cy); + runInference(); + }, + + randomise: () => { + if (!activeIml) return; + activeIml.randomiseWeights(state.spreadLevel); + runInference(); + }, + + clearExamples: () => { + activeIml?.clearDataset(); + }, + + train: () => { + if (!activeIml) return null; + const loss = activeIml.train(); + runInference(); + return loss; + }, + + trainAsync: () => { + if (!activeIml) return Promise.resolve(null); + return activeIml.trainAsync(() => runInference()); + }, + + thumbsUp: () => { + if (!activeIml) return Promise.resolve(null); + activeIml.addExample( + activeIml.inputState.slice(), + activeIml.outputState.slice() + ); + return activeIml.trainAsync(() => runInference()); + }, + + thumbsDown: (speed?: number) => { + if (!activeIml) return; + const spread = state.spreadLevel; + const s = speed ?? (0.15 * (1 - spread) + 0.05 * spread); + activeIml.moveWeights(s, spread); + runInference(); + }, + + evalLoss: () => activeIml?.evalLoss() ?? null, + + inferBatch: (points: number[][]) => { + if (!activeIml) return []; + return activeIml.inferBatch(points); + }, + + getLayerStats: () => activeIml?.getLayerStats() ?? [], + + getWeights: () => activeIml?._getFlatWeights() ?? [], + + getExampleCount: () => activeIml?.exampleCount ?? 0, + + getLoss: () => activeIml?.lastLoss ?? null, + + saveState: () => { + if (!activeIml) return; + const data = { + weights: activeIml._getFlatWeights(), + inputState: activeIml.inputState.slice(), + outputState: activeIml.outputState.slice(), + exampleCount: activeIml.exampleCount, + lossHistory: activeIml.lossHistory.slice(), + outputMode: state.outputMode, + spreadLevel: state.spreadLevel, + }; + localStorage.setItem('nisps-a-immersive', JSON.stringify(data)); + }, + + dispose: () => { + imlJoy?.destroy(); + imlHand?.destroy(); + imlJoy = null; + imlHand = null; + activeIml = null; + }, + }; + + return store; +} diff --git a/tests/solidjs/ml-store.spec.ts b/tests/solidjs/ml-store.spec.ts new file mode 100644 index 0000000..117508e --- /dev/null +++ b/tests/solidjs/ml-store.spec.ts @@ -0,0 +1,241 @@ +/** + * e2e tests for f04-ml-store-dual-iml. + * Validates: + * VAL-ML-001: Initial outputs bounded [0,1] + * VAL-ML-002: Initial state is untrained + * VAL-ML-022: Output count changes with mode + */ +import { test, expect } from '@playwright/test'; +import { loadSolidApp } from './helpers'; + +test.describe('ML Store — Dual IML', () => { + + // ─── VAL-ML-001: Initial outputs bounded [0,1] ─── + + test('VAL-ML-001a: getOutputs() returns 126 values on fresh load', async ({ page }) => { + await loadSolidApp(page); + + const result = await page.evaluate(() => { + const probe = (window as any).__nisps; + const outputs = probe.getOutputs(); + return { + length: outputs.length, + isArray: Array.isArray(outputs), + }; + }); + + expect(result.isArray).toBe(true); + expect(result.length).toBe(126); + }); + + test('VAL-ML-001b: all initial outputs are in [0,1]', async ({ page }) => { + await loadSolidApp(page); + + const result = await page.evaluate(() => { + const outputs = (window as any).__nisps.getOutputs(); + return { + min: Math.min(...outputs), + max: Math.max(...outputs), + allBounded: outputs.every((v: number) => v >= 0 && v <= 1), + allFinite: outputs.every((v: number) => Number.isFinite(v)), + }; + }); + + expect(result.allBounded).toBe(true); + expect(result.allFinite).toBe(true); + expect(result.min).toBeGreaterThanOrEqual(0); + expect(result.max).toBeLessThanOrEqual(1); + }); + + // ─── VAL-ML-002: Initial state is untrained ─── + + test('VAL-ML-002a: getExampleCount() returns 0 on fresh load', async ({ page }) => { + await loadSolidApp(page); + + const count = await page.evaluate(() => (window as any).__nisps.getExampleCount()); + expect(count).toBe(0); + }); + + test('VAL-ML-002b: getLoss() returns null on fresh load', async ({ page }) => { + await loadSolidApp(page); + + const loss = await page.evaluate(() => (window as any).__nisps.getLoss()); + expect(loss).toBeNull(); + }); + + // ─── VAL-ML-022: Output count changes with mode ─── + + test('VAL-ML-022a: default output count is 126 (synth mode default)', async ({ page }) => { + await loadSolidApp(page); + + const length = await page.evaluate(() => (window as any).__nisps.getOutputs().length); + expect(length).toBe(126); + }); + + test('VAL-ML-022b: outputCount signal tracks current count', async ({ page }) => { + await loadSolidApp(page); + + const result = await page.evaluate(() => { + const store = (window as any).__nispsStore; + return store.outputCount(); + }); + + // Default is 126 (synth mode is default) + expect(result).toBe(126); + }); + + test('VAL-ML-022c: mode change to visual produces 20 outputs', async ({ page }) => { + await loadSolidApp(page); + + const result = await page.evaluate(async () => { + const store = (window as any).__nispsStore; + const probe = (window as any).__nisps; + + // Switch mode via store + await store.setOutputMode('visual'); + + return { + outputCount: probe.getOutputs().length, + allBounded: probe.getOutputs().every((v: number) => v >= 0 && v <= 1), + }; + }); + + expect(result.outputCount).toBe(20); + expect(result.allBounded).toBe(true); + }); + + test('VAL-ML-022d: mode change back to synth produces 126 outputs', async ({ page }) => { + await loadSolidApp(page); + + const result = await page.evaluate(async () => { + const store = (window as any).__nispsStore; + const probe = (window as any).__nisps; + + // Switch to visual first + await store.setOutputMode('visual'); + + // Now switch to synth + await store.setOutputMode('synth'); + + return { + outputCount: probe.getOutputs().length, + allBounded: probe.getOutputs().every((v: number) => v >= 0 && v <= 1), + }; + }); + + expect(result.outputCount).toBe(126); + expect(result.allBounded).toBe(true); + }); + + test('VAL-ML-022e: mode change to midi-cc produces dynamic output count (default 8)', async ({ page }) => { + await loadSolidApp(page); + + const result = await page.evaluate(async () => { + const store = (window as any).__nispsStore; + const probe = (window as any).__nisps; + + // Switch to midi-cc mode + await store.setOutputMode('midi-cc'); + + return { + outputCount: probe.getOutputs().length, + allBounded: probe.getOutputs().every((v: number) => v >= 0 && v <= 1), + }; + }); + + expect(result.outputCount).toBe(8); // default MIDI CC count + expect(result.allBounded).toBe(true); + }); + + test('VAL-ML-022f: mode change to audio-canvas produces 36 outputs', async ({ page }) => { + await loadSolidApp(page); + + const result = await page.evaluate(async () => { + const store = (window as any).__nispsStore; + const probe = (window as any).__nisps; + + // Switch to audio-canvas mode + await store.setOutputMode('audio-canvas'); + + return { + outputCount: probe.getOutputs().length, + allBounded: probe.getOutputs().every((v: number) => v >= 0 && v <= 1), + }; + }); + + expect(result.outputCount).toBe(36); + expect(result.allBounded).toBe(true); + }); + + // ─── ML Store state after mode switch ─── + + test('after mode switch to visual, outputs are still valid for inference', async ({ page }) => { + await loadSolidApp(page); + + const result = await page.evaluate(async () => { + const store = (window as any).__nispsStore; + const probe = (window as any).__nisps; + + // Switch to visual + await store.setOutputMode('visual'); + + // Set inputs and verify inference works + probe.setInputs(0.3, 0.7); + const outputs = probe.getOutputs(); + return { + length: outputs.length, + allBounded: outputs.every((v: number) => v >= 0 && v <= 1), + allFinite: outputs.every((v: number) => Number.isFinite(v)), + }; + }); + + expect(result.length).toBe(20); + expect(result.allBounded).toBe(true); + expect(result.allFinite).toBe(true); + }); + + test('dual IML instances exist for joystick and hand tracking', async ({ page }) => { + await loadSolidApp(page); + + const result = await page.evaluate(() => { + const store = (window as any).__nispsStore; + const imlJoy = store.getImlJoy(); + const imlHand = store.getImlHand(); + return { + hasJoyIml: imlJoy !== null, + hasHandIml: imlHand !== null, + joyInputs: imlJoy?.nInputs, + handInputs: imlHand?.nInputs, + joyOutputs: imlJoy?.nOutputs, + handOutputs: imlHand?.nOutputs, + }; + }); + + expect(result.hasJoyIml).toBe(true); + expect(result.hasHandIml).toBe(true); + expect(result.joyInputs).toBe(2); + expect(result.handInputs).toBe(14); + expect(result.joyOutputs).toBe(126); + expect(result.handOutputs).toBe(126); + }); + + test('loss remains null and exampleCount remains 0 after mode switch', async ({ page }) => { + await loadSolidApp(page); + + const result = await page.evaluate(async () => { + const store = (window as any).__nispsStore; + const probe = (window as any).__nisps; + + // Switch modes + await store.setOutputMode('visual'); + + return { + exampleCount: probe.getExampleCount(), + loss: probe.getLoss(), + }; + }); + + expect(result.exampleCount).toBe(0); + expect(result.loss).toBeNull(); + }); +});