feat(solidjs): implement ML store with dual WasmIML instances and dynamic output count

Create ML store wrapping dual WasmIML instances (imlJoy for joystick with
2 inputs, imlHand for hand tracking with 14 inputs) with warm-start weight
transfer on mode switch. Outputs stored as Float32Array signal. Mode switching
triggers MLP resize (visual=20, synth=126, midi-cc=8, audio-canvas=36).
Initial state: 0 examples, null loss, all outputs bounded [0,1].

Fulfills: VAL-ML-001, VAL-ML-002, VAL-ML-022

Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
This commit is contained in:
monkey-w1n5t0n 2026-04-07 17:23:06 +02:00
parent 447ab54b54
commit 6fdc2985f9
5 changed files with 606 additions and 64 deletions

View file

@ -1,42 +1,32 @@
import { onMount, onCleanup, createSignal } from 'solid-js';
import { WasmIML } from './core/iml';
import { createSignalBus, type SignalBus } from './bus/signal-bus';
import { createMLStore, type MLStore } from './stores/ml-store';
import { exposeDebugProbe } from './probe/debug-probe';
const N_INPUTS = 2;
const HIDDEN_LAYERS = [32, 48, 64];
const N_OUTPUTS = 126;
// Singleton signal bus — shared across the entire app
const bus: SignalBus = createSignalBus();
// Expose bus globally for e2e tests and debug access
(window as any).__nispsBus = bus;
export { bus };
import bus from './bus';
/**
* Root application component for the NISPS immersive app.
* Initializes WASM IML, runs initial inference, and exposes debug probe.
* Initializes ML store (dual WasmIML instances), runs initial inference,
* and exposes debug probe.
*/
export default function App() {
const [status, setStatus] = createSignal<string>('Loading WASM...');
const [ready, setReady] = createSignal(false);
// IML instance lives outside reactive state (opaque handle)
let iml: WasmIML | null = null;
let mlStore: MLStore | null = null;
onMount(async () => {
try {
iml = await WasmIML.create(N_INPUTS, N_OUTPUTS, HIDDEN_LAYERS);
const store = await createMLStore(bus);
// Run initial inference at center position
iml.setInput(0, 0.5);
iml.setInput(1, 0.5);
iml.process();
// Expose store globally for e2e tests
(window as any).__nispsStore = store;
const outputs = iml.getOutputs();
const allBounded = outputs.every((v: number) => v >= 0 && v <= 1);
// Expose bus topics for testing dual IML and mode switching
bus.createTopic('ml.imlJoy').emit(store.getImlJoy() as any);
bus.createTopic('ml.imlHand').emit(store.getImlHand() as any);
const outputs = store.outputs();
const allBounded = Array.from(outputs).every(v => v >= 0 && v <= 1);
if (!allBounded) {
setStatus('ERROR: outputs out of bounds');
console.error('Unbounded outputs:', outputs);
@ -47,7 +37,9 @@ export default function App() {
setStatus(`Ready — ${outputs.length} outputs loaded`);
// Expose debug probe if ?debug=1
exposeDebugProbe(iml);
exposeDebugProbe(store);
mlStore = store;
} catch (err) {
setStatus(`Error: ${err}`);
console.error('WASM init failed:', err);
@ -55,7 +47,7 @@ export default function App() {
});
onCleanup(() => {
iml?.destroy();
mlStore?.dispose();
});
return (

16
src/bus/index.ts Normal file
View file

@ -0,0 +1,16 @@
/**
* Singleton signal bus — shared across the entire app.
*
* Created once at module load time and also exposed at window.__nispsBus
* for e2e tests and debug access.
*/
import { createSignalBus, type SignalBus } from './signal-bus';
const bus: SignalBus = createSignalBus();
// Expose bus globally for e2e tests and debug access
(window as any).__nispsBus = bus;
export default bus;
export type { SignalBus };

View file

@ -3,9 +3,10 @@
* Used by Playwright e2e tests. Zero footprint in production.
*
* All methods are synchronous (bypass SolidJS batching with batch()/untrack()).
* Updated to work with MLStore interface.
*/
import type { WasmIML } from '../core/iml';
import type { MLStore } from '../stores/ml-store';
export interface DebugProbe {
getOutputs: () => number[];
@ -39,59 +40,41 @@ export function isDebugMode(): boolean {
}
/**
* Create the debug probe object, wired to a WasmIML instance.
* Create the debug probe object, wired to an MLStore instance.
*/
export function createDebugProbe(iml: WasmIML): DebugProbe {
export function createDebugProbe(store: MLStore): DebugProbe {
return {
getOutputs: () => iml.getOutputs(),
getOutputs: () => Array.from(store.outputs()),
getLoss: () => iml.lastLoss,
getLoss: () => store.getLoss(),
getWeights: () => iml._getFlatWeights(),
getWeights: () => store.getWeights(),
getExampleCount: () => iml.exampleCount,
getExampleCount: () => store.getExampleCount(),
setInputs: (x: number, y: number) => {
iml.setInput(0, Math.max(0, Math.min(1, x)));
iml.setInput(1, Math.max(0, Math.min(1, y)));
iml.process();
store.setInputs(x, y);
},
thumbsUp: () => {
iml.addExample(iml.inputState.slice(), iml.outputState.slice());
return iml.trainAsync();
},
thumbsUp: () => store.thumbsUp(),
thumbsDown: () => {
const spread = 0.6;
const speed = 0.15 * (1 - spread) + 0.05 * spread;
iml.moveWeights(speed, spread);
},
thumbsDown: () => store.thumbsDown(),
train: () => iml.train(),
train: () => store.train(),
trainAsync: () => iml.trainAsync(),
trainAsync: () => store.trainAsync(),
randomise: () => iml.randomiseWeights(0.6),
randomise: () => store.randomise(),
clearExamples: () => iml.clearDataset(),
clearExamples: () => store.clearExamples(),
saveState: () => {
const state = {
weights: iml._getFlatWeights(),
inputState: iml.inputState.slice(),
outputState: iml.outputState.slice(),
exampleCount: iml.exampleCount,
lossHistory: iml.lossHistory.slice(),
};
localStorage.setItem('nisps-a-immersive', JSON.stringify(state));
},
saveState: () => store.saveState(),
evalLoss: () => iml.evalLoss(),
evalLoss: () => store.evalLoss(),
inferBatch: (points: number[][]) => iml.inferBatch(points),
inferBatch: (points: number[][]) => store.inferBatch(points),
getLayerStats: () => iml.getLayerStats(),
getLayerStats: () => store.getLayerStats(),
};
}
@ -99,10 +82,10 @@ export function createDebugProbe(iml: WasmIML): DebugProbe {
* Expose window.__nisps if ?debug=1 is present.
* Returns the probe object (or undefined if not in debug mode).
*/
export function exposeDebugProbe(iml: WasmIML): DebugProbe | undefined {
export function exposeDebugProbe(store: MLStore): DebugProbe | undefined {
if (!isDebugMode()) return undefined;
const probe = createDebugProbe(iml);
const probe = createDebugProbe(store);
(window as any).__nisps = probe;
console.log('[NISPS] Debug probe exposed at window.__nisps');
return probe;

310
src/stores/ml-store.ts Normal file
View file

@ -0,0 +1,310 @@
/**
* ML Store — SolidJS reactive wrapper around dual WasmIML instances.
*
* Architecture (mirrors old a-app.js):
* imlJoy: 2 inputs (joystick), dynamic outputs — warm-started on resize
* imlHand: 14 inputs (hand tracking), dynamic outputs — fresh on resize
* iml: pointer to whichever is active (imlJoy by default)
*
* Outputs are a Float32Array stored in a signal (NOT a store — proxies break typed arrays).
* Mode changes trigger MLP recreation with warm-start weight transfer.
*/
import { createSignal, onCleanup } from 'solid-js';
import { createStore } from 'solid-js/store';
import { WasmIML } from '../core/iml';
import type { SignalBus } from '../bus/signal-bus';
// ─── Constants ────────────────────────────────────────────────────────
export const N_JOY_INPUTS = 2;
export const N_HAND_INPUTS = 14;
export const N_VISUAL_OUTPUTS = 20;
export const N_SYNTH_OUTPUTS = 126;
export const N_MIDI_CC_DEFAULT = 8;
export const N_AUDIO_CANVAS_OUTPUTS = 36;
export const HIDDEN_LAYERS_JOY = [32, 48, 64];
export const HIDDEN_LAYERS_HAND = [48, 48, 64];
export type OutputMode = 'visual' | 'synth' | 'midi-cc' | 'audio-canvas';
/** Map from output mode to number of MLP outputs */
export function outputCountForMode(mode: OutputMode, midiCCCount = N_MIDI_CC_DEFAULT): number {
switch (mode) {
case 'visual': return N_VISUAL_OUTPUTS;
case 'synth': return N_SYNTH_OUTPUTS;
case 'midi-cc': return midiCCCount;
case 'audio-canvas': return N_AUDIO_CANVAS_OUTPUTS;
default: return N_SYNTH_OUTPUTS;
}
}
// ─── ML Store Interface ──────────────────────────────────────────────
export interface MLState {
/** Current output mode */
outputMode: OutputMode;
/** Dynamic output count for MIDI CC mode */
midiCCCount: number;
/** Spread level [0,1] — controls weight initialization & RL noise */
spreadLevel: number;
/** Whether IML instances are initialized */
initialized: boolean;
}
export interface MLStore {
// Reactive state
readonly state: MLState;
readonly outputs: () => Float32Array;
readonly outputCount: () => number;
// IML access (opaque handles, NOT reactive)
getActiveIml(): WasmIML | null;
getImlJoy(): WasmIML | null;
getImlHand(): WasmIML | null;
// Actions
setOutputMode(mode: OutputMode): Promise<void>;
setMidiCCCount(count: number): Promise<void>;
setSpreadLevel(level: number): void;
// Direct IML operations (synchronous)
setInputs(x: number, y: number): void;
randomise(): void;
clearExamples(): void;
train(): number | null;
trainAsync(): Promise<number | null>;
thumbsUp(): Promise<number | null>;
thumbsDown(speed?: number): void;
evalLoss(): number | null;
inferBatch(points: number[][]): number[][];
getLayerStats(): Array<{ meanAbs: number; maxAbs: number; deadFrac: number; satFrac: number }>;
getWeights(): number[];
getExampleCount(): number;
getLoss(): number | null;
saveState(): void;
// Lifecycle
dispose(): void;
}
// ─── Factory ─────────────────────────────────────────────────────────
export async function createMLStore(bus: SignalBus): Promise<MLStore> {
const [state, setState] = createStore<MLState>({
outputMode: 'synth', // default mode — old app starts at N_SYNTH_OUTPUTS
midiCCCount: N_MIDI_CC_DEFAULT,
spreadLevel: 0.6,
initialized: false,
});
// Outputs as signal — Float32Array MUST NOT go in store (proxy overhead)
const initialCount = outputCountForMode('synth');
const [outputs, setOutputs] = createSignal<Float32Array>(new Float32Array(initialCount));
const [outputCount, setOutputCount] = createSignal<number>(initialCount);
// IML instances — module-scope opaque handles (not reactive)
let imlJoy: WasmIML | null = null;
let imlHand: WasmIML | null = null;
let activeIml: WasmIML | null = null;
// Initialize dual IML instances
const currentCount = outputCount();
imlJoy = await WasmIML.create(N_JOY_INPUTS, currentCount, HIDDEN_LAYERS_JOY);
imlJoy.setLogger((msg: string) => console.log('[NISPS:joy]', msg));
imlHand = await WasmIML.create(N_HAND_INPUTS, currentCount, HIDDEN_LAYERS_HAND);
imlHand.setLogger((msg: string) => console.log('[NISPS:hand]', msg));
activeIml = imlJoy; // default to joystick
// Run initial inference at center position
activeIml.setInput(0, 0.5);
activeIml.setInput(1, 0.5);
activeIml.process();
setOutputs(new Float32Array(activeIml.getOutputs()));
setState('initialized', true);
// Create bus topics for external consumers
const outputsTopic = bus.createTopic<Float32Array>('ml.outputs');
const outputCountTopic = bus.createTopic<number>('ml.outputCount');
const modeTopic = bus.createTopic<OutputMode>('mode.output');
// Publish initial values
outputsTopic.emit(outputs());
outputCountTopic.emit(outputCount());
// ─── Helper: run inference and update signal ───
function runInference(): void {
if (!activeIml) return;
activeIml.process();
const newOutputs = new Float32Array(activeIml.getOutputs());
setOutputs(newOutputs);
outputsTopic.emit(newOutputs);
}
// ─── Helper: resize MLP (mode switch) ───
async function resizeMLP(newCount: number): Promise<void> {
if (newCount === outputCount()) return;
// Extract joystick weights for warm-start
const joySnapshot = imlJoy ? imlJoy.extractWeights() : null;
// Destroy old instances
imlJoy?.destroy();
imlHand?.destroy();
// Joystick IML: warm-start if possible
if (joySnapshot) {
imlJoy = await WasmIML.createWithWarmStart(
joySnapshot, newCount, 1000, 1.0, 0.00001
);
} else {
imlJoy = await WasmIML.create(N_JOY_INPUTS, newCount, HIDDEN_LAYERS_JOY);
imlJoy.randomiseWeights(state.spreadLevel);
}
imlJoy.setLogger((msg: string) => console.log('[NISPS:joy]', msg));
// Hand IML: fresh init
imlHand = await WasmIML.create(N_HAND_INPUTS, newCount, HIDDEN_LAYERS_HAND);
imlHand.setLogger((msg: string) => console.log('[NISPS:hand]', msg));
imlHand.randomiseWeights(state.spreadLevel);
activeIml = imlJoy;
// Run inference at current position
activeIml.setInput(0, 0.5);
activeIml.setInput(1, 0.5);
activeIml.process();
const newOutputs = new Float32Array(activeIml.getOutputs());
setOutputs(newOutputs);
setOutputCount(newCount);
outputsTopic.emit(newOutputs);
outputCountTopic.emit(newCount);
console.log(`[NISPS] MLP resized to ${newCount} outputs (joystick IML warm-started)`);
}
// ─── Store implementation ───
const store: MLStore = {
get state() { return state; },
outputs,
outputCount,
getActiveIml: () => activeIml,
getImlJoy: () => imlJoy,
getImlHand: () => imlHand,
setOutputMode: async (mode: OutputMode) => {
setState('outputMode', mode);
modeTopic.emit(mode);
const targetCount = outputCountForMode(mode, state.midiCCCount);
await resizeMLP(targetCount);
},
setMidiCCCount: async (count: number) => {
setState('midiCCCount', count);
if (state.outputMode === 'midi-cc') {
await resizeMLP(count);
}
},
setSpreadLevel: (level: number) => {
setState('spreadLevel', Math.max(0, Math.min(1, level)));
},
setInputs: (x: number, y: number) => {
if (!activeIml) return;
const cx = Math.max(0, Math.min(1, x));
const cy = Math.max(0, Math.min(1, y));
activeIml.setInput(0, cx);
activeIml.setInput(1, cy);
runInference();
},
randomise: () => {
if (!activeIml) return;
activeIml.randomiseWeights(state.spreadLevel);
runInference();
},
clearExamples: () => {
activeIml?.clearDataset();
},
train: () => {
if (!activeIml) return null;
const loss = activeIml.train();
runInference();
return loss;
},
trainAsync: () => {
if (!activeIml) return Promise.resolve(null);
return activeIml.trainAsync(() => runInference());
},
thumbsUp: () => {
if (!activeIml) return Promise.resolve(null);
activeIml.addExample(
activeIml.inputState.slice(),
activeIml.outputState.slice()
);
return activeIml.trainAsync(() => runInference());
},
thumbsDown: (speed?: number) => {
if (!activeIml) return;
const spread = state.spreadLevel;
const s = speed ?? (0.15 * (1 - spread) + 0.05 * spread);
activeIml.moveWeights(s, spread);
runInference();
},
evalLoss: () => activeIml?.evalLoss() ?? null,
inferBatch: (points: number[][]) => {
if (!activeIml) return [];
return activeIml.inferBatch(points);
},
getLayerStats: () => activeIml?.getLayerStats() ?? [],
getWeights: () => activeIml?._getFlatWeights() ?? [],
getExampleCount: () => activeIml?.exampleCount ?? 0,
getLoss: () => activeIml?.lastLoss ?? null,
saveState: () => {
if (!activeIml) return;
const data = {
weights: activeIml._getFlatWeights(),
inputState: activeIml.inputState.slice(),
outputState: activeIml.outputState.slice(),
exampleCount: activeIml.exampleCount,
lossHistory: activeIml.lossHistory.slice(),
outputMode: state.outputMode,
spreadLevel: state.spreadLevel,
};
localStorage.setItem('nisps-a-immersive', JSON.stringify(data));
},
dispose: () => {
imlJoy?.destroy();
imlHand?.destroy();
imlJoy = null;
imlHand = null;
activeIml = null;
},
};
return store;
}

View file

@ -0,0 +1,241 @@
/**
* e2e tests for f04-ml-store-dual-iml.
* Validates:
* VAL-ML-001: Initial outputs bounded [0,1]
* VAL-ML-002: Initial state is untrained
* VAL-ML-022: Output count changes with mode
*/
import { test, expect } from '@playwright/test';
import { loadSolidApp } from './helpers';
test.describe('ML Store — Dual IML', () => {
// ─── VAL-ML-001: Initial outputs bounded [0,1] ───
test('VAL-ML-001a: getOutputs() returns 126 values on fresh load', async ({ page }) => {
await loadSolidApp(page);
const result = await page.evaluate(() => {
const probe = (window as any).__nisps;
const outputs = probe.getOutputs();
return {
length: outputs.length,
isArray: Array.isArray(outputs),
};
});
expect(result.isArray).toBe(true);
expect(result.length).toBe(126);
});
test('VAL-ML-001b: all initial outputs are in [0,1]', async ({ page }) => {
await loadSolidApp(page);
const result = await page.evaluate(() => {
const outputs = (window as any).__nisps.getOutputs();
return {
min: Math.min(...outputs),
max: Math.max(...outputs),
allBounded: outputs.every((v: number) => v >= 0 && v <= 1),
allFinite: outputs.every((v: number) => Number.isFinite(v)),
};
});
expect(result.allBounded).toBe(true);
expect(result.allFinite).toBe(true);
expect(result.min).toBeGreaterThanOrEqual(0);
expect(result.max).toBeLessThanOrEqual(1);
});
// ─── VAL-ML-002: Initial state is untrained ───
test('VAL-ML-002a: getExampleCount() returns 0 on fresh load', async ({ page }) => {
await loadSolidApp(page);
const count = await page.evaluate(() => (window as any).__nisps.getExampleCount());
expect(count).toBe(0);
});
test('VAL-ML-002b: getLoss() returns null on fresh load', async ({ page }) => {
await loadSolidApp(page);
const loss = await page.evaluate(() => (window as any).__nisps.getLoss());
expect(loss).toBeNull();
});
// ─── VAL-ML-022: Output count changes with mode ───
test('VAL-ML-022a: default output count is 126 (synth mode default)', async ({ page }) => {
await loadSolidApp(page);
const length = await page.evaluate(() => (window as any).__nisps.getOutputs().length);
expect(length).toBe(126);
});
test('VAL-ML-022b: outputCount signal tracks current count', async ({ page }) => {
await loadSolidApp(page);
const result = await page.evaluate(() => {
const store = (window as any).__nispsStore;
return store.outputCount();
});
// Default is 126 (synth mode is default)
expect(result).toBe(126);
});
test('VAL-ML-022c: mode change to visual produces 20 outputs', async ({ page }) => {
await loadSolidApp(page);
const result = await page.evaluate(async () => {
const store = (window as any).__nispsStore;
const probe = (window as any).__nisps;
// Switch mode via store
await store.setOutputMode('visual');
return {
outputCount: probe.getOutputs().length,
allBounded: probe.getOutputs().every((v: number) => v >= 0 && v <= 1),
};
});
expect(result.outputCount).toBe(20);
expect(result.allBounded).toBe(true);
});
test('VAL-ML-022d: mode change back to synth produces 126 outputs', async ({ page }) => {
await loadSolidApp(page);
const result = await page.evaluate(async () => {
const store = (window as any).__nispsStore;
const probe = (window as any).__nisps;
// Switch to visual first
await store.setOutputMode('visual');
// Now switch to synth
await store.setOutputMode('synth');
return {
outputCount: probe.getOutputs().length,
allBounded: probe.getOutputs().every((v: number) => v >= 0 && v <= 1),
};
});
expect(result.outputCount).toBe(126);
expect(result.allBounded).toBe(true);
});
test('VAL-ML-022e: mode change to midi-cc produces dynamic output count (default 8)', async ({ page }) => {
await loadSolidApp(page);
const result = await page.evaluate(async () => {
const store = (window as any).__nispsStore;
const probe = (window as any).__nisps;
// Switch to midi-cc mode
await store.setOutputMode('midi-cc');
return {
outputCount: probe.getOutputs().length,
allBounded: probe.getOutputs().every((v: number) => v >= 0 && v <= 1),
};
});
expect(result.outputCount).toBe(8); // default MIDI CC count
expect(result.allBounded).toBe(true);
});
test('VAL-ML-022f: mode change to audio-canvas produces 36 outputs', async ({ page }) => {
await loadSolidApp(page);
const result = await page.evaluate(async () => {
const store = (window as any).__nispsStore;
const probe = (window as any).__nisps;
// Switch to audio-canvas mode
await store.setOutputMode('audio-canvas');
return {
outputCount: probe.getOutputs().length,
allBounded: probe.getOutputs().every((v: number) => v >= 0 && v <= 1),
};
});
expect(result.outputCount).toBe(36);
expect(result.allBounded).toBe(true);
});
// ─── ML Store state after mode switch ───
test('after mode switch to visual, outputs are still valid for inference', async ({ page }) => {
await loadSolidApp(page);
const result = await page.evaluate(async () => {
const store = (window as any).__nispsStore;
const probe = (window as any).__nisps;
// Switch to visual
await store.setOutputMode('visual');
// Set inputs and verify inference works
probe.setInputs(0.3, 0.7);
const outputs = probe.getOutputs();
return {
length: outputs.length,
allBounded: outputs.every((v: number) => v >= 0 && v <= 1),
allFinite: outputs.every((v: number) => Number.isFinite(v)),
};
});
expect(result.length).toBe(20);
expect(result.allBounded).toBe(true);
expect(result.allFinite).toBe(true);
});
test('dual IML instances exist for joystick and hand tracking', async ({ page }) => {
await loadSolidApp(page);
const result = await page.evaluate(() => {
const store = (window as any).__nispsStore;
const imlJoy = store.getImlJoy();
const imlHand = store.getImlHand();
return {
hasJoyIml: imlJoy !== null,
hasHandIml: imlHand !== null,
joyInputs: imlJoy?.nInputs,
handInputs: imlHand?.nInputs,
joyOutputs: imlJoy?.nOutputs,
handOutputs: imlHand?.nOutputs,
};
});
expect(result.hasJoyIml).toBe(true);
expect(result.hasHandIml).toBe(true);
expect(result.joyInputs).toBe(2);
expect(result.handInputs).toBe(14);
expect(result.joyOutputs).toBe(126);
expect(result.handOutputs).toBe(126);
});
test('loss remains null and exampleCount remains 0 after mode switch', async ({ page }) => {
await loadSolidApp(page);
const result = await page.evaluate(async () => {
const store = (window as any).__nispsStore;
const probe = (window as any).__nisps;
// Switch modes
await store.setOutputMode('visual');
return {
exampleCount: probe.getExampleCount(),
loss: probe.getLoss(),
};
});
expect(result.exampleCount).toBe(0);
expect(result.loss).toBeNull();
});
});