From 54821fffd09ca0b9af324de34a586d53dff73fc7 Mon Sep 17 00:00:00 2001 From: "Claude (stream-10)" Date: Wed, 29 Apr 2026 17:33:01 +0300 Subject: [PATCH] playground/stream-10: wire phase 1-4 features into modes (meml-5wg) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds the SolidJS-side feature parity with the legacy a-immersive playground. Every Phase 1–4 feature called out in .local/recon/04-playground.md now flows through the runtime → stores → primitives pipeline. Phase 1 (compound axes / zoom / drawer / trail): - control-routing.ts: resolves Boldness/Memory/Precision into input/output/exploration store writes via createEffect. - SettingsDrawer (new): tabbed sections for Input / Training / Exploration / Output / Per-param overrides / Advanced. Per-param ParamEditor list wires through modeStore.setOverride. - JoyMap navigator added to ModeShell; tap-to-return + long-press region pin handlers wired into the runtime. - Anchor mode toggle (auto/sticky/center) exposed via drawer. Phase 2 (pinning / history / A/B): - features/snapshots.ts: bridge sessionStore push/pop to actual Float32Array weights. Auto-snapshot before train/randomize/thumbs-down. - Undo button + long-press snapshot list popup wired in ModeShell. - A/B capture/toggle/accept/revert in SettingsDrawer. - features/overrides.ts buildPinMask() composes override `pinned` flags + sessionStore param pins; passed to mlStore.moveWeights. - Region pinning long-press → addRegionPin + auto-snapshot tag "pinned baseline". Phase 3 (refinement / exploration): - features/heatmap-sampler.ts: 16x16 grid via mlStore.inferBatch, three colour modes, throttled 5/sec, refreshed on ml.trained / ml.delta_update bus events. - exploration-store.ts (new): RL noise level, floor/cap/growth/decay, spread, learning rate, weight decay, auto-explore, pressure. - Auto-explore timer in mode-runtime drives moveWeights at user interval, zoom-scaled intensity. - Pressure feedback: window pointerdown/up timestamps mapped to explorationStore.setPressure; feeds growNoise/decayNoise. Phase 4 (output / persistence / polish): - features/weight-health.ts: histogram, status, per-layer L2 deltas, vanishing/exploding/converged classification. - features/session-preset.ts: full state capture/restore + base64url URL sharing; main.tsx applies on load. - SettingsDrawer Advanced tab renders WeightHealth, GradientFlow, LayerStats, Heatmap, session preset save/load/share. Override application: - features/overrides.ts applies per-param mute/freeze/curve/range between MLP outputs and the engine. Mode runtime exposes `paramOutputs` (length = schema.params) which all modes now bind to OutputDisplay so ranges and mutes are visible. - Freeze flags push into outputStore.freezeMask reactively. Mic input: - features/mic-input.ts: getUserMedia + AnalyserNode-derived {energy, brightness, pitch, aperiodicity}. Mode runtime feeds them into channels 2..(input_size-1) when active. ModeShell shows a Mic toggle for audio_in modes. Debug probe (probe.ts): - Synchronous bypass for snapshot, A/B, pins, overrides, axes, spread, output freeze, heatmap, weight health, session presets, URL params, bus emit/on. Untracked reads/writes throughout to avoid SolidJS reactivity surprises in tests. Modes: - All 8 firmware mode TSX files now use the default SettingsDrawer (no per-mode SliderBank scaffolding). They bind OutputDisplay to runtime.paramOutputs (post-override) instead of processedOutputs. Build status: - bun run typecheck: clean - bun run build: clean - dev server smoke: index, /modes, mode-runtime, SettingsDrawer, features/* all serve. Note: bd close meml-5wg failed because Dolt server unreachable from this worktree. Issue should be closed manually by orchestrator. --- playground/src/debug/probe.ts | 388 ++++++++++- playground/src/dev/probe-smoke.ts | 121 ++++ playground/src/features/control-routing.ts | 77 ++ playground/src/features/heatmap-sampler.ts | 160 +++++ playground/src/features/mic-input.ts | 138 ++++ playground/src/features/overrides.ts | 131 ++++ playground/src/features/session-preset.ts | 288 ++++++++ playground/src/features/snapshots.ts | 123 ++++ playground/src/features/trail.ts | 55 ++ playground/src/features/weight-health.ts | 124 ++++ playground/src/main.tsx | 14 +- playground/src/modes/BreakOrMode.tsx | 18 +- playground/src/modes/ChannelStripMode.tsx | 23 +- playground/src/modes/ElysiamorfMode.tsx | 18 +- playground/src/modes/MEMLCeliumMode.tsx | 18 +- playground/src/modes/ModeShell.tsx | 181 ++++- playground/src/modes/PAFSynthMode.tsx | 19 +- .../src/modes/SettingsDrawer.module.css | 250 +++++++ playground/src/modes/SettingsDrawer.tsx | 656 ++++++++++++++++++ .../src/modes/SoundAnalysisMIDIMode.tsx | 55 +- playground/src/modes/VerbFXMode.tsx | 18 +- playground/src/modes/XIASRIMode.tsx | 29 +- playground/src/modes/mode-helpers.ts | 14 +- playground/src/modes/mode-runtime.ts | 400 ++++++++++- playground/src/stores/exploration-store.ts | 254 +++++++ playground/src/stores/index.ts | 5 + playground/src/stores/session-store.ts | 38 +- 27 files changed, 3346 insertions(+), 269 deletions(-) create mode 100644 playground/src/dev/probe-smoke.ts create mode 100644 playground/src/features/control-routing.ts create mode 100644 playground/src/features/heatmap-sampler.ts create mode 100644 playground/src/features/mic-input.ts create mode 100644 playground/src/features/overrides.ts create mode 100644 playground/src/features/session-preset.ts create mode 100644 playground/src/features/snapshots.ts create mode 100644 playground/src/features/trail.ts create mode 100644 playground/src/features/weight-health.ts create mode 100644 playground/src/modes/SettingsDrawer.module.css create mode 100644 playground/src/modes/SettingsDrawer.tsx create mode 100644 playground/src/stores/exploration-store.ts diff --git a/playground/src/debug/probe.ts b/playground/src/debug/probe.ts index 78e5255..ca1b991 100644 --- a/playground/src/debug/probe.ts +++ b/playground/src/debug/probe.ts @@ -1,25 +1,71 @@ /** * Debug probe: window.__nisps * - * Stream 7 wires this to the real WasmIML via mlStore. Methods are - * synchronous (or return immediately-resolved promises). The probe - * deliberately bypasses Solid reactivity so tests get deterministic, - * imperative semantics. + * Synchronous-or-immediate Promise API for Playwright tests and dev console + * use. Bypasses SolidJS reactivity by reading/writing the underlying stores + * and WASM directly — values propagate to the UI on the next reactive tick, + * but the probe always returns the freshest state. * - * The probe self-initialises the ML engine on first use that needs it - * — Playwright tests can `await window.__nisps.__init()` before driving - * inference, or just call methods and tolerate a few no-ops while the - * lazy init resolves. While the init is in flight, `__ready` is false; - * synchronous methods that need ML are best-effort no-ops. + * Stream 10 expanded the surface to cover stream 8/9 features: + * - Snapshot stack (push, pop, list) + * - A/B compare (capture, toggle, accept, revert) + * - Region pinning (add, remove, clear) + * - Auto-explore (toggle, interval, intensity) + * - Heatmap (refresh, color mode, getCells) + * - Weight health (histogram, status, layer stats) + * - Session presets (save, load, list, share URL) + * - Override application (read, write, clear) + * + * Test contract: every method either returns a value, returns null/empty, + * or returns an immediately-resolved Promise. None of them throw — bad + * input is silently ignored. */ +import { batch, untrack } from 'solid-js'; import { mlStore } from '../stores/ml-store'; +import { modeStore } from '../stores/mode-store'; +import { sessionStore } from '../stores/session-store'; +import { explorationStore } from '../stores/exploration-store'; +import { controlStore } from '../stores/control-store'; +import { inputStore } from '../stores/input-store'; +import { outputStore } from '../stores/output-store'; +import { coreBus } from '../stores/bus'; +import { + autoSnapshot, + undoLastSnapshot, + captureA as snapCaptureA, + toggleAB as snapToggleAB, + acceptB as snapAcceptB, + revertToA as snapRevertToA, +} from '../features/snapshots'; +import { HeatmapSampler } from '../features/heatmap-sampler'; +import { + weightHistogram, + weightStatus as healthStatus, + layerNorms, + gradientStatuses, + layerWeightCounts, +} from '../features/weight-health'; +import { + captureSessionPreset, + restoreSessionPreset, + saveNamedPreset, + loadNamedPreset, + buildShareUrl, + applyUrlParams, + type SessionPresetPayload, +} from '../features/session-preset'; +import type { HeatmapColorMode } from '../features/heatmap-sampler'; export interface DebugProbe { /** Current 126-element output vector (Float32Array). */ getOutputs(): Float32Array; + /** Current per-param post-override values (length = active mode params). */ + getParamOutputs(): Float32Array; /** Last training loss, or null if no training has occurred. */ getLoss(): number | null; + /** Loss history of last training run. */ + getLossHistory(): ReadonlyArray; /** Flat weight array. */ getWeights(): Float32Array; /** Number of training examples currently in the dataset. */ @@ -46,6 +92,68 @@ export interface DebugProbe { inferBatch(points: ReadonlyArray): Float32Array; /** Per-layer weight statistics: layerCount * 4 floats (mean|w|, max|w|, dead%, sat%). */ getLayerStats(): Float32Array; + + // Snapshot / undo / A/B + pushSnapshot(tag: string): void; + popSnapshot(): boolean; + listSnapshots(): ReadonlyArray<{ id: string; tag: string; timestamp: number; noiseLevel: number }>; + clearSnapshots(): void; + captureA(): void; + toggleAB(): 'A' | 'B'; + acceptB(): void; + revertToA(): void; + + // Pins + addRegionPin(pin: { x: number; y: number; width: number; height: number }): string; + removeRegionPin(id: string): void; + toggleParamPin(modeId: string, paramName: string, outputIndex: number): boolean; + isParamPinned(modeId: string, paramName: string): boolean; + + // Auto-explore + setAutoExplore(enabled: boolean): void; + setAutoExploreInterval(ms: number): void; + setAutoExploreIntensity(v: number): void; + + // Pressure + setPressure(force: number, holdMs?: number): void; + + // Heatmap + refreshHeatmap(force?: boolean): boolean; + getHeatmapCells(): Float32Array; + setHeatmapColorMode(mode: HeatmapColorMode): void; + + // Weight health + getWeightHistogram(): number[]; + getWeightStatus(): 'dead' | 'saturating' | 'healthy'; + getGradientFlow(beforeWeights: Float32Array, afterWeights: Float32Array): { norms: number[]; status: ReturnType }; + + // Overrides + setOverride(modeId: string, paramName: string, override: Partial<{ min: number; max: number; curve: string; muted: boolean; pinned: boolean; frozen: boolean; fixedValue: number }>): void; + clearOverride(modeId: string, paramName: string): void; + clearAllOverrides(modeId: string): void; + getOverride(modeId: string, paramName: string): unknown; + + // Compound axes + setAxis(axis: 'boldness' | 'memory' | 'precision', value: number): void; + applyControlPreset(id: string): boolean; + + // Pipelines + setZoom(zoom: number): void; + setSpread(spread: number): void; + setOutputFreeze(frozen: boolean): void; + + // Session presets + captureSessionPreset(withWeights?: boolean): SessionPresetPayload; + restoreSessionPreset(payload: SessionPresetPayload): boolean; + saveSessionPreset(name: string, withWeights?: boolean): void; + loadSessionPreset(id: string): boolean; + buildShareUrl(): string; + applyUrlParams(search?: string): boolean; + + // Bus + emit(topic: string, data: unknown): void; + on(topic: string, handler: (data: unknown) => void): () => void; + /** True once the WASM is fully initialised. */ readonly __ready: boolean; /** Force initialisation. Returns a promise that resolves when the WASM is ready. */ @@ -60,9 +168,6 @@ declare global { const EMPTY_F32 = new Float32Array(0); -// We auto-initialise lazily so a test that immediately calls `.train()` -// after page load doesn't silently no-op. The promise is shared across -// calls so we don't kick off two simultaneous loads. let lazyInitPromise: Promise | null = null; function lazyInit(): Promise { if (mlStore.iml) return Promise.resolve(); @@ -72,6 +177,9 @@ function lazyInit(): Promise { return lazyInitPromise; } +// Probe-local heatmap sampler (independent of any active mode runtime). +const probeHeatmap = new HeatmapSampler({ resolution: 16 }); + const probe: DebugProbe = { get __ready(): boolean { return !!mlStore.iml && mlStore.state.ready; @@ -85,10 +193,24 @@ const probe: DebugProbe = { return mlStore.outputs(); }, + getParamOutputs(): Float32Array { + // The probe doesn't have a runtime reference; recompute from raw outputs. + const raw = mlStore.outputs(); + if (raw.length === 0) return EMPTY_F32; + // Just return the raw outputs — actual override application requires + // schema knowledge that the probe doesn't track. Tests can read + // overrides via `getOverride` instead. + return raw; + }, + getLoss(): number | null { return mlStore.state.lastLoss; }, + getLossHistory(): ReadonlyArray { + return mlStore.state.lossHistory; + }, + getWeights(): Float32Array { return mlStore.getWeights(); }, @@ -102,22 +224,27 @@ const probe: DebugProbe = { void lazyInit(); return; } - mlStore.iml.inferXY(x, y); + untrack(() => mlStore.iml!.inferXY(x, y)); }, thumbsUp(): void { if (!mlStore.iml) return; - // Stream 10 will replace this with the full RL controller; the - // legacy probe behaviour is "train, then settle". For now we run - // a sync training step. - mlStore.iml.train(); + untrack(() => { + autoSnapshot('before thumbs-up (probe)'); + mlStore.iml!.train(explorationStore.state.learningRate); + explorationStore.decayNoise(0.5); + }); }, thumbsDown(): void { if (!mlStore.iml) return; - // Default RL noise burst at the playground's typical spread. Stream - // 10 will hook the noise cap from the control surface state. - mlStore.iml.moveWeights(0.1, 0.6); + untrack(() => { + autoSnapshot('before thumbs-down (probe)'); + const cap = explorationStore.state.noiseCap; + const spread = explorationStore.state.spread; + mlStore.iml!.moveWeights(cap, spread); + explorationStore.growNoise(0.5); + }); }, train(): number { @@ -125,18 +252,25 @@ const probe: DebugProbe = { void lazyInit(); return 0; } - return mlStore.iml.train(); + return untrack(() => { + autoSnapshot('before train (probe)'); + return mlStore.iml!.train(explorationStore.state.learningRate); + }); }, async trainAsync(): Promise { await lazyInit(); if (!mlStore.iml) return 0; - return mlStore.iml.trainAsync(); + autoSnapshot('before trainAsync (probe)'); + return mlStore.iml.trainAsync(explorationStore.state.learningRate); }, randomise(): void { if (!mlStore.iml) return; - mlStore.iml.randomiseWeights(0.6); + untrack(() => { + autoSnapshot('before randomize (probe)'); + mlStore.iml!.randomiseWeights(explorationStore.state.spread); + }); }, clearExamples(): void { @@ -161,6 +295,214 @@ const probe: DebugProbe = { if (!mlStore.iml) return EMPTY_F32; return mlStore.iml.getLayerStatsFlat(); }, + + // ----- Snapshot / undo / A/B --------------------------------------------- + + pushSnapshot(tag: string): void { + autoSnapshot(tag); + }, + + popSnapshot(): boolean { + return undoLastSnapshot(); + }, + + listSnapshots() { + return sessionStore.listSnapshots().map((s) => ({ + id: s.id, + tag: s.tag, + timestamp: s.timestamp, + noiseLevel: s.noiseLevel, + })); + }, + + clearSnapshots(): void { + sessionStore.clearSnapshots(); + }, + + captureA(): void { + snapCaptureA(); + }, + + toggleAB(): 'A' | 'B' { + return snapToggleAB(); + }, + + acceptB(): void { + snapAcceptB(); + }, + + revertToA(): void { + snapRevertToA(); + }, + + // ----- Pins --------------------------------------------------------------- + + addRegionPin(pin) { + const created = sessionStore.addRegionPin(pin); + return created.id; + }, + + removeRegionPin(id: string): void { + sessionStore.removeRegionPin(id); + }, + + toggleParamPin(modeId: string, paramName: string, outputIndex: number): boolean { + return sessionStore.toggleParamPin(`${modeId}:${paramName}`, outputIndex); + }, + + isParamPinned(modeId: string, paramName: string): boolean { + return sessionStore.isParamPinned(`${modeId}:${paramName}`); + }, + + // ----- Auto-explore ------------------------------------------------------- + + setAutoExplore(enabled: boolean): void { + explorationStore.setAutoExplore(enabled); + }, + + setAutoExploreInterval(ms: number): void { + explorationStore.setAutoExploreInterval(ms); + }, + + setAutoExploreIntensity(v: number): void { + explorationStore.setAutoExploreIntensity(v); + }, + + setPressure(force: number, holdMs: number = 0): void { + explorationStore.setPressure(force, holdMs); + }, + + // ----- Heatmap ------------------------------------------------------------ + + refreshHeatmap(force = false): boolean { + if (!mlStore.iml) return false; + const cfg = inputStore.config; + const z = cfg.zoom; + const cx = cfg.anchorMode === 'center' ? 0.5 : cfg.anchorX; + const cy = cfg.anchorMode === 'center' ? 0.5 : cfg.anchorY; + return probeHeatmap.update({ cx, cy, zoom: z }, force); + }, + + getHeatmapCells(): Float32Array { + return probeHeatmap.getCells(); + }, + + setHeatmapColorMode(mode: HeatmapColorMode): void { + probeHeatmap.setColorMode(mode); + }, + + // ----- Weight health ------------------------------------------------------ + + getWeightHistogram(): number[] { + return weightHistogram(mlStore.weights()); + }, + + getWeightStatus() { + return healthStatus(mlStore.weights()); + }, + + getGradientFlow(beforeWeights: Float32Array, afterWeights: Float32Array) { + if (!mlStore.iml) return { norms: [], status: [] }; + const arch = mlStore.iml.architecture; + const sizes = layerWeightCounts({ + inputSize: arch.inputSize, + hidden: arch.hidden, + outputSize: arch.outputSize, + }); + const norms = layerNorms(beforeWeights, afterWeights, sizes); + return { norms, status: gradientStatuses(norms) }; + }, + + // ----- Overrides ---------------------------------------------------------- + + setOverride(modeId: string, paramName: string, patch): void { + const existing = modeStore.getOverride(modeId, paramName); + const next = { + min: existing?.min ?? 0, + max: existing?.max ?? 1, + curve: existing?.curve ?? 'linear', + curveParam: existing?.curveParam, + muted: existing?.muted ?? false, + pinned: existing?.pinned ?? false, + frozen: existing?.frozen ?? false, + fixedValue: existing?.fixedValue ?? 0.5, + ...patch, + } as Parameters[2]; + modeStore.setOverride(modeId, paramName, next); + }, + + clearOverride(modeId: string, paramName: string): void { + modeStore.clearOverride(modeId, paramName); + }, + + clearAllOverrides(modeId: string): void { + modeStore.clearAllOverrides(modeId); + }, + + getOverride(modeId: string, paramName: string) { + return modeStore.getOverride(modeId, paramName); + }, + + // ----- Compound axes ------------------------------------------------------ + + setAxis(axis: 'boldness' | 'memory' | 'precision', value: number): void { + controlStore.setAxis(axis, value); + }, + + applyControlPreset(id: string): boolean { + return controlStore.applyPreset(id); + }, + + // ----- Pipelines ---------------------------------------------------------- + + setZoom(zoom: number): void { + inputStore.setZoom(zoom); + }, + + setSpread(spread: number): void { + explorationStore.setSpread(spread); + }, + + setOutputFreeze(frozen: boolean): void { + outputStore.setFreezeOutput(frozen); + }, + + // ----- Session presets ---------------------------------------------------- + + captureSessionPreset(withWeights = false): SessionPresetPayload { + return captureSessionPreset(withWeights); + }, + + restoreSessionPreset(payload: SessionPresetPayload): boolean { + return batch(() => restoreSessionPreset(payload)); + }, + + saveSessionPreset(name: string, withWeights = false): void { + saveNamedPreset(name, withWeights); + }, + + loadSessionPreset(id: string): boolean { + return batch(() => loadNamedPreset(id)); + }, + + buildShareUrl(): string { + return buildShareUrl(); + }, + + applyUrlParams(search?: string): boolean { + return batch(() => applyUrlParams(search)); + }, + + // ----- Bus ---------------------------------------------------------------- + + emit(topic: string, data: unknown): void { + // Lossy cast — the probe is intentionally weakly typed. + coreBus.emit(topic as never, data as never); + }, + + on(topic: string, handler): () => void { + return coreBus.onPrefix(topic, handler); + }, }; /** diff --git a/playground/src/dev/probe-smoke.ts b/playground/src/dev/probe-smoke.ts new file mode 100644 index 0000000..3cafbc9 --- /dev/null +++ b/playground/src/dev/probe-smoke.ts @@ -0,0 +1,121 @@ +/** + * Tiny smoke check that exercises the debug probe API. Run from the dev + * console after the page loads: + * + * import('./dev/probe-smoke.ts').then((m) => m.smokeCheck()); + * + * Returns a list of pass/fail results. Useful for manual verification when + * Playwright isn't available. + * + * This file is NOT bundled by default — Vite only includes referenced + * modules and `dev/probe-smoke.ts` is opt-in. + */ + +export interface SmokeResult { + name: string; + ok: boolean; + detail?: string; +} + +export async function smokeCheck(): Promise { + const results: SmokeResult[] = []; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const probe = (window as unknown as { __nisps?: any }).__nisps; + if (!probe) { + results.push({ name: 'probe installed', ok: false, detail: 'window.__nisps missing' }); + return results; + } + results.push({ name: 'probe installed', ok: true }); + + try { + await probe.__init(); + results.push({ name: '__init', ok: true }); + } catch (err) { + results.push({ name: '__init', ok: false, detail: String(err) }); + } + + try { + probe.setInputs(0.5, 0.5); + const outs = probe.getOutputs(); + results.push({ + name: 'setInputs(0.5, 0.5)', + ok: outs.length > 0, + detail: `outputs.length = ${outs.length}`, + }); + } catch (err) { + results.push({ name: 'setInputs', ok: false, detail: String(err) }); + } + + try { + probe.thumbsUp(); + results.push({ name: 'thumbsUp', ok: true }); + } catch (err) { + results.push({ name: 'thumbsUp', ok: false, detail: String(err) }); + } + + try { + probe.thumbsDown(); + results.push({ name: 'thumbsDown', ok: true }); + } catch (err) { + results.push({ name: 'thumbsDown', ok: false, detail: String(err) }); + } + + try { + const before = probe.getWeights(); + probe.pushSnapshot('smoke'); + probe.randomise(); + probe.popSnapshot(); + const after = probe.getWeights(); + const restored = before.length > 0 && after.length === before.length; + results.push({ name: 'snapshot push/pop', ok: restored }); + } catch (err) { + results.push({ name: 'snapshot push/pop', ok: false, detail: String(err) }); + } + + try { + probe.setAxis('boldness', 0.7); + probe.setAxis('memory', 0.3); + probe.setAxis('precision', 0.5); + results.push({ name: 'compound axes', ok: true }); + } catch (err) { + results.push({ name: 'compound axes', ok: false, detail: String(err) }); + } + + try { + const ok = probe.refreshHeatmap(true); + const cells = probe.getHeatmapCells(); + results.push({ + name: 'heatmap', + ok: cells.length > 0, + detail: `took=${ok} cells=${cells.length}`, + }); + } catch (err) { + results.push({ name: 'heatmap', ok: false, detail: String(err) }); + } + + try { + const hist = probe.getWeightHistogram(); + const status = probe.getWeightStatus(); + results.push({ + name: 'weight health', + ok: hist.length === 10, + detail: `status=${status} bins=${hist.length}`, + }); + } catch (err) { + results.push({ name: 'weight health', ok: false, detail: String(err) }); + } + + try { + const url = probe.buildShareUrl(); + const ok = typeof url === 'string' && url.startsWith('?session='); + results.push({ name: 'buildShareUrl', ok, detail: url.slice(0, 60) }); + } catch (err) { + results.push({ name: 'buildShareUrl', ok: false, detail: String(err) }); + } + + return results; +} + +if (typeof window !== 'undefined') { + (window as unknown as { smokeCheck?: typeof smokeCheck }).smokeCheck = smokeCheck; +} diff --git a/playground/src/features/control-routing.ts b/playground/src/features/control-routing.ts new file mode 100644 index 0000000..55e6a7a --- /dev/null +++ b/playground/src/features/control-routing.ts @@ -0,0 +1,77 @@ +/** + * Compound-axis → store routing. + * + * Converts the resolved parameter map from `controlStore.resolveParams()` + * into concrete writes against: + * - `inputStore` (zoom, deadzone, inputCurve, smoothing, momentumZoom) + * - `outputStore` (slewRate) + * - `explorationStore` (noiseCap, noiseGrowth, noiseDecay, learningRate, weightDecay) + * + * Called as a Solid `createEffect` from the mode runtime; whenever any axis + * changes, this re-applies. Per-store setters internally clamp the values. + * + * Note: `inputCurve` and `slewRate` from the Precision axis intentionally + * use the centered_power exponent and slew-rate-per-frame contracts; the + * resolver mirrors the legacy table values. + */ + +import { controlStore } from '../stores/control-store'; +import { inputStore } from '../stores/input-store'; +import { outputStore } from '../stores/output-store'; +import { explorationStore } from '../stores/exploration-store'; +import type { MomentumZoomMode } from '../input/pipeline'; + +let lastApplied = ''; + +/** + * Apply the resolved compound-axis params. Returns true if any store was + * modified. + */ +export function applyControlRouting(): boolean { + const params = controlStore.resolveParams(); + // Coalesce: only re-apply if any value changed. + const sig = JSON.stringify({ + z: params['zoom'], + nc: params['noiseCap'], + ng: params['noiseGrowth'], + nd: params['noiseDecay'], + lr: params['learningRate'], + wd: params['weightDecay'], + ic: params['inputCurve'], + dz: params['deadzone'], + sm: params['smoothing'], + sr: params['slewRate'], + mz: params['momentumZoom'], + }); + if (sig === lastApplied) return false; + lastApplied = sig; + + const num = (k: string): number | undefined => + typeof params[k] === 'number' ? (params[k] as number) : undefined; + + // Input pipeline + const z = num('zoom'); + if (z !== undefined) inputStore.setZoom(z); + const dz = num('deadzone'); + if (dz !== undefined) inputStore.setDeadzone(dz); + const ic = num('inputCurve'); + if (ic !== undefined) inputStore.setInputCurve(ic); + const ism = num('smoothing'); + if (ism !== undefined) inputStore.setSmoothing(ism); + const mz = params['momentumZoom']; + if (typeof mz === 'string') { + const valid: MomentumZoomMode[] = ['off', 'gentle', 'strong']; + if (valid.includes(mz as MomentumZoomMode)) { + inputStore.setMomentumZoom(mz as MomentumZoomMode); + } + } + + // Output pipeline + const sr = num('slewRate'); + if (sr !== undefined) outputStore.setSlewRate(sr); + + // Exploration / RL + explorationStore.applyCompoundParams(params); + + return true; +} diff --git a/playground/src/features/heatmap-sampler.ts b/playground/src/features/heatmap-sampler.ts new file mode 100644 index 0000000..a908a1d --- /dev/null +++ b/playground/src/features/heatmap-sampler.ts @@ -0,0 +1,160 @@ +/** + * Heatmap sampler — samples MLP across a 2D input window, reduces outputs + * to a scalar per cell. + * + * Three color modes: + * - 'luminance' → mean output magnitude + * - 'variance' → output variance + * - 'divergence' → distance from a reference (center) output + * + * Sampler is throttled: `update` no-ops if called more than once per + * `MIN_INTERVAL_MS`. Caller drives updates via `coreBus.on('ml.delta_update')` + * and `coreBus.on('ml.trained')` events plus the input `zoom` accessor. + * + * The sampler returns a `Float32Array` of length `resolution * resolution` + * directly consumable by the `Heatmap` primitive. Stale grid values stay + * around between updates so the canvas doesn't blank. + */ + +import { mlStore } from '../stores/ml-store'; + +export type HeatmapColorMode = 'luminance' | 'variance' | 'divergence'; + +const MIN_INTERVAL_MS = 200; // 5 updates / sec + +export interface HeatmapSamplerOptions { + /** Grid size N×N. Default 16. */ + resolution?: number; + /** Initial color mode. */ + colorMode?: HeatmapColorMode; +} + +export interface HeatmapWindow { + /** Anchor point in input space [0,1]. */ + cx: number; + cy: number; + /** Side length of the sampled window (zoom level). */ + zoom: number; +} + +export class HeatmapSampler { + resolution: number; + colorMode: HeatmapColorMode; + private cells: Float32Array; + private lastSampleMs = 0; + private pending = false; + + constructor(opts: HeatmapSamplerOptions = {}) { + this.resolution = Math.max(2, Math.min(64, opts.resolution ?? 16)); + this.colorMode = opts.colorMode ?? 'luminance'; + this.cells = new Float32Array(this.resolution * this.resolution); + } + + /** Current grid for binding to the Heatmap primitive. */ + getCells(): Float32Array { + return this.cells; + } + + setColorMode(mode: HeatmapColorMode): void { + this.colorMode = mode; + } + + setResolution(n: number): void { + const r = Math.max(2, Math.min(64, n | 0)); + if (r === this.resolution) return; + this.resolution = r; + this.cells = new Float32Array(r * r); + } + + /** + * Refresh the grid. Throttled to MIN_INTERVAL_MS. + * + * Returns `true` if a fresh sample was taken, `false` if throttled. + * Reads MLP via `mlStore.inferBatch` (no-op if WASM not ready). + */ + update(window: HeatmapWindow, force: boolean = false): boolean { + const now = performance.now(); + if (!force && now - this.lastSampleMs < MIN_INTERVAL_MS) { + this.pending = true; + return false; + } + this.lastSampleMs = now; + this.pending = false; + + const N = this.resolution; + const total = N * N; + const points: Array = new Array(total); + + const z = Math.max(0.01, Math.min(1, window.zoom)); + const half = z * 0.5; + for (let y = 0; y < N; y++) { + for (let x = 0; x < N; x++) { + const u = N > 1 ? x / (N - 1) : 0.5; + const v = N > 1 ? y / (N - 1) : 0.5; + const px = window.cx + (u - 0.5) * z; // could go negative; the sampler clamps + const py = window.cy + (v - 0.5) * z; + points[y * N + x] = [ + Math.max(0, Math.min(1, px)), + Math.max(0, Math.min(1, py)), + ]; + // Note: we'd love to compute half-based bounds but the existing input + // pipeline already clamps, so just clamp here. + void half; + } + } + + const flat = mlStore.inferBatch(points); + if (flat.length === 0) { + // WASM not ready — cells remain at last value. + return false; + } + const outSize = mlStore.state.outputSize; + + // Reduce per cell. + let referenceMean = 0; + if (this.colorMode === 'divergence') { + // Reference = center cell (closest to N/2, N/2). + const cy = Math.floor(N / 2); + const cx = Math.floor(N / 2); + const base = (cy * N + cx) * outSize; + let sum = 0; + for (let i = 0; i < outSize; i++) sum += flat[base + i] ?? 0; + referenceMean = sum / outSize; + } + + for (let i = 0; i < total; i++) { + const base = i * outSize; + let sum = 0; + let sumSq = 0; + for (let j = 0; j < outSize; j++) { + const v = flat[base + j] ?? 0; + sum += v; + sumSq += v * v; + } + const mean = sum / outSize; + const variance = Math.max(0, sumSq / outSize - mean * mean); + switch (this.colorMode) { + case 'luminance': + this.cells[i] = mean; + break; + case 'variance': + this.cells[i] = variance; + break; + case 'divergence': + this.cells[i] = Math.abs(mean - referenceMean); + break; + } + } + return true; + } + + /** True if a call is queued behind throttling. */ + hasPendingUpdate(): boolean { + return this.pending; + } + + reset(): void { + this.cells.fill(0); + this.lastSampleMs = 0; + } +} diff --git a/playground/src/features/mic-input.ts b/playground/src/features/mic-input.ts new file mode 100644 index 0000000..a5fac47 --- /dev/null +++ b/playground/src/features/mic-input.ts @@ -0,0 +1,138 @@ +/** + * Microphone input — captures user audio and emits audio analysis features + * for modes that take audio_in as primary input (XIASRI, SoundAnalysisMIDI). + * + * Uses an `AudioWorkletNode` for the analysis. Keeps things simple by + * computing four lightweight features in the main thread via AnalyserNode: + * + * - energy: RMS amplitude (0..1) + * - brightness: spectral centroid normalised to 0..1 + * - pitch: dominant peak frequency normalised log scale + * - aperiodicity: 1 - peak_strength (0..1) + * + * Stream 10 doesn't ship a fully featured DSP analyser — this is a + * pragmatic pipeline that gets values into the MLP. Firmware-equivalent + * XIASRI analysis is a deeper port (out of scope here). + * + * Usage: call `start()` from a user gesture; subscribe via `getFeatures()` + * (Float32Array of length 4). `stop()` releases the mic. + */ + +const MIN_FREQ = 80; +const MAX_FREQ = 1200; +const FFT_SIZE = 1024; + +export interface MicFeatures { + energy: number; + brightness: number; + pitch: number; + aperiodicity: number; +} + +export interface MicInputOptions { + /** Optional shared AudioContext. If absent, creates one. */ + audioContext?: AudioContext; +} + +export class MicInput { + private ctx: AudioContext | null = null; + private stream: MediaStream | null = null; + private analyser: AnalyserNode | null = null; + private freqBuf: Float32Array = new Float32Array(new ArrayBuffer(FFT_SIZE / 2 * 4)); + private timeBuf: Float32Array = new Float32Array(new ArrayBuffer(FFT_SIZE * 4)); + private features: MicFeatures = { energy: 0, brightness: 0, pitch: 0, aperiodicity: 1 }; + private running = false; + + constructor(private opts: MicInputOptions = {}) {} + + /** Start mic capture. Resolves once audio is flowing. */ + async start(): Promise { + if (this.running) return; + if (typeof navigator === 'undefined' || !navigator.mediaDevices) { + throw new Error('[mic-input] navigator.mediaDevices unavailable'); + } + this.ctx = this.opts.audioContext ?? new AudioContext(); + if (this.ctx.state === 'suspended') await this.ctx.resume(); + this.stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + const src = this.ctx.createMediaStreamSource(this.stream); + this.analyser = this.ctx.createAnalyser(); + this.analyser.fftSize = FFT_SIZE; + this.analyser.smoothingTimeConstant = 0.6; + src.connect(this.analyser); + this.running = true; + } + + async stop(): Promise { + if (!this.running) return; + this.running = false; + if (this.stream) { + for (const t of this.stream.getTracks()) t.stop(); + this.stream = null; + } + if (this.ctx && !this.opts.audioContext) { + try { await this.ctx.close(); } catch { /* ignore */ } + } + this.ctx = null; + this.analyser = null; + } + + isRunning(): boolean { + return this.running; + } + + /** Compute and return the latest feature vector. */ + getFeatures(): MicFeatures { + if (!this.running || !this.analyser || !this.ctx) return this.features; + const sr = this.ctx.sampleRate; + const a = this.analyser; + a.getFloatFrequencyData(this.freqBuf); + a.getFloatTimeDomainData(this.timeBuf); + + // RMS energy + let sumSq = 0; + for (let i = 0; i < this.timeBuf.length; ++i) { + const s = this.timeBuf[i]!; + sumSq += s * s; + } + const rms = Math.sqrt(sumSq / this.timeBuf.length); + const energy = Math.min(1, rms * 4); + + // Convert dB → linear, find spectral centroid + peak + let sumMag = 0; + let sumWeighted = 0; + let peakBin = 0; + let peakMag = -Infinity; + const binHz = sr / FFT_SIZE; + const minBin = Math.floor(MIN_FREQ / binHz); + const maxBin = Math.min(this.freqBuf.length - 1, Math.ceil(MAX_FREQ / binHz)); + for (let i = 0; i < this.freqBuf.length; ++i) { + const db = this.freqBuf[i]!; + const lin = Math.pow(10, db / 20); + sumMag += lin; + sumWeighted += i * binHz * lin; + if (i >= minBin && i <= maxBin && db > peakMag) { + peakMag = db; + peakBin = i; + } + } + const centroidHz = sumMag > 1e-12 ? sumWeighted / sumMag : 0; + const brightness = Math.min(1, centroidHz / (sr * 0.25)); + + // Pitch (log scale 80–1200 Hz) + const pitchHz = peakBin * binHz; + const pitchNorm = pitchHz <= MIN_FREQ + ? 0 + : pitchHz >= MAX_FREQ + ? 1 + : (Math.log(pitchHz / MIN_FREQ) / Math.log(MAX_FREQ / MIN_FREQ)); + + // Aperiodicity: 1 - relative peak height (vs spectral mean) + const meanLin = sumMag / this.freqBuf.length; + const peakLin = Math.pow(10, peakMag / 20); + const ratio = meanLin > 1e-12 ? peakLin / meanLin : 0; + const aperiodicity = Math.max(0, Math.min(1, 1 - Math.min(1, ratio / 30))); + + this.features = { energy, brightness, pitch: pitchNorm, aperiodicity }; + return this.features; + } +} diff --git a/playground/src/features/overrides.ts b/playground/src/features/overrides.ts new file mode 100644 index 0000000..e9c134b --- /dev/null +++ b/playground/src/features/overrides.ts @@ -0,0 +1,131 @@ +/** + * Override application — apply per-parameter overrides between MLP outputs + * and the engine's parameter destination. + * + * The MLP emits values in [0, 1]. The mode schema specifies a per-parameter + * `min`, `max`, and `curve`. Modes can layer overrides on top via + * `modeStore.setOverride` to: + * - Re-curve the value (`curve` and `curveParam`). + * - Mute it (replace with `fixedValue` ∈ [hardMin, hardMax]). + * - Freeze it (hold last processed value). + * - Tighten the active range (`min`/`max` ⊂ schema bounds). + * + * `applyOverrides` walks the schema params in order, looks up overrides for + * each, and writes results into a target Float32Array of the same length as + * the MLP slice. The function is pure — caller manages buffer reuse. + * + * The output buffer values are NORMALISED to [0, 1] in their effective + * range so the mode's audio engine, MIDI router, or visualiser can scale + * uniformly. Engines/MIDI consumers that need actual min/max units can use + * `paramsToSliderValues` from `mode-helpers`. + */ + +import type { ParamOverride } from '../stores/mode-store'; +import type { Param } from '../modes/generated/types'; +import { applyCurve, type CurveName, clamp01 } from '../output/curves'; + +/** + * Result of applying overrides: + * - `values` is a Float32Array of length `params.length`, each in [0, 1] + * representing the post-override normalised parameter value. + * - `freezeMask` (optional) marks per-output frozen flags. Only allocated + * when at least one param is frozen — saves work on the typical case. + */ +export interface OverrideResult { + values: Float32Array; + freezeMask: Uint8Array | null; + /** Number of muted/frozen entries (for telemetry). */ + affectedCount: number; +} + +/** + * Apply per-param overrides to the slice of MLP outputs corresponding to + * the schema's params. + * + * @param raw processed (post output-pipeline) MLP outputs in [0,1] + * @param prev previous post-override outputs (for freeze hold) + * @param params mode schema params + * @param overrides per-name override map (sparse; missing keys fall through) + * @param dest optional destination Float32Array; allocated if absent + */ +export function applyOverrides( + raw: Float32Array, + prev: Float32Array | null, + params: ReadonlyArray, + overrides: Record, + dest?: Float32Array, +): OverrideResult { + const n = params.length; + const out = dest && dest.length === n ? dest : new Float32Array(n); + let affected = 0; + let freezeMask: Uint8Array | null = null; + + for (let i = 0; i < n; ++i) { + const p = params[i]!; + const ov = overrides[p.name]; + const r = clamp01(raw[i] ?? 0); + + if (!ov) { + // No override → identity (raw normalised value). + out[i] = r; + continue; + } + + if (ov.frozen) { + // Freeze: hold last processed value. Track in freezeMask so the + // output pipeline can also honor it next frame. + if (!freezeMask) freezeMask = new Uint8Array(n); + freezeMask[i] = 1; + out[i] = prev?.[i] ?? r; + affected++; + continue; + } + + if (ov.muted) { + // Mute: replace with fixedValue. fixedValue is stored in schema units + // [min, max] — normalise back to [0, 1] in the override's effective range. + const range = ov.max - ov.min; + const norm = range > 0 ? (ov.fixedValue - ov.min) / range : 0.5; + out[i] = clamp01(norm); + affected++; + continue; + } + + // Apply curve (if any), then clamp. + let v = r; + if (ov.curve && ov.curve !== 'linear') { + v = applyCurve(ov.curve as CurveName, r, ov.curveParam); + } + out[i] = clamp01(v); + } + + return { values: out, freezeMask, affectedCount: affected }; +} + +/** + * Build a Uint8Array of length `outputSize` marking every pinned output + * with a 1. Used by `WasmIML.moveWeights` to skip pinned weights during RL + * perturbation. Combines: + * - explicit param-pins from sessionStore + * - per-param `pinned` flag in modeStore overrides + */ +export function buildPinMask( + outputSize: number, + modeId: string, + params: ReadonlyArray, + overrides: Record, + paramPins: ReadonlyArray<{ key: string; outputIndex: number }>, +): Uint8Array { + const mask = new Uint8Array(outputSize); + for (let i = 0; i < params.length && i < outputSize; ++i) { + const ov = overrides[params[i]!.name]; + if (ov && ov.pinned) mask[i] = 1; + } + for (const pin of paramPins) { + if (!pin.key.startsWith(modeId + ':')) continue; + if (pin.outputIndex >= 0 && pin.outputIndex < outputSize) { + mask[pin.outputIndex] = 1; + } + } + return mask; +} diff --git a/playground/src/features/session-preset.ts b/playground/src/features/session-preset.ts new file mode 100644 index 0000000..81a4849 --- /dev/null +++ b/playground/src/features/session-preset.ts @@ -0,0 +1,288 @@ +/** + * Session preset — full app-state snapshot for save/load and URL sharing. + * + * A preset captures: control axes + offsets, exploration config, output + * pipeline settings, input pipeline settings, mode override map, the + * active mode id. ML weights are NOT included by default (large), but + * `serializeWithWeights` is provided for power users. + * + * URL sharing uses a compact encoding: + * ?session= + * ?boldness=...&memory=...&precision=... + * Either form rehydrates on next load. + */ + +import { controlStore } from '../stores/control-store'; +import { explorationStore } from '../stores/exploration-store'; +import { modeStore } from '../stores/mode-store'; +import { inputStore } from '../stores/input-store'; +import { outputStore } from '../stores/output-store'; +import { mlStore } from '../stores/ml-store'; +import { sessionStore } from '../stores/session-store'; + +export interface SessionPresetPayload { + v: 1; + modeId: string | null; + control: { + boldness: number; + memory: number; + precision: number; + presetId: string | null; + offsets: typeof controlStore.state.offsets; + }; + exploration: { + spread: number; + noiseFloor: number; + noiseCap: number; + noiseGrowth: number; + noiseDecay: number; + learningRate: number; + weightDecay: number; + }; + output: { + globalCurve: number; + smoothing: number; + slewRate: number | null; // null encodes Infinity + freezeOutput: boolean; + }; + input: { + zoom: number; + anchorMode: 'auto' | 'sticky' | 'center'; + deadzone: number; + inputCurve: number; + smoothing: number; + momentumZoom: 'off' | 'gentle' | 'strong'; + invertX: boolean; + invertY: boolean; + }; + modeOverrides: typeof modeStore.state.overrides; + /** Base64-encoded weights, optional. */ + weights?: string; +} + +function encode(s: string): string { + // base64url encoding + const b64 = btoa(s); + return b64.replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, ''); +} + +function decode(s: string): string | null { + try { + let b64 = s.replace(/-/g, '+').replace(/_/g, '/'); + while (b64.length % 4 !== 0) b64 += '='; + return atob(b64); + } catch { + return null; + } +} + +function f32ToBase64(arr: Float32Array): string { + const u8 = new Uint8Array(arr.buffer, arr.byteOffset, arr.byteLength); + let s = ''; + for (let i = 0; i < u8.length; i++) s += String.fromCharCode(u8[i]!); + return encode(s); +} + +function base64ToF32(s: string): Float32Array | null { + const raw = decode(s); + if (!raw) return null; + const u8 = new Uint8Array(raw.length); + for (let i = 0; i < raw.length; i++) u8[i] = raw.charCodeAt(i); + // Round to 4 bytes + const len = u8.byteLength - (u8.byteLength % 4); + return new Float32Array(u8.buffer, u8.byteOffset, len / 4).slice(); +} + +/** + * Read every store and assemble a SessionPresetPayload. If + * `withWeights = true`, also encodes the current MLP weight blob. + */ +export function captureSessionPreset(withWeights: boolean = false): SessionPresetPayload { + const c = controlStore.state; + const e = explorationStore.state; + const o = outputStore.config; + const i = inputStore.config; + + const payload: SessionPresetPayload = { + v: 1, + modeId: modeStore.state.activeModeId, + control: { + boldness: c.boldness, + memory: c.memory, + precision: c.precision, + presetId: c.presetId, + offsets: JSON.parse(JSON.stringify(c.offsets)), + }, + exploration: { + spread: e.spread, + noiseFloor: e.noiseFloor, + noiseCap: e.noiseCap, + noiseGrowth: e.noiseGrowth, + noiseDecay: e.noiseDecay, + learningRate: e.learningRate, + weightDecay: e.weightDecay, + }, + output: { + globalCurve: o.globalCurve, + smoothing: o.smoothing, + slewRate: isFinite(o.slewRate) ? o.slewRate : null, + freezeOutput: o.freezeOutput, + }, + input: { + zoom: i.zoom, + anchorMode: i.anchorMode, + deadzone: i.deadzone, + inputCurve: i.inputCurve, + smoothing: i.smoothing, + momentumZoom: i.momentumZoom, + invertX: i.invertX, + invertY: i.invertY, + }, + modeOverrides: JSON.parse(JSON.stringify(modeStore.state.overrides)), + }; + + if (withWeights) { + const w = mlStore.getWeights(); + if (w.length > 0) { + payload.weights = f32ToBase64(w); + } + } + return payload; +} + +/** + * Restore a SessionPresetPayload into all stores. Best-effort: missing + * fields fall back to current values. Returns true if at least one store + * was modified. + */ +export function restoreSessionPreset(payload: SessionPresetPayload): boolean { + if (!payload || payload.v !== 1) return false; + + // Mode first (so override map is keyed correctly). + if (payload.modeId && payload.modeId !== modeStore.state.activeModeId) { + modeStore.switchMode(payload.modeId); + } + + // Control axes & offsets. + if (payload.control) { + controlStore.setAxis('boldness', payload.control.boldness); + controlStore.setAxis('memory', payload.control.memory); + controlStore.setAxis('precision', payload.control.precision); + if (payload.control.presetId) controlStore.applyPreset(payload.control.presetId); + // Manually rewrite offsets — the store doesn't have a public bulk setter + // (control-store doesn't expose it because it's a private field). + // Use clearOffsets + setOffset roundtrip. + for (const axis of ['boldness', 'memory', 'precision'] as const) { + controlStore.clearOffsets(axis); + const off = payload.control.offsets[axis] ?? {}; + for (const [k, v] of Object.entries(off)) { + if (typeof v === 'number') controlStore.setOffset(axis, k, v); + } + } + } + + if (payload.exploration) { + explorationStore.setSpread(payload.exploration.spread); + explorationStore.setNoiseFloor(payload.exploration.noiseFloor); + explorationStore.setNoiseCap(payload.exploration.noiseCap); + explorationStore.setNoiseGrowth(payload.exploration.noiseGrowth); + explorationStore.setNoiseDecay(payload.exploration.noiseDecay); + explorationStore.setLearningRate(payload.exploration.learningRate); + explorationStore.setWeightDecay(payload.exploration.weightDecay); + } + + if (payload.output) { + outputStore.setGlobalCurve(payload.output.globalCurve); + outputStore.setSmoothing(payload.output.smoothing); + outputStore.setSlewRate(payload.output.slewRate ?? Infinity); + outputStore.setFreezeOutput(payload.output.freezeOutput); + } + + if (payload.input) { + inputStore.setZoom(payload.input.zoom); + inputStore.setAnchorMode(payload.input.anchorMode); + inputStore.setDeadzone(payload.input.deadzone); + inputStore.setInputCurve(payload.input.inputCurve); + inputStore.setSmoothing(payload.input.smoothing); + inputStore.setMomentumZoom(payload.input.momentumZoom); + inputStore.setInvert(payload.input.invertX, payload.input.invertY); + } + + if (payload.modeOverrides) { + for (const modeId of Object.keys(payload.modeOverrides)) { + const map = payload.modeOverrides[modeId] ?? {}; + modeStore.clearAllOverrides(modeId); + for (const [name, ov] of Object.entries(map)) { + modeStore.setOverride(modeId, name, ov); + } + } + } + + if (payload.weights) { + const w = base64ToF32(payload.weights); + if (w && mlStore.iml && w.length > 0) { + try { mlStore.setWeights(w); } catch { /* ignore size mismatch */ } + } + } + + return true; +} + +/** Save preset to sessionStore (named blob in localStorage). */ +export function saveNamedPreset(name: string, withWeights: boolean = false): void { + const payload = captureSessionPreset(withWeights); + sessionStore.savePreset(name, payload); +} + +/** Load a named preset by id from sessionStore. */ +export function loadNamedPreset(id: string): boolean { + const preset = sessionStore.state.presets.find((p) => p.id === id); + if (!preset) return false; + return restoreSessionPreset(preset.payload as SessionPresetPayload); +} + +// --------------------------------------------------------------------------- +// URL sharing +// --------------------------------------------------------------------------- + +/** Build a query-string fragment from current state (without weights). */ +export function buildShareUrl(): string { + const payload = captureSessionPreset(false); + const json = JSON.stringify(payload); + const enc = encode(json); + return `?session=${enc}`; +} + +/** Parse `window.location.search` and apply a session if present. */ +export function applyUrlParams(search: string = window.location.search): boolean { + const sp = new URLSearchParams(search); + // Compact ?boldness=...&memory=... form. + const cb = sp.get('boldness'); + const cm = sp.get('memory'); + const cp = sp.get('precision'); + let applied = false; + if (cb !== null || cm !== null || cp !== null) { + if (cb !== null) controlStore.setAxis('boldness', clamp01(parseFloat(cb))); + if (cm !== null) controlStore.setAxis('memory', clamp01(parseFloat(cm))); + if (cp !== null) controlStore.setAxis('precision', clamp01(parseFloat(cp))); + applied = true; + } + const sess = sp.get('session'); + if (sess) { + const json = decode(sess); + if (json) { + try { + const payload = JSON.parse(json) as SessionPresetPayload; + applied = restoreSessionPreset(payload) || applied; + } catch { + // Ignore malformed + } + } + } + return applied; +} + +function clamp01(v: number): number { + if (!isFinite(v)) return 0; + return v < 0 ? 0 : v > 1 ? 1 : v; +} diff --git a/playground/src/features/snapshots.ts b/playground/src/features/snapshots.ts new file mode 100644 index 0000000..7e1db3f --- /dev/null +++ b/playground/src/features/snapshots.ts @@ -0,0 +1,123 @@ +/** + * Snapshot helpers — bridge sessionStore.pushSnapshot/popSnapshot to the + * actual MLP weights via mlStore. Auto-snapshot before train/randomize/ + * thumbs-down. + */ + +import { mlStore } from '../stores/ml-store'; +import { sessionStore } from '../stores/session-store'; +import { explorationStore } from '../stores/exploration-store'; +import { inputStore } from '../stores/input-store'; + +/** Capture current weights + noise level + zoom level. */ +export function autoSnapshot(tag: string): void { + if (!mlStore.iml) return; + const w = mlStore.getWeights(); + if (w.length === 0) return; + sessionStore.pushSnapshot(tag, { + noiseLevel: explorationStore.state.noiseLevel, + zoomLevel: inputStore.config.zoom, + weights: w, + }); +} + +/** + * Pop the most recent snapshot and restore weights. Returns true on success. + */ +export function undoLastSnapshot(): boolean { + const snap = sessionStore.popSnapshot(); + if (!snap || !snap.weights) return false; + if (!mlStore.iml) return false; + try { + mlStore.setWeights(snap.weights); + explorationStore.setNoiseLevel(snap.noiseLevel); + return true; + } catch { + return false; + } +} + +/** Restore a specific snapshot by id. */ +export function restoreSnapshotById(id: string): boolean { + const snap = sessionStore.jumpToSnapshot(id); + if (!snap || !snap.weights) return false; + if (!mlStore.iml) return false; + try { + mlStore.setWeights(snap.weights); + explorationStore.setNoiseLevel(snap.noiseLevel); + return true; + } catch { + return false; + } +} + +/** + * A/B compare helpers — capture the current weights as A, the next state + * becomes B. Toggle swaps in O(1). + */ +export function captureA(): void { + if (!mlStore.iml) return; + const w = mlStore.getWeights(); + if (w.length === 0) return; + sessionStore.captureA({ + id: `a-${Date.now().toString(36)}`, + tag: 'A', + timestamp: Date.now(), + noiseLevel: explorationStore.state.noiseLevel, + zoomLevel: inputStore.config.zoom, + weights: w, + }); +} + +export function toggleAB(): 'A' | 'B' { + if (!mlStore.iml) return sessionStore.state.ab.live; + // Capture the current live state on first toggle. + const cur = mlStore.getWeights(); + const live = sessionStore.state.ab.live; + if (live === 'B' && !sessionStore.state.ab.b) { + // First toggle from B → A: capture the current state as B. + sessionStore.captureB({ + id: `b-${Date.now().toString(36)}`, + tag: 'B', + timestamp: Date.now(), + noiseLevel: explorationStore.state.noiseLevel, + zoomLevel: inputStore.config.zoom, + weights: cur, + }); + } else if (live === 'A') { + // Going back from A → B: update B with the current edits before flipping. + sessionStore.captureB({ + id: `b-${Date.now().toString(36)}`, + tag: 'B', + timestamp: Date.now(), + noiseLevel: explorationStore.state.noiseLevel, + zoomLevel: inputStore.config.zoom, + weights: cur, + }); + } + const next = sessionStore.toggleAB(); + // Apply the side we just switched to. + const target = next === 'A' ? sessionStore.state.ab.a : sessionStore.state.ab.b; + if (target?.weights) { + try { mlStore.setWeights(target.weights); } catch { /* ignore */ } + explorationStore.setNoiseLevel(target.noiseLevel); + } + return next; +} + +export function acceptB(): void { + sessionStore.acceptB(); +} + +export function revertToA(): void { + if (!mlStore.iml) { + sessionStore.revertToA(); + return; + } + const a = sessionStore.state.ab.a; + if (a?.weights) { + try { mlStore.setWeights(a.weights); } catch { /* ignore */ } + explorationStore.setNoiseLevel(a.noiseLevel); + } + sessionStore.revertToA(); +} diff --git a/playground/src/features/trail.ts b/playground/src/features/trail.ts new file mode 100644 index 0000000..77612e7 --- /dev/null +++ b/playground/src/features/trail.ts @@ -0,0 +1,55 @@ +/** + * Trail tracking — fixed-size ring of recent input positions for the JoyMap + * vanishing trail and tap-to-return. + * + * Uses `performance.now()` for timestamps. The JoyMap primitive filters by + * age internally (5-second window), so the buffer can be larger than the + * displayed window. + */ + +import { createSignal, type Accessor } from 'solid-js'; + +export interface TrailPoint { + x: number; + y: number; + t: number; // performance.now() +} + +const MAX_POINTS = 300; +const MIN_DELTA_PX = 0.005; // skip near-duplicate points (in 0..1 space) + +export interface TrailRing { + /** Reactive accessor for current points. */ + points: Accessor>; + push(x: number, y: number): void; + clear(): void; +} + +export function createTrailRing(): TrailRing { + const [points, setPoints] = createSignal>([], { equals: false }); + + let buf: TrailPoint[] = []; + + return { + points, + push(x, y) { + const now = performance.now(); + // Skip if too close to last point. + const last = buf.length > 0 ? buf[buf.length - 1] : null; + if (last) { + const dx = x - last.x; + const dy = y - last.y; + if (dx * dx + dy * dy < MIN_DELTA_PX * MIN_DELTA_PX && now - last.t < 50) { + return; + } + } + buf.push({ x, y, t: now }); + if (buf.length > MAX_POINTS) buf.shift(); + setPoints(buf.slice()); + }, + clear() { + buf = []; + setPoints([]); + }, + }; +} diff --git a/playground/src/features/weight-health.ts b/playground/src/features/weight-health.ts new file mode 100644 index 0000000..31ff255 --- /dev/null +++ b/playground/src/features/weight-health.ts @@ -0,0 +1,124 @@ +/** + * Weight health + gradient flow analysis. + * + * Pure functions over Float32Array weight buffers and `LayerStats[]` from + * the WasmIML. Mode UI binds these to the WeightHealth and GradientFlow + * primitives. + */ + +import type { LayerStats } from '../ml/types'; +import type { WeightStatus } from '../primitives/WeightHealth'; +import type { GradientStatus } from '../primitives/GradientFlow'; + +const HISTOGRAM_BINS = 10; +/** Magnitude bin edges: [0, 0.01, 0.05, 0.15, 0.4, 0.8, 1.5, 2.5, 4, 7, ∞] */ +const BIN_EDGES = [0, 0.01, 0.05, 0.15, 0.4, 0.8, 1.5, 2.5, 4, 7]; + +/** Build a 10-bin histogram of weight magnitudes from a flat weight array. */ +export function weightHistogram(weights: Float32Array): number[] { + const bins = new Array(HISTOGRAM_BINS).fill(0); + if (weights.length === 0) return bins; + for (let i = 0; i < weights.length; ++i) { + const m = Math.abs(weights[i]!); + let bin = HISTOGRAM_BINS - 1; + for (let b = 0; b < HISTOGRAM_BINS - 1; ++b) { + if (m < BIN_EDGES[b + 1]!) { + bin = b; + break; + } + } + bins[bin]++; + } + return bins; +} + +/** Determine overall network health status. */ +export function weightStatus(weights: Float32Array): WeightStatus { + if (weights.length === 0) return 'healthy'; + let dead = 0; + let saturating = 0; + for (let i = 0; i < weights.length; ++i) { + const m = Math.abs(weights[i]!); + if (m < 0.01) dead++; + else if (m > 3.0) saturating++; + } + const deadFrac = dead / weights.length; + const satFrac = saturating / weights.length; + if (satFrac > 0.4) return 'saturating'; + if (deadFrac > 0.3) return 'dead'; + return 'healthy'; +} + +/** + * Per-layer L2 norm of weight delta (after - before). Used to populate + * GradientFlow primitive. + */ +export function layerNorms(beforeWeights: Float32Array, afterWeights: Float32Array, layerSizes: ReadonlyArray): number[] { + const out: number[] = []; + let offset = 0; + for (const sz of layerSizes) { + let sum = 0; + const end = Math.min(beforeWeights.length, afterWeights.length, offset + sz); + for (let i = offset; i < end; ++i) { + const d = (afterWeights[i] ?? 0) - (beforeWeights[i] ?? 0); + sum += d * d; + } + out.push(Math.sqrt(sum)); + offset += sz; + } + return out; +} + +/** + * Detect vanishing/exploding/converged gradient flow given per-layer norms. + */ +export function gradientStatuses(norms: ReadonlyArray): GradientStatus[] { + if (norms.length === 0) return []; + // Converged check first: all near-zero. + const allTiny = norms.every((n) => n < 1e-6); + if (allTiny) return norms.map(() => 'converged' as GradientStatus); + + const out: GradientStatus[] = []; + for (let i = 0; i < norms.length; ++i) { + if (i === 0) { + out.push('healthy'); + continue; + } + const prev = norms[i - 1] ?? 0; + const curr = norms[i] ?? 0; + if (prev <= 1e-12) { + out.push('healthy'); + } else if (curr < 0.5 * prev) { + out.push('vanishing'); + } else if (curr > 2.0 * prev) { + out.push('exploding'); + } else { + out.push('healthy'); + } + } + return out; +} + +/** + * Estimate per-layer weight count given an MLP architecture. + * + * MLP layout: layer i has weights = inputs * outputs + outputs (bias). + * `[inputSize, ...hidden, outputSize]` — N+1 layer sizes, N layers. + */ +export function layerWeightCounts(arch: { inputSize: number; hidden: ReadonlyArray; outputSize: number }): number[] { + const sizes = [arch.inputSize, ...arch.hidden, arch.outputSize]; + const counts: number[] = []; + for (let i = 0; i < sizes.length - 1; ++i) { + const ins = sizes[i]!; + const outs = sizes[i + 1]!; + counts.push(ins * outs + outs); + } + return counts; +} + +/** Map LayerStats to a healthy/dead/saturating per-layer status. */ +export function layerStatsToStatus(stats: LayerStats): WeightStatus { + if (stats.saturatingFrac > 0.4) return 'saturating'; + if (stats.deadFrac > 0.3) return 'dead'; + return 'healthy'; +} diff --git a/playground/src/main.tsx b/playground/src/main.tsx index 3fdf670..69850f7 100644 --- a/playground/src/main.tsx +++ b/playground/src/main.tsx @@ -3,15 +3,23 @@ import { render } from 'solid-js/web'; import App from './App'; import './styles/tokens.css'; import { installDebugProbe } from './debug/probe'; +import { applyUrlParams } from './features/session-preset'; const root = document.getElementById('root'); if (!root) { throw new Error('Root element #root not found'); } -// Install debug probe early. It is a stub for now; stream 10 fills it in -// once WASM ML is wired up. Keeping the install path stable from day one -// makes Playwright tests insensitive to ordering. +// Install debug probe early. Playwright tests use it to drive the ML +// engine programmatically. installDebugProbe(); +// Apply URL params (?session=..., ?boldness=..., etc) before mounting so +// stores read the resolved values on first render. +try { + applyUrlParams(); +} catch (err) { + console.warn('[main] applyUrlParams failed:', err); +} + render(() => , root); diff --git a/playground/src/modes/BreakOrMode.tsx b/playground/src/modes/BreakOrMode.tsx index 9b1ce63..34a543c 100644 --- a/playground/src/modes/BreakOrMode.tsx +++ b/playground/src/modes/BreakOrMode.tsx @@ -7,32 +7,18 @@ import { ModeShell } from './ModeShell'; import { useModeRuntime } from './mode-runtime'; import { XYPad } from '../primitives/XYPad'; import { OutputDisplay } from '../primitives/OutputDisplay'; -import { SliderBank } from '../primitives/SliderBank'; import { LossPlot } from '../primitives/LossPlot'; -import { paramsToSliderConfig, outputsToSliderValues } from './mode-helpers'; import { BreakorSchema } from './generated/breakor_schema'; export const BreakOrMode: Component = () => { const schema = BreakorSchema; const runtime = useModeRuntime(schema); - const sliderConfig = paramsToSliderConfig(schema.params); - const sliderValues = () => outputsToSliderValues(runtime.processedOutputs(), schema.params); return ( ( - { - /* read-only */ - }} - /> - )} + drawerTitle="Breakor settings" primaryInput={() => ( <> { outputArea={() => ( <> { const schema = ChannelStripSchema; const runtime = useModeRuntime(schema); - const sliderConfig = paramsToSliderConfig(schema.params); - const sliderValues = () => outputsToSliderValues(runtime.processedOutputs(), schema.params); - return ( ( - { - /* read-only for now */ - }} - /> - )} + drawerTitle="Channel strip settings" primaryInput={() => ( <> { outputArea={() => ( <> { const schema = ElysiamorfSchema; const runtime = useModeRuntime(schema); - const sliderConfig = paramsToSliderConfig(schema.params); - const sliderValues = () => outputsToSliderValues(runtime.processedOutputs(), schema.params); return ( ( - { - /* read-only */ - }} - /> - )} + drawerTitle="Elysiamorf settings" primaryInput={() => ( <> { outputArea={() => ( <> { const schema = MemlceliumSchema; const runtime = useModeRuntime(schema); const [voiceSpace, setVoiceSpace] = createSignal(0); - const sliderConfig = paramsToSliderConfig(schema.params); - const sliderValues = () => outputsToSliderValues(runtime.processedOutputs(), schema.params); return ( { runtime={runtime} activeVoiceSpace={voiceSpace} onVoiceSpaceChange={setVoiceSpace} - drawerTitle="MEMLCelium voice + CV" - drawerContent={() => ( - { - /* read-only */ - }} - /> - )} + drawerTitle="MEMLCelium settings" primaryInput={() => ( <> { outputArea={() => ( <> JSX.Element; /** Output / visualisation area. */ outputArea: () => JSX.Element; - /** Optional drawer body for mode-specific settings. */ + /** Optional drawer body for mode-specific settings (replaces default). */ drawerContent?: () => JSX.Element; drawerTitle?: string; /** Active voice space index (only used if schema.voice_spaces is non-empty). */ @@ -40,6 +45,7 @@ export interface ModeShellProps { export const ModeShell: Component = (props) => { const [drawerOpen, setDrawerOpen] = createSignal(false); + const [snapshotPopupOpen, setSnapshotPopupOpen] = createSignal(false); const showVoiceSpaces = () => props.schema.ui.show_voice_space_selector && props.schema.voice_spaces.length > 0; @@ -49,6 +55,14 @@ export const ModeShell: Component = (props) => { label, })); + const noiseRings = createMemo(() => { + // Outer = noiseCap, inner = noiseLevel — both as fractions of half-window. + const cap = explorationStore.state.noiseCap; + const cur = explorationStore.state.noiseLevel; + if (cap <= 0) return null; + return [cap * 0.4, cur * 0.4] as const; + }); + return (
@@ -74,6 +88,28 @@ export const ModeShell: Component = (props) => {
+ + void props.runtime.mic.start()} + aria-label="Enable microphone" + >🎤 Mic + } + > + + + = (props) => { class={styles.audioBtn} onClick={() => void props.runtime.audio.start()} aria-label="Start audio engine" - > - ▶ Start audio - + >▶ Start audio } > - - - + >⏹ Stop audio +
-
{props.primaryInput()}
+
+ {props.primaryInput()} + inputStore.config.zoom} + anchor={() => [ + inputStore.config.anchorMode === 'center' ? 0.5 : inputStore.config.anchorX, + inputStore.config.anchorMode === 'center' ? 0.5 : inputStore.config.anchorY, + ]} + position={props.runtime.pipedInput} + trail={props.runtime.trail} + regionPins={() => sessionStore.state.regionPins} + noiseRings={noiseRings as () => readonly [number, number] | null} + frozen={props.runtime.frozen} + onTrailTap={(p) => props.runtime.snapToTrail(p)} + onLongPress={() => props.runtime.pinCurrentRegion()} + ariaLabel="Joy map navigator" + /> +
{props.outputArea()}
@@ -136,15 +182,24 @@ export const ModeShell: Component = (props) => { onThumbsUp={() => props.runtime.thumbsUp()} onThumbsDown={() => props.runtime.thumbsDown()} onUndo={() => { - // Stream 9 ships without undo wiring — no-op until session-store - // gets a snapshot/pop method exposed via the runtime. Stubbed - // so the button still appears. + // Click = pop one snapshot. Long-press = open list popup. + props.runtime.undo(); }} exampleCount={() => props.runtime.training.examples()} lastLoss={() => props.runtime.training.lastLoss()} busy={() => props.runtime.training.busy()} - canUndo={() => false} + canUndo={() => props.runtime.canUndo()} /> + 0}> + + @@ -159,20 +214,72 @@ export const ModeShell: Component = (props) => { input ({props.runtime.pipedInput()[0].toFixed(2)}, {' '} {props.runtime.pipedInput()[1].toFixed(2)}) + {' · '} + noise {explorationStore.state.noiseLevel.toFixed(3)}

- - setDrawerOpen(false)} - side="right" - title={props.drawerTitle ?? 'Mode settings'} - width={420} + setDrawerOpen(false)} + side="right" + title={props.drawerTitle ?? 'Mode settings'} + width={460} + > + + } > {props.drawerContent!()} - - + + + + {/* Snapshot list popup (long-press undo equivalent) */} + setSnapshotPopupOpen(false)} + side="right" + title="Snapshot history" + width={320} + > +
    + {(s) => ( +
  • + +
    • + )}     +
+
); }; diff --git a/playground/src/modes/PAFSynthMode.tsx b/playground/src/modes/PAFSynthMode.tsx index a268762..bdeeaca 100644 --- a/playground/src/modes/PAFSynthMode.tsx +++ b/playground/src/modes/PAFSynthMode.tsx @@ -11,9 +11,7 @@ import { ModeShell } from './ModeShell'; import { useModeRuntime } from './mode-runtime'; import { XYPad } from '../primitives/XYPad'; import { OutputDisplay } from '../primitives/OutputDisplay'; -import { SliderBank } from '../primitives/SliderBank'; import { LossPlot } from '../primitives/LossPlot'; -import { paramsToSliderConfig, outputsToSliderValues } from './mode-helpers'; import { PafSynthSchema } from './generated/paf_synth_schema'; export const PAFSynthMode: Component = () => { @@ -21,8 +19,6 @@ export const PAFSynthMode: Component = () => { const runtime = useModeRuntime(schema); const [voiceSpace, setVoiceSpace] = createSignal(0); - const sliderConfig = paramsToSliderConfig(schema.params); - const sliderValues = () => outputsToSliderValues(runtime.processedOutputs(), schema.params); return ( { runtime={runtime} activeVoiceSpace={voiceSpace} onVoiceSpaceChange={setVoiceSpace} - drawerTitle="PAF synth params" - drawerContent={() => ( - { - // Sliders are display-only here. Stream 10 wires the per-param - // override editor which writes through modeStore.setOverride. - }} - /> - )} + drawerTitle="PAF synth settings" primaryInput={() => ( <> { outputArea={() => ( <> span { + flex: 1; + font-size: var(--fs-sm); +} + +.paramList { + display: flex; + flex-direction: column; + gap: var(--sp-2); + max-height: 360px; + overflow-y: auto; +} + +.layerStats { + display: flex; + flex-direction: column; + gap: var(--sp-1); + font-family: var(--font-mono); + font-size: var(--fs-xs); +} + +.layerStatRow { + display: flex; + align-items: center; + gap: var(--sp-2); + padding: 2px var(--sp-2); + background: var(--bg-2); + border-radius: var(--r-1); +} + +.layerStatLabel { + color: var(--accent); + font-weight: 600; + min-width: 24px; +} + +.layerStatValue { + flex: 1; + color: var(--fg-mute); +} + +.layerStatStatus { + font-size: 10px; + text-transform: uppercase; + padding: 1px 6px; + border-radius: var(--r-1); + background: var(--bg-3); +} + +.layerStatStatus[data-status="dead"] { color: #999; } +.layerStatStatus[data-status="saturating"] { color: var(--warn); } +.layerStatStatus[data-status="healthy"] { color: var(--good); } + +.textInput { + background: var(--bg-2); + border: 1px solid var(--line); + border-radius: var(--r-1); + color: var(--fg); + font-family: var(--font-mono); + font-size: var(--fs-sm); + padding: var(--sp-1) var(--sp-2); + flex: 1; +} + +.hr { + border: none; + border-top: 1px dashed var(--line); + margin: var(--sp-2) 0; +} diff --git a/playground/src/modes/SettingsDrawer.tsx b/playground/src/modes/SettingsDrawer.tsx new file mode 100644 index 0000000..1f409b0 --- /dev/null +++ b/playground/src/modes/SettingsDrawer.tsx @@ -0,0 +1,656 @@ +/** + * SettingsDrawer — settings panel rendered inside ModeShell's drawer. + * + * Sections: + * - Input: deadzone, curve, smoothing, momentum, anchor mode, invert + * - Training: learning rate, weight decay + * - Exploration: spread, noise floor/cap/growth/decay, + * auto-explore (toggle, interval, intensity) + * - Output: global curve, smoothing, slew rate, freeze + * - Per-param overrides: ParamEditor list + * - Advanced: weight health histogram, gradient flow, session presets + * + * Sections are collapsible. Per-param-override editor renders inline (also + * accessible via double-tap on slider in the live readout — wired in + * ModeShell). The drawer is a child of `ModeShell`'s `drawerContent` slot. + */ + +import { Component, createSignal, For, Show, createMemo, createEffect } from 'solid-js'; +import { ParamEditor, type ParamOverride as PrimitiveParamOverride } from '../primitives/ParamEditor'; +import { Slider } from '../primitives/Slider'; +import { PillToggle } from '../primitives/PillToggle'; +import { WeightHealth } from '../primitives/WeightHealth'; +import { GradientFlow } from '../primitives/GradientFlow'; +import { Heatmap } from '../primitives/Heatmap'; +import { ProgressRing } from '../primitives/ProgressRing'; +import { LossPlot } from '../primitives/LossPlot'; + +import { inputStore } from '../stores/input-store'; +import { outputStore } from '../stores/output-store'; +import { explorationStore } from '../stores/exploration-store'; +import { modeStore, defaultParamOverride, type ParamOverride } from '../stores/mode-store'; +import { sessionStore } from '../stores/session-store'; +import { mlStore } from '../stores/ml-store'; +import { coreBus } from '../stores/bus'; +import { + ZOOM_MIN, + ZOOM_MAX, + DEADZONE_MAX, + INPUT_CURVE_MIN, + INPUT_CURVE_MAX, + SMOOTHING_MAX, +} from '../input/pipeline'; +import { GLOBAL_CURVE_MIN, GLOBAL_CURVE_MAX } from '../output/pipeline'; +import { + layerNorms, + gradientStatuses, + weightHistogram, + weightStatus, + layerStatsToStatus, + layerWeightCounts, +} from '../features/weight-health'; +import { saveNamedPreset, loadNamedPreset, buildShareUrl } from '../features/session-preset'; +import { captureA, toggleAB, acceptB, revertToA } from '../features/snapshots'; +import type { ModeRuntime } from './mode-runtime'; +import type { ModeSchema } from './generated'; +import type { Param } from './generated/types'; +import type { CurveName } from '../output/curves'; +import type { HeatmapColorMode } from '../features/heatmap-sampler'; + +import styles from './SettingsDrawer.module.css'; + +export interface SettingsDrawerProps { + schema: ModeSchema; + runtime: ModeRuntime; + /** Show advanced section (weight health, gradient flow, session presets). */ + showAdvanced?: boolean; +} + +interface Section { + id: string; + title: string; + defaultOpen?: boolean; + render: () => any; +} + +export const SettingsDrawer: Component = (props) => { + const [open, setOpen] = createSignal>({ + input: false, + training: false, + exploration: false, + output: false, + overrides: false, + advanced: false, + snapshots: false, + }); + const toggle = (id: string) => setOpen((s) => ({ ...s, [id]: !s[id] })); + + // ----- Per-param overrides UI ---------------------------------------- + const overrides = () => modeStore.state.overrides[props.schema.mode_id] ?? {}; + + const getOverride = (param: Param): PrimitiveParamOverride => { + const cur = overrides()[param.name]; + if (cur) { + return { + min: cur.min, + max: cur.max, + curve: cur.curve as CurveName, + curveParam: cur.curveParam, + muted: cur.muted, + pinned: cur.pinned, + fixedValue: cur.fixedValue, + }; + } + return { + min: param.min, + max: param.max, + curve: param.curve as CurveName, + muted: false, + pinned: false, + fixedValue: param.default, + }; + }; + + const writeOverride = (param: Param, next: PrimitiveParamOverride) => { + const stored: ParamOverride = { + min: next.min, + max: next.max, + curve: next.curve, + curveParam: next.curveParam, + muted: next.muted, + pinned: next.pinned, + fixedValue: next.fixedValue, + // Frozen flag is local to the runtime (set via output-store freezeMask) + // — keep it false in the persistent override unless the user explicitly + // toggles it. + frozen: overrides()[param.name]?.frozen ?? false, + }; + modeStore.setOverride(props.schema.mode_id, param.name, stored); + }; + + // ----- Per-output health (advanced section) ----------------------------- + const [advancedExpanded, setAdvancedExpanded] = createSignal(props.showAdvanced ?? false); + const histogram = createMemo(() => { + const w = mlStore.weights(); + return weightHistogram(w); + }); + const status = createMemo(() => weightStatus(mlStore.weights())); + + // Gradient flow: capture before/after weights on training events. + const [normHistory, setNormHistory] = createSignal([]); + const [gradStatuses, setGradStatuses] = createSignal>([]); + let lastBeforeWeights: Float32Array | null = null; + + const offTrainStart = coreBus.on('snap.push', (e) => { + if (e.tag === 'before train' && mlStore.iml) { + lastBeforeWeights = mlStore.getWeights(); + } + }); + const offTrainEnd = coreBus.on('ml.trained', () => { + if (lastBeforeWeights && mlStore.iml) { + const after = mlStore.getWeights(); + const arch = mlStore.iml.architecture; + const sizes = layerWeightCounts({ + inputSize: arch.inputSize, + hidden: arch.hidden, + outputSize: arch.outputSize, + }); + const norms = layerNorms(lastBeforeWeights, after, sizes); + setNormHistory(norms); + setGradStatuses(gradientStatuses(norms)); + lastBeforeWeights = null; + } + }); + + // Session-preset save / share UI state + const [presetName, setPresetName] = createSignal(''); + const [shareUrl, setShareUrl] = createSignal(''); + + // Snapshot list (long-press-undo equivalent) + const snapshots = () => sessionStore.listSnapshots(); + + // Cleanup on unmount (Solid handles via owner; we only need to detach bus + // subs explicitly because they live across renders). + // We can't use onCleanup here because this component is recreated per-mode, + // but bus.on returns an unsubscribe. Defer cleanup to when the component + // unmounts via createEffect cleanup function: + createEffect(() => { + return () => { offTrainStart(); offTrainEnd(); }; + }); + + // ----- A/B compare visibility ------------------------------------------ + const ab = () => sessionStore.state.ab; + const live = () => ab().live; + + // --------------------------------------------------------------------- + // Render helpers + // --------------------------------------------------------------------- + + const SectionHeader = (h: { id: string; title: string; count?: number }) => ( + + ); + + return ( +
+ {/* Snapshots / undo / A/B */} +
+ + +
+
+ + +
+ 0}> +
    + {(s) => ( +
  • + +
    • + )}     +
+ +
+

A / B compare

+
+ active: {live()} + }> + no capture yet + +
+
+ + + + +
+
+ +
+ + {/* Input pipeline */} +
+ + +
+ inputStore.setZoom(v)} + /> + inputStore.setDeadzone(v)} + /> + inputStore.setInputCurve(v)} + /> + inputStore.setSmoothing(v)} + /> +
+ + inputStore.config.anchorMode} + onChange={(v) => inputStore.setAnchorMode(v as 'auto' | 'sticky' | 'center')} + ariaLabel="Anchor mode" + /> +
+
+ + inputStore.config.momentumZoom} + onChange={(v) => inputStore.setMomentumZoom(v as 'off' | 'gentle' | 'strong')} + ariaLabel="Momentum zoom" + /> +
+
+ + +
+
+ +
+ + {/* Training */} +
+ + +
+ explorationStore.setLearningRate(v)} + /> + explorationStore.setWeightDecay(v)} + /> + mlStore.state.lossHistory} width={320} height={70} /> +
+ +
+ + {/* Exploration */} +
+ + +
+ explorationStore.setSpread(v)} + /> + explorationStore.setNoiseFloor(v)} + /> + explorationStore.setNoiseCap(v)} + /> + explorationStore.setNoiseGrowth(v)} + /> + explorationStore.setNoiseDecay(v)} + /> +
+

Auto-explore

+
+ + + 1} + ariaLabel="Auto-explore active" + /> + +
+ explorationStore.setAutoExploreInterval(v)} + /> + explorationStore.setAutoExploreIntensity(v)} + /> +
+ +
+ + {/* Output */} +
+ + +
+ outputStore.setGlobalCurve(v)} + /> + outputStore.setSmoothing(v)} + /> + outputStore.setSlewRate(v)} + /> + +
+ +
+ + {/* Per-param overrides */} +
+ + +
+
+ +
+
+ {(p) => ( + getOverride(p)} + onChange={(next) => writeOverride(p, next)} + compact + /> + )} +
+
+ +
+ + {/* Advanced: weight health, gradient flow, heatmap */} +
+ + +
+ + +

Weight health

+ +

Gradient flow (last train)

+ 0} + fallback={

No training run yet.

} + > + + +

Per-layer stats

+
+ {(stats, idx) => ( +
+ L{idx()} + + mean|w| {stats.meanAbs.toFixed(3)} + · max|w| {stats.maxAbs.toFixed(3)} + · {(stats.deadFrac * 100).toFixed(0)}% dead + · {(stats.saturatingFrac * 100).toFixed(0)}% sat + + {layerStatsToStatus(stats)} +
+ )} +
+

Input heatmap

+
+ + props.runtime.heatmap.colorMode()} + onChange={(v) => props.runtime.heatmap.setColorMode(v as HeatmapColorMode)} + ariaLabel="Heatmap color mode" + /> + +
+ + +
+

Session preset

+
+ setPresetName(e.currentTarget.value)} + /> + +
+ 0}> +
    + {(p) => ( +
  • + {p.name} + + +
    • + )}     +
+ +
+ + + e.currentTarget.select()} + /> + +
+
+ +
+
+ ); +}; + +export default SettingsDrawer; diff --git a/playground/src/modes/SoundAnalysisMIDIMode.tsx b/playground/src/modes/SoundAnalysisMIDIMode.tsx index d4296ee..5fc102c 100644 --- a/playground/src/modes/SoundAnalysisMIDIMode.tsx +++ b/playground/src/modes/SoundAnalysisMIDIMode.tsx @@ -2,10 +2,15 @@ * SoundAnalysisMIDIMode — sound analysis → MIDI CC output (audio_in input). * * Schema declares `primary_input: 'audio_in'` and `engine_id: 'thru'` (no - * synthesis). The full firmware pipeline feeds audio analysis features - * (pitch / aperiodicity / energy / brightness / etc.) into the first 6 - * input channels and joystick coords into the last 4. Stream 9 ships a - * scaffold UI; mic capture + analysis wiring is a stream-10 task. + * synthesis). Stream 10 wires the mic into channels 2..5 via ModeShell's + * Mic button + the runtime's MicInput. The joystick is used as a fallback + * for channels 0..1 and during testing without mic permissions. + * + * MIDI output routing is read-only here — the SettingsDrawer's per-param + * override editor lets users tighten the active range / mute params; the + * actual MIDI device wiring is left for the engine host to handle (or for + * a future stream that adds WebMIDI sender). For now CC values are + * visible in the live OutputDisplay. */ import { Component, Show } from 'solid-js'; @@ -13,44 +18,18 @@ import { ModeShell } from './ModeShell'; import { useModeRuntime } from './mode-runtime'; import { VirtualJoystick } from '../primitives/VirtualJoystick'; import { OutputDisplay } from '../primitives/OutputDisplay'; -import { SliderBank } from '../primitives/SliderBank'; import { LossPlot } from '../primitives/LossPlot'; -import { paramsToSliderConfig, outputsToSliderValues } from './mode-helpers'; import { SoundAnalysisMidiSchema } from './generated/sound_analysis_midi_schema'; export const SoundAnalysisMIDIMode: Component = () => { const schema = SoundAnalysisMidiSchema; const runtime = useModeRuntime(schema); - const sliderConfig = paramsToSliderConfig(schema.params); - const sliderValues = () => outputsToSliderValues(runtime.processedOutputs(), schema.params); return ( ( -
- { - /* read-only */ - }} - /> -

- WebMIDI routing is wired up in stream 10. For now CC values are - visible in the live readout. -

-
- )} + drawerTitle="Sound analysis → MIDI settings" primaryInput={() => ( <> { gap: 'var(--sp-3)', padding: 'var(--sp-4)', background: 'var(--bg-2)', - border: '1px dashed var(--line-strong)', + border: `1px ${runtime.mic.started() ? 'solid var(--accent-2)' : 'dashed var(--line-strong)'}`, 'border-radius': 'var(--r-2)', 'min-width': '260px', 'min-height': '200px', @@ -81,11 +60,13 @@ export const SoundAnalysisMIDIMode: Component = () => { 'text-align': 'center', }} > - Mic input — TODO + + {runtime.mic.started() ? '🎤 Mic active' : 'Mic input'} + - Stream 10 will request `getUserMedia` and feed audio analysis - features into the MLP. For now you can still drive the model - manually with the joystick below. + {runtime.mic.started() + ? 'Audio features → channels 2..5. Joystick below drives channels 0..1.' + : 'Tap "Mic" in the header to feed mic features into the model.'} { outputArea={() => ( <> { const schema = VerbFxSchema; const runtime = useModeRuntime(schema); const [voiceSpace, setVoiceSpace] = createSignal(0); - const sliderConfig = paramsToSliderConfig(schema.params); - const sliderValues = () => outputsToSliderValues(runtime.processedOutputs(), schema.params); return ( { runtime={runtime} activeVoiceSpace={voiceSpace} onVoiceSpaceChange={setVoiceSpace} - drawerTitle="Verb / FX params" - drawerContent={() => ( - { - /* read-only */ - }} - /> - )} + drawerTitle="Verb / FX settings" primaryInput={() => ( <> { outputArea={() => ( <> { const schema = XiasriSchema; const runtime = useModeRuntime(schema); - const sliderConfig = paramsToSliderConfig(schema.params); - const sliderValues = () => outputsToSliderValues(runtime.processedOutputs(), schema.params); return ( ( - { - /* read-only */ - }} - /> - )} + drawerTitle="XIASRI settings" primaryInput={() => ( <> { position={runtime.pipedInput} /> - Joystick → verb / pitch space. Mic input wiring is a stream-10 task. + Joystick → verb / pitch space. Enable mic for audio-reactive features. )} outputArea={() => ( <> ): SliderConfig /** * Map a Float32Array (length = N) of normalized values [0,1] to a flat * array sized to match the slider config (already in min/max range). + * + * If `overrides` is supplied, per-param min/max overrides are honoured + * instead of the schema defaults — this is what the mode UI sliders + * display when the user has tightened the active range. */ export function outputsToSliderValues( outputs: Float32Array, params: ReadonlyArray, + overrides?: Record, ): number[] { const out: number[] = []; for (let i = 0; i < params.length; ++i) { const v = outputs[i] ?? 0; const p = params[i]!; - out.push(p.min + v * (p.max - p.min)); + const ov = overrides?.[p.name]; + if (ov && ov.muted && ov.fixedValue !== undefined) { + out.push(ov.fixedValue); + continue; + } + const min = ov?.min ?? p.min; + const max = ov?.max ?? p.max; + out.push(min + v * (max - min)); } return out; } diff --git a/playground/src/modes/mode-runtime.ts b/playground/src/modes/mode-runtime.ts index bc30713..191a0f3 100644 --- a/playground/src/modes/mode-runtime.ts +++ b/playground/src/modes/mode-runtime.ts @@ -6,23 +6,30 @@ * 1. Hold a primary 2D input position (joystick / xy-pad / external feed). * 2. Push it through the input pipeline. * 3. Forward the processed (x, y) to the WASM MLP as input channels [0..N]. - * Modes with input_size > 2 zero-pad the unused channels. + * Modes with input_size > 2 zero-pad the unused channels (or the mic + * analyser fills 0..3 if active). * 4. Pull the WASM outputs (Float32Array of 126), slice to the schema's * `output_size`, and run them through the output pipeline. - * 5. Throttle + ship the processed slice to the AudioWorklet engine. + * 5. Apply per-param overrides (modeStore.activeOverrides). + * 6. Throttle + ship the processed slice to the AudioWorklet engine. * - * To keep mode TSX files small and consistent, this module exposes a hook - * `useModeRuntime(schema)` that owns the lifecycle and exposes reactive - * accessors plus the `setInput(x, y)` driver. Modes only have to render a - * primary input that calls `runtime.setInput(x, y)` and the runtime takes - * care of everything downstream. + * Stream 10 wires: + * - Compound axes → underlying stores (input/output/exploration). + * - Per-param overrides between MLP outputs and engine. + * - Snapshot + undo + A/B compare on RL events. + * - Pin mask passed to moveWeights. + * - Auto-explore interval timer. + * - Trail tracking for JoyMap. + * - Heatmap sampler invalidated on weight-change events. + * - Mic features pushed into MLP for audio-input modes. */ -import { createEffect, createSignal, onCleanup, onMount } from 'solid-js'; +import { batch, createEffect, createMemo, createSignal, onCleanup, onMount, untrack } from 'solid-js'; -import { mlStore, modeStore, controlStore } from '../stores'; +import { mlStore, modeStore, controlStore, sessionStore, explorationStore } from '../stores'; import { inputStore } from '../stores/input-store'; import { outputStore } from '../stores/output-store'; +import { coreBus } from '../stores/bus'; import { processInput, defaultInputState, type InputState } from '../input/pipeline'; import { processOutput, @@ -33,6 +40,13 @@ import { EngineHost } from '../audio/engine-host'; import type { EngineId } from '../ml/types'; import type { ModeSchema } from './generated'; +import { applyOverrides, buildPinMask } from '../features/overrides'; +import { applyControlRouting } from '../features/control-routing'; +import { createTrailRing, type TrailPoint } from '../features/trail'; +import { autoSnapshot, undoLastSnapshot } from '../features/snapshots'; +import { HeatmapSampler, type HeatmapColorMode } from '../features/heatmap-sampler'; +import { MicInput } from '../features/mic-input'; + /** * Throttle interval for engine param updates (ms). 50ms ≈ 20fps which * matches the legacy playground's C15 update cadence. @@ -62,6 +76,12 @@ export interface ModeRuntime { /** Output-sliced + pipeline-processed vector (length = schema.output_size). */ processedOutputs: () => Float32Array; + /** + * Final per-param values after override application (length = params.length). + * These are what get shipped to the engine and visualised in sliders. + */ + paramOutputs: () => Float32Array; + /** True iff WASM has loaded and the MLP is ready. */ ready: () => boolean; @@ -73,6 +93,13 @@ export interface ModeRuntime { setMuted: (muted: boolean) => void; }; + /** Mic input control (for audio_in modes; safe to call on others — no-op). */ + mic: { + started: () => boolean; + start: () => Promise; + stop: () => Promise; + }; + /** Loss / training plumbing surfaced from mlStore. */ training: { busy: () => boolean; @@ -88,6 +115,28 @@ export interface ModeRuntime { thumbsUp: () => void; thumbsDown: () => void; randomize: () => void; + /** Pop snapshot stack and restore weights. Returns true on success. */ + undo: () => boolean; + + /** Whether the snapshot stack has anything to undo. */ + canUndo: () => boolean; + + /** Trail ring for JoyMap binding. */ + trail: () => ReadonlyArray; + /** Snap input back to a trail point. */ + snapToTrail: (p: { x: number; y: number }) => void; + + /** Heatmap sampler. Returns the underlying cells; modes pass to . */ + heatmap: { + cells: () => Float32Array; + setColorMode: (m: HeatmapColorMode) => void; + colorMode: () => HeatmapColorMode; + refresh: (force?: boolean) => void; + resolution: () => number; + }; + + /** Region pin: pin the current zoom window (long-press handler). */ + pinCurrentRegion: () => void; } interface RuntimeOptions { @@ -124,12 +173,33 @@ export function useModeRuntime( modeStore.switchMode(schema.mode_id); } + // ----- Compound axis routing ------------------------------------------- + // Whenever any axis or offset changes, re-derive the underlying store + // values. Track axis reads explicitly; do the writes inside untrack so + // we don't pick up stale dependencies on write-targets (input/output/ + // exploration stores). + createEffect(() => { + void controlStore.state.boldness; + void controlStore.state.memory; + void controlStore.state.precision; + void controlStore.state.offsets.boldness; + void controlStore.state.offsets.memory; + void controlStore.state.offsets.precision; + untrack(() => applyControlRouting()); + }); + // ----- Input pipeline state -------------------------------------------- const [pipedInput, setPipedInput] = createSignal([0.5, 0.5]); const [frozen, setFrozen] = createSignal(false); let inputState: InputState = defaultInputState(); let lastFrameMs = performance.now(); + // Trail ring + const trailRing = createTrailRing(); + + // Pressure tracking (set externally via setPressure on touch). + let pressDownAt: number | null = null; + const setInput = (rawX: number, rawY: number): void => { const now = performance.now(); const dt = Math.max(0.001, (now - lastFrameMs) / 1000); @@ -140,24 +210,47 @@ export function useModeRuntime( inputStore.__setLiveState(result.state); setPipedInput([result.x, result.y]); setFrozen(result.frozen); + trailRing.push(result.x, result.y); if (!ready()) return; - // Push input to the MLP. Channels beyond [x,y] are zeroed out — modes - // with input_size > 2 currently aren't fed extra inputs (audio analysis - // wiring is a stream-10 task). + // Push input to the MLP. Channels beyond [x,y] are zeroed out (or + // overridden with mic features in audio-input modes). const inSz = schema.ml.input_size; mlStore.setInput(0, result.x); if (inSz > 1) mlStore.setInput(1, result.y); - for (let i = 2; i < inSz; ++i) mlStore.setInput(i, 0); + // Mic features (if active) take channels 2..5. + if (mic.isRunning() && inSz > 2) { + const f = mic.getFeatures(); + const fv = [f.energy, f.brightness, f.pitch, f.aperiodicity]; + for (let i = 2; i < inSz; ++i) { + mlStore.setInput(i, fv[i - 2] ?? 0); + } + } else { + for (let i = 2; i < inSz; ++i) mlStore.setInput(i, 0); + } mlStore.process(); + + // Update pressure-feedback hold timer. + if (pressDownAt !== null) { + explorationStore.setPressure( + explorationStore.state.pressureForce, + now - pressDownAt, + ); + } }; - // ----- Output pipeline state ------------------------------------------- + // ----- Output pipeline + override application -------------------------- let outputState: OutputState = defaultOutputState(); const sliceLen = schema.ml.output_size; const [processedOutputs, setProcessedOutputs] = createSignal( new Float32Array(sliceLen), - { equals: false }, // always notify even when buffer is reused in-place + { equals: false }, + ); + const paramCount = schema.params.length; + let prevParamOutputs: Float32Array | null = null; + const [paramOutputs, setParamOutputs] = createSignal( + new Float32Array(paramCount), + { equals: false }, ); // Run the output pipeline whenever raw outputs change. @@ -175,12 +268,52 @@ export function useModeRuntime( const result = processOutput(slice as Float32Array, outputStore.config, outputState, dtMs); outputState = result.state; setProcessedOutputs(result.processed); + + // Apply per-param overrides for the per-param consumer (engine, sliders). + // Only the first `params.length` outputs are user-visible parameters; the + // remainder is reserved for engine internals (none right now). + const paramSlice = result.processed.length === paramCount + ? result.processed + : result.processed.subarray(0, paramCount); + const overrides = modeStore.state.overrides[schema.mode_id] ?? {}; + const applied = applyOverrides( + paramSlice as Float32Array, + prevParamOutputs, + schema.params, + overrides, + ); + prevParamOutputs = applied.values; + setParamOutputs(applied.values); }; - // Trigger recompute on any raw-output change. + // Trigger recompute on any raw-output change. Wrap in untrack so reading + // outputStore.config inside processOutput doesn't add a tracked dep here. createEffect(() => { rawOutputsAccessor(); - recomputeOutputs(); + untrack(recomputeOutputs); + }); + + // Re-run override application when overrides change (no new ML output). + createEffect(() => { + void modeStore.state.overrides[schema.mode_id]; + untrack(() => { + const raw = rawOutputsAccessor(); + if (raw.length > 0) recomputeOutputs(); + }); + }); + + // Push the freeze mask into outputStore whenever overrides change. + createEffect(() => { + const ovs = modeStore.state.overrides[schema.mode_id] ?? {}; + let mask: Uint8Array | null = null; + for (let i = 0; i < schema.params.length; ++i) { + const ov = ovs[schema.params[i]!.name]; + if (ov && ov.frozen) { + if (!mask) mask = new Uint8Array(schema.params.length); + mask[i] = 1; + } + } + untrack(() => outputStore.setFreezeMask(mask)); }); // ----- Engine wiring (audio) ------------------------------------------- @@ -195,7 +328,6 @@ export function useModeRuntime( pendingParams = null; return; } - // Copy because EngineHost transfers the buffer. const copy = new Float32Array(pendingParams); pendingParams = null; try { @@ -213,9 +345,9 @@ export function useModeRuntime( } }; - // Pipe processedOutputs into the engine host whenever they change. + // Pipe paramOutputs into the engine host whenever they change. createEffect(() => { - const out = processedOutputs(); + const out = paramOutputs(); if (out.length === 0) return; if (!host.isStarted) return; scheduleParamFlush(out); @@ -228,8 +360,7 @@ export function useModeRuntime( try { await host.start(engineId); setAudioStarted(true); - // Push the current outputs immediately on start. - const out = processedOutputs(); + const out = paramOutputs(); if (out.length > 0) host.setParams(new Float32Array(out)); } catch (err) { // eslint-disable-next-line no-console @@ -258,56 +389,228 @@ export function useModeRuntime( pendingParams = null; }); + // ----- Mic input ------------------------------------------------------- + const mic = new MicInput(); + const [micStarted, setMicStarted] = createSignal(false); + const startMic = async () => { + try { + await mic.start(); + setMicStarted(true); + } catch (err) { + // eslint-disable-next-line no-console + console.error('[mode-runtime] mic start failed:', err); + } + }; + const stopMic = async () => { + try { + await mic.stop(); + } finally { + setMicStarted(false); + } + }; + onCleanup(() => { void stopMic(); }); + + // ----- Snapshot stack helpers ------------------------------------------ + const canUndoMemo = createMemo(() => sessionStore.state.snapshots.length > 0); + + // ----- Heatmap sampler ------------------------------------------------- + const sampler = new HeatmapSampler({ resolution: 16 }); + const [heatmapColorMode, setHeatmapColorMode] = createSignal('luminance'); + const [heatmapCells, setHeatmapCells] = createSignal(sampler.getCells(), { equals: false }); + const refreshHeatmap = (force = false): void => { + untrack(() => { + if (!ready()) return; + const cfg = inputStore.config; + const z = cfg.zoom; + const cx = cfg.anchorMode === 'center' ? 0.5 : cfg.anchorX; + const cy = cfg.anchorMode === 'center' ? 0.5 : cfg.anchorY; + const took = sampler.update({ cx, cy, zoom: z }, force); + if (took) { + setHeatmapCells(new Float32Array(sampler.getCells())); + } + }); + }; + + // Refresh heatmap on weight changes (training, RL feedback, randomize). + const offTrained = coreBus.on('ml.trained', () => refreshHeatmap()); + const offDelta = coreBus.on('ml.delta_update', () => refreshHeatmap()); + onCleanup(() => { offTrained(); offDelta(); }); + + // Initial heatmap refresh once WASM is ready. + createEffect(() => { + if (ready()) refreshHeatmap(true); + }); + + // ----- Auto-explore loop ----------------------------------------------- + let autoExploreTimer: number | null = null; + const tickAutoExplore = () => { + untrack(() => { + if (!ready()) return; + if (!explorationStore.state.autoExploreEnabled) return; + const intensity = explorationStore.state.autoExploreIntensity; + // Zoom-scaled intensity: smaller zoom = gentler. + const zoom = inputStore.config.zoom; + const scaledIntensity = intensity * (0.3 + 0.7 * zoom); + const cap = explorationStore.state.noiseCap * scaledIntensity; + autoSnapshot('before auto-explore'); + const spread = explorationStore.state.spread; + const overrides = modeStore.state.overrides[schema.mode_id] ?? {}; + const pinMask = buildPinMask( + mlStore.state.outputSize, + schema.mode_id, + schema.params, + overrides, + sessionStore.state.paramPins, + ); + mlStore.moveWeights(cap, spread, pinMask); + explorationStore.growNoise(0.5); + // Re-run inference to update the heatmap and visuals. + const [x, y] = pipedInput(); + setInput(x, y); + }); + }; + createEffect(() => { + const enabled = explorationStore.state.autoExploreEnabled; + const interval = explorationStore.state.autoExploreIntervalMs; + if (autoExploreTimer !== null) { + window.clearInterval(autoExploreTimer); + autoExploreTimer = null; + } + if (enabled) { + autoExploreTimer = window.setInterval(tickAutoExplore, interval); + } + }); + onCleanup(() => { + if (autoExploreTimer !== null) { + window.clearInterval(autoExploreTimer); + autoExploreTimer = null; + } + }); + + // ----- Touch / pressure feedback --------------------------------------- + const onPointerDown = (e: PointerEvent) => { + pressDownAt = performance.now(); + explorationStore.setPressure( + // Force is 0..1 on supported devices; default 0.5 elsewhere. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (e as any).pressure ?? 0.5, + 0, + ); + }; + const onPointerUp = () => { + pressDownAt = null; + explorationStore.setPressure(0, 0); + }; + if (typeof window !== 'undefined') { + window.addEventListener('pointerdown', onPointerDown, { passive: true }); + window.addEventListener('pointerup', onPointerUp); + window.addEventListener('pointercancel', onPointerUp); + onCleanup(() => { + window.removeEventListener('pointerdown', onPointerDown); + window.removeEventListener('pointerup', onPointerUp); + window.removeEventListener('pointercancel', onPointerUp); + }); + } + // ----- Training helpers ------------------------------------------------- const trainOnCurrent = () => { if (!ready()) return; - const lr = controlStore.resolveParams()['learningRate']; - const lrNum = typeof lr === 'number' ? lr : schema.ml.default_learning_rate; - mlStore.train(lrNum, schema.ml.default_max_iterations, 0.001); + autoSnapshot('before train'); + const lr = explorationStore.state.learningRate; + mlStore.train(lr, schema.ml.default_max_iterations, 0.001); }; const thumbsUp = () => { if (!ready()) return; - // Push a label = current pipeline-processed slice as the target at the - // current input. This matches the legacy "thumbs up = remember the - // current sound at this position" semantics. const [x, y] = pipedInput(); - const out = processedOutputs(); + const out = paramOutputs(); if (out.length === 0) return; const features = new Array(schema.ml.input_size).fill(0); features[0] = x; if (features.length > 1) features[1] = y; - const labels = Array.from(out); + // Mic features into the example too, if active. + if (mic.isRunning() && schema.ml.input_size > 2) { + const f = mic.getFeatures(); + const fv = [f.energy, f.brightness, f.pitch, f.aperiodicity]; + for (let i = 2; i < schema.ml.input_size; ++i) { + features[i] = fv[i - 2] ?? 0; + } + } + // Labels: the raw 126-vector targets (not the override-applied; the MLP + // doesn't know about overrides). + const labels = Array.from(processedOutputs()); + autoSnapshot('before thumbs-up'); mlStore.addExample(features, labels); + explorationStore.decayNoise(explorationStore.state.pressureForce); trainOnCurrent(); }; const thumbsDown = () => { if (!ready()) return; - const params = controlStore.resolveParams(); - const cap = typeof params['noiseCap'] === 'number' - ? (params['noiseCap'] as number) - : 0.12; - const spread = schema.ml.default_spread; - mlStore.moveWeights(cap, spread); - // Re-run inference at current input so the visual updates. + autoSnapshot('before thumbs-down'); + const cap = explorationStore.state.noiseCap; + const spread = explorationStore.state.spread; + const overrides = modeStore.state.overrides[schema.mode_id] ?? {}; + const pinMask = buildPinMask( + mlStore.state.outputSize, + schema.mode_id, + schema.params, + overrides, + sessionStore.state.paramPins, + ); + mlStore.moveWeights(cap, spread, pinMask); + explorationStore.growNoise(explorationStore.state.pressureForce); const [x, y] = pipedInput(); setInput(x, y); }; const randomize = () => { if (!ready()) return; - mlStore.drawWeights(schema.ml.default_spread); + autoSnapshot('before randomize'); + mlStore.drawWeights(explorationStore.state.spread); const [x, y] = pipedInput(); setInput(x, y); }; + const undo = (): boolean => { + const ok = undoLastSnapshot(); + if (ok) { + const [x, y] = pipedInput(); + setInput(x, y); + } + return ok; + }; + + // ----- Region pins ------------------------------------------------------ + const pinCurrentRegion = () => { + const cfg = inputStore.config; + const z = cfg.zoom; + const cx = cfg.anchorMode === 'center' ? 0.5 : cfg.anchorX; + const cy = cfg.anchorMode === 'center' ? 0.5 : cfg.anchorY; + const halfZ = z * 0.5; + sessionStore.addRegionPin({ + x: Math.max(0, cx - halfZ), + y: Math.max(0, cy - halfZ), + width: Math.min(z, 1), + height: Math.min(z, 1), + }); + autoSnapshot('pinned baseline'); + }; + + // Snap-to-trail + const snapToTrail = (p: { x: number; y: number }) => { + batch(() => { + setInput(p.x, p.y); + }); + }; + return { setInput, pipedInput, frozen, rawOutputs: rawOutputsAccessor, processedOutputs, + paramOutputs, ready, audio: { started: audioStarted, @@ -315,6 +618,11 @@ export function useModeRuntime( stop: stopAudio, setMuted: (muted) => host.setMuted(muted), }, + mic: { + started: micStarted, + start: startMic, + stop: stopMic, + }, training: { busy: () => mlStore.state.training, examples: () => mlStore.state.exampleCount, @@ -325,6 +633,22 @@ export function useModeRuntime( thumbsUp, thumbsDown, randomize, + undo, + canUndo: () => canUndoMemo(), + trail: trailRing.points, + snapToTrail, + heatmap: { + cells: heatmapCells, + setColorMode: (m) => { + sampler.setColorMode(m); + setHeatmapColorMode(m); + refreshHeatmap(true); + }, + colorMode: heatmapColorMode, + refresh: refreshHeatmap, + resolution: () => sampler.resolution, + }, + pinCurrentRegion, }; } diff --git a/playground/src/stores/exploration-store.ts b/playground/src/stores/exploration-store.ts new file mode 100644 index 0000000..bb6b985 --- /dev/null +++ b/playground/src/stores/exploration-store.ts @@ -0,0 +1,254 @@ +/** + * Exploration store — RL feedback state and auto-explore configuration. + * + * Holds: + * - `noiseLevel` — current RL exploration noise (mutated by thumbs-up/down). + * - `noiseFloor`/`noiseCap` — bounds; cap maps from compound-axis Boldness. + * - `noiseGrowth`/`noiseDecay` — multiplicative factors per RL feedback step. + * - `spread` — sigmoid-saturation regime [0,1]. + * - `autoExplore` — interval, intensity, last-tick. + * - `pressure` — last touch force/hold sample for feedback scaling. + * + * Stream 10 owns this store; the runtime reads from it on every thumbs-down + * and the auto-explore interval timer. Mutations via store actions; reads via + * signals. + */ + +import { createStore, produce } from 'solid-js/store'; +import { schedulePersist, loadPersisted } from './persistence'; +import { clamp } from '../output/curves'; + +const STORAGE_KEY = 'nisps:exploration'; + +export interface ExplorationState { + /** Live RL noise level (always in [floor, cap]). */ + noiseLevel: number; + /** Lower bound. Compound axes can move it. */ + noiseFloor: number; + /** Upper bound. Mapped from Boldness. */ + noiseCap: number; + /** Per-thumbs-down growth factor (>1). */ + noiseGrowth: number; + /** Per-thumbs-up decay factor (<1). */ + noiseDecay: number; + /** Master sigmoid-saturation regime [0,1]. */ + spread: number; + /** Learning rate for trainings (compound axes can override). */ + learningRate: number; + /** Weight decay applied during moveWeights (Boldness/Memory). */ + weightDecay: number; + + /** Auto-explore: enabled flag. */ + autoExploreEnabled: boolean; + /** Auto-explore tick interval, ms. [500, 10000]. */ + autoExploreIntervalMs: number; + /** Intensity scaling for auto-explore noise [0.1, 1.0]. */ + autoExploreIntensity: number; + + /** Last touch pressure-sensitive sample (0..1). */ + pressureForce: number; + /** Hold duration of current touch (ms). 0 if no touch. */ + holdMs: number; +} + +interface PersistedExploration { + noiseFloor: number; + noiseCap: number; + noiseGrowth: number; + noiseDecay: number; + spread: number; + learningRate: number; + weightDecay: number; + autoExploreEnabled: boolean; + autoExploreIntervalMs: number; + autoExploreIntensity: number; +} + +function defaults(): ExplorationState { + return { + noiseLevel: 0.05, + noiseFloor: 0.005, + noiseCap: 0.12, + noiseGrowth: 1.5, + noiseDecay: 0.97, + spread: 0.6, + learningRate: 1.0, + weightDecay: 0.06, + + autoExploreEnabled: false, + autoExploreIntervalMs: 2000, + autoExploreIntensity: 0.5, + + pressureForce: 0, + holdMs: 0, + }; +} + +function loadInitial(): ExplorationState { + const base = defaults(); + const persisted = loadPersisted>(STORAGE_KEY, {}); + return { + ...base, + noiseFloor: persisted.noiseFloor ?? base.noiseFloor, + noiseCap: persisted.noiseCap ?? base.noiseCap, + noiseGrowth: persisted.noiseGrowth ?? base.noiseGrowth, + noiseDecay: persisted.noiseDecay ?? base.noiseDecay, + spread: persisted.spread ?? base.spread, + learningRate: persisted.learningRate ?? base.learningRate, + weightDecay: persisted.weightDecay ?? base.weightDecay, + autoExploreEnabled: persisted.autoExploreEnabled ?? base.autoExploreEnabled, + autoExploreIntervalMs: persisted.autoExploreIntervalMs ?? base.autoExploreIntervalMs, + autoExploreIntensity: persisted.autoExploreIntensity ?? base.autoExploreIntensity, + }; +} + +const [state, setState] = createStore(loadInitial()); + +function persist(): void { + schedulePersist(STORAGE_KEY, () => ({ + noiseFloor: state.noiseFloor, + noiseCap: state.noiseCap, + noiseGrowth: state.noiseGrowth, + noiseDecay: state.noiseDecay, + spread: state.spread, + learningRate: state.learningRate, + weightDecay: state.weightDecay, + autoExploreEnabled: state.autoExploreEnabled, + autoExploreIntervalMs: state.autoExploreIntervalMs, + autoExploreIntensity: state.autoExploreIntensity, + })); +} + +export const explorationStore = { + state, + + /** Bulk update from compound-axis resolution. */ + applyCompoundParams(params: Record): void { + setState(produce((s) => { + const get = (k: string): number | undefined => { + const v = params[k]; + return typeof v === 'number' ? v : undefined; + }; + const cap = get('noiseCap'); + if (cap !== undefined) s.noiseCap = clamp(cap, 0.005, 1); + const growth = get('noiseGrowth'); + if (growth !== undefined) s.noiseGrowth = clamp(growth, 1, 4); + const decay = get('noiseDecay'); + if (decay !== undefined) s.noiseDecay = clamp(decay, 0, 1); + const lr = get('learningRate'); + if (lr !== undefined) s.learningRate = clamp(lr, 0.01, 10); + const wd = get('weightDecay'); + if (wd !== undefined) s.weightDecay = clamp(wd, 0, 0.5); + // Keep noiseLevel in [floor, cap] + if (s.noiseLevel > s.noiseCap) s.noiseLevel = s.noiseCap; + if (s.noiseLevel < s.noiseFloor) s.noiseLevel = s.noiseFloor; + })); + persist(); + }, + + setNoiseLevel(v: number): void { + setState(produce((s) => { + s.noiseLevel = clamp(v, s.noiseFloor, s.noiseCap); + })); + }, + + setSpread(v: number): void { + setState(produce((s) => { + s.spread = clamp(v, 0, 1); + })); + persist(); + }, + + setNoiseFloor(v: number): void { + setState(produce((s) => { + s.noiseFloor = clamp(v, 0, s.noiseCap); + if (s.noiseLevel < s.noiseFloor) s.noiseLevel = s.noiseFloor; + })); + persist(); + }, + + setNoiseCap(v: number): void { + setState(produce((s) => { + s.noiseCap = clamp(v, s.noiseFloor, 1); + if (s.noiseLevel > s.noiseCap) s.noiseLevel = s.noiseCap; + })); + persist(); + }, + + setNoiseGrowth(v: number): void { + setState(produce((s) => { s.noiseGrowth = clamp(v, 1, 4); })); + persist(); + }, + + setNoiseDecay(v: number): void { + setState(produce((s) => { s.noiseDecay = clamp(v, 0, 1); })); + persist(); + }, + + setLearningRate(v: number): void { + setState(produce((s) => { s.learningRate = clamp(v, 0.001, 10); })); + persist(); + }, + + setWeightDecay(v: number): void { + setState(produce((s) => { s.weightDecay = clamp(v, 0, 0.5); })); + persist(); + }, + + /** + * Mutate noise after a thumbs-down/up. Pressure ∈ [0,1] modulates the + * effective growth/decay; >0.5 = harder, <0.5 = softer. + */ + growNoise(pressure: number = 0.5): number { + let next = state.noiseLevel; + setState(produce((s) => { + const p = clamp(pressure, 0, 1); + // Pressure 0 = baseline, 1 = stronger. Reduces effect of growth. + const factor = 1 + (s.noiseGrowth - 1) * (0.5 + p); + next = clamp(s.noiseLevel * factor, s.noiseFloor, s.noiseCap); + s.noiseLevel = next; + })); + return next; + }, + + decayNoise(pressure: number = 0.5): number { + let next = state.noiseLevel; + setState(produce((s) => { + const p = clamp(pressure, 0, 1); + // Stronger pressure → faster decay. + const factor = s.noiseDecay - (1 - s.noiseDecay) * (p - 0.5) * 0.4; + next = clamp(s.noiseLevel * clamp(factor, 0, 1), s.noiseFloor, s.noiseCap); + s.noiseLevel = next; + })); + return next; + }, + + setAutoExplore(enabled: boolean): void { + setState(produce((s) => { s.autoExploreEnabled = enabled; })); + persist(); + }, + + setAutoExploreInterval(ms: number): void { + setState(produce((s) => { s.autoExploreIntervalMs = clamp(ms, 500, 10000); })); + persist(); + }, + + setAutoExploreIntensity(v: number): void { + setState(produce((s) => { s.autoExploreIntensity = clamp(v, 0.1, 1); })); + persist(); + }, + + setPressure(force: number, holdMs: number): void { + setState(produce((s) => { + s.pressureForce = clamp(force, 0, 1); + s.holdMs = Math.max(0, holdMs); + })); + }, + + reset(): void { + setState(defaults()); + persist(); + }, +}; + +export type ExplorationStore = typeof explorationStore; diff --git a/playground/src/stores/index.ts b/playground/src/stores/index.ts index 5bbbfc8..8cac6df 100644 --- a/playground/src/stores/index.ts +++ b/playground/src/stores/index.ts @@ -30,4 +30,9 @@ export { type ABState, type SessionPreset, } from './session-store'; +export { + explorationStore, + type ExplorationStore, + type ExplorationState, +} from './exploration-store'; export { schedulePersist, flushPersist, loadPersisted, clearPersisted } from './persistence'; diff --git a/playground/src/stores/session-store.ts b/playground/src/stores/session-store.ts index 90d8c01..c1ae573 100644 --- a/playground/src/stores/session-store.ts +++ b/playground/src/stores/session-store.ts @@ -1,9 +1,16 @@ /** * Session store — snapshot stack, A/B compare, region pins, named session presets. * - * Stream 8 (this stream) provides the API and a working in-memory - * implementation with stub data shapes. Stream 10 wires the snapshots to - * real ML weights and surfaces the data through UI. + * Stream 10 wires real ML weights into the snapshot stack and A/B state. + * Snapshots store a Float32Array copy of the weights at the moment they + * were pushed; restoring pops + writes back via mlStore.setWeights. + * + * Snapshot stack capacity is bounded (MAX_SNAPSHOTS); the oldest is dropped + * when full. The stack is in-memory only — losing it across reloads is + * acceptable (matches the legacy playground). + * + * Region pins, param pins, and named session presets ARE persisted because + * they're explicit user intent. */ import { createStore, produce } from 'solid-js/store'; @@ -13,15 +20,15 @@ import { schedulePersist, loadPersisted } from './persistence'; const STORAGE_KEY = 'nisps:session'; const MAX_SNAPSHOTS = 20; -/** A single weights snapshot (placeholder until ML stream is live). */ +/** A single weights snapshot. */ export interface Snapshot { id: string; tag: string; timestamp: number; noiseLevel: number; zoomLevel: number | null; - /** Stream 7+ stores actual weights here (Float32Array via ArrayBuffer in JSON). */ - weightsRef: string | null; + /** Float32Array weight copy. Null if not captured (e.g. before WASM ready). */ + weights: Float32Array | null; } export interface RegionPin { @@ -106,14 +113,17 @@ export const sessionStore = { // ----- Snapshots ----- - pushSnapshot(tag: string, opts: { noiseLevel?: number; zoomLevel?: number | null; weightsRef?: string | null } = {}): Snapshot { + pushSnapshot( + tag: string, + opts: { noiseLevel?: number; zoomLevel?: number | null; weights?: Float32Array | null } = {}, + ): Snapshot { const snap: Snapshot = { id: nextSnapshotId(), tag, timestamp: Date.now(), noiseLevel: opts.noiseLevel ?? 0, zoomLevel: opts.zoomLevel ?? null, - weightsRef: opts.weightsRef ?? null, + weights: opts.weights ? new Float32Array(opts.weights) : null, }; setState(produce((s) => { s.snapshots.push(snap); @@ -125,6 +135,18 @@ export const sessionStore = { return snap; }, + /** Convenience peek at the most recent snapshot. */ + peekSnapshot(): Snapshot | null { + return state.snapshots.length > 0 + ? state.snapshots[state.snapshots.length - 1]! + : null; + }, + + /** Get a copy of all snapshots (for UI rendering). */ + listSnapshots(): ReadonlyArray { + return state.snapshots; + }, + popSnapshot(): Snapshot | null { let popped: Snapshot | null = null; setState(produce((s) => {