From ebd5dba4c04b51aaf00074e92238af665e001315 Mon Sep 17 00:00:00 2001 From: w1n5t0n Date: Wed, 29 Apr 2026 17:00:43 +0300 Subject: [PATCH] feat(playground/modes): ModeShell + runtime + switcher scaffolding (meml-yt7) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stream 9 (part 1/2) — shared infrastructure for the per-mode TSX layer. - ModeShell.tsx: common chrome (header, voice-space PillToggle, audio start/stop, control axes, training controls, drawer, status line). Modes only have to author primary input + output JSX. - mode-runtime.ts: useModeRuntime(schema) hook. Owns input pipeline, WASM ML calls, output pipeline, and a 50ms-throttled engine-host bridge. Exposes setInput/processedOutputs/training/audio surfaces so mode TSX stays declarative. - mode-helpers.ts: schema → SliderBank config + Float32Array → slider-range value mapping. Pure functions. - ModeSwitcher.tsx: top-of-page modeStore.switchMode(e.currentTarget.value)} + aria-label="Select active mode" + > + + {(m) => ( + + )} + + + + {active().description} + + + + + + ); +}; + +export default ModeSwitcher; diff --git a/playground/src/modes/mode-helpers.ts b/playground/src/modes/mode-helpers.ts new file mode 100644 index 0000000..652d90d --- /dev/null +++ b/playground/src/modes/mode-helpers.ts @@ -0,0 +1,54 @@ +/** + * Helpers shared across mode TSX files. Pure functions; no Solid state. + */ + +import type { Param } from './generated/types'; +import type { SliderConfig } from '../primitives/SliderBank'; +import type { CurveName } from '../output/curves'; + +/** + * Convert a schema param list into SliderConfig entries that the SliderBank + * primitive understands. Sliders are grouped by the schema's `group` field + * so the bank renders collapsible sections. + */ +export function paramsToSliderConfig(params: ReadonlyArray): SliderConfig[] { + let lastGroup: string | null = null; + return params.map((p) => { + const isNewGroup = p.group !== lastGroup; + lastGroup = p.group; + const cfg: SliderConfig = { + id: p.name, + label: p.label, + min: p.min, + max: p.max, + curve: p.curve as CurveName, + }; + if (isNewGroup) cfg.section = formatGroupName(p.group); + return cfg; + }); +} + +/** + * Map a Float32Array (length = N) of normalized values [0,1] to a flat + * array sized to match the slider config (already in min/max range). + */ +export function outputsToSliderValues( + outputs: Float32Array, + params: ReadonlyArray, +): number[] { + const out: number[] = []; + for (let i = 0; i < params.length; ++i) { + const v = outputs[i] ?? 0; + const p = params[i]!; + out.push(p.min + v * (p.max - p.min)); + } + return out; +} + +function formatGroupName(group: string): string { + if (!group) return ''; + return group + .split(/[_\s]+/) + .map((p) => p.charAt(0).toUpperCase() + p.slice(1)) + .join(' '); +} diff --git a/playground/src/modes/mode-runtime.ts b/playground/src/modes/mode-runtime.ts new file mode 100644 index 0000000..bc30713 --- /dev/null +++ b/playground/src/modes/mode-runtime.ts @@ -0,0 +1,339 @@ +/** + * Mode runtime — shared wiring between mode TSX components and the + * playground's stores / WASM ML / audio engine host. + * + * Every mode does the same dance: + * 1. Hold a primary 2D input position (joystick / xy-pad / external feed). + * 2. Push it through the input pipeline. + * 3. Forward the processed (x, y) to the WASM MLP as input channels [0..N]. + * Modes with input_size > 2 zero-pad the unused channels. + * 4. Pull the WASM outputs (Float32Array of 126), slice to the schema's + * `output_size`, and run them through the output pipeline. + * 5. Throttle + ship the processed slice to the AudioWorklet engine. + * + * To keep mode TSX files small and consistent, this module exposes a hook + * `useModeRuntime(schema)` that owns the lifecycle and exposes reactive + * accessors plus the `setInput(x, y)` driver. Modes only have to render a + * primary input that calls `runtime.setInput(x, y)` and the runtime takes + * care of everything downstream. + */ + +import { createEffect, createSignal, onCleanup, onMount } from 'solid-js'; + +import { mlStore, modeStore, controlStore } from '../stores'; +import { inputStore } from '../stores/input-store'; +import { outputStore } from '../stores/output-store'; +import { processInput, defaultInputState, type InputState } from '../input/pipeline'; +import { + processOutput, + defaultOutputState, + type OutputState, +} from '../output/pipeline'; +import { EngineHost } from '../audio/engine-host'; +import type { EngineId } from '../ml/types'; +import type { ModeSchema } from './generated'; + +/** + * Throttle interval for engine param updates (ms). 50ms ≈ 20fps which + * matches the legacy playground's C15 update cadence. + */ +const ENGINE_PARAM_THROTTLE_MS = 50; + +/** A single shared EngineHost. Audio only starts on user gesture. */ +let engineHost: EngineHost | null = null; +function getEngineHost(): EngineHost { + if (!engineHost) engineHost = new EngineHost(); + return engineHost; +} + +export interface ModeRuntime { + /** Driver — call from joystick/xy-pad/etc. */ + setInput: (x: number, y: number) => void; + + /** Most recent processed input (after pipeline). */ + pipedInput: () => readonly [number, number]; + + /** Whether the input is currently frozen by zoom. */ + frozen: () => boolean; + + /** Raw 126-output ML vector (live). */ + rawOutputs: () => Float32Array; + + /** Output-sliced + pipeline-processed vector (length = schema.output_size). */ + processedOutputs: () => Float32Array; + + /** True iff WASM has loaded and the MLP is ready. */ + ready: () => boolean; + + /** Audio host control. */ + audio: { + started: () => boolean; + start: () => Promise; + stop: () => Promise; + setMuted: (muted: boolean) => void; + }; + + /** Loss / training plumbing surfaced from mlStore. */ + training: { + busy: () => boolean; + examples: () => number; + lastLoss: () => number | null; + lossHistory: () => ReadonlyArray; + }; + + /** Trigger a sync train + push the current pipeline-processed sample. */ + trainOnCurrent: () => void; + + /** RL callbacks. */ + thumbsUp: () => void; + thumbsDown: () => void; + randomize: () => void; +} + +interface RuntimeOptions { + /** Override the engine id (defaults to schema.engine_id). */ + engineOverride?: EngineId; + /** Skip starting the audio engine even on user gesture. */ + audioDisabled?: boolean; +} + +/** + * Create the runtime for a given mode schema. Call from inside a Solid + * component (uses createSignal/onCleanup). + */ +export function useModeRuntime( + schema: ModeSchema, + opts: RuntimeOptions = {}, +): ModeRuntime { + // ----- WASM init --------------------------------------------------------- + const [ready, setReady] = createSignal(mlStore.state.ready); + + // Lazy initialise WASM. Idempotent across remounts. + void mlStore + .initialize(schema.ml.input_size, schema.ml.output_size) + .then(() => setReady(true)) + .catch((err) => { + // Best-effort; UI keeps running without ML. + // eslint-disable-next-line no-console + console.error('[mode-runtime] mlStore.initialize failed:', err); + }); + + // ----- Mode-store side effects ------------------------------------------ + // Make sure the active mode in the store matches what's actually rendered. + if (modeStore.state.activeModeId !== schema.mode_id) { + modeStore.switchMode(schema.mode_id); + } + + // ----- Input pipeline state -------------------------------------------- + const [pipedInput, setPipedInput] = createSignal([0.5, 0.5]); + const [frozen, setFrozen] = createSignal(false); + let inputState: InputState = defaultInputState(); + let lastFrameMs = performance.now(); + + const setInput = (rawX: number, rawY: number): void => { + const now = performance.now(); + const dt = Math.max(0.001, (now - lastFrameMs) / 1000); + lastFrameMs = now; + + const result = processInput([rawX, rawY], inputStore.config, inputState, dt); + inputState = result.state; + inputStore.__setLiveState(result.state); + setPipedInput([result.x, result.y]); + setFrozen(result.frozen); + + if (!ready()) return; + // Push input to the MLP. Channels beyond [x,y] are zeroed out — modes + // with input_size > 2 currently aren't fed extra inputs (audio analysis + // wiring is a stream-10 task). + const inSz = schema.ml.input_size; + mlStore.setInput(0, result.x); + if (inSz > 1) mlStore.setInput(1, result.y); + for (let i = 2; i < inSz; ++i) mlStore.setInput(i, 0); + mlStore.process(); + }; + + // ----- Output pipeline state ------------------------------------------- + let outputState: OutputState = defaultOutputState(); + const sliceLen = schema.ml.output_size; + const [processedOutputs, setProcessedOutputs] = createSignal( + new Float32Array(sliceLen), + { equals: false }, // always notify even when buffer is reused in-place + ); + + // Run the output pipeline whenever raw outputs change. + const rawOutputsAccessor = mlStore.outputs; + let lastOutFrameMs = performance.now(); + + const recomputeOutputs = () => { + const raw = rawOutputsAccessor(); + if (raw.length === 0) return; + const now = performance.now(); + const dtMs = Math.max(1, now - lastOutFrameMs); + lastOutFrameMs = now; + // Slice to mode's output_size up front. + const slice = raw.length === sliceLen ? raw : raw.subarray(0, sliceLen); + const result = processOutput(slice as Float32Array, outputStore.config, outputState, dtMs); + outputState = result.state; + setProcessedOutputs(result.processed); + }; + + // Trigger recompute on any raw-output change. + createEffect(() => { + rawOutputsAccessor(); + recomputeOutputs(); + }); + + // ----- Engine wiring (audio) ------------------------------------------- + const host = getEngineHost(); + const [audioStarted, setAudioStarted] = createSignal(host.isStarted); + let pendingParams: Float32Array | null = null; + let throttleTimer: number | null = null; + + const flushParams = () => { + throttleTimer = null; + if (!pendingParams || !host.isStarted) { + pendingParams = null; + return; + } + // Copy because EngineHost transfers the buffer. + const copy = new Float32Array(pendingParams); + pendingParams = null; + try { + host.setParams(copy); + } catch (err) { + // eslint-disable-next-line no-console + console.warn('[mode-runtime] setParams failed:', err); + } + }; + + const scheduleParamFlush = (params: Float32Array) => { + pendingParams = params; + if (throttleTimer === null) { + throttleTimer = window.setTimeout(flushParams, ENGINE_PARAM_THROTTLE_MS); + } + }; + + // Pipe processedOutputs into the engine host whenever they change. + createEffect(() => { + const out = processedOutputs(); + if (out.length === 0) return; + if (!host.isStarted) return; + scheduleParamFlush(out); + }); + + const engineId = (opts.engineOverride ?? (schema.engine_id as EngineId)); + + const startAudio = async (): Promise => { + if (opts.audioDisabled) return; + try { + await host.start(engineId); + setAudioStarted(true); + // Push the current outputs immediately on start. + const out = processedOutputs(); + if (out.length > 0) host.setParams(new Float32Array(out)); + } catch (err) { + // eslint-disable-next-line no-console + console.error('[mode-runtime] audio start failed:', err); + } + }; + + const stopAudio = async (): Promise => { + try { + await host.stop(); + } finally { + setAudioStarted(false); + } + }; + + // Switch engine if mode changes engine_id (e.g. on remount). + onMount(() => { + if (host.isStarted) host.setEngine(engineId); + }); + + onCleanup(() => { + if (throttleTimer !== null) { + clearTimeout(throttleTimer); + throttleTimer = null; + } + pendingParams = null; + }); + + // ----- Training helpers ------------------------------------------------- + const trainOnCurrent = () => { + if (!ready()) return; + const lr = controlStore.resolveParams()['learningRate']; + const lrNum = typeof lr === 'number' ? lr : schema.ml.default_learning_rate; + mlStore.train(lrNum, schema.ml.default_max_iterations, 0.001); + }; + + const thumbsUp = () => { + if (!ready()) return; + // Push a label = current pipeline-processed slice as the target at the + // current input. This matches the legacy "thumbs up = remember the + // current sound at this position" semantics. + const [x, y] = pipedInput(); + const out = processedOutputs(); + if (out.length === 0) return; + const features = new Array(schema.ml.input_size).fill(0); + features[0] = x; + if (features.length > 1) features[1] = y; + const labels = Array.from(out); + mlStore.addExample(features, labels); + trainOnCurrent(); + }; + + const thumbsDown = () => { + if (!ready()) return; + const params = controlStore.resolveParams(); + const cap = typeof params['noiseCap'] === 'number' + ? (params['noiseCap'] as number) + : 0.12; + const spread = schema.ml.default_spread; + mlStore.moveWeights(cap, spread); + // Re-run inference at current input so the visual updates. + const [x, y] = pipedInput(); + setInput(x, y); + }; + + const randomize = () => { + if (!ready()) return; + mlStore.drawWeights(schema.ml.default_spread); + const [x, y] = pipedInput(); + setInput(x, y); + }; + + return { + setInput, + pipedInput, + frozen, + rawOutputs: rawOutputsAccessor, + processedOutputs, + ready, + audio: { + started: audioStarted, + start: startAudio, + stop: stopAudio, + setMuted: (muted) => host.setMuted(muted), + }, + training: { + busy: () => mlStore.state.training, + examples: () => mlStore.state.exampleCount, + lastLoss: () => mlStore.state.lastLoss, + lossHistory: () => mlStore.state.lossHistory, + }, + trainOnCurrent, + thumbsUp, + thumbsDown, + randomize, + }; +} + +/** + * Disposes the shared EngineHost. Test helper — production never calls this. + */ +export function __disposeEngineHost(): void { + if (engineHost) { + engineHost.dispose(); + engineHost = null; + } +}