From 57c9edee3a872146a08afa35a2cf50d2fb276eb8 Mon Sep 17 00:00:00 2001 From: monkey-w1n5t0n Date: Sun, 28 Jun 2026 22:21:45 +0200 Subject: [PATCH] feat(slp-workshop): wire interactive browser Jolt + Explore controls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Playground parity for the two learning gestures, TS-only (reuses the existing nisps_ml_get/set_weights bindings; no C++/wasm change): - playground/src/ml/jolt.ts — TS port of nisps/ml/jolt.hpp. - playground/src/output/ou-explore.ts — TS port of nisps/ml/ou_noise.hpp. - mode-runtime.ts — jolt (~200Hz weight-morph timer via mlStore weights) + explore (OU stage in recomputeOutputs, ~30Hz roam timer). Both inert by default and cleaned up on unmount, so other modes are unaffected. - SLPWorkshopMode.tsx — hold-to-Jolt button + Explore slider. Uses Math.random() (documented); stochastic exploration aids don't need firmware-parity. Verified: playground tsc --noEmit clean. --- CLAUDE.md | 2 +- playground/src/ml/jolt.ts | 122 +++++++++++++++++++++++ playground/src/modes/SLPWorkshopMode.tsx | 56 +++++++++++ playground/src/modes/mode-runtime.ts | 122 +++++++++++++++++++++++ playground/src/output/ou-explore.ts | 122 +++++++++++++++++++++++ playground/src/stores/bus.ts | 2 +- 6 files changed, 424 insertions(+), 2 deletions(-) create mode 100644 playground/src/ml/jolt.ts create mode 100644 playground/src/output/ou-explore.ts diff --git a/CLAUDE.md b/CLAUDE.md index cab2a6d..3e3d944 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -128,7 +128,7 @@ The WASM target is fixed at `MLP<2, 10, 14, 18, 126>`. Modes with smaller `outpu - Engine MLP architecture is fixed at compile time — supporting per-mode hidden-layer shapes would need either multiple WASM modules or runtime variation. - Mic input through the worklet for XIASRI / SoundAnalysisMIDI is not wired; UI scaffolds render but feature is TODO. - C15 voice space integration in C15Mode is a placeholder. -- Jolt + OU-explore learning gestures are wired in firmware (`ModeBase`, surfaced on TogB1 / RVX1) and the SLP-Workshop mode runs in the browser, but the *interactive* browser controls for Jolt/OU are not yet wired into the playground UI (the WASM weight bindings exist; the runtime/output-stage hooks are the remaining work). +- The browser Jolt/OU controls (`playground/src/ml/jolt.ts`, `playground/src/output/ou-explore.ts`) reimplement the gesture math in TS rather than calling the C++ `ml::Jolt`/`ml::OUNoise` through WASM. They drive weights via the existing `nisps_ml_get/set_weights` bindings and use `Math.random()` (not the deterministic `Rng`) — fine for stochastic exploration aids, but firmware↔browser bit-parity of the noise itself is intentionally not guaranteed. ## URL parameters (playground) diff --git a/playground/src/ml/jolt.ts b/playground/src/ml/jolt.ts new file mode 100644 index 0000000..8b10f34 --- /dev/null +++ b/playground/src/ml/jolt.ts @@ -0,0 +1,122 @@ +/** + * jolt.ts — "Jolt": held-button continuous weight morph. + * + * Faithful TS port of `nisps/ml/jolt.hpp` (which itself ports upstream + * memllib InterfaceRL "jolts"). While a button/pedal is held, pick a + * handful of random weights scattered across the whole network and + * EMA-glide each toward a bounded random target; on arrival, re-roll the + * target so the motion never stops. Releasing freezes the weights where + * they landed (the change is permanent). + * + * This operates on the MLP's FLAT weight buffer (the same buffer reached + * via `mlStore.getWeights()` / `setWeights()` / weight_count), so it is + * architecture-agnostic. + * + * RNG note: the C++ owns a per-instance deterministic xoshiro256+ for + * firmware↔browser parity. This is a browser-only exploration aid that + * never round-trips through WASM, so we use `Math.random()` — the exact + * sequence does not need to match firmware. Constants are reproduced + * verbatim from JoltParams. + * + * DEFAULT STATE IS INERT: a freshly constructed Jolt is inactive and + * `step()` is a no-op until `press()` is called. + */ + +/** Upstream JoltParams defaults (kJolt* constants). */ +export interface JoltParams { + /** kJoltNumWeights — number of weights morphed at once. */ + numWeights: number; + /** kJoltMorphRate — EMA glide per tick (~1s @200Hz). */ + morphRate: number; + /** kJoltWeightMin — lower bound of a random target. */ + targetMin: number; + /** kJoltWeightMax — upper bound of a random target. */ + targetMax: number; + /** kJoltTargetEpsilon — re-roll target once within this distance. */ + targetEpsilon: number; +} + +export const DEFAULT_JOLT_PARAMS: JoltParams = { + numWeights: 40, + morphRate: 0.017, + targetMin: -1.2, + targetMax: 0.9, + targetEpsilon: 0.05, +}; + +export class Jolt { + private params_: JoltParams = { ...DEFAULT_JOLT_PARAMS }; + private active_ = false; + private n_ = 0; + private idx_: Int32Array = new Int32Array(0); + private target_: Float32Array = new Float32Array(0); + + constructor(params?: Partial) { + if (params) this.params_ = { ...this.params_, ...params }; + } + + setParams(p: Partial): void { + this.params_ = { ...this.params_, ...p }; + } + + get params(): Readonly { + return this.params_; + } + + active(): boolean { + return this.active_; + } + + /** + * Begin a jolt over a flat weight buffer of `weightCount` entries: pick + * `numWeights` random global indices and a bounded random target each. + */ + press(weightCount: number): void { + this.active_ = true; + this.n_ = this.params_.numWeights; + if (weightCount <= 0) { + this.n_ = 0; + return; + } + if (this.idx_.length < this.n_) this.idx_ = new Int32Array(this.n_); + if (this.target_.length < this.n_) this.target_ = new Float32Array(this.n_); + for (let i = 0; i < this.n_; ++i) { + this.idx_[i] = Math.floor(Math.random() * weightCount) % weightCount; + this.target_[i] = this.rollTarget_(); + } + } + + /** + * Per control tick while held: EMA-glide each selected weight toward its + * target, re-rolling targets that have been reached. No-op when inactive. + * Mutates `weights` in place. + */ + step(weights: Float32Array): void { + if (!this.active_) return; + const wc = weights.length; + const rate = this.params_.morphRate; + const eps = this.params_.targetEpsilon; + for (let i = 0; i < this.n_; ++i) { + const k = this.idx_[i]!; + if (k < 0 || k >= wc) continue; + let w = weights[k]!; + w += rate * (this.target_[i]! - w); + weights[k] = w; + if (Math.abs(this.target_[i]! - w) < eps) { + this.target_[i] = this.rollTarget_(); + } + } + } + + /** Release: freeze weights where they are (permanent). */ + release(): void { + this.active_ = false; + } + + private rollTarget_(): number { + return ( + this.params_.targetMin + + Math.random() * (this.params_.targetMax - this.params_.targetMin) + ); + } +} diff --git a/playground/src/modes/SLPWorkshopMode.tsx b/playground/src/modes/SLPWorkshopMode.tsx index cbcb6be..7541c5b 100644 --- a/playground/src/modes/SLPWorkshopMode.tsx +++ b/playground/src/modes/SLPWorkshopMode.tsx @@ -16,12 +16,19 @@ import { useModeRuntime } from './mode-runtime'; import { XYPad } from '../primitives/XYPad'; import { OutputDisplay } from '../primitives/OutputDisplay'; import { LossPlot } from '../primitives/LossPlot'; +import { Slider } from '../primitives/Slider'; import { SlpWorkshopSchema } from './generated/slp_workshop_schema'; export const SLPWorkshopMode: Component = () => { const schema = SlpWorkshopSchema; const runtime = useModeRuntime(schema); const [voiceSpace, setVoiceSpace] = createSignal(0); + const [exploreLevel, setExploreLevel] = createSignal(0); + + const setExplore = (v: number): void => { + setExploreLevel(v); + runtime.explore.setIntensity(v); + }; return ( { Synth Library Portland workshop instrument — MEMLCelium voice with live Jolt + Explore learning controls. + + {/* Adaptive-learning gestures. */} +
+ {/* Jolt — hold to morph weights live, release to freeze. */} + + + {/* Explore — OU random-walk exploration intensity. */} + +
)} outputArea={() => ( diff --git a/playground/src/modes/mode-runtime.ts b/playground/src/modes/mode-runtime.ts index 191a0f3..31404be 100644 --- a/playground/src/modes/mode-runtime.ts +++ b/playground/src/modes/mode-runtime.ts @@ -42,6 +42,8 @@ import type { ModeSchema } from './generated'; import { applyOverrides, buildPinMask } from '../features/overrides'; import { applyControlRouting } from '../features/control-routing'; +import { Jolt } from '../ml/jolt'; +import { OUExplore } from '../output/ou-explore'; import { createTrailRing, type TrailPoint } from '../features/trail'; import { autoSnapshot, undoLastSnapshot } from '../features/snapshots'; import { HeatmapSampler, type HeatmapColorMode } from '../features/heatmap-sampler'; @@ -137,6 +139,32 @@ export interface ModeRuntime { /** Region pin: pin the current zoom window (long-press handler). */ pinCurrentRegion: () => void; + + /** + * Jolt — held-button continuous weight morph (port of nisps/ml/jolt.hpp). + * `press()` starts morphing the network's weights; a control-rate timer + * glides them while held; `release()` freezes them in place. Inert until + * pressed; modes that never call `press()` are unaffected. + */ + jolt: { + /** Begin the jolt (button press / toggle on). */ + press: () => void; + /** Freeze the jolt (button release / toggle off). */ + release: () => void; + /** Whether the jolt is currently active. */ + active: () => boolean; + }; + + /** + * Explore — OU random-walk exploration intensity in [0,1] (port of + * nisps/ml/ou_noise.hpp). Added to the mode's output vector before audio. + * Intensity 0 = disabled passthrough; modes that never set it are + * unaffected. + */ + explore: { + setIntensity: (level: number) => void; + intensity: () => number; + }; } interface RuntimeOptions { @@ -253,6 +281,14 @@ export function useModeRuntime( { equals: false }, ); + // ----- Explore (OU exploration noise) ---------------------------------- + // Inert by default (intensity 0). Applied to the processed slice below, + // after the output pipeline and before override application, so it rides + // on top of the mode's mapping output. Modes that never set an intensity + // leave the output untouched (parity-safe). + const ouExplore = new OUExplore(); + const [exploreIntensity, setExploreIntensitySig] = createSignal(0); + // Run the output pipeline whenever raw outputs change. const rawOutputsAccessor = mlStore.outputs; let lastOutFrameMs = performance.now(); @@ -267,6 +303,9 @@ export function useModeRuntime( const slice = raw.length === sliceLen ? raw : raw.subarray(0, sliceLen); const result = processOutput(slice as Float32Array, outputStore.config, outputState, dtMs); outputState = result.state; + // Explore (OU) noise: temporally-correlated random walk added on top of + // the processed output, clamped to [0,1]. No-op when intensity is 0. + ouExplore.apply(result.processed); setProcessedOutputs(result.processed); // Apply per-param overrides for the per-param consumer (engine, sliders). @@ -487,6 +526,80 @@ export function useModeRuntime( } }); + // ----- Jolt (held-button continuous weight morph) ---------------------- + // Inert until press(). While active, a ~200Hz control-rate timer glides a + // scatter of the network's weights toward random targets (port of + // nisps/ml/jolt.hpp). release() freezes them where they landed. + const jolt = new Jolt(); + let joltTimer: number | null = null; + // ~200Hz to match the upstream firmware control rate the constants assume. + const JOLT_TICK_MS = 5; + + const tickJolt = () => { + if (!ready() || !jolt.active()) return; + const w = mlStore.getWeights(); + if (w.length === 0) return; + jolt.step(w); + mlStore.setWeights(w); + coreBus.emit('ml.delta_update', { reason: 'jolt' }); + // Re-run inference so the audio + visuals reflect the morphed weights. + const [x, y] = pipedInput(); + setInput(x, y); + }; + + const joltPress = () => { + if (!ready()) return; + autoSnapshot('before jolt'); + jolt.press(mlStore.iml?.weightCount ?? mlStore.getWeights().length); + if (joltTimer === null) { + joltTimer = window.setInterval(tickJolt, JOLT_TICK_MS); + } + }; + + const joltRelease = () => { + jolt.release(); + if (joltTimer !== null) { + window.clearInterval(joltTimer); + joltTimer = null; + } + }; + onCleanup(() => { + if (joltTimer !== null) { + window.clearInterval(joltTimer); + joltTimer = null; + } + }); + + // ----- Explore (OU) control-rate driver -------------------------------- + // The OU walk advances inside recomputeOutputs (driven by setInput). When + // the input is static the walk would stall, so while Explore is active we + // keep ticking so the sound keeps roaming. Inert when intensity is 0. + let exploreTimer: number | null = null; + const EXPLORE_TICK_MS = 30; + const setExploreIntensity = (level: number) => { + ouExplore.setIntensity(level); + setExploreIntensitySig(ouExplore.intensity()); + if (ouExplore.enabled() && exploreTimer === null) { + exploreTimer = window.setInterval(() => { + untrack(() => { + if (!ready()) return; + // Re-run the output pipeline (advances the OU state) and reship. + recomputeOutputs(); + }); + }, EXPLORE_TICK_MS); + } else if (!ouExplore.enabled() && exploreTimer !== null) { + window.clearInterval(exploreTimer); + exploreTimer = null; + ouExplore.reset(); + } + }; + onCleanup(() => { + if (exploreTimer !== null) { + window.clearInterval(exploreTimer); + exploreTimer = null; + } + }); + // ----- Touch / pressure feedback --------------------------------------- const onPointerDown = (e: PointerEvent) => { pressDownAt = performance.now(); @@ -649,6 +762,15 @@ export function useModeRuntime( resolution: () => sampler.resolution, }, pinCurrentRegion, + jolt: { + press: joltPress, + release: joltRelease, + active: () => jolt.active(), + }, + explore: { + setIntensity: setExploreIntensity, + intensity: exploreIntensity, + }, }; } diff --git a/playground/src/output/ou-explore.ts b/playground/src/output/ou-explore.ts new file mode 100644 index 0000000..ccd6fc4 --- /dev/null +++ b/playground/src/output/ou-explore.ts @@ -0,0 +1,122 @@ +/** + * ou-explore.ts — Ornstein-Uhlenbeck exploration noise on the output. + * + * Faithful TS port of `nisps/ml/ou_noise.hpp`. Adds a per-output-channel + * Ornstein-Uhlenbeck random walk to the network's action (output) vector + * before it reaches audio. Unlike i.i.d. per-frame noise, an OU process is + * temporally correlated: each output drifts in long, smooth sweeps and is + * gently pulled back toward the mapping output (mean reversion). Because + * learning stays active while the noise roams, "likes" registered during + * the wander steer the network toward sounds the player wants. + * + * Discrete Euler-Maruyama update, per output channel x (mu = 0): + * x += theta * (-x) * dt + noiseScale * N(0,1) + * out = clamp(out + x, 0, 1) + * where, to make the stationary std equal a requested `std`: + * noiseScale = std * sqrt(2 * theta * dt) + * and the exploration knob [0,1] maps std = level * kMaxAmplitude (0.65). + * + * RNG note: the C++ owns a per-instance deterministic xoshiro256+ for + * firmware↔browser parity. This is a browser-only exploration aid that + * never round-trips through WASM, so `Math.random()` is fine. The gaussian + * is reproduced as the same sum-of-three-uniforms shape the C++ `Rng` uses + * (`next_float_gaussian`). + * + * DEFAULT STATE IS INERT: intensity defaults to 0, `enabled()` is false, and + * `apply()` neither advances the RNG nor touches the output — so the output + * passes through bit-identically when intensity is 0 (parity-safe). + */ + +/** Upstream kMaxAmplitude: the exploration knob's full-scale stationary std. */ +export const OU_MAX_AMPLITUDE = 0.65; + +/** Sum-of-three-uniforms gaussian (matches nisps::Rng::next_float_gaussian). */ +function gaussian(stddev = 1): number { + // Three uniforms in [-1, 1): variance 1/3 each ⇒ sum has variance 1. + const a = Math.random() * 2 - 1; + const b = Math.random() * 2 - 1; + const c = Math.random() * 2 - 1; + return (a + b + c) * stddev; +} + +export class OUExplore { + private theta_ = 0.02; + private dt_ = 0.001; + private stationaryStd_ = 0; + private noiseScale_ = 0; + private state_: Float32Array = new Float32Array(0); + + constructor() { + this.recomputeScale_(); + } + + /** + * Exploration amount in [0,1]; 0 disables (inert). Maps to the OU + * stationary std = level * kMaxAmplitude. + */ + setIntensity(level: number): void { + let l = level; + if (l < 0) l = 0; + else if (l > 1) l = 1; + this.stationaryStd_ = l * OU_MAX_AMPLITUDE; + this.recomputeScale_(); + } + + intensity(): number { + return this.stationaryStd_ / OU_MAX_AMPLITUDE; + } + + enabled(): boolean { + return this.stationaryStd_ > 0; + } + + setTheta(theta: number): void { + this.theta_ = theta; + this.recomputeScale_(); + } + + setDt(dt: number): void { + this.dt_ = dt; + this.recomputeScale_(); + } + + /** + * Advance the per-channel OU state and add it (clamped) to `out`, mutating + * `out` in place. No-op (no state advance, no output change) when disabled. + * `out` is the post-inference parameter vector. + */ + apply(out: Float32Array): void { + if (!this.enabled()) return; + const n = out.length; + if (this.state_.length < n) { + // Grow the per-channel state, preserving existing drift. + const next = new Float32Array(n); + next.set(this.state_); + this.state_ = next; + } + const theta = this.theta_; + const dt = this.dt_; + const scale = this.noiseScale_; + for (let i = 0; i < n; ++i) { + // mu = 0: the walk is an offset that mean-reverts to zero, so the + // network's own mapping output stays the anchor. + let s = this.state_[i]!; + s += theta * -s * dt + scale * gaussian(1); + this.state_[i] = s; + let v = out[i]! + s; + if (v < 0) v = 0; + else if (v > 1) v = 1; + out[i] = v; + } + } + + reset(): void { + this.state_.fill(0); + } + + private recomputeScale_(): void { + let k = 2 * this.theta_ * this.dt_; + if (k < 0) k = 0; + this.noiseScale_ = this.stationaryStd_ * Math.sqrt(k); + } +} diff --git a/playground/src/stores/bus.ts b/playground/src/stores/bus.ts index 84007a5..123ef9f 100644 --- a/playground/src/stores/bus.ts +++ b/playground/src/stores/bus.ts @@ -100,7 +100,7 @@ export function createBus(): Bus { export type CoreEvents = { // ML 'ml.trained': { loss: number }; - 'ml.delta_update': { reason: 'thumbs_up' | 'thumbs_down' | 'randomize' | 'undo' }; + 'ml.delta_update': { reason: 'thumbs_up' | 'thumbs_down' | 'randomize' | 'undo' | 'jolt' }; 'ml.example_added': { count: number }; 'ml.examples_cleared': void;