feat(slp-workshop): wire interactive browser Jolt + Explore controls

Playground parity for the two learning gestures, TS-only (reuses the
existing nisps_ml_get/set_weights bindings; no C++/wasm change):

- playground/src/ml/jolt.ts — TS port of nisps/ml/jolt.hpp.
- playground/src/output/ou-explore.ts — TS port of nisps/ml/ou_noise.hpp.
- mode-runtime.ts — jolt (~200Hz weight-morph timer via mlStore weights)
  + explore (OU stage in recomputeOutputs, ~30Hz roam timer). Both inert
  by default and cleaned up on unmount, so other modes are unaffected.
- SLPWorkshopMode.tsx — hold-to-Jolt button + Explore slider.

Uses Math.random() (documented); stochastic exploration aids don't need
firmware-parity. Verified: playground tsc --noEmit clean.
This commit is contained in:
monkey-w1n5t0n 2026-06-28 22:21:45 +02:00
parent 4e60d0192a
commit 57c9edee3a
6 changed files with 424 additions and 2 deletions

View file

@ -128,7 +128,7 @@ The WASM target is fixed at `MLP<2, 10, 14, 18, 126>`. Modes with smaller `outpu
- Engine MLP architecture is fixed at compile time — supporting per-mode hidden-layer shapes would need either multiple WASM modules or runtime variation.
- Mic input through the worklet for XIASRI / SoundAnalysisMIDI is not wired; UI scaffolds render but feature is TODO.
- C15 voice space integration in C15Mode is a placeholder.
- Jolt + OU-explore learning gestures are wired in firmware (`ModeBase`, surfaced on TogB1 / RVX1) and the SLP-Workshop mode runs in the browser, but the *interactive* browser controls for Jolt/OU are not yet wired into the playground UI (the WASM weight bindings exist; the runtime/output-stage hooks are the remaining work).
- The browser Jolt/OU controls (`playground/src/ml/jolt.ts`, `playground/src/output/ou-explore.ts`) reimplement the gesture math in TS rather than calling the C++ `ml::Jolt`/`ml::OUNoise` through WASM. They drive weights via the existing `nisps_ml_get/set_weights` bindings and use `Math.random()` (not the deterministic `Rng`) — fine for stochastic exploration aids, but firmware↔browser bit-parity of the noise itself is intentionally not guaranteed.
## URL parameters (playground)

122
playground/src/ml/jolt.ts Normal file
View file

@ -0,0 +1,122 @@
/**
* jolt.ts "Jolt": held-button continuous weight morph.
*
* Faithful TS port of `nisps/ml/jolt.hpp` (which itself ports upstream
* memllib InterfaceRL "jolts"). While a button/pedal is held, pick a
* handful of random weights scattered across the whole network and
* EMA-glide each toward a bounded random target; on arrival, re-roll the
* target so the motion never stops. Releasing freezes the weights where
* they landed (the change is permanent).
*
* This operates on the MLP's FLAT weight buffer (the same buffer reached
* via `mlStore.getWeights()` / `setWeights()` / weight_count), so it is
* architecture-agnostic.
*
* RNG note: the C++ owns a per-instance deterministic xoshiro256+ for
* firmwarebrowser parity. This is a browser-only exploration aid that
* never round-trips through WASM, so we use `Math.random()` the exact
* sequence does not need to match firmware. Constants are reproduced
* verbatim from JoltParams.
*
* DEFAULT STATE IS INERT: a freshly constructed Jolt is inactive and
* `step()` is a no-op until `press()` is called.
*/
/** Upstream JoltParams defaults (kJolt* constants). */
export interface JoltParams {
/** kJoltNumWeights — number of weights morphed at once. */
numWeights: number;
/** kJoltMorphRate — EMA glide per tick (~1s @200Hz). */
morphRate: number;
/** kJoltWeightMin — lower bound of a random target. */
targetMin: number;
/** kJoltWeightMax — upper bound of a random target. */
targetMax: number;
/** kJoltTargetEpsilon — re-roll target once within this distance. */
targetEpsilon: number;
}
export const DEFAULT_JOLT_PARAMS: JoltParams = {
numWeights: 40,
morphRate: 0.017,
targetMin: -1.2,
targetMax: 0.9,
targetEpsilon: 0.05,
};
export class Jolt {
private params_: JoltParams = { ...DEFAULT_JOLT_PARAMS };
private active_ = false;
private n_ = 0;
private idx_: Int32Array = new Int32Array(0);
private target_: Float32Array = new Float32Array(0);
constructor(params?: Partial<JoltParams>) {
if (params) this.params_ = { ...this.params_, ...params };
}
setParams(p: Partial<JoltParams>): void {
this.params_ = { ...this.params_, ...p };
}
get params(): Readonly<JoltParams> {
return this.params_;
}
active(): boolean {
return this.active_;
}
/**
* Begin a jolt over a flat weight buffer of `weightCount` entries: pick
* `numWeights` random global indices and a bounded random target each.
*/
press(weightCount: number): void {
this.active_ = true;
this.n_ = this.params_.numWeights;
if (weightCount <= 0) {
this.n_ = 0;
return;
}
if (this.idx_.length < this.n_) this.idx_ = new Int32Array(this.n_);
if (this.target_.length < this.n_) this.target_ = new Float32Array(this.n_);
for (let i = 0; i < this.n_; ++i) {
this.idx_[i] = Math.floor(Math.random() * weightCount) % weightCount;
this.target_[i] = this.rollTarget_();
}
}
/**
* Per control tick while held: EMA-glide each selected weight toward its
* target, re-rolling targets that have been reached. No-op when inactive.
* Mutates `weights` in place.
*/
step(weights: Float32Array): void {
if (!this.active_) return;
const wc = weights.length;
const rate = this.params_.morphRate;
const eps = this.params_.targetEpsilon;
for (let i = 0; i < this.n_; ++i) {
const k = this.idx_[i]!;
if (k < 0 || k >= wc) continue;
let w = weights[k]!;
w += rate * (this.target_[i]! - w);
weights[k] = w;
if (Math.abs(this.target_[i]! - w) < eps) {
this.target_[i] = this.rollTarget_();
}
}
}
/** Release: freeze weights where they are (permanent). */
release(): void {
this.active_ = false;
}
private rollTarget_(): number {
return (
this.params_.targetMin +
Math.random() * (this.params_.targetMax - this.params_.targetMin)
);
}
}

View file

@ -16,12 +16,19 @@ import { useModeRuntime } from './mode-runtime';
import { XYPad } from '../primitives/XYPad';
import { OutputDisplay } from '../primitives/OutputDisplay';
import { LossPlot } from '../primitives/LossPlot';
import { Slider } from '../primitives/Slider';
import { SlpWorkshopSchema } from './generated/slp_workshop_schema';
export const SLPWorkshopMode: Component = () => {
const schema = SlpWorkshopSchema;
const runtime = useModeRuntime(schema);
const [voiceSpace, setVoiceSpace] = createSignal(0);
const [exploreLevel, setExploreLevel] = createSignal(0);
const setExplore = (v: number): void => {
setExploreLevel(v);
runtime.explore.setIntensity(v);
};
return (
<ModeShell
@ -42,6 +49,55 @@ export const SLPWorkshopMode: Component = () => {
Synth Library Portland workshop instrument MEMLCelium voice with
live Jolt + Explore learning controls.
</span>
{/* Adaptive-learning gestures. */}
<div
style={{
display: 'flex',
'flex-direction': 'column',
gap: 'var(--sp-2)',
width: '280px',
'margin-top': 'var(--sp-2)',
}}
>
{/* Jolt — hold to morph weights live, release to freeze. */}
<button
type="button"
aria-label="Jolt — hold to morph the network's weights"
aria-pressed={runtime.jolt.active()}
onPointerDown={(e) => {
e.preventDefault();
runtime.jolt.press();
}}
onPointerUp={() => runtime.jolt.release()}
onPointerLeave={() => runtime.jolt.release()}
onPointerCancel={() => runtime.jolt.release()}
style={{
padding: 'var(--sp-2)',
'border-radius': 'var(--r-1)',
border: '1px solid var(--line)',
background: runtime.jolt.active() ? 'var(--accent, #ef9e5b)' : 'var(--bg-2)',
color: runtime.jolt.active() ? '#000' : 'var(--fg)',
'font-weight': 600,
cursor: 'pointer',
'touch-action': 'none',
'user-select': 'none',
}}
>
Jolt {runtime.jolt.active() ? '(hold…)' : '(hold)'}
</button>
{/* Explore — OU random-walk exploration intensity. */}
<Slider
label="Explore"
ariaLabel="Explore — exploration random-walk intensity"
min={0}
max={1}
value={exploreLevel()}
onChange={setExplore}
decimals={2}
/>
</div>
</>
)}
outputArea={() => (

View file

@ -42,6 +42,8 @@ import type { ModeSchema } from './generated';
import { applyOverrides, buildPinMask } from '../features/overrides';
import { applyControlRouting } from '../features/control-routing';
import { Jolt } from '../ml/jolt';
import { OUExplore } from '../output/ou-explore';
import { createTrailRing, type TrailPoint } from '../features/trail';
import { autoSnapshot, undoLastSnapshot } from '../features/snapshots';
import { HeatmapSampler, type HeatmapColorMode } from '../features/heatmap-sampler';
@ -137,6 +139,32 @@ export interface ModeRuntime {
/** Region pin: pin the current zoom window (long-press handler). */
pinCurrentRegion: () => void;
/**
* Jolt held-button continuous weight morph (port of nisps/ml/jolt.hpp).
* `press()` starts morphing the network's weights; a control-rate timer
* glides them while held; `release()` freezes them in place. Inert until
* pressed; modes that never call `press()` are unaffected.
*/
jolt: {
/** Begin the jolt (button press / toggle on). */
press: () => void;
/** Freeze the jolt (button release / toggle off). */
release: () => void;
/** Whether the jolt is currently active. */
active: () => boolean;
};
/**
* Explore OU random-walk exploration intensity in [0,1] (port of
* nisps/ml/ou_noise.hpp). Added to the mode's output vector before audio.
* Intensity 0 = disabled passthrough; modes that never set it are
* unaffected.
*/
explore: {
setIntensity: (level: number) => void;
intensity: () => number;
};
}
interface RuntimeOptions {
@ -253,6 +281,14 @@ export function useModeRuntime(
{ equals: false },
);
// ----- Explore (OU exploration noise) ----------------------------------
// Inert by default (intensity 0). Applied to the processed slice below,
// after the output pipeline and before override application, so it rides
// on top of the mode's mapping output. Modes that never set an intensity
// leave the output untouched (parity-safe).
const ouExplore = new OUExplore();
const [exploreIntensity, setExploreIntensitySig] = createSignal(0);
// Run the output pipeline whenever raw outputs change.
const rawOutputsAccessor = mlStore.outputs;
let lastOutFrameMs = performance.now();
@ -267,6 +303,9 @@ export function useModeRuntime(
const slice = raw.length === sliceLen ? raw : raw.subarray(0, sliceLen);
const result = processOutput(slice as Float32Array, outputStore.config, outputState, dtMs);
outputState = result.state;
// Explore (OU) noise: temporally-correlated random walk added on top of
// the processed output, clamped to [0,1]. No-op when intensity is 0.
ouExplore.apply(result.processed);
setProcessedOutputs(result.processed);
// Apply per-param overrides for the per-param consumer (engine, sliders).
@ -487,6 +526,80 @@ export function useModeRuntime(
}
});
// ----- Jolt (held-button continuous weight morph) ----------------------
// Inert until press(). While active, a ~200Hz control-rate timer glides a
// scatter of the network's weights toward random targets (port of
// nisps/ml/jolt.hpp). release() freezes them where they landed.
const jolt = new Jolt();
let joltTimer: number | null = null;
// ~200Hz to match the upstream firmware control rate the constants assume.
const JOLT_TICK_MS = 5;
const tickJolt = () => {
if (!ready() || !jolt.active()) return;
const w = mlStore.getWeights();
if (w.length === 0) return;
jolt.step(w);
mlStore.setWeights(w);
coreBus.emit('ml.delta_update', { reason: 'jolt' });
// Re-run inference so the audio + visuals reflect the morphed weights.
const [x, y] = pipedInput();
setInput(x, y);
};
const joltPress = () => {
if (!ready()) return;
autoSnapshot('before jolt');
jolt.press(mlStore.iml?.weightCount ?? mlStore.getWeights().length);
if (joltTimer === null) {
joltTimer = window.setInterval(tickJolt, JOLT_TICK_MS);
}
};
const joltRelease = () => {
jolt.release();
if (joltTimer !== null) {
window.clearInterval(joltTimer);
joltTimer = null;
}
};
onCleanup(() => {
if (joltTimer !== null) {
window.clearInterval(joltTimer);
joltTimer = null;
}
});
// ----- Explore (OU) control-rate driver --------------------------------
// The OU walk advances inside recomputeOutputs (driven by setInput). When
// the input is static the walk would stall, so while Explore is active we
// keep ticking so the sound keeps roaming. Inert when intensity is 0.
let exploreTimer: number | null = null;
const EXPLORE_TICK_MS = 30;
const setExploreIntensity = (level: number) => {
ouExplore.setIntensity(level);
setExploreIntensitySig(ouExplore.intensity());
if (ouExplore.enabled() && exploreTimer === null) {
exploreTimer = window.setInterval(() => {
untrack(() => {
if (!ready()) return;
// Re-run the output pipeline (advances the OU state) and reship.
recomputeOutputs();
});
}, EXPLORE_TICK_MS);
} else if (!ouExplore.enabled() && exploreTimer !== null) {
window.clearInterval(exploreTimer);
exploreTimer = null;
ouExplore.reset();
}
};
onCleanup(() => {
if (exploreTimer !== null) {
window.clearInterval(exploreTimer);
exploreTimer = null;
}
});
// ----- Touch / pressure feedback ---------------------------------------
const onPointerDown = (e: PointerEvent) => {
pressDownAt = performance.now();
@ -649,6 +762,15 @@ export function useModeRuntime(
resolution: () => sampler.resolution,
},
pinCurrentRegion,
jolt: {
press: joltPress,
release: joltRelease,
active: () => jolt.active(),
},
explore: {
setIntensity: setExploreIntensity,
intensity: exploreIntensity,
},
};
}

View file

@ -0,0 +1,122 @@
/**
* ou-explore.ts Ornstein-Uhlenbeck exploration noise on the output.
*
* Faithful TS port of `nisps/ml/ou_noise.hpp`. Adds a per-output-channel
* Ornstein-Uhlenbeck random walk to the network's action (output) vector
* before it reaches audio. Unlike i.i.d. per-frame noise, an OU process is
* temporally correlated: each output drifts in long, smooth sweeps and is
* gently pulled back toward the mapping output (mean reversion). Because
* learning stays active while the noise roams, "likes" registered during
* the wander steer the network toward sounds the player wants.
*
* Discrete Euler-Maruyama update, per output channel x (mu = 0):
* x += theta * (-x) * dt + noiseScale * N(0,1)
* out = clamp(out + x, 0, 1)
* where, to make the stationary std equal a requested `std`:
* noiseScale = std * sqrt(2 * theta * dt)
* and the exploration knob [0,1] maps std = level * kMaxAmplitude (0.65).
*
* RNG note: the C++ owns a per-instance deterministic xoshiro256+ for
* firmwarebrowser parity. This is a browser-only exploration aid that
* never round-trips through WASM, so `Math.random()` is fine. The gaussian
* is reproduced as the same sum-of-three-uniforms shape the C++ `Rng` uses
* (`next_float_gaussian`).
*
* DEFAULT STATE IS INERT: intensity defaults to 0, `enabled()` is false, and
* `apply()` neither advances the RNG nor touches the output so the output
* passes through bit-identically when intensity is 0 (parity-safe).
*/
/** Upstream kMaxAmplitude: the exploration knob's full-scale stationary std. */
export const OU_MAX_AMPLITUDE = 0.65;
/** Sum-of-three-uniforms gaussian (matches nisps::Rng::next_float_gaussian). */
function gaussian(stddev = 1): number {
// Three uniforms in [-1, 1): variance 1/3 each ⇒ sum has variance 1.
const a = Math.random() * 2 - 1;
const b = Math.random() * 2 - 1;
const c = Math.random() * 2 - 1;
return (a + b + c) * stddev;
}
export class OUExplore {
private theta_ = 0.02;
private dt_ = 0.001;
private stationaryStd_ = 0;
private noiseScale_ = 0;
private state_: Float32Array = new Float32Array(0);
constructor() {
this.recomputeScale_();
}
/**
* Exploration amount in [0,1]; 0 disables (inert). Maps to the OU
* stationary std = level * kMaxAmplitude.
*/
setIntensity(level: number): void {
let l = level;
if (l < 0) l = 0;
else if (l > 1) l = 1;
this.stationaryStd_ = l * OU_MAX_AMPLITUDE;
this.recomputeScale_();
}
intensity(): number {
return this.stationaryStd_ / OU_MAX_AMPLITUDE;
}
enabled(): boolean {
return this.stationaryStd_ > 0;
}
setTheta(theta: number): void {
this.theta_ = theta;
this.recomputeScale_();
}
setDt(dt: number): void {
this.dt_ = dt;
this.recomputeScale_();
}
/**
* Advance the per-channel OU state and add it (clamped) to `out`, mutating
* `out` in place. No-op (no state advance, no output change) when disabled.
* `out` is the post-inference parameter vector.
*/
apply(out: Float32Array): void {
if (!this.enabled()) return;
const n = out.length;
if (this.state_.length < n) {
// Grow the per-channel state, preserving existing drift.
const next = new Float32Array(n);
next.set(this.state_);
this.state_ = next;
}
const theta = this.theta_;
const dt = this.dt_;
const scale = this.noiseScale_;
for (let i = 0; i < n; ++i) {
// mu = 0: the walk is an offset that mean-reverts to zero, so the
// network's own mapping output stays the anchor.
let s = this.state_[i]!;
s += theta * -s * dt + scale * gaussian(1);
this.state_[i] = s;
let v = out[i]! + s;
if (v < 0) v = 0;
else if (v > 1) v = 1;
out[i] = v;
}
}
reset(): void {
this.state_.fill(0);
}
private recomputeScale_(): void {
let k = 2 * this.theta_ * this.dt_;
if (k < 0) k = 0;
this.noiseScale_ = this.stationaryStd_ * Math.sqrt(k);
}
}

View file

@ -100,7 +100,7 @@ export function createBus<E extends EventMap>(): Bus<E> {
export type CoreEvents = {
// ML
'ml.trained': { loss: number };
'ml.delta_update': { reason: 'thumbs_up' | 'thumbs_down' | 'randomize' | 'undo' };
'ml.delta_update': { reason: 'thumbs_up' | 'thumbs_down' | 'randomize' | 'undo' | 'jolt' };
'ml.example_added': { count: number };
'ml.examples_cleared': void;