feat(manifold): port Jolt + OU explore gestures as UI shells

Bring the two playground-only exploration UIs into manifold ahead of the
playground's retirement (one-core-engine refactor §P1):

- Jolt: press-and-hold continuous weight-morph, release to freeze.
- Explore: Ornstein-Uhlenbeck exploration intensity on the output vector.

Interim TS maths ported verbatim from the retired playground modules
(engine/jolt.ts, engine/ou-explore.ts). The ExplorationController
(engine/exploration.ts) is the single P3 swap boundary: it drives Jolt via the
existing EngineApi get/set-weights + process route, and OU via a new inert-by-
default output-morph hook on the spine. In §P3 only that module changes to call
nisps_ml_jolt_press/release + nisps_ml_explore_intensity.

UI lands in the Learning drawer (Jolt hold-button + Explore slider), monochrome-
consistent, British copy. Gates green: typecheck, build, Playwright smoke.
This commit is contained in:
monkey-w1n5t0n 2026-07-13 23:27:40 +02:00
parent 29dc88be3a
commit 9056ac3f5e
8 changed files with 517 additions and 1 deletions

View file

@ -23,7 +23,7 @@
*/
import { useEffect, useMemo, useRef, useState } from 'react';
import type { CSSProperties } from 'react';
import { useEngine, useEngineVersion } from '../engine';
import { useEngine, useEngineVersion, ExplorationController } from '../engine';
import { MF_MODES, modeEngineId, seededGradient, shapeValues } from './model';
import type { MFParam } from './model';
import { CompositeStage } from './CompositeStage';
@ -125,6 +125,11 @@ export function ConsoleApp({ focus: initialFocus = 'composite' }: ConsoleAppProp
const [learningRate, setLearningRate] = useState(0.00001);
const [decay, setDecay] = useState(0.97);
const [spreadLevel, setSpreadLevel] = useState(0.6);
// Exploration gestures (Jolt held weight-morph + OU explore-intensity). The
// maths lives in the ExplorationController (engine/exploration.ts); these are
// the React-visible reflections the Learning drawer renders.
const [joltActive, setJoltActive] = useState(false);
const [exploreIntensity, setExploreIntensityState] = useState(0);
// Active output MODE (TOP dock selector) — default Particle System. The dock
// backend + audio backend derive from this.
const [outputMode, setOutputModeState] = useState<OutputMode>(DEFAULT_OUTPUT_MODE);
@ -180,6 +185,34 @@ export function ConsoleApp({ focus: initialFocus = 'composite' }: ConsoleAppProp
setUndoDepth(s.undoDepth);
};
// The exploration controller (Jolt weight-morph + OU explore-intensity). Same
// lazy-once-per-engine pattern as the feedback controller; its interim TS
// maths becomes WASM calls in P3 (engine/exploration.ts P3 SWAP POINT).
const explorationRef = useRef<ExplorationController | null>(null);
if (engine && !explorationRef.current) {
explorationRef.current = new ExplorationController(engine);
}
useEffect(
() => () => {
explorationRef.current?.dispose();
explorationRef.current = null;
},
[],
);
const onJoltPress = () => {
explorationRef.current?.joltPress();
setJoltActive(explorationRef.current?.joltActive() ?? false);
};
const onJoltRelease = () => {
explorationRef.current?.joltRelease();
setJoltActive(false);
};
const setExploreIntensity = (v: number) => {
explorationRef.current?.setExploreIntensity(v);
setExploreIntensityState(explorationRef.current?.exploreIntensity() ?? v);
};
const setFeedbackMode = (m: FeedbackModeUI) => {
setFeedbackModeState(m);
controllerRef.current?.setMode(m as ProtoFeedbackMode);
@ -739,6 +772,12 @@ export function ConsoleApp({ focus: initialFocus = 'composite' }: ConsoleAppProp
setDecay,
spreadLevel,
setSpreadLevel,
// exploration gestures (Jolt + OU explore)
joltActive,
onJoltPress,
onJoltRelease,
exploreIntensity,
setExploreIntensity,
// synth
audioStarted,
onToggleAudio,

View file

@ -168,6 +168,45 @@ function LearningDrawer(ctx: ConsoleCtx, depth: DrawerDepth) {
</p>
)}
<SectionLabel>Exploration</SectionLabel>
<div style={{ display: 'flex', gap: 8, alignItems: 'center', flexWrap: 'wrap' }}>
<Button
size="sm"
variant="secondary"
active={ctx.joltActive}
aria-label="Jolt — hold to morph the network's weights, release to freeze"
aria-pressed={ctx.joltActive}
onPointerDown={(e) => {
e.preventDefault();
ctx.onJoltPress();
}}
onPointerUp={ctx.onJoltRelease}
onPointerLeave={ctx.onJoltRelease}
onPointerCancel={ctx.onJoltRelease}
style={{ touchAction: 'none', userSelect: 'none' }}
>
Jolt {ctx.joltActive ? '(morphing…)' : '(hold)'}
</Button>
<span style={{ fontSize: 'var(--fs-xs)', color: 'var(--fg-dim)' }}>
hold to morph the whole net live · release to freeze
</span>
</div>
<Slider
label="explore · output wander"
value={ctx.exploreIntensity}
min={0}
max={1}
step={0.01}
onChange={ctx.setExploreIntensity}
/>
{depth === 'expanded' && (
<p style={{ fontSize: 'var(--fs-xs)', color: 'var(--fg-dim)', margin: 0, lineHeight: 1.6 }}>
Explore adds a slow random walk (Ornstein-Uhlenbeck) on the outputs so the sound roams;
likes and dislikes registered mid-wander steer the net toward what you want. 0 = off. Jolt
(firmware TogB1) and Explore (RVX1) drive the same core gestures as the MEMLNaut hardware.
</p>
)}
<SectionLabel>Recorded examples</SectionLabel>
<div style={{ display: 'flex', gap: 8, alignItems: 'center' }}>
<Chip>

View file

@ -159,6 +159,17 @@ export interface ConsoleCtx {
spreadLevel: number;
setSpreadLevel: (v: number) => void;
// ---- Exploration gestures (one-core-engine §P1; interim TS shells) ----
/** True while the Jolt press-and-hold weight-morph is engaged. */
joltActive: boolean;
/** Begin the Jolt morph (button press). */
onJoltPress: () => void;
/** Freeze the Jolt morph where it landed (button release). */
onJoltRelease: () => void;
/** OU exploration amount on the output vector, [0,1]; 0 = off. */
exploreIntensity: number;
setExploreIntensity: (v: number) => void;
// ---- Synth engine (dock-spec §5) ----
audioStarted: boolean;
onToggleAudio: () => void;

View file

@ -0,0 +1,143 @@
/**
* exploration.ts ExplorationController: wires the two playground exploration
* gestures onto manifold's EngineApi.
*
* Jolt hold to continuously morph the network's weights live, release to
* freeze (nisps/ml/jolt.hpp; firmware TogB1).
* Explore an Ornstein-Uhlenbeck random walk on the output that makes the
* sound slowly roam so likes/dislikes can steer it (nisps/ml/
* ou_noise.hpp; firmware RVX1 = exploration amount).
*
* The gestures the playground drove from `mode-runtime.ts` are re-homed here as a
* plain framework-neutral class (no Solid stores, no React) that the Console owns
* one of, once the engine resolves. The Learning drawer's Jolt button + Explore
* slider call these methods.
*
* P3 SWAP POINT
* The maths currently runs in TS (`jolt.ts` / `ou-explore.ts`, ported from the
* retired playground). In §P3 of docs/specs/plans/one-core-engine-refactor.md it
* moves into the C++/WASM core:
*
* joltPress(count) nisps_ml_jolt_press
* joltRelease() nisps_ml_jolt_release
* setExploreIntensity(l) nisps_ml_explore_intensity
*
* When that lands, ONLY THIS MODULE changes: `jolt.ts` + `ou-explore.ts` are
* deleted, the per-tick stepping below (getWeights step setWeights process,
* and the OU output-morph) is removed because the core owns the control-rate
* morph + OU advance, and the calls above target the bindings. The UI keeps
* calling the same `joltPress/joltRelease/joltActive/setExploreIntensity/
* exploreIntensity` surface — no drawer changes.
*
*/
import type { EngineApi } from './engine-api';
import { Jolt } from './jolt';
import { OUExplore } from './ou-explore';
// Control-rate cadences, reproduced from the playground's mode-runtime.ts.
// Manifold has no always-on control-rate loop to hook (the spine is push-driven;
// the input-layer rAF only runs while a poll-based source is active), so — like
// the playground — each gesture owns a scoped interval that exists ONLY while it
// is engaged and is torn down on release / at intensity 0. Not a global timer.
/** ~200Hz — matches the upstream firmware control rate the Jolt constants assume. */
const JOLT_TICK_MS = 5;
/** ~33Hz — keeps the OU walk roaming when the input is static. */
const EXPLORE_TICK_MS = 30;
export class ExplorationController {
private readonly engine: EngineApi;
private readonly jolt = new Jolt();
private readonly ou = new OUExplore();
private joltTimer: ReturnType<typeof setInterval> | null = null;
private exploreTimer: ReturnType<typeof setInterval> | null = null;
constructor(engine: EngineApi) {
this.engine = engine;
// Register the OU walk as the spine's post-output morph. It is inert while
// intensity is 0 (apply() early-returns), so the spine stays parity-safe
// until Explore is turned up. Applied on EVERY spine tick (user input or the
// EXPLORE_TICK_MS driver), exactly as the playground applied it in
// recomputeOutputs. In P3 this becomes a core-side step, not a JS morph.
engine.spine.setOutputMorph((routed) => this.ou.apply(routed));
}
// ---- Jolt (held-button continuous weight morph) --------------------------
joltActive(): boolean {
return this.jolt.active();
}
/** Press-and-hold on: begin morphing a scatter of weights toward random targets. */
joltPress(): void {
if (!this.engine.getState().ready) return;
const weightCount = this.engine.getWeights().length;
this.jolt.press(weightCount);
if (this.joltTimer === null) {
this.joltTimer = setInterval(() => this.tickJolt_(), JOLT_TICK_MS);
}
}
/** Release: freeze the weights where they landed (permanent) and stop ticking. */
joltRelease(): void {
this.jolt.release();
if (this.joltTimer !== null) {
clearInterval(this.joltTimer);
this.joltTimer = null;
}
}
private tickJolt_(): void {
if (!this.jolt.active()) return;
const w = this.engine.getWeights();
if (w.length === 0) return;
this.jolt.step(w);
this.engine.setWeights(w);
// Re-run inference so audio + visuals reflect the morphed weights without
// the user having to move the controller.
this.engine.process();
}
// ---- Explore (OU exploration noise on the output) ------------------------
exploreIntensity(): number {
return this.ou.intensity();
}
/**
* Set the exploration amount in [0,1]. >0 starts the roaming driver so the
* sound keeps wandering even when the input is static; 0 stops it and resets
* the walk so the output passes through cleanly again.
*/
setExploreIntensity(level: number): void {
this.ou.setIntensity(level);
if (this.ou.enabled() && this.exploreTimer === null) {
this.exploreTimer = setInterval(() => {
if (!this.engine.getState().ready) return;
// Re-tick the last input through the spine; the registered output morph
// advances the OU state and reships the routed vector.
this.engine.process();
}, EXPLORE_TICK_MS);
} else if (!this.ou.enabled() && this.exploreTimer !== null) {
clearInterval(this.exploreTimer);
this.exploreTimer = null;
this.ou.reset();
// Flush the now-clean output (no residual drift) to audio + visuals.
this.engine.process();
}
}
// ---- Lifecycle -----------------------------------------------------------
dispose(): void {
this.joltRelease();
if (this.exploreTimer !== null) {
clearInterval(this.exploreTimer);
this.exploreTimer = null;
}
this.ou.setIntensity(0);
this.ou.reset();
this.engine.spine.setOutputMorph(null);
}
}

View file

@ -15,6 +15,12 @@ export type {
export { Spine } from './spine';
export type { SpineState, BackendSend } from './spine';
// Exploration gestures (interim TS shells; maths moves to WASM in P3).
export { ExplorationController } from './exploration';
export { Jolt, DEFAULT_JOLT_PARAMS } from './jolt';
export type { JoltParams } from './jolt';
export { OUExplore, OU_MAX_AMPLITUDE } from './ou-explore';
export { WasmIML } from './wasm-iml';
export type { WasmIMLOptions } from './wasm-iml';

130
manifold/src/engine/jolt.ts Normal file
View file

@ -0,0 +1,130 @@
/**
* jolt.ts "Jolt": held-button continuous weight morph.
*
* Faithful TS port of `nisps/ml/jolt.hpp` (which itself ports upstream memllib
* InterfaceRL "jolts"). While a button/pedal is held, pick a handful of random
* weights scattered across the whole network and EMA-glide each toward a bounded
* random target; on arrival, re-roll the target so the motion never stops.
* Releasing freezes the weights where they landed (the change is permanent).
*
* This operates on the MLP's FLAT weight buffer (the same buffer reached via
* `EngineApi.getWeights()` / `setWeights()`), so it is architecture-agnostic.
*
* Ported verbatim from the retired `playground/src/ml/jolt.ts` (one-core-engine
* refactor §P1). This is an INTERIM TS shell in §P3 the maths moves into the
* C++/WASM core (`nisps_ml_jolt_press/release`) and this file is deleted; the
* swap is localised to `exploration.ts` (see the P3 SWAP POINT note there).
*
* Firmware note: the canonical `nisps/ml/jolt.hpp` also owns a post-release
* learning-rate ramp (`lr_scale()` / `tick_lr_ramp()`, the RVX-driven feel). The
* playground TS port never wired that (learning stays caller-driven), so this
* interim shell matches the playground; the full ramp arrives with the P3 WASM
* binding.
*
* RNG note: the C++ owns a per-instance deterministic xoshiro256+ for
* firmwarebrowser parity. This is a browser-only exploration aid that never
* round-trips through WASM, so we use `Math.random()` the exact sequence does
* not need to match firmware. Constants are reproduced verbatim from JoltParams.
*
* DEFAULT STATE IS INERT: a freshly constructed Jolt is inactive and `step()` is
* a no-op until `press()` is called.
*/
/** Upstream JoltParams defaults (kJolt* constants). */
export interface JoltParams {
/** kJoltNumWeights — number of weights morphed at once. */
numWeights: number;
/** kJoltMorphRate — EMA glide per tick (~1s @200Hz). */
morphRate: number;
/** kJoltWeightMin — lower bound of a random target. */
targetMin: number;
/** kJoltWeightMax — upper bound of a random target. */
targetMax: number;
/** kJoltTargetEpsilon — re-roll target once within this distance. */
targetEpsilon: number;
}
export const DEFAULT_JOLT_PARAMS: JoltParams = {
numWeights: 40,
morphRate: 0.017,
targetMin: -1.2,
targetMax: 0.9,
targetEpsilon: 0.05,
};
export class Jolt {
private params_: JoltParams = { ...DEFAULT_JOLT_PARAMS };
private active_ = false;
private n_ = 0;
private idx_: Int32Array = new Int32Array(0);
private target_: Float32Array = new Float32Array(0);
constructor(params?: Partial<JoltParams>) {
if (params) this.params_ = { ...this.params_, ...params };
}
setParams(p: Partial<JoltParams>): void {
this.params_ = { ...this.params_, ...p };
}
get params(): Readonly<JoltParams> {
return this.params_;
}
active(): boolean {
return this.active_;
}
/**
* Begin a jolt over a flat weight buffer of `weightCount` entries: pick
* `numWeights` random global indices and a bounded random target each.
*/
press(weightCount: number): void {
this.active_ = true;
this.n_ = this.params_.numWeights;
if (weightCount <= 0) {
this.n_ = 0;
return;
}
if (this.idx_.length < this.n_) this.idx_ = new Int32Array(this.n_);
if (this.target_.length < this.n_) this.target_ = new Float32Array(this.n_);
for (let i = 0; i < this.n_; ++i) {
this.idx_[i] = Math.floor(Math.random() * weightCount) % weightCount;
this.target_[i] = this.rollTarget_();
}
}
/**
* Per control tick while held: EMA-glide each selected weight toward its
* target, re-rolling targets that have been reached. No-op when inactive.
* Mutates `weights` in place.
*/
step(weights: Float32Array): void {
if (!this.active_) return;
const wc = weights.length;
const rate = this.params_.morphRate;
const eps = this.params_.targetEpsilon;
for (let i = 0; i < this.n_; ++i) {
const k = this.idx_[i]!;
if (k < 0 || k >= wc) continue;
let w = weights[k]!;
w += rate * (this.target_[i]! - w);
weights[k] = w;
if (Math.abs(this.target_[i]! - w) < eps) {
this.target_[i] = this.rollTarget_();
}
}
}
/** Release: freeze weights where they are (permanent). */
release(): void {
this.active_ = false;
}
private rollTarget_(): number {
return (
this.params_.targetMin +
Math.random() * (this.params_.targetMax - this.params_.targetMin)
);
}
}

View file

@ -0,0 +1,128 @@
/**
* ou-explore.ts Ornstein-Uhlenbeck exploration noise on the output.
*
* Faithful TS port of `nisps/ml/ou_noise.hpp`. Adds a per-output-channel
* Ornstein-Uhlenbeck random walk to the network's action (output) vector before
* it reaches audio. Unlike i.i.d. per-frame noise, an OU process is temporally
* correlated: each output drifts in long, smooth sweeps and is gently pulled back
* toward the mapping output (mean reversion). Because learning stays active while
* the noise roams, "likes" registered during the wander steer the network toward
* sounds the player wants.
*
* Discrete Euler-Maruyama update, per output channel x (mu = 0):
* x += theta * (-x) * dt + noiseScale * N(0,1)
* out = clamp(out + x, 0, 1)
* where, to make the stationary std equal a requested `std`:
* noiseScale = std * sqrt(2 * theta * dt)
* and the exploration knob [0,1] maps std = level * kMaxAmplitude (0.65).
*
* Ported verbatim from the retired `playground/src/output/ou-explore.ts`
* (one-core-engine refactor §P1). This is an INTERIM TS shell in §P3 the maths
* moves into the C++/WASM core (`nisps_ml_explore_intensity`) and this file is
* deleted; the swap is localised to `exploration.ts` (see its P3 SWAP POINT).
* Firmware maps RVX1 to this exploration amount.
*
* RNG note: the C++ owns a per-instance deterministic xoshiro256+ for
* firmwarebrowser parity. This is a browser-only exploration aid that never
* round-trips through WASM, so `Math.random()` is fine. The gaussian is
* reproduced as the same sum-of-three-uniforms shape the C++ `Rng` uses
* (`next_float_gaussian`).
*
* DEFAULT STATE IS INERT: intensity defaults to 0, `enabled()` is false, and
* `apply()` neither advances the RNG nor touches the output so the output
* passes through bit-identically when intensity is 0 (parity-safe).
*/
/** Upstream kMaxAmplitude: the exploration knob's full-scale stationary std. */
export const OU_MAX_AMPLITUDE = 0.65;
/** Sum-of-three-uniforms gaussian (matches nisps::Rng::next_float_gaussian). */
function gaussian(stddev = 1): number {
// Three uniforms in [-1, 1): variance 1/3 each ⇒ sum has variance 1.
const a = Math.random() * 2 - 1;
const b = Math.random() * 2 - 1;
const c = Math.random() * 2 - 1;
return (a + b + c) * stddev;
}
export class OUExplore {
private theta_ = 0.02;
private dt_ = 0.001;
private stationaryStd_ = 0;
private noiseScale_ = 0;
private state_: Float32Array = new Float32Array(0);
constructor() {
this.recomputeScale_();
}
/**
* Exploration amount in [0,1]; 0 disables (inert). Maps to the OU stationary
* std = level * kMaxAmplitude.
*/
setIntensity(level: number): void {
let l = level;
if (l < 0) l = 0;
else if (l > 1) l = 1;
this.stationaryStd_ = l * OU_MAX_AMPLITUDE;
this.recomputeScale_();
}
intensity(): number {
return this.stationaryStd_ / OU_MAX_AMPLITUDE;
}
enabled(): boolean {
return this.stationaryStd_ > 0;
}
setTheta(theta: number): void {
this.theta_ = theta;
this.recomputeScale_();
}
setDt(dt: number): void {
this.dt_ = dt;
this.recomputeScale_();
}
/**
* Advance the per-channel OU state and add it (clamped) to `out`, mutating
* `out` in place. No-op (no state advance, no output change) when disabled.
* `out` is the post-inference parameter vector.
*/
apply(out: Float32Array): void {
if (!this.enabled()) return;
const n = out.length;
if (this.state_.length < n) {
// Grow the per-channel state, preserving existing drift.
const next = new Float32Array(n);
next.set(this.state_);
this.state_ = next;
}
const theta = this.theta_;
const dt = this.dt_;
const scale = this.noiseScale_;
for (let i = 0; i < n; ++i) {
// mu = 0: the walk is an offset that mean-reverts to zero, so the
// network's own mapping output stays the anchor.
let s = this.state_[i]!;
s += theta * -s * dt + scale * gaussian(1);
this.state_[i] = s;
let v = out[i]! + s;
if (v < 0) v = 0;
else if (v > 1) v = 1;
out[i] = v;
}
}
reset(): void {
this.state_.fill(0);
}
private recomputeScale_(): void {
let k = 2 * this.theta_ * this.dt_;
if (k < 0) k = 0;
this.noiseScale_ = this.stationaryStd_ * Math.sqrt(k);
}
}

View file

@ -103,6 +103,12 @@ export class Spine implements EngineSink {
private liveOutputs: F32 = new Float32Array(126);
private liveWeights: F32 = new Float32Array(0);
// Optional post-output transform, applied to the routed buffer AFTER the
// output pipeline and BEFORE backend.send (exploration noise — see
// engine/exploration.ts). Inert (null) by default so the spine stays
// parity-safe; the OU morph itself is a no-op until intensity > 0.
private outputMorph: ((routed: Float32Array) => void) | null = null;
private lastTickMs = 0;
// ---- EngineSink ----------------------------------------------------
@ -163,6 +169,16 @@ export class Spine implements EngineSink {
this.backendSend = backendSend;
}
/**
* Register (or clear with null) a post-output transform applied to the routed
* buffer in place, after the output pipeline and before backend.send. Used for
* exploration noise (engine/exploration.ts). Interim seam in P3 the OU walk
* runs in the WASM core and this hook can retire.
*/
setOutputMorph(fn: ((routed: Float32Array) => void) | null): void {
this.outputMorph = fn;
}
// ---- The hot action ------------------------------------------------
/**
@ -230,6 +246,10 @@ export class Spine implements EngineSink {
this.routedBuf = routed;
}
// 4b. optional exploration morph on the routed vector (OU noise). Inert
// unless a controller has registered one AND it is turned up.
if (this.outputMorph && this.routedBuf) this.outputMorph(this.routedBuf);
// 5. single backend.send at the tail (off React render).
if (this.backendSend && this.routedBuf) this.backendSend(this.routedBuf);