From 9056ac3f5e5652f45d948ca216e3108ea4e7c777 Mon Sep 17 00:00:00 2001 From: monkey-w1n5t0n Date: Mon, 13 Jul 2026 23:27:40 +0200 Subject: [PATCH] feat(manifold): port Jolt + OU explore gestures as UI shells MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bring the two playground-only exploration UIs into manifold ahead of the playground's retirement (one-core-engine refactor §P1): - Jolt: press-and-hold continuous weight-morph, release to freeze. - Explore: Ornstein-Uhlenbeck exploration intensity on the output vector. Interim TS maths ported verbatim from the retired playground modules (engine/jolt.ts, engine/ou-explore.ts). The ExplorationController (engine/exploration.ts) is the single P3 swap boundary: it drives Jolt via the existing EngineApi get/set-weights + process route, and OU via a new inert-by- default output-morph hook on the spine. In §P3 only that module changes to call nisps_ml_jolt_press/release + nisps_ml_explore_intensity. UI lands in the Learning drawer (Jolt hold-button + Explore slider), monochrome- consistent, British copy. Gates green: typecheck, build, Playwright smoke. --- manifold/src/console/ConsoleApp.tsx | 41 +++++++- manifold/src/console/Drawers.tsx | 39 ++++++++ manifold/src/console/types.ts | 11 +++ manifold/src/engine/exploration.ts | 143 ++++++++++++++++++++++++++++ manifold/src/engine/index.ts | 6 ++ manifold/src/engine/jolt.ts | 130 +++++++++++++++++++++++++ manifold/src/engine/ou-explore.ts | 128 +++++++++++++++++++++++++ manifold/src/engine/spine.ts | 20 ++++ 8 files changed, 517 insertions(+), 1 deletion(-) create mode 100644 manifold/src/engine/exploration.ts create mode 100644 manifold/src/engine/jolt.ts create mode 100644 manifold/src/engine/ou-explore.ts diff --git a/manifold/src/console/ConsoleApp.tsx b/manifold/src/console/ConsoleApp.tsx index 3089cbe..6038767 100644 --- a/manifold/src/console/ConsoleApp.tsx +++ b/manifold/src/console/ConsoleApp.tsx @@ -23,7 +23,7 @@ */ import { useEffect, useMemo, useRef, useState } from 'react'; import type { CSSProperties } from 'react'; -import { useEngine, useEngineVersion } from '../engine'; +import { useEngine, useEngineVersion, ExplorationController } from '../engine'; import { MF_MODES, modeEngineId, seededGradient, shapeValues } from './model'; import type { MFParam } from './model'; import { CompositeStage } from './CompositeStage'; @@ -125,6 +125,11 @@ export function ConsoleApp({ focus: initialFocus = 'composite' }: ConsoleAppProp const [learningRate, setLearningRate] = useState(0.00001); const [decay, setDecay] = useState(0.97); const [spreadLevel, setSpreadLevel] = useState(0.6); + // Exploration gestures (Jolt held weight-morph + OU explore-intensity). The + // maths lives in the ExplorationController (engine/exploration.ts); these are + // the React-visible reflections the Learning drawer renders. + const [joltActive, setJoltActive] = useState(false); + const [exploreIntensity, setExploreIntensityState] = useState(0); // Active output MODE (TOP dock selector) — default Particle System. The dock // backend + audio backend derive from this. const [outputMode, setOutputModeState] = useState(DEFAULT_OUTPUT_MODE); @@ -180,6 +185,34 @@ export function ConsoleApp({ focus: initialFocus = 'composite' }: ConsoleAppProp setUndoDepth(s.undoDepth); }; + // The exploration controller (Jolt weight-morph + OU explore-intensity). Same + // lazy-once-per-engine pattern as the feedback controller; its interim TS + // maths becomes WASM calls in P3 (engine/exploration.ts P3 SWAP POINT). + const explorationRef = useRef(null); + if (engine && !explorationRef.current) { + explorationRef.current = new ExplorationController(engine); + } + useEffect( + () => () => { + explorationRef.current?.dispose(); + explorationRef.current = null; + }, + [], + ); + + const onJoltPress = () => { + explorationRef.current?.joltPress(); + setJoltActive(explorationRef.current?.joltActive() ?? false); + }; + const onJoltRelease = () => { + explorationRef.current?.joltRelease(); + setJoltActive(false); + }; + const setExploreIntensity = (v: number) => { + explorationRef.current?.setExploreIntensity(v); + setExploreIntensityState(explorationRef.current?.exploreIntensity() ?? v); + }; + const setFeedbackMode = (m: FeedbackModeUI) => { setFeedbackModeState(m); controllerRef.current?.setMode(m as ProtoFeedbackMode); @@ -739,6 +772,12 @@ export function ConsoleApp({ focus: initialFocus = 'composite' }: ConsoleAppProp setDecay, spreadLevel, setSpreadLevel, + // exploration gestures (Jolt + OU explore) + joltActive, + onJoltPress, + onJoltRelease, + exploreIntensity, + setExploreIntensity, // synth audioStarted, onToggleAudio, diff --git a/manifold/src/console/Drawers.tsx b/manifold/src/console/Drawers.tsx index 457f767..80719da 100644 --- a/manifold/src/console/Drawers.tsx +++ b/manifold/src/console/Drawers.tsx @@ -168,6 +168,45 @@ function LearningDrawer(ctx: ConsoleCtx, depth: DrawerDepth) {

)} + Exploration +
+ + + hold to morph the whole net live · release to freeze + +
+ + {depth === 'expanded' && ( +

+ Explore adds a slow random walk (Ornstein-Uhlenbeck) on the outputs so the sound roams; + likes and dislikes registered mid-wander steer the net toward what you want. 0 = off. Jolt + (firmware TogB1) and Explore (RVX1) drive the same core gestures as the MEMLNaut hardware. +

+ )} + Recorded examples
diff --git a/manifold/src/console/types.ts b/manifold/src/console/types.ts index 2756b00..3b742a1 100644 --- a/manifold/src/console/types.ts +++ b/manifold/src/console/types.ts @@ -159,6 +159,17 @@ export interface ConsoleCtx { spreadLevel: number; setSpreadLevel: (v: number) => void; + // ---- Exploration gestures (one-core-engine §P1; interim TS shells) ---- + /** True while the Jolt press-and-hold weight-morph is engaged. */ + joltActive: boolean; + /** Begin the Jolt morph (button press). */ + onJoltPress: () => void; + /** Freeze the Jolt morph where it landed (button release). */ + onJoltRelease: () => void; + /** OU exploration amount on the output vector, [0,1]; 0 = off. */ + exploreIntensity: number; + setExploreIntensity: (v: number) => void; + // ---- Synth engine (dock-spec §5) ---- audioStarted: boolean; onToggleAudio: () => void; diff --git a/manifold/src/engine/exploration.ts b/manifold/src/engine/exploration.ts new file mode 100644 index 0000000..1e2f327 --- /dev/null +++ b/manifold/src/engine/exploration.ts @@ -0,0 +1,143 @@ +/** + * exploration.ts — ExplorationController: wires the two playground exploration + * gestures onto manifold's EngineApi. + * + * • Jolt — hold to continuously morph the network's weights live, release to + * freeze (nisps/ml/jolt.hpp; firmware TogB1). + * • Explore — an Ornstein-Uhlenbeck random walk on the output that makes the + * sound slowly roam so likes/dislikes can steer it (nisps/ml/ + * ou_noise.hpp; firmware RVX1 = exploration amount). + * + * The gestures the playground drove from `mode-runtime.ts` are re-homed here as a + * plain framework-neutral class (no Solid stores, no React) that the Console owns + * one of, once the engine resolves. The Learning drawer's Jolt button + Explore + * slider call these methods. + * + * ─── P3 SWAP POINT ──────────────────────────────────────────────────────────── + * The maths currently runs in TS (`jolt.ts` / `ou-explore.ts`, ported from the + * retired playground). In §P3 of docs/specs/plans/one-core-engine-refactor.md it + * moves into the C++/WASM core: + * + * joltPress(count) → nisps_ml_jolt_press + * joltRelease() → nisps_ml_jolt_release + * setExploreIntensity(l) → nisps_ml_explore_intensity + * + * When that lands, ONLY THIS MODULE changes: `jolt.ts` + `ou-explore.ts` are + * deleted, the per-tick stepping below (getWeights → step → setWeights → process, + * and the OU output-morph) is removed because the core owns the control-rate + * morph + OU advance, and the calls above target the bindings. The UI keeps + * calling the same `joltPress/joltRelease/joltActive/setExploreIntensity/ + * exploreIntensity` surface — no drawer changes. + * ────────────────────────────────────────────────────────────────────────────── + */ + +import type { EngineApi } from './engine-api'; +import { Jolt } from './jolt'; +import { OUExplore } from './ou-explore'; + +// Control-rate cadences, reproduced from the playground's mode-runtime.ts. +// Manifold has no always-on control-rate loop to hook (the spine is push-driven; +// the input-layer rAF only runs while a poll-based source is active), so — like +// the playground — each gesture owns a scoped interval that exists ONLY while it +// is engaged and is torn down on release / at intensity 0. Not a global timer. +/** ~200Hz — matches the upstream firmware control rate the Jolt constants assume. */ +const JOLT_TICK_MS = 5; +/** ~33Hz — keeps the OU walk roaming when the input is static. */ +const EXPLORE_TICK_MS = 30; + +export class ExplorationController { + private readonly engine: EngineApi; + private readonly jolt = new Jolt(); + private readonly ou = new OUExplore(); + + private joltTimer: ReturnType | null = null; + private exploreTimer: ReturnType | null = null; + + constructor(engine: EngineApi) { + this.engine = engine; + // Register the OU walk as the spine's post-output morph. It is inert while + // intensity is 0 (apply() early-returns), so the spine stays parity-safe + // until Explore is turned up. Applied on EVERY spine tick (user input or the + // EXPLORE_TICK_MS driver), exactly as the playground applied it in + // recomputeOutputs. In P3 this becomes a core-side step, not a JS morph. + engine.spine.setOutputMorph((routed) => this.ou.apply(routed)); + } + + // ---- Jolt (held-button continuous weight morph) -------------------------- + + joltActive(): boolean { + return this.jolt.active(); + } + + /** Press-and-hold on: begin morphing a scatter of weights toward random targets. */ + joltPress(): void { + if (!this.engine.getState().ready) return; + const weightCount = this.engine.getWeights().length; + this.jolt.press(weightCount); + if (this.joltTimer === null) { + this.joltTimer = setInterval(() => this.tickJolt_(), JOLT_TICK_MS); + } + } + + /** Release: freeze the weights where they landed (permanent) and stop ticking. */ + joltRelease(): void { + this.jolt.release(); + if (this.joltTimer !== null) { + clearInterval(this.joltTimer); + this.joltTimer = null; + } + } + + private tickJolt_(): void { + if (!this.jolt.active()) return; + const w = this.engine.getWeights(); + if (w.length === 0) return; + this.jolt.step(w); + this.engine.setWeights(w); + // Re-run inference so audio + visuals reflect the morphed weights without + // the user having to move the controller. + this.engine.process(); + } + + // ---- Explore (OU exploration noise on the output) ------------------------ + + exploreIntensity(): number { + return this.ou.intensity(); + } + + /** + * Set the exploration amount in [0,1]. >0 starts the roaming driver so the + * sound keeps wandering even when the input is static; 0 stops it and resets + * the walk so the output passes through cleanly again. + */ + setExploreIntensity(level: number): void { + this.ou.setIntensity(level); + if (this.ou.enabled() && this.exploreTimer === null) { + this.exploreTimer = setInterval(() => { + if (!this.engine.getState().ready) return; + // Re-tick the last input through the spine; the registered output morph + // advances the OU state and reships the routed vector. + this.engine.process(); + }, EXPLORE_TICK_MS); + } else if (!this.ou.enabled() && this.exploreTimer !== null) { + clearInterval(this.exploreTimer); + this.exploreTimer = null; + this.ou.reset(); + // Flush the now-clean output (no residual drift) to audio + visuals. + this.engine.process(); + } + } + + // ---- Lifecycle ----------------------------------------------------------- + + dispose(): void { + this.joltRelease(); + if (this.exploreTimer !== null) { + clearInterval(this.exploreTimer); + this.exploreTimer = null; + } + this.ou.setIntensity(0); + this.ou.reset(); + this.engine.spine.setOutputMorph(null); + } +} diff --git a/manifold/src/engine/index.ts b/manifold/src/engine/index.ts index dfe95f5..b42bc8b 100644 --- a/manifold/src/engine/index.ts +++ b/manifold/src/engine/index.ts @@ -15,6 +15,12 @@ export type { export { Spine } from './spine'; export type { SpineState, BackendSend } from './spine'; +// Exploration gestures (interim TS shells; maths moves to WASM in P3). +export { ExplorationController } from './exploration'; +export { Jolt, DEFAULT_JOLT_PARAMS } from './jolt'; +export type { JoltParams } from './jolt'; +export { OUExplore, OU_MAX_AMPLITUDE } from './ou-explore'; + export { WasmIML } from './wasm-iml'; export type { WasmIMLOptions } from './wasm-iml'; diff --git a/manifold/src/engine/jolt.ts b/manifold/src/engine/jolt.ts new file mode 100644 index 0000000..b5305af --- /dev/null +++ b/manifold/src/engine/jolt.ts @@ -0,0 +1,130 @@ +/** + * jolt.ts — "Jolt": held-button continuous weight morph. + * + * Faithful TS port of `nisps/ml/jolt.hpp` (which itself ports upstream memllib + * InterfaceRL "jolts"). While a button/pedal is held, pick a handful of random + * weights scattered across the whole network and EMA-glide each toward a bounded + * random target; on arrival, re-roll the target so the motion never stops. + * Releasing freezes the weights where they landed (the change is permanent). + * + * This operates on the MLP's FLAT weight buffer (the same buffer reached via + * `EngineApi.getWeights()` / `setWeights()`), so it is architecture-agnostic. + * + * Ported verbatim from the retired `playground/src/ml/jolt.ts` (one-core-engine + * refactor §P1). This is an INTERIM TS shell — in §P3 the maths moves into the + * C++/WASM core (`nisps_ml_jolt_press/release`) and this file is deleted; the + * swap is localised to `exploration.ts` (see the P3 SWAP POINT note there). + * + * Firmware note: the canonical `nisps/ml/jolt.hpp` also owns a post-release + * learning-rate ramp (`lr_scale()` / `tick_lr_ramp()`, the RVX-driven feel). The + * playground TS port never wired that (learning stays caller-driven), so this + * interim shell matches the playground; the full ramp arrives with the P3 WASM + * binding. + * + * RNG note: the C++ owns a per-instance deterministic xoshiro256+ for + * firmware↔browser parity. This is a browser-only exploration aid that never + * round-trips through WASM, so we use `Math.random()` — the exact sequence does + * not need to match firmware. Constants are reproduced verbatim from JoltParams. + * + * DEFAULT STATE IS INERT: a freshly constructed Jolt is inactive and `step()` is + * a no-op until `press()` is called. + */ + +/** Upstream JoltParams defaults (kJolt* constants). */ +export interface JoltParams { + /** kJoltNumWeights — number of weights morphed at once. */ + numWeights: number; + /** kJoltMorphRate — EMA glide per tick (~1s @200Hz). */ + morphRate: number; + /** kJoltWeightMin — lower bound of a random target. */ + targetMin: number; + /** kJoltWeightMax — upper bound of a random target. */ + targetMax: number; + /** kJoltTargetEpsilon — re-roll target once within this distance. */ + targetEpsilon: number; +} + +export const DEFAULT_JOLT_PARAMS: JoltParams = { + numWeights: 40, + morphRate: 0.017, + targetMin: -1.2, + targetMax: 0.9, + targetEpsilon: 0.05, +}; + +export class Jolt { + private params_: JoltParams = { ...DEFAULT_JOLT_PARAMS }; + private active_ = false; + private n_ = 0; + private idx_: Int32Array = new Int32Array(0); + private target_: Float32Array = new Float32Array(0); + + constructor(params?: Partial) { + if (params) this.params_ = { ...this.params_, ...params }; + } + + setParams(p: Partial): void { + this.params_ = { ...this.params_, ...p }; + } + + get params(): Readonly { + return this.params_; + } + + active(): boolean { + return this.active_; + } + + /** + * Begin a jolt over a flat weight buffer of `weightCount` entries: pick + * `numWeights` random global indices and a bounded random target each. + */ + press(weightCount: number): void { + this.active_ = true; + this.n_ = this.params_.numWeights; + if (weightCount <= 0) { + this.n_ = 0; + return; + } + if (this.idx_.length < this.n_) this.idx_ = new Int32Array(this.n_); + if (this.target_.length < this.n_) this.target_ = new Float32Array(this.n_); + for (let i = 0; i < this.n_; ++i) { + this.idx_[i] = Math.floor(Math.random() * weightCount) % weightCount; + this.target_[i] = this.rollTarget_(); + } + } + + /** + * Per control tick while held: EMA-glide each selected weight toward its + * target, re-rolling targets that have been reached. No-op when inactive. + * Mutates `weights` in place. + */ + step(weights: Float32Array): void { + if (!this.active_) return; + const wc = weights.length; + const rate = this.params_.morphRate; + const eps = this.params_.targetEpsilon; + for (let i = 0; i < this.n_; ++i) { + const k = this.idx_[i]!; + if (k < 0 || k >= wc) continue; + let w = weights[k]!; + w += rate * (this.target_[i]! - w); + weights[k] = w; + if (Math.abs(this.target_[i]! - w) < eps) { + this.target_[i] = this.rollTarget_(); + } + } + } + + /** Release: freeze weights where they are (permanent). */ + release(): void { + this.active_ = false; + } + + private rollTarget_(): number { + return ( + this.params_.targetMin + + Math.random() * (this.params_.targetMax - this.params_.targetMin) + ); + } +} diff --git a/manifold/src/engine/ou-explore.ts b/manifold/src/engine/ou-explore.ts new file mode 100644 index 0000000..811aab2 --- /dev/null +++ b/manifold/src/engine/ou-explore.ts @@ -0,0 +1,128 @@ +/** + * ou-explore.ts — Ornstein-Uhlenbeck exploration noise on the output. + * + * Faithful TS port of `nisps/ml/ou_noise.hpp`. Adds a per-output-channel + * Ornstein-Uhlenbeck random walk to the network's action (output) vector before + * it reaches audio. Unlike i.i.d. per-frame noise, an OU process is temporally + * correlated: each output drifts in long, smooth sweeps and is gently pulled back + * toward the mapping output (mean reversion). Because learning stays active while + * the noise roams, "likes" registered during the wander steer the network toward + * sounds the player wants. + * + * Discrete Euler-Maruyama update, per output channel x (mu = 0): + * x += theta * (-x) * dt + noiseScale * N(0,1) + * out = clamp(out + x, 0, 1) + * where, to make the stationary std equal a requested `std`: + * noiseScale = std * sqrt(2 * theta * dt) + * and the exploration knob [0,1] maps std = level * kMaxAmplitude (0.65). + * + * Ported verbatim from the retired `playground/src/output/ou-explore.ts` + * (one-core-engine refactor §P1). This is an INTERIM TS shell — in §P3 the maths + * moves into the C++/WASM core (`nisps_ml_explore_intensity`) and this file is + * deleted; the swap is localised to `exploration.ts` (see its P3 SWAP POINT). + * Firmware maps RVX1 to this exploration amount. + * + * RNG note: the C++ owns a per-instance deterministic xoshiro256+ for + * firmware↔browser parity. This is a browser-only exploration aid that never + * round-trips through WASM, so `Math.random()` is fine. The gaussian is + * reproduced as the same sum-of-three-uniforms shape the C++ `Rng` uses + * (`next_float_gaussian`). + * + * DEFAULT STATE IS INERT: intensity defaults to 0, `enabled()` is false, and + * `apply()` neither advances the RNG nor touches the output — so the output + * passes through bit-identically when intensity is 0 (parity-safe). + */ + +/** Upstream kMaxAmplitude: the exploration knob's full-scale stationary std. */ +export const OU_MAX_AMPLITUDE = 0.65; + +/** Sum-of-three-uniforms gaussian (matches nisps::Rng::next_float_gaussian). */ +function gaussian(stddev = 1): number { + // Three uniforms in [-1, 1): variance 1/3 each ⇒ sum has variance 1. + const a = Math.random() * 2 - 1; + const b = Math.random() * 2 - 1; + const c = Math.random() * 2 - 1; + return (a + b + c) * stddev; +} + +export class OUExplore { + private theta_ = 0.02; + private dt_ = 0.001; + private stationaryStd_ = 0; + private noiseScale_ = 0; + private state_: Float32Array = new Float32Array(0); + + constructor() { + this.recomputeScale_(); + } + + /** + * Exploration amount in [0,1]; 0 disables (inert). Maps to the OU stationary + * std = level * kMaxAmplitude. + */ + setIntensity(level: number): void { + let l = level; + if (l < 0) l = 0; + else if (l > 1) l = 1; + this.stationaryStd_ = l * OU_MAX_AMPLITUDE; + this.recomputeScale_(); + } + + intensity(): number { + return this.stationaryStd_ / OU_MAX_AMPLITUDE; + } + + enabled(): boolean { + return this.stationaryStd_ > 0; + } + + setTheta(theta: number): void { + this.theta_ = theta; + this.recomputeScale_(); + } + + setDt(dt: number): void { + this.dt_ = dt; + this.recomputeScale_(); + } + + /** + * Advance the per-channel OU state and add it (clamped) to `out`, mutating + * `out` in place. No-op (no state advance, no output change) when disabled. + * `out` is the post-inference parameter vector. + */ + apply(out: Float32Array): void { + if (!this.enabled()) return; + const n = out.length; + if (this.state_.length < n) { + // Grow the per-channel state, preserving existing drift. + const next = new Float32Array(n); + next.set(this.state_); + this.state_ = next; + } + const theta = this.theta_; + const dt = this.dt_; + const scale = this.noiseScale_; + for (let i = 0; i < n; ++i) { + // mu = 0: the walk is an offset that mean-reverts to zero, so the + // network's own mapping output stays the anchor. + let s = this.state_[i]!; + s += theta * -s * dt + scale * gaussian(1); + this.state_[i] = s; + let v = out[i]! + s; + if (v < 0) v = 0; + else if (v > 1) v = 1; + out[i] = v; + } + } + + reset(): void { + this.state_.fill(0); + } + + private recomputeScale_(): void { + let k = 2 * this.theta_ * this.dt_; + if (k < 0) k = 0; + this.noiseScale_ = this.stationaryStd_ * Math.sqrt(k); + } +} diff --git a/manifold/src/engine/spine.ts b/manifold/src/engine/spine.ts index 12ceda3..47bc705 100644 --- a/manifold/src/engine/spine.ts +++ b/manifold/src/engine/spine.ts @@ -103,6 +103,12 @@ export class Spine implements EngineSink { private liveOutputs: F32 = new Float32Array(126); private liveWeights: F32 = new Float32Array(0); + // Optional post-output transform, applied to the routed buffer AFTER the + // output pipeline and BEFORE backend.send (exploration noise — see + // engine/exploration.ts). Inert (null) by default so the spine stays + // parity-safe; the OU morph itself is a no-op until intensity > 0. + private outputMorph: ((routed: Float32Array) => void) | null = null; + private lastTickMs = 0; // ---- EngineSink ---------------------------------------------------- @@ -163,6 +169,16 @@ export class Spine implements EngineSink { this.backendSend = backendSend; } + /** + * Register (or clear with null) a post-output transform applied to the routed + * buffer in place, after the output pipeline and before backend.send. Used for + * exploration noise (engine/exploration.ts). Interim seam — in P3 the OU walk + * runs in the WASM core and this hook can retire. + */ + setOutputMorph(fn: ((routed: Float32Array) => void) | null): void { + this.outputMorph = fn; + } + // ---- The hot action ------------------------------------------------ /** @@ -230,6 +246,10 @@ export class Spine implements EngineSink { this.routedBuf = routed; } + // 4b. optional exploration morph on the routed vector (OU noise). Inert + // unless a controller has registered one AND it is turned up. + if (this.outputMorph && this.routedBuf) this.outputMorph(this.routedBuf); + // 5. single backend.send at the tail (off React render). if (this.backendSend && this.routedBuf) this.backendSend(this.routedBuf);