2026-03-21 23:57:58 +01:00
// NISPS Immersive — Design A
// Full-viewport flow field with floating overlays
2026-03-23 22:23:07 +01:00
import { WasmIML } from './nisps/nisps-wasm.js' ;
2026-03-21 23:57:58 +01:00
import { FlowFieldVisualizer } from './ui/visualizer.js' ;
2026-04-03 21:36:24 +02:00
import { AudioCanvas } from './audio/audio-canvas.js' ;
2026-04-03 18:30:26 +02:00
import { C15Adapter } from './synth/c15-adapter.js' ;
2026-03-24 10:54:13 +01:00
import { Arpeggiator } from './synth/arpeggiator.js' ;
2026-03-22 10:01:50 +01:00
import { MIDIInput } from './synth/midi-input.js' ;
2026-03-22 22:59:08 +01:00
import { SYNTH _PARAM _MAP , SYNTH _PARAM _NAMES , SYNTH _PARAM _COLORS , applyCurve , applyGroupOverride } from './synth/param-map.js' ;
2026-03-30 16:05:45 +02:00
import { MIDIOutput } from './midi/midi-output.js' ;
import { loadCCMap , saveCCMap , createCCParam , exportCCMap , importCCMap , CC _NAMES } from './midi/midi-cc-map.js' ;
2026-04-02 21:35:38 +02:00
import { listPresets as listMidiPresets , loadPreset as loadMidiPreset , loadPresetFromFile as loadMidiPresetFromFile } from './midi/midi-cc-presets.js' ;
2026-03-22 10:01:50 +01:00
import { GamepadInput } from './ui/gamepad.js' ;
feat(playground): add MediaPipe hand tracking input with dev panel
Hand tracking via webcam as alternative to joystick input:
- Right hand tracks 14 derived features (palm XY, finger curls, spread,
roll, pitch, pinch) through a separate 14-input MLP
- Left hand gesture recognition (1 finger = thumbs up, 2 = thumbs down)
with 400ms debounce hold
- Split-zone PIP display (no camera feed, skeleton only) with dashed
divider and cross-zone dimming
- Dual IML architecture: independent imlJoy (2 inputs) and imlHand
(14 inputs) with pointer swap, preserving training data per mode
- Dev panel (?devmode=true): draggable/collapsible floating panel with
sliders for MediaPipe confidence thresholds, smoothing, gesture hold
time, world landmarks toggle, and live feature bar monitor
- Visual presets always route to joystick IML (prevents dimension mismatch)
- Race condition guard on input mode switching
2026-03-23 22:28:58 +01:00
import { HandTracker } from './ui/hand-tracker.js' ;
import { createDevPanel } from './ui/dev-panel.js' ;
2026-03-22 23:50:56 +01:00
import { SYNTH _PRESETS , PRESET _TIERS } from './synth/presets.js' ;
2026-04-03 19:26:12 +02:00
import { ADDITIVE _PRESETS , ADDITIVE _PRESET _TIERS } from './synth/additive-presets.js' ;
import { FM _PRESETS , FM _PRESET _TIERS } from './synth/fm-presets.js' ;
2026-04-03 18:42:59 +02:00
import { EOCChain } from './eoc/index.js' ;
import { EOCChainUI , moduleFactory } from './ui/eoc-chain-ui.js' ;
2026-04-03 18:49:07 +02:00
import { EngineSwitcher } from './ui/engine-switcher.js' ;
2026-04-03 19:21:59 +02:00
import { EOCJoystick } from './ui/eoc-joystick.js' ;
feat(playground): add Modular audio mode with shared mod pool
New "Modular" engine in a-immersive with three hot-swappable Faust
sub-engines (subtractive/additive/fm) sharing a common modulation pool:
16 ADSR slots + 32 LFO slots (single-knob sine->tri->square->saw
wavemorph) routed through a 48-source x 10-destination matrix per
engine. Per-connection scalar amounts in [-1, 1], summed at each
destination. Default MLP output count is 512 (32 mod-source params +
480 matrix cells); model reinits on sub-engine swap, count change, or
engine-param exposure toggle.
Faust layer:
- mod-pool.lib: shared ADSR/LFO/source-bus library
- gen-modular-dsp.py: byte-reproducible generator (source of truth)
- modular-subtractive: faithful Minimoog (3 osc, ladder filter, no envs)
- modular-additive: 64-partial, spectral shape + formants, no envs/LFOs
- modular-fm: 4-op matrix + self-feedback, no envs
- All three share d08=amp, d09=pan conventions
- MODULAR_DESTINATIONS.md: authoritative destination table
JS layer:
- ModularEngine: self-contained SynthEngine with getState/setState,
setSubEngine, setModSourceCount, setExposeEngineParam
- modular-ui: drawer with sub-engine toggle, ADSR/LFO count steppers,
per-slot enable switches, matrix grid editor (tap-cycle, long-press
precise, right-click menu, negative amounts), preset overlay
- modular-presets: 6 named presets (Slow pad, Plucky bass, Crystal,
DX bell, Morphing drone + default)
- a-app.js: Modular mode registered, paramMeta:change -> resizeMLP,
modular DSP state persisted under modularDspState, window.__nisps
debug hooks for programmatic control
Tests: tests/e2e/modular-mode.spec.js (11 Playwright tests, all passing
including DSP state survives reload, sub-engine swap keeps paramCount,
preset apply verification).
Also fixes a pre-existing build.sh bug where the -e flag caused faust
to overwrite .wasm outputs with expanded DSP source text, leaving
additive/fm-matrix/eoc-* committed as invalid WebAssembly. Rebuilt all
affected engines with the corrected script. Added an early-message
buffer to faust-worklet-processor.js so setParam calls arriving before
wasm instantiation are queued rather than dropped (needed when the user
configures modular state before clicking Start Audio).
2026-04-11 07:34:41 +02:00
import { initModularUI } from './ui/modular-ui.js' ;
2026-04-03 18:56:35 +02:00
import { AdditiveEngine } from './synth/additive-engine.js' ;
import { FMEngine } from './synth/fm-engine.js' ;
feat(playground): add Modular audio mode with shared mod pool
New "Modular" engine in a-immersive with three hot-swappable Faust
sub-engines (subtractive/additive/fm) sharing a common modulation pool:
16 ADSR slots + 32 LFO slots (single-knob sine->tri->square->saw
wavemorph) routed through a 48-source x 10-destination matrix per
engine. Per-connection scalar amounts in [-1, 1], summed at each
destination. Default MLP output count is 512 (32 mod-source params +
480 matrix cells); model reinits on sub-engine swap, count change, or
engine-param exposure toggle.
Faust layer:
- mod-pool.lib: shared ADSR/LFO/source-bus library
- gen-modular-dsp.py: byte-reproducible generator (source of truth)
- modular-subtractive: faithful Minimoog (3 osc, ladder filter, no envs)
- modular-additive: 64-partial, spectral shape + formants, no envs/LFOs
- modular-fm: 4-op matrix + self-feedback, no envs
- All three share d08=amp, d09=pan conventions
- MODULAR_DESTINATIONS.md: authoritative destination table
JS layer:
- ModularEngine: self-contained SynthEngine with getState/setState,
setSubEngine, setModSourceCount, setExposeEngineParam
- modular-ui: drawer with sub-engine toggle, ADSR/LFO count steppers,
per-slot enable switches, matrix grid editor (tap-cycle, long-press
precise, right-click menu, negative amounts), preset overlay
- modular-presets: 6 named presets (Slow pad, Plucky bass, Crystal,
DX bell, Morphing drone + default)
- a-app.js: Modular mode registered, paramMeta:change -> resizeMLP,
modular DSP state persisted under modularDspState, window.__nisps
debug hooks for programmatic control
Tests: tests/e2e/modular-mode.spec.js (11 Playwright tests, all passing
including DSP state survives reload, sub-engine swap keeps paramCount,
preset apply verification).
Also fixes a pre-existing build.sh bug where the -e flag caused faust
to overwrite .wasm outputs with expanded DSP source text, leaving
additive/fm-matrix/eoc-* committed as invalid WebAssembly. Rebuilt all
affected engines with the corrected script. Added an early-message
buffer to faust-worklet-processor.js so setParam calls arriving before
wasm instantiation are queued rather than dropped (needed when the user
configures modular state before clicking Start Audio).
2026-04-11 07:34:41 +02:00
import { ModularEngine } from './synth/modular-engine.js' ;
import { MODULAR _PRESETS , applyPreset as applyModularPreset , findPreset as findModularPreset } from './synth/modular-presets.js' ;
2026-03-21 23:57:58 +01:00
// ---- Constants ----
feat(playground): add MediaPipe hand tracking input with dev panel
Hand tracking via webcam as alternative to joystick input:
- Right hand tracks 14 derived features (palm XY, finger curls, spread,
roll, pitch, pinch) through a separate 14-input MLP
- Left hand gesture recognition (1 finger = thumbs up, 2 = thumbs down)
with 400ms debounce hold
- Split-zone PIP display (no camera feed, skeleton only) with dashed
divider and cross-zone dimming
- Dual IML architecture: independent imlJoy (2 inputs) and imlHand
(14 inputs) with pointer swap, preserving training data per mode
- Dev panel (?devmode=true): draggable/collapsible floating panel with
sliders for MediaPipe confidence thresholds, smoothing, gesture hold
time, world landmarks toggle, and live feature bar monitor
- Visual presets always route to joystick IML (prevents dimension mismatch)
- Race condition guard on input mode switching
2026-03-23 22:28:58 +01:00
const N _JOY _INPUTS = 2 ;
const N _HAND _INPUTS = 14 ;
const N _INPUTS = N _JOY _INPUTS ; // default (joystick)
2026-03-21 23:57:58 +01:00
const N _VISUAL _OUTPUTS = 20 ;
const N _SYNTH _OUTPUTS = SYNTH _PARAM _MAP . length ; // 126
2026-04-03 21:36:24 +02:00
// Audio canvas output count is dynamic — queried from audioCanvas.getOutputCount()
2026-03-30 16:05:45 +02:00
let N _OUTPUTS = N _SYNTH _OUTPUTS ; // Dynamic — changes with output mode
2026-04-02 23:12:18 +02:00
let audioCanvas = null ; // lazily created on first switch to audio-canvas mode
2026-03-21 23:57:58 +01:00
const STORAGE _KEY = 'nisps-a-immersive' ;
feat(playground): add Modular audio mode with shared mod pool
New "Modular" engine in a-immersive with three hot-swappable Faust
sub-engines (subtractive/additive/fm) sharing a common modulation pool:
16 ADSR slots + 32 LFO slots (single-knob sine->tri->square->saw
wavemorph) routed through a 48-source x 10-destination matrix per
engine. Per-connection scalar amounts in [-1, 1], summed at each
destination. Default MLP output count is 512 (32 mod-source params +
480 matrix cells); model reinits on sub-engine swap, count change, or
engine-param exposure toggle.
Faust layer:
- mod-pool.lib: shared ADSR/LFO/source-bus library
- gen-modular-dsp.py: byte-reproducible generator (source of truth)
- modular-subtractive: faithful Minimoog (3 osc, ladder filter, no envs)
- modular-additive: 64-partial, spectral shape + formants, no envs/LFOs
- modular-fm: 4-op matrix + self-feedback, no envs
- All three share d08=amp, d09=pan conventions
- MODULAR_DESTINATIONS.md: authoritative destination table
JS layer:
- ModularEngine: self-contained SynthEngine with getState/setState,
setSubEngine, setModSourceCount, setExposeEngineParam
- modular-ui: drawer with sub-engine toggle, ADSR/LFO count steppers,
per-slot enable switches, matrix grid editor (tap-cycle, long-press
precise, right-click menu, negative amounts), preset overlay
- modular-presets: 6 named presets (Slow pad, Plucky bass, Crystal,
DX bell, Morphing drone + default)
- a-app.js: Modular mode registered, paramMeta:change -> resizeMLP,
modular DSP state persisted under modularDspState, window.__nisps
debug hooks for programmatic control
Tests: tests/e2e/modular-mode.spec.js (11 Playwright tests, all passing
including DSP state survives reload, sub-engine swap keeps paramCount,
preset apply verification).
Also fixes a pre-existing build.sh bug where the -e flag caused faust
to overwrite .wasm outputs with expanded DSP source text, leaving
additive/fm-matrix/eoc-* committed as invalid WebAssembly. Rebuilt all
affected engines with the corrected script. Added an early-message
buffer to faust-worklet-processor.js so setParam calls arriving before
wasm instantiation are queued rather than dropped (needed when the user
configures modular state before clicking Start Audio).
2026-04-11 07:34:41 +02:00
// Phase E — deferred modular DSP state. loadState() stashes this; the
// engine-switch handler consumes it as soon as a ModularEngine exists.
let _pendingModularDspState = null ;
2026-03-21 23:57:58 +01:00
const VISUAL _PARAM _NAMES = [
'Flow' , 'Scale' , 'Speed' , 'Hue' , 'Spread' , 'Size' , 'Trail' , 'Turb' ,
'Attract' , 'Radius' , 'DispRate' , 'DispAmt' , 'Lifetime' , 'Respawn' ,
'Advection' , 'Inertia' , 'Drag' , 'Repulse' , 'RepCnt' , 'RepRate' ,
] ;
const VISUAL _PARAM _COLORS = [
2026-03-22 00:36:00 +01:00
'#ff6a00' , '#00ccff' , '#ff6600' , '#ff00cc' , '#ffcc00' , '#88ff00' ,
2026-03-21 23:57:58 +01:00
'#0088ff' , '#ff3366' , '#9bff5f' , '#59d3ff' , '#ff8f3f' , '#a0b7ff' ,
'#f4ff7a' , '#ffa8db' , '#7dffc8' , '#ffd166' , '#8ad4ff' , '#ff5f5f' ,
'#ffc15f' , '#ff8a3d' ,
] ;
// ---- Presets ----
const PRESETS = {
'calm-to-chaotic' : [
{ input : [ 0.1 , 0.9 ] , output : [ 0.25 , 0.3 , 0.1 , 0.55 , 0.2 , 0.3 , 0.02 , 0.05 , 0.9 , 0.45 , 0.25 , 0.2 , 0.9 , 0.0 , 0.0 , 0.2 , 0.05 , 0.0 , 0.0 , 0.2 ] } ,
{ input : [ 0.9 , 0.1 ] , output : [ 0.75 , 0.7 , 0.9 , 0.05 , 0.8 , 0.7 , 0.9 , 0.95 , 0.3 , 0.2 , 0.85 , 0.7 , 0.25 , 0.55 , 1.0 , 0.92 , 0.02 , 0.95 , 0.8 , 0.85 ] } ,
{ input : [ 0.5 , 0.5 ] , output : [ 0.5 , 0.5 , 0.5 , 0.3 , 0.5 , 0.5 , 0.4 , 0.5 , 0.7 , 0.6 , 0.5 , 0.45 , 0.55 , 0.9 , 0.5 , 0.65 , 0.08 , 0.45 , 0.4 , 0.5 ] } ,
] ,
'rainbow-sweep' : [
{ input : [ 0.0 , 0.5 ] , output : [ 0.5 , 0.5 , 0.4 , 0.0 , 0.3 , 0.4 , 0.05 , 0.3 , 0.8 , 0.55 , 0.4 , 0.3 , 0.8 , 0.0 , 0.0 , 0.45 , 0.08 , 0.2 , 0.25 , 0.45 ] } ,
{ input : [ 0.5 , 0.5 ] , output : [ 0.5 , 0.5 , 0.4 , 0.5 , 0.3 , 0.4 , 0.05 , 0.3 , 0.8 , 0.55 , 0.55 , 0.35 , 0.7 , 0.0 , 0.4 , 0.45 , 0.08 , 0.45 , 0.4 , 0.6 ] } ,
{ input : [ 1.0 , 0.5 ] , output : [ 0.5 , 0.5 , 0.4 , 1.0 , 0.3 , 0.4 , 0.05 , 0.3 , 0.8 , 0.55 , 0.75 , 0.45 , 0.6 , 0.0 , 0.8 , 0.45 , 0.08 , 0.7 , 0.55 , 0.75 ] } ,
] ,
'vortex' : [
{ input : [ 0.5 , 0.5 ] , output : [ 0.0 , 0.8 , 0.8 , 0.6 , 0.1 , 0.15 , 0.02 , 1.0 , 1.0 , 0.3 , 0.95 , 0.85 , 0.25 , 1.0 , 0.5 , 0.95 , 0.01 , 1.0 , 1.0 , 1.0 ] } ,
{ input : [ 0.0 , 0.0 ] , output : [ 0.5 , 0.2 , 0.3 , 0.8 , 0.9 , 0.6 , 0.08 , 0.1 , 0.35 , 0.8 , 0.25 , 0.15 , 0.8 , 0.5 , 0.2 , 0.35 , 0.2 , 0.15 , 0.2 , 0.25 ] } ,
{ input : [ 1.0 , 1.0 ] , output : [ 0.5 , 0.2 , 0.3 , 0.2 , 0.9 , 0.6 , 0.08 , 0.1 , 0.35 , 0.8 , 0.25 , 0.15 , 0.8 , 0.5 , 0.8 , 0.35 , 0.2 , 0.15 , 0.2 , 0.25 ] } ,
{ input : [ 0.0 , 1.0 ] , output : [ 0.3 , 0.4 , 0.5 , 0.4 , 0.5 , 0.4 , 0.05 , 0.5 , 0.65 , 0.5 , 0.55 , 0.45 , 0.45 , 0.2 , 0.4 , 0.7 , 0.1 , 0.5 , 0.4 , 0.55 ] } ,
{ input : [ 1.0 , 0.0 ] , output : [ 0.7 , 0.4 , 0.5 , 0.0 , 0.5 , 0.4 , 0.05 , 0.5 , 0.65 , 0.5 , 0.55 , 0.45 , 0.45 , 0.2 , 0.9 , 0.7 , 0.1 , 0.5 , 0.4 , 0.55 ] } ,
] ,
'spiral' : [
{ input : [ 0.5 , 0.5 ] , output : [ 0.0 , 0.6 , 0.6 , 0.3 , 0.15 , 0.2 , 0.03 , 0.7 , 0.9 , 0.25 , 0.7 , 0.6 , 0.35 , 0.8 , 0.65 , 0.9 , 0.02 , 0.3 , 0.5 , 0.7 ] } ,
{ input : [ 0.0 , 0.0 ] , output : [ 0.8 , 0.3 , 0.4 , 0.7 , 0.8 , 0.5 , 0.06 , 0.2 , 0.5 , 0.7 , 0.3 , 0.2 , 0.7 , 0.3 , 0.3 , 0.4 , 0.15 , 0.1 , 0.1 , 0.3 ] } ,
{ input : [ 1.0 , 1.0 ] , output : [ 0.2 , 0.3 , 0.4 , 0.1 , 0.8 , 0.5 , 0.06 , 0.2 , 0.5 , 0.7 , 0.3 , 0.2 , 0.7 , 0.3 , 0.9 , 0.4 , 0.15 , 0.1 , 0.1 , 0.3 ] } ,
] ,
'embers' : [
{ input : [ 0.5 , 0.5 ] , output : [ 0.1 , 0.4 , 0.2 , 0.05 , 0.05 , 0.6 , 0.02 , 0.15 , 0.6 , 0.3 , 0.15 , 0.9 , 0.5 , 0.7 , 0.1 , 0.3 , 0.2 , 0.0 , 0.0 , 0.2 ] } ,
{ input : [ 0.2 , 0.8 ] , output : [ 0.5 , 0.6 , 0.15 , 0.08 , 0.08 , 0.8 , 0.015 , 0.1 , 0.8 , 0.5 , 0.1 , 0.5 , 0.8 , 0.4 , 0.05 , 0.5 , 0.1 , 0.0 , 0.0 , 0.15 ] } ,
{ input : [ 0.8 , 0.2 ] , output : [ 0.9 , 0.3 , 0.35 , 0.02 , 0.12 , 0.35 , 0.04 , 0.3 , 0.4 , 0.2 , 0.3 , 1.0 , 0.3 , 0.9 , 0.2 , 0.15 , 0.3 , 0.0 , 0.0 , 0.3 ] } ,
] ,
} ;
2026-04-03 20:01:14 +02:00
// ---- ShapeSeq feature flag ----
const shapeSeqEnabled = new URLSearchParams ( location . search ) . get ( 'shapeseq' ) === '1' ;
// Lazy-loaded ShapeSeq modules (only when shapeSeqEnabled)
let ShapeSeqEngine , StepVisualizer , ChainBuilderUI , EventBus , getDefaultBus ;
// ShapeSeq state
let shapeSeqEngine = null ;
let shapeSeqViz = null ;
let shapeSeqChainUI = null ;
let shapeSeqBus = null ;
let _shapeSeqInited = false ;
2026-03-21 23:57:58 +01:00
// ---- App state ----
feat(playground): add MediaPipe hand tracking input with dev panel
Hand tracking via webcam as alternative to joystick input:
- Right hand tracks 14 derived features (palm XY, finger curls, spread,
roll, pitch, pinch) through a separate 14-input MLP
- Left hand gesture recognition (1 finger = thumbs up, 2 = thumbs down)
with 400ms debounce hold
- Split-zone PIP display (no camera feed, skeleton only) with dashed
divider and cross-zone dimming
- Dual IML architecture: independent imlJoy (2 inputs) and imlHand
(14 inputs) with pointer swap, preserving training data per mode
- Dev panel (?devmode=true): draggable/collapsible floating panel with
sliders for MediaPipe confidence thresholds, smoothing, gesture hold
time, world landmarks toggle, and live feature bar monitor
- Visual presets always route to joystick IML (prevents dimension mismatch)
- Race condition guard on input mode switching
2026-03-23 22:28:58 +01:00
let iml ; // active IML (points to imlJoy or imlHand)
let imlJoy ; // IML for joystick mode (2 inputs)
let imlHand ; // IML for hand tracking mode (14 inputs)
let inputMode = 'joystick' ; // 'joystick' | 'hands'
let handTracker = null ;
2026-03-21 23:57:58 +01:00
let visualizer ;
let synthVisualizer ;
2026-04-03 18:30:26 +02:00
let activeEngine = null ; // active SynthEngine (currently always a C15Adapter)
let c15 = null ; // alias kept for MIDIInput and volume controls that still reference it
2026-03-24 10:54:13 +01:00
let arpeggiator = null ;
2026-03-22 10:01:50 +01:00
let midiInput = null ;
2026-03-21 23:57:58 +01:00
let outputMode = 'visual' ;
2026-03-30 16:05:45 +02:00
2026-04-03 18:49:07 +02:00
// EOC Effects Chain — module-scoped so audio-start and setActiveEngine can reference it
let eocChain = null ;
let _eocInited = false ; // guard: init once per AudioContext lifetime
2026-04-03 19:21:59 +02:00
// EOC Linked/Independent mode — second IML driven by joystick(s), independent training
let imlEoc = null ; // second IML for EOC params (Linked/Independent mode)
2026-04-03 19:03:12 +02:00
let eocTrainingTarget = 'synth' ; // 'synth' | 'eoc' — which MLP RL feedback targets
2026-04-03 19:21:59 +02:00
let eocJoystick = null ; // EOCJoystick instance for Independent mode
2026-04-03 19:03:12 +02:00
feat(playground): add Modular audio mode with shared mod pool
New "Modular" engine in a-immersive with three hot-swappable Faust
sub-engines (subtractive/additive/fm) sharing a common modulation pool:
16 ADSR slots + 32 LFO slots (single-knob sine->tri->square->saw
wavemorph) routed through a 48-source x 10-destination matrix per
engine. Per-connection scalar amounts in [-1, 1], summed at each
destination. Default MLP output count is 512 (32 mod-source params +
480 matrix cells); model reinits on sub-engine swap, count change, or
engine-param exposure toggle.
Faust layer:
- mod-pool.lib: shared ADSR/LFO/source-bus library
- gen-modular-dsp.py: byte-reproducible generator (source of truth)
- modular-subtractive: faithful Minimoog (3 osc, ladder filter, no envs)
- modular-additive: 64-partial, spectral shape + formants, no envs/LFOs
- modular-fm: 4-op matrix + self-feedback, no envs
- All three share d08=amp, d09=pan conventions
- MODULAR_DESTINATIONS.md: authoritative destination table
JS layer:
- ModularEngine: self-contained SynthEngine with getState/setState,
setSubEngine, setModSourceCount, setExposeEngineParam
- modular-ui: drawer with sub-engine toggle, ADSR/LFO count steppers,
per-slot enable switches, matrix grid editor (tap-cycle, long-press
precise, right-click menu, negative amounts), preset overlay
- modular-presets: 6 named presets (Slow pad, Plucky bass, Crystal,
DX bell, Morphing drone + default)
- a-app.js: Modular mode registered, paramMeta:change -> resizeMLP,
modular DSP state persisted under modularDspState, window.__nisps
debug hooks for programmatic control
Tests: tests/e2e/modular-mode.spec.js (11 Playwright tests, all passing
including DSP state survives reload, sub-engine swap keeps paramCount,
preset apply verification).
Also fixes a pre-existing build.sh bug where the -e flag caused faust
to overwrite .wasm outputs with expanded DSP source text, leaving
additive/fm-matrix/eoc-* committed as invalid WebAssembly. Rebuilt all
affected engines with the corrected script. Added an early-message
buffer to faust-worklet-processor.js so setParam calls arriving before
wasm instantiation are queued rather than dropped (needed when the user
configures modular state before clicking Start Audio).
2026-04-11 07:34:41 +02:00
// Modular-mode UI (Phase C) — created once during init(), toggled via show/hide
// by setActiveEngine() when the modular engine becomes active.
let modularUI = null ;
2026-03-30 16:05:45 +02:00
// ---- MIDI CC state ----
let midiOutput = null ;
2026-04-03 18:55:06 +02:00
// Storage key scoped per engine so CC assignments don't bleed between engines.
function midiCCStorageKey ( ) {
return ` nisps-midi-cc-map: ${ activeEngine ? . id ? ? 'shaper-feedback' } ` ;
}
let midiCCMap = loadCCMap ( midiCCStorageKey ( ) ) ;
2026-03-30 16:05:45 +02:00
// Per-param overrides for MIDI CC mode (same shape as visual overrides)
let midiCCOverrides = midiCCMap . map ( p => ( {
min : p . min , max : p . max , curve : p . curve , frozen : p . muted , fixedValue : p . fixedValue ,
} ) ) ;
const MIDI _CC _PARAM _COLORS = [ ] ;
// Generate distinct colors for CC params
function _generateCCColors ( n ) {
MIDI _CC _PARAM _COLORS . length = 0 ;
for ( let i = 0 ; i < n ; i ++ ) {
const hue = ( i * 137.508 ) % 360 ; // golden angle
MIDI _CC _PARAM _COLORS . push ( ` hsl( ${ hue } , 70%, 60%) ` ) ;
}
}
_generateCCColors ( midiCCMap . length ) ;
2026-04-03 18:55:06 +02:00
/** Reload the MIDI CC map from localStorage for the current engine. */
function reloadMidiCCMap ( ) {
const saved = loadCCMap ( midiCCStorageKey ( ) ) ;
midiCCMap . length = 0 ;
midiCCMap . push ( ... saved ) ;
midiCCOverrides . length = 0 ;
midiCCOverrides . push ( ... saved . map ( p => ( {
min : p . min , max : p . max , curve : p . curve , frozen : p . muted , fixedValue : p . fixedValue ,
} ) ) ) ;
_generateCCColors ( midiCCMap . length ) ;
}
2026-03-22 22:59:08 +01:00
// tame level is now baked into groupOverrides defaults at init time (see ?tame URL param)
2026-03-22 00:36:00 +01:00
let spreadLevel = 0.6 ;
2026-03-21 23:57:58 +01:00
let noiseLevel = 0.05 ;
2026-03-30 16:05:45 +02:00
let rlExplorationDecay = 0.97 ;
feat(playground): implement Phases 2-4 of control surface spec
Phase 2 — Pinning + History:
- snapshot-stack.js: ring buffer (20 max) with auto-snapshot on
train/randomize/thumbs-down, multi-level undo, tagged entries
- ab-compare.js: A/B weight state comparison with capture/toggle/accept/revert
- region-pin.js: pin rectangular input-space regions (Approach A: example
pinning), pinned examples always included in training
- param-pin.js: per-output pin flags, pin mask skips pinned nodes in moveWeights
- phase2-ui.js: undo button with history popup, A/B toggle, long-press region
pin, double-tap param pin
- Modified mlp.js/iml.js/nisps-wasm.js to accept outputPinMask in moveWeights
Phase 3 — Input Refinement + Exploration:
- pressure-feedback.js: touch force + hold duration → intensity multiplier
- auto-explore.js: automated thumbs-down at configurable interval, zoom-scaled
- input-heatmap.js: 16×16 MLP sampling, 3 color modes (luminance/variance/
divergence), zoom-aware resampling, offscreen canvas rendering
- phase3-ui.js: auto-explore toggle with progress ring, heatmap eye icon,
pressure indicators, settings drawer section
- joy-map-enhanced.js: added setHeatmap() for background layer rendering
Phase 4 — Output Pipeline + Visualization + Polish:
- output-pipeline.js: global curve → smoothing → slew rate → freeze gate
- weight-health.js: weight magnitude histogram, dead/saturating/healthy status
- gradient-flow.js: per-layer weight-delta analysis, vanishing/exploding detection
- session-presets.js: save/load full state, URL sharing via compact params
- phase4-ui.js: freeze button, network health panel, session preset UI
All phases merged into a-app.js with proper integration: auto-snapshots,
pressure-modulated RL, heatmap triggers, output pipeline in routeOutputs,
gradient capture around training, persistence for all new state.
2026-03-26 09:48:12 +01:00
2026-03-21 23:57:58 +01:00
// Joystick state
let joyX = 0.5 ;
let joyY = 0.5 ;
let joyDragging = false ;
let joyFollowMode = false ;
let joyTrail = [ ] ;
// Gamepad
2026-03-22 10:01:50 +01:00
let gamepad ;
2026-03-21 23:57:58 +01:00
// Raw param slider values (for examples mode advanced editing)
let rawParamValues = new Array ( N _OUTPUTS ) . fill ( 0.5 ) ;
// ---- DOM refs ----
let $canvas , $heatmapCells , $heatmapTooltip ;
let $joystickContainer , $joyMap , $joyMapCtx ;
let $noiseRing , $followBadge ;
let $rlButtons , $btnThumbsUp , $btnThumbsDown ;
2026-03-30 16:05:45 +02:00
let $statusText ;
let $lossCanvas , $lossCtx ;
let $engineParams ;
let $followPill ;
2026-03-21 23:57:58 +01:00
let $synthVisCanvas ;
2026-03-30 16:05:45 +02:00
// Undo stack — stores weight snapshots before RL actions
const undoStack = [ ] ;
const MAX _UNDO = 20 ;
// ---- Visual Overrides: per-param curve/min/max/freeze for visual mode ----
const visualOverrides = VISUAL _PARAM _NAMES . map ( ( ) => ( {
min : 0 ,
max : 1 ,
curve : 0.5 ,
frozen : false ,
fixedValue : 0.5 ,
} ) ) ;
2026-04-02 23:12:18 +02:00
// ---- Audio Canvas Overrides ----
2026-04-03 21:36:24 +02:00
// Dynamic array — resized when AudioCanvas output count changes
let audioCanvasOverrides = [ ] ;
function _ensureAudioCanvasOverrides ( count ) {
while ( audioCanvasOverrides . length < count ) {
audioCanvasOverrides . push ( { min : 0 , max : 1 , curve : 0.5 , frozen : false , fixedValue : 0.5 } ) ;
}
if ( audioCanvasOverrides . length > count ) audioCanvasOverrides . length = count ;
}
2026-04-02 23:12:18 +02:00
2026-03-21 23:57:58 +01:00
// ---- Synth Sections (for SynthVisualizer) ----
const SYNTH _SECTIONS = [
{ name : 'Env A' , count : 7 , color : '#4488ff' } ,
{ name : 'Env B' , count : 7 , color : '#4488ff' } ,
{ name : 'Env C' , count : 6 , color : '#6688dd' } ,
{ name : 'Osc A' , count : 5 , color : '#ff8844' } ,
{ name : 'Osc B' , count : 5 , color : '#ff8844' } ,
{ name : 'Shp A' , count : 6 , color : '#ff4466' } ,
{ name : 'Shp B' , count : 6 , color : '#ff4466' } ,
{ name : 'Comb' , count : 8 , color : '#44ddaa' } ,
2026-03-22 22:59:08 +01:00
{ name : 'SVF' , count : 9 , color : '#44ddaa' } ,
{ name : 'Gap' , count : 6 , color : '#44bbaa' } ,
{ name : 'FB Mix' , count : 9 , color : '#ddaa44' } ,
2026-03-21 23:57:58 +01:00
{ name : 'Out Mix' , count : 14 , color : '#ddaa44' } ,
{ name : 'Cabinet' , count : 8 , color : '#aa88dd' } ,
{ name : 'Flanger' , count : 13 , color : '#dd88aa' } ,
{ name : 'Echo' , count : 7 , color : '#88aadd' } ,
{ name : 'Reverb' , count : 6 , color : '#88ddaa' } ,
{ name : 'Unison' , count : 3 , color : '#cccccc' } ,
{ name : 'Mono' , count : 1 , color : '#999999' } ,
] ;
2026-03-22 20:57:21 +01:00
// ---- Group Overrides: per-group curve + per-param min/max/curve/mute ----
// groupOverrides[sectionIndex] = { curve: 0.5, params: [{ min, max, curve, muted, fixedValue }, ...] }
2026-03-22 22:59:08 +01:00
// Defaults are seeded from safeMin/safeMax in SYNTH_PARAM_MAP, scaled by the
// ?tame URL parameter (0 = unconstrained, 1 = full safe limits).
// localStorage overrides on load.
2026-03-22 23:50:56 +01:00
// Module-level tame so applyPreset() can use it
const _urlTame = parseFloat ( new URLSearchParams ( window . location . search ) . get ( 'tame' ) ? ? '1' ) ;
const tameLevel = isNaN ( _urlTame ) ? 1 : Math . max ( 0 , Math . min ( 1 , _urlTame ) ) ;
2026-03-22 22:59:08 +01:00
const groupOverrides = ( ( ) => {
2026-03-22 23:50:56 +01:00
const t = tameLevel ;
2026-03-22 22:59:08 +01:00
let flatIdx = 0 ;
return SYNTH _SECTIONS . map ( sec => ( {
curve : 0.5 ,
params : new Array ( sec . count ) . fill ( null ) . map ( ( ) => {
const p = SYNTH _PARAM _MAP [ flatIdx ++ ] ;
// Interpolate: at tame=0 use full [0,1]; at tame=1 use [safeMin, safeMax]
const safeMin = p ? . safeMin ? ? 0 ;
const safeMax = p ? . safeMax ? ? 1 ;
return {
min : safeMin * t ,
max : 1 - ( 1 - safeMax ) * t ,
curve : 0.5 ,
muted : false ,
fixedValue : 0.5 ,
} ;
} ) ,
} ) ) ;
} ) ( ) ;
2026-03-22 20:57:21 +01:00
2026-03-22 23:50:56 +01:00
// Lookup: param name -> flat index (for preset application)
const paramNameToIndex = new Map ( ) ;
SYNTH _PARAM _MAP . forEach ( ( p , i ) => paramNameToIndex . set ( p . name , i ) ) ;
// Currently active synth preset id (null = no preset / manual)
let activeSynthPresetId = null ;
2026-03-22 20:57:21 +01:00
// Build a flat lookup: paramIndex -> { sectionIndex, localIndex }
const paramToSection = [ ] ;
{
let idx = 0 ;
for ( let si = 0 ; si < SYNTH _SECTIONS . length ; si ++ ) {
for ( let li = 0 ; li < SYNTH _SECTIONS [ si ] . count ; li ++ ) {
paramToSection . push ( { si , li } ) ;
idx ++ ;
}
}
// Pad for any params beyond sections
while ( paramToSection . length < N _SYNTH _OUTPUTS ) {
paramToSection . push ( null ) ;
}
}
2026-04-03 19:41:05 +02:00
/ * *
* Rebuild paramToSection from an engine ' s paramMeta groups .
* For C15 , restores the static SYNTH _SECTIONS - based mapping .
* For non - C15 engines , derives sections from paramMeta group fields .
* /
function rebuildParamToSection ( paramMeta ) {
paramToSection . length = 0 ;
let si = 0 , li = 0 ;
let currentGroup = null ;
for ( const pm of paramMeta ) {
const group = pm . group ? ? 'Other' ;
if ( group !== currentGroup ) {
if ( currentGroup !== null ) si ++ ;
currentGroup = group ;
li = 0 ;
}
paramToSection . push ( { si , li } ) ;
li ++ ;
}
}
2026-04-11 08:31:19 +02:00
/ * *
* Dynamic section metadata for non - C15 engines ( parallel to SYNTH _SECTIONS
* for C15 ) . Indexed by section index ( the ` si ` field in paramToSection ) .
*
* nonC15Sections [ si ] = { name , color , count , startIndex }
* nonC15GroupCurves [ si ] = scalar group - master curve [ 0 , 1 ]
*
* Group curves persist across engine switches keyed by section NAME , so a
* user ' s "Filter" curve survives a sub - engine swap that keeps the same
* group label .
* /
let nonC15Sections = [ ] ;
let nonC15GroupCurves = [ ] ;
const _nonC15GroupCurveMemory = new Map ( ) ; // groupName -> curve
function rebuildNonC15Sections ( paramMeta ) {
nonC15Sections = [ ] ;
nonC15GroupCurves = [ ] ;
let currentGroup = null ;
let si = - 1 ;
for ( let i = 0 ; i < paramMeta . length ; i ++ ) {
const g = paramMeta [ i ] . group ? ? 'Other' ;
if ( g !== currentGroup ) {
si ++ ;
currentGroup = g ;
nonC15Sections . push ( {
name : g ,
color : _colorFromGroup ( g ) ,
count : 0 ,
startIndex : i ,
} ) ;
nonC15GroupCurves . push ( _nonC15GroupCurveMemory . get ( g ) ? ? 0.5 ) ;
}
nonC15Sections [ si ] . count ++ ;
}
}
2026-04-03 19:41:05 +02:00
/ * *
* Restore paramToSection to the static C15 layout ( from SYNTH _SECTIONS ) .
* /
function restoreC15ParamToSection ( ) {
paramToSection . length = 0 ;
let idx = 0 ;
for ( let si = 0 ; si < SYNTH _SECTIONS . length ; si ++ ) {
for ( let li = 0 ; li < SYNTH _SECTIONS [ si ] . count ; li ++ ) {
paramToSection . push ( { si , li } ) ;
idx ++ ;
}
}
while ( paramToSection . length < N _SYNTH _OUTPUTS ) {
paramToSection . push ( null ) ;
}
}
2026-04-03 19:26:12 +02:00
// ---- Engine-aware preset helpers ----
/ * *
* Return the preset array for the given engine id .
* /
function getPresetsForEngine ( engineId ) {
switch ( engineId ) {
case 'shaper-feedback' : return SYNTH _PRESETS ;
case 'additive' : return ADDITIVE _PRESETS ;
case 'fm' : return FM _PRESETS ;
default : return [ ] ;
}
}
/ * *
* Return the tier labels for the given engine id .
* /
function getPresetTiersForEngine ( engineId ) {
switch ( engineId ) {
case 'shaper-feedback' : return PRESET _TIERS ;
case 'additive' : return ADDITIVE _PRESET _TIERS ;
case 'fm' : return FM _PRESET _TIERS ;
default : return [ ] ;
}
}
// ---- Engine param overrides (for non-C15 engines) ----
// Flat array of per-param overrides for the current Faust engine.
// null when C15 is active (uses groupOverrides instead).
// Each entry: { min, max, curve, muted, fixedValue }
let engineParamOverrides = null ;
/ * *
* Build a fresh engineParamOverrides array from the current engine ' s paramMeta .
* All params default to unmuted with [ 0 , 1 ] range and linear curve .
* /
function buildEngineParamOverrides ( ) {
const meta = activeEngine ? . paramMeta ;
if ( ! meta ) { engineParamOverrides = null ; return ; }
engineParamOverrides = meta . map ( p => ( {
min : 0 ,
max : 1 ,
curve : 0.5 ,
muted : false ,
fixedValue : 0.5 ,
} ) ) ;
}
2026-03-22 20:57:21 +01:00
/ * *
* Apply group overrides ( per - param curve + min / max ) to a single ML output value .
* Returns the remapped value , or fixedValue if the param is muted .
2026-04-03 19:26:12 +02:00
*
* For non - C15 engines , uses engineParamOverrides ( flat array ) .
* For C15 , uses the nested groupOverrides / paramToSection system .
2026-03-22 20:57:21 +01:00
* /
function applyGroupOverrides ( rawValue , paramIndex ) {
2026-04-03 19:26:12 +02:00
// Non-C15 engine: use flat engine param overrides
if ( engineParamOverrides && paramIndex < engineParamOverrides . length ) {
const p = engineParamOverrides [ paramIndex ] ;
if ( p . muted ) return p . fixedValue ;
return applyGroupOverride ( rawValue , p . curve , p . min , p . max ) ;
}
// C15 path: nested section/group overrides
2026-03-22 20:57:21 +01:00
const mapping = paramToSection [ paramIndex ] ;
if ( ! mapping ) return rawValue ;
const ov = groupOverrides [ mapping . si ] ;
const p = ov . params [ mapping . li ] ;
if ( p . muted ) return p . fixedValue ;
// Per-param curve overrides group curve (if param curve != 0.5, use it; else use group curve)
const curve = p . curve !== 0.5 ? p . curve : ov . curve ;
return applyGroupOverride ( rawValue , curve , p . min , p . max ) ;
}
/** Check if param at given index is muted */
function isParamMuted ( paramIndex ) {
2026-04-03 19:26:12 +02:00
// Non-C15 engine: use flat engine param overrides
if ( engineParamOverrides && paramIndex < engineParamOverrides . length ) {
return engineParamOverrides [ paramIndex ] . muted ;
}
// C15 path
2026-03-22 20:57:21 +01:00
const mapping = paramToSection [ paramIndex ] ;
if ( ! mapping ) return false ;
return groupOverrides [ mapping . si ] . params [ mapping . li ] . muted ;
}
2026-03-22 23:50:56 +01:00
// ---- Synth Preset Application ----
/ * *
* Apply a synth preset by id .
2026-04-03 19:26:12 +02:00
* Engine - aware : uses groupOverrides for C15 , engineParamOverrides for Faust engines .
* Sets muted / active , min / max / curve / fixedValue for all params ,
2026-03-22 23:50:56 +01:00
* re - routes outputs , saves state , and updates the UI dropdown .
* /
function applyPreset ( presetId ) {
2026-04-03 19:26:12 +02:00
const engineId = activeEngine ? . id ? ? 'shaper-feedback' ;
const presets = getPresetsForEngine ( engineId ) ;
const preset = presets . find ( p => p . id === presetId ) ;
2026-03-22 23:50:56 +01:00
if ( ! preset ) {
2026-04-03 19:26:12 +02:00
console . warn ( ` [NISPS] Unknown preset: ${ presetId } (engine: ${ engineId } ) ` ) ;
2026-03-22 23:50:56 +01:00
return ;
}
2026-04-03 19:26:12 +02:00
const allActive = preset . active === null ; // null = all params active
2026-03-22 23:50:56 +01:00
const activeSet = allActive ? null : new Set ( preset . active ) ;
const overrides = preset . overrides || { } ;
const mutedOv = preset . mutedOverrides || { } ;
2026-04-03 19:26:12 +02:00
if ( engineId === 'shaper-feedback' ) {
// ---- C15 path: uses groupOverrides / paramToSection ----
const t = tameLevel ;
for ( let i = 0 ; i < N _SYNTH _OUTPUTS ; i ++ ) {
const param = SYNTH _PARAM _MAP [ i ] ;
const mapping = paramToSection [ i ] ;
if ( ! mapping ) continue ;
const gp = groupOverrides [ mapping . si ] . params [ mapping . li ] ;
const paramName = param . name ;
const safeMin = param . safeMin ? ? 0 ;
const safeMax = param . safeMax ? ? 1 ;
const tameMin = safeMin * t ;
const tameMax = 1 - ( 1 - safeMax ) * t ;
const isActive = allActive || activeSet . has ( paramName ) ;
if ( isActive ) {
gp . muted = false ;
const ov = overrides [ paramName ] ;
if ( ov ) {
gp . min = ov . min !== undefined ? ov . min : tameMin ;
gp . max = ov . max !== undefined ? ov . max : tameMax ;
gp . curve = ov . curve !== undefined ? ov . curve : 0.5 ;
gp . fixedValue = ov . fixedValue !== undefined ? ov . fixedValue : param . defaultValue ;
} else {
gp . min = tameMin ;
gp . max = tameMax ;
gp . curve = 0.5 ;
gp . fixedValue = param . defaultValue ;
}
2026-03-22 23:50:56 +01:00
} else {
2026-04-03 19:26:12 +02:00
gp . muted = true ;
const mov = mutedOv [ paramName ] ;
gp . fixedValue = ( mov && mov . fixedValue !== undefined ) ? mov . fixedValue : param . defaultValue ;
2026-03-22 23:50:56 +01:00
gp . min = tameMin ;
gp . max = tameMax ;
gp . curve = 0.5 ;
}
}
2026-04-03 19:26:12 +02:00
// Apply group curves
for ( let si = 0 ; si < SYNTH _SECTIONS . length ; si ++ ) {
const gc = preset . groupCurves || { } ;
const secName = SYNTH _SECTIONS [ si ] . name ;
groupOverrides [ si ] . curve = ( gc [ secName ] !== undefined ) ? gc [ secName ] : 0.5 ;
}
} else {
// ---- Faust engine path: uses engineParamOverrides (flat array) ----
const meta = activeEngine ? . paramMeta ? ? [ ] ;
// Build id -> index lookup
const idToIndex = new Map ( ) ;
meta . forEach ( ( p , i ) => idToIndex . set ( p . id , i ) ) ;
// Ensure engineParamOverrides exists with correct length
if ( ! engineParamOverrides || engineParamOverrides . length !== meta . length ) {
buildEngineParamOverrides ( ) ;
}
for ( let i = 0 ; i < meta . length ; i ++ ) {
const paramId = meta [ i ] . id ;
const ep = engineParamOverrides [ i ] ;
const isActive = allActive || activeSet . has ( paramId ) ;
if ( isActive ) {
ep . muted = false ;
const ov = overrides [ paramId ] ;
if ( ov ) {
ep . min = ov . min !== undefined ? ov . min : 0 ;
ep . max = ov . max !== undefined ? ov . max : 1 ;
ep . curve = ov . curve !== undefined ? ov . curve : 0.5 ;
ep . fixedValue = ov . fixedValue !== undefined ? ov . fixedValue : 0.5 ;
} else {
ep . min = 0 ;
ep . max = 1 ;
ep . curve = 0.5 ;
ep . fixedValue = 0.5 ;
}
} else {
ep . muted = true ;
const mov = mutedOv [ paramId ] ;
ep . fixedValue = ( mov && mov . fixedValue !== undefined ) ? mov . fixedValue : 0.5 ;
ep . min = 0 ;
ep . max = 1 ;
ep . curve = 0.5 ;
}
}
2026-03-22 23:50:56 +01:00
}
activeSynthPresetId = presetId ;
// Close the group drawer if open
hideGroupDrawer ( ) ;
// Re-route outputs through updated overrides
if ( iml ) {
routeOutputs ( iml . getOutputs ( ) ) ;
}
// Update the preset dropdown
const $presetSelect = document . getElementById ( 'synth-preset-select' ) ;
if ( $presetSelect ) $presetSelect . value = presetId ;
// Save to storage
saveState ( ) ;
2026-04-03 19:26:12 +02:00
console . log ( ` [NISPS] Applied synth preset: ${ preset . name } (engine: ${ engineId } ) ` ) ;
2026-03-22 23:50:56 +01:00
}
2026-03-21 23:57:58 +01:00
// ---- SynthVisualizer class ----
class SynthVisualizer {
constructor ( canvas ) {
this . canvas = canvas ;
this . ctx = canvas . getContext ( '2d' ) ;
this . params = new Array ( N _OUTPUTS ) . fill ( 0.5 ) ;
this . displayParams = new Array ( N _OUTPUTS ) . fill ( 0.5 ) ;
this . lerpSpeed = 0.12 ;
this . topPadding = 40 ;
this . bottomPadding = 100 ;
// Interaction state
this . _dragging = false ;
this . _dragBarIndex = - 1 ;
this . _interactionEnabled = false ;
2026-03-22 20:57:21 +01:00
// Hover tooltip state
this . _hoveredBar = - 1 ;
this . _tooltipEl = null ;
2026-03-21 23:57:58 +01:00
// Build section map
this . sectionMap = [ ] ;
let idx = 0 ;
for ( const sec of SYNTH _SECTIONS ) {
for ( let i = 0 ; i < sec . count && idx < N _OUTPUTS ; i ++ , idx ++ ) {
this . sectionMap . push ( sec ) ;
}
}
// Fill remainder with generic
while ( this . sectionMap . length < N _OUTPUTS ) {
this . sectionMap . push ( { name : 'Other' , count : 1 , color : '#666666' } ) ;
}
2026-03-22 20:57:21 +01:00
// Always-on hover tracking for tooltip (independent of enableInteraction)
this . canvas . addEventListener ( 'pointermove' , ( e ) => {
if ( this . _dragging ) return ; // interaction handler takes over
const idx = this . hitTest ( e . clientX , e . clientY ) ;
this . _hoveredBar = idx ;
// Store mouse position in canvas pixels for tooltip
const rect = this . canvas . getBoundingClientRect ( ) ;
const dpr = window . devicePixelRatio || 1 ;
this . _mouseCanvasX = ( e . clientX - rect . left ) * dpr ;
this . _mouseCanvasY = ( e . clientY - rect . top ) * dpr ;
} ) ;
this . canvas . addEventListener ( 'pointerleave' , ( ) => {
this . _hoveredBar = - 1 ;
} ) ;
2026-03-21 23:57:58 +01:00
this . resize ( ) ;
}
2026-04-03 19:41:05 +02:00
/ * *
* Rebuild the visualizer for a new engine ' s param layout .
* Derives sections from paramMeta ' s group field so it works with any engine .
* @ param { Array < { group ? : string } > } paramMeta
* /
rebuild ( paramMeta ) {
if ( ! paramMeta || paramMeta . length === 0 ) return ;
// Derive sections from paramMeta groups
const sections = [ ] ;
let currentGroup = null ;
let currentCount = 0 ;
for ( const pm of paramMeta ) {
const group = pm . group ? ? 'Other' ;
if ( group !== currentGroup ) {
if ( currentGroup !== null ) {
sections . push ( { name : currentGroup , count : currentCount , color : _colorFromGroup ( currentGroup ) } ) ;
}
currentGroup = group ;
currentCount = 0 ;
}
currentCount ++ ;
}
if ( currentGroup !== null ) {
sections . push ( { name : currentGroup , count : currentCount , color : _colorFromGroup ( currentGroup ) } ) ;
}
// Rebuild sectionMap
this . sectionMap = [ ] ;
for ( const sec of sections ) {
for ( let i = 0 ; i < sec . count ; i ++ ) {
this . sectionMap . push ( sec ) ;
}
}
// Resize param arrays
this . params = new Array ( paramMeta . length ) . fill ( 0.5 ) ;
this . displayParams = new Array ( paramMeta . length ) . fill ( 0.5 ) ;
}
2026-03-21 23:57:58 +01:00
resize ( ) {
this . canvas . width = window . innerWidth * ( window . devicePixelRatio || 1 ) ;
this . canvas . height = window . innerHeight * ( window . devicePixelRatio || 1 ) ;
this . canvas . style . width = '100%' ;
this . canvas . style . height = '100%' ;
}
setParams ( outputs ) {
for ( let i = 0 ; i < N _OUTPUTS && i < outputs . length ; i ++ ) {
this . params [ i ] = outputs [ i ] ;
}
}
draw ( ) {
const ctx = this . ctx ;
const W = this . canvas . width ;
const H = this . canvas . height ;
const dpr = window . devicePixelRatio || 1 ;
const n = N _OUTPUTS ;
// Lerp display values toward target
for ( let i = 0 ; i < n ; i ++ ) {
this . displayParams [ i ] += ( this . params [ i ] - this . displayParams [ i ] ) * this . lerpSpeed ;
}
// Background
ctx . fillStyle = '#0a0a0a' ;
ctx . fillRect ( 0 , 0 , W , H ) ;
2026-03-22 20:57:21 +01:00
// Build list of visible (non-muted) param indices
const visibleIndices = [ ] ;
for ( let i = 0 ; i < n ; i ++ ) {
if ( ! isParamMuted ( i ) ) visibleIndices . push ( i ) ;
}
const nVisible = visibleIndices . length || 1 ;
// Calculate section gap positions (among visible params only)
2026-03-21 23:57:58 +01:00
const sectionGaps = new Set ( ) ;
2026-03-22 20:57:21 +01:00
let prevSi = - 1 ;
for ( const vi of visibleIndices ) {
const mapping = paramToSection [ vi ] ;
const curSi = mapping ? mapping . si : - 1 ;
if ( prevSi >= 0 && curSi !== prevSi ) {
sectionGaps . add ( vi ) ;
}
prevSi = curSi ;
2026-03-21 23:57:58 +01:00
}
const topPad = this . topPadding * dpr ;
const bottomPad = this . bottomPadding * dpr ;
const totalGapPx = sectionGaps . size * 2 * dpr ;
const barAreaWidth = W - totalGapPx ;
2026-03-22 20:57:21 +01:00
const barWidth = barAreaWidth / nVisible ;
2026-03-21 23:57:58 +01:00
const usableHeight = H - topPad - bottomPad ;
const maxBarHeight = usableHeight ;
// Store layout for interaction hit-testing
2026-03-22 20:57:21 +01:00
this . _layout = { W , H , n , barWidth , totalGapPx , sectionGaps , topPad , bottomPad , usableHeight , dpr , visibleIndices } ;
2026-03-21 23:57:58 +01:00
// Draw bars
let x = 0 ;
2026-03-22 20:57:21 +01:00
let prevSection = null ;
2026-03-21 23:57:58 +01:00
let sectionStartX = 0 ;
2026-03-22 20:57:21 +01:00
let sectionIndex = - 1 ;
// Store bar x positions for hit testing (indexed by original param index)
this . _barXPositions = new Array ( n ) . fill ( - 1 ) ;
this . _barWidths = new Array ( n ) . fill ( 0 ) ;
// Store section label regions (in CSS pixels) for drawer hit testing
this . _sectionLabelRegions = [ ] ;
for ( let vi = 0 ; vi < visibleIndices . length ; vi ++ ) {
const i = visibleIndices [ vi ] ;
2026-04-03 20:01:14 +02:00
const sec = this . sectionMap [ i ] ? ? { name : 'Other' , count : 1 , color : '#666666' } ;
2026-03-22 20:57:21 +01:00
const mapping = paramToSection [ i ] ;
const curSi = mapping ? mapping . si : - 1 ;
2026-03-21 23:57:58 +01:00
// Section divider gap
if ( sectionGaps . has ( i ) ) {
// Draw section label for the previous section at top
2026-03-22 20:57:21 +01:00
if ( prevSection ) {
this . _drawSectionLabel ( ctx , prevSection . name , sectionStartX , x , topPad , prevSection . color ) ;
this . _sectionLabelRegions . push ( {
index : sectionIndex ,
left : sectionStartX / dpr ,
right : x / dpr ,
top : 0 ,
bottom : topPad / dpr ,
name : prevSection . name ,
} ) ;
}
sectionIndex = curSi ;
2026-03-21 23:57:58 +01:00
x += 2 * dpr ;
sectionStartX = x ;
prevSection = sec ;
}
2026-03-22 20:57:21 +01:00
if ( vi === 0 ) {
2026-03-21 23:57:58 +01:00
sectionStartX = x ;
prevSection = sec ;
2026-03-22 20:57:21 +01:00
sectionIndex = curSi ;
2026-03-21 23:57:58 +01:00
}
this . _barXPositions [ i ] = x ;
2026-03-22 20:57:21 +01:00
this . _barWidths [ i ] = barWidth ;
2026-03-21 23:57:58 +01:00
const val = this . displayParams [ i ] ;
const barH = val * maxBarHeight ;
const barY = topPad + usableHeight - barH ;
2026-03-30 16:05:45 +02:00
// Bar with slight transparency
ctx . fillStyle = sec . color + 'cc' ;
2026-03-21 23:57:58 +01:00
ctx . fillRect ( x , barY , Math . max ( barWidth - 0.5 , 1 ) , barH ) ;
// Bright top edge
2026-03-30 16:05:45 +02:00
ctx . fillStyle = sec . color ;
2026-03-21 23:57:58 +01:00
ctx . fillRect ( x , barY , Math . max ( barWidth - 0.5 , 1 ) , Math . min ( 2 * dpr , barH ) ) ;
x += barWidth ;
}
// Draw final section label at top
if ( prevSection ) {
this . _drawSectionLabel ( ctx , prevSection . name , sectionStartX , x , topPad , prevSection . color ) ;
2026-03-22 20:57:21 +01:00
this . _sectionLabelRegions . push ( {
index : sectionIndex ,
left : sectionStartX / dpr ,
right : x / dpr ,
top : 0 ,
bottom : topPad / dpr ,
name : prevSection . name ,
} ) ;
2026-03-21 23:57:58 +01:00
}
2026-03-22 20:57:21 +01:00
// Draw tooltip for hovered bar
this . _drawTooltip ( ctx , dpr ) ;
2026-03-21 23:57:58 +01:00
}
_drawSectionLabel ( ctx , name , startX , endX , topPad , color ) {
const dpr = window . devicePixelRatio || 1 ;
const fontSize = 9 * dpr ;
ctx . save ( ) ;
2026-03-22 00:36:00 +01:00
ctx . font = ` ${ fontSize } px 'JetBrains Mono', monospace ` ;
2026-03-21 23:57:58 +01:00
ctx . fillStyle = color + '88' ;
ctx . textAlign = 'center' ;
ctx . textBaseline = 'bottom' ;
const cx = ( startX + endX ) / 2 ;
ctx . fillText ( name , cx , topPad - 4 * dpr ) ;
ctx . restore ( ) ;
}
2026-03-22 20:57:21 +01:00
// Returns section region at client coordinates, or null
hitTestSection ( clientX , clientY ) {
if ( ! this . _sectionLabelRegions ) return null ;
const rect = this . canvas . getBoundingClientRect ( ) ;
const px = clientX - rect . left ;
const py = clientY - rect . top ;
for ( const region of this . _sectionLabelRegions ) {
if ( px >= region . left && px <= region . right && py >= region . top && py <= region . bottom ) {
return region ;
}
}
return null ;
}
2026-03-21 23:57:58 +01:00
// Returns bar index at canvas-relative pixel x, or -1
hitTest ( clientX , clientY ) {
if ( ! this . _barXPositions || ! this . _layout ) return - 1 ;
const rect = this . canvas . getBoundingClientRect ( ) ;
const dpr = this . _layout . dpr ;
const px = ( clientX - rect . left ) * dpr ;
const n = this . _layout . n ;
for ( let i = 0 ; i < n ; i ++ ) {
const bx = this . _barXPositions [ i ] ;
2026-03-22 20:57:21 +01:00
if ( bx < 0 ) continue ; // muted
const bw = this . _barWidths [ i ] || this . _layout . barWidth ;
if ( px >= bx && px < bx + bw ) return i ;
2026-03-21 23:57:58 +01:00
}
return - 1 ;
}
// Returns 0-1 value from clientY (top of bar area = 1, bottom = 0)
yToValue ( clientY ) {
if ( ! this . _layout ) return 0.5 ;
const rect = this . canvas . getBoundingClientRect ( ) ;
const dpr = this . _layout . dpr ;
const py = ( clientY - rect . top ) * dpr ;
const { topPad , usableHeight } = this . _layout ;
const val = 1 - ( py - topPad ) / usableHeight ;
return Math . max ( 0 , Math . min ( 1 , val ) ) ;
}
enableInteraction ( enabled ) {
if ( enabled && ! this . _interactionEnabled ) {
this . _interactionEnabled = true ;
this . canvas . style . cursor = 'crosshair' ;
this . _onPointerDown = ( e ) => {
const idx = this . hitTest ( e . clientX , e . clientY ) ;
if ( idx < 0 ) return ;
e . preventDefault ( ) ;
this . _dragging = true ;
this . _dragBarIndex = idx ;
this . canvas . setPointerCapture ( e . pointerId ) ;
const val = this . yToValue ( e . clientY ) ;
rawParamValues [ idx ] = val ;
this . params [ idx ] = val ;
routeOutputs ( rawParamValues ) ;
updateHeatmap ( rawParamValues ) ;
} ;
this . _onPointerMove = ( e ) => {
if ( ! this . _dragging ) return ;
e . preventDefault ( ) ;
// Allow sliding to adjacent bars
let idx = this . hitTest ( e . clientX , e . clientY ) ;
if ( idx < 0 ) idx = this . _dragBarIndex ;
this . _dragBarIndex = idx ;
const val = this . yToValue ( e . clientY ) ;
rawParamValues [ idx ] = val ;
this . params [ idx ] = val ;
routeOutputs ( rawParamValues ) ;
updateHeatmap ( rawParamValues ) ;
} ;
this . _onPointerUp = ( e ) => {
this . _dragging = false ;
this . _dragBarIndex = - 1 ;
} ;
this . canvas . addEventListener ( 'pointerdown' , this . _onPointerDown ) ;
this . canvas . addEventListener ( 'pointermove' , this . _onPointerMove ) ;
this . canvas . addEventListener ( 'pointerup' , this . _onPointerUp ) ;
this . canvas . addEventListener ( 'pointercancel' , this . _onPointerUp ) ;
} else if ( ! enabled && this . _interactionEnabled ) {
this . _interactionEnabled = false ;
this . canvas . style . cursor = '' ;
this . _dragging = false ;
this . _dragBarIndex = - 1 ;
2026-03-22 20:57:21 +01:00
this . _hoveredBar = - 1 ;
2026-03-21 23:57:58 +01:00
if ( this . _onPointerDown ) {
this . canvas . removeEventListener ( 'pointerdown' , this . _onPointerDown ) ;
this . canvas . removeEventListener ( 'pointermove' , this . _onPointerMove ) ;
this . canvas . removeEventListener ( 'pointerup' , this . _onPointerUp ) ;
this . canvas . removeEventListener ( 'pointercancel' , this . _onPointerUp ) ;
}
}
}
2026-03-22 20:57:21 +01:00
_drawTooltip ( ctx , dpr ) {
const i = this . _hoveredBar ;
if ( i < 0 || ! this . _layout ) return ;
if ( this . _barXPositions [ i ] < 0 ) return ; // muted
2026-04-03 18:55:06 +02:00
const name = ( activeEngine ? . paramMeta ? . [ i ] ? . name ) || SYNTH _PARAM _NAMES [ i ] || ` p ${ i } ` ;
2026-03-22 20:57:21 +01:00
const val = this . displayParams [ i ] ;
let rangeStr = '0.00 – 1.00' ;
let curveStr = '0.50' ;
2026-04-03 19:41:05 +02:00
if ( engineParamOverrides && i < engineParamOverrides . length ) {
// Non-C15 engine: use flat engine param overrides
const p = engineParamOverrides [ i ] ;
2026-03-22 20:57:21 +01:00
rangeStr = ` ${ p . min . toFixed ( 2 ) } – ${ p . max . toFixed ( 2 ) } ` ;
2026-04-03 19:41:05 +02:00
curveStr = p . curve . toFixed ( 2 ) ;
} else {
// C15 path: nested section/group overrides
const mapping = paramToSection [ i ] ;
if ( mapping ) {
const ov = groupOverrides [ mapping . si ] ;
const p = ov . params [ mapping . li ] ;
rangeStr = ` ${ p . min . toFixed ( 2 ) } – ${ p . max . toFixed ( 2 ) } ` ;
const curve = p . curve !== 0.5 ? p . curve : ov . curve ;
curveStr = curve . toFixed ( 2 ) ;
}
2026-03-22 20:57:21 +01:00
}
const lines = [ name , ` Val: ${ val . toFixed ( 2 ) } ` , ` Range: ${ rangeStr } ` , ` Curve: ${ curveStr } ` ] ;
const fontSize = 10 * dpr ;
const lineHeight = fontSize * 1.4 ;
const padX = 8 * dpr ;
const padY = 6 * dpr ;
ctx . save ( ) ;
ctx . font = ` ${ fontSize } px 'JetBrains Mono', monospace ` ;
// Measure text
let maxW = 0 ;
for ( const line of lines ) {
const m = ctx . measureText ( line ) ;
if ( m . width > maxW ) maxW = m . width ;
}
const boxW = maxW + padX * 2 ;
const boxH = lines . length * lineHeight + padY * 2 ;
// Position near the mouse cursor
const mx = this . _mouseCanvasX || 0 ;
const my = this . _mouseCanvasY || 0 ;
const offset = 12 * dpr ;
let tx = mx + offset ;
let ty = my - boxH - offset ;
// Clamp to canvas
if ( tx + boxW > this . _layout . W - 2 * dpr ) tx = mx - boxW - offset ;
if ( ty < 2 * dpr ) ty = my + offset ;
if ( tx < 2 * dpr ) tx = 2 * dpr ;
// Background
ctx . fillStyle = 'rgba(0, 0, 0, 0.88)' ;
ctx . strokeStyle = 'rgba(255, 255, 255, 0.15)' ;
ctx . lineWidth = 1 ;
const r = 4 * dpr ;
ctx . beginPath ( ) ;
ctx . roundRect ( tx , ty , boxW , boxH , r ) ;
ctx . fill ( ) ;
ctx . stroke ( ) ;
// Text
ctx . textAlign = 'left' ;
ctx . textBaseline = 'top' ;
for ( let li = 0 ; li < lines . length ; li ++ ) {
ctx . fillStyle = li === 0 ? '#ffffff' : 'rgba(255,255,255,0.6)' ;
if ( li === 0 ) ctx . font = ` bold ${ fontSize } px 'JetBrains Mono', monospace ` ;
else ctx . font = ` ${ fontSize } px 'JetBrains Mono', monospace ` ;
ctx . fillText ( lines [ li ] , tx + padX , ty + padY + li * lineHeight ) ;
}
ctx . restore ( ) ;
}
2026-03-21 23:57:58 +01:00
}
// ---- Preset padding ----
function padPresetOutputs ( outputs ) {
const padded = new Array ( N _OUTPUTS ) . fill ( 0.5 ) ;
for ( let i = 0 ; i < outputs . length && i < padded . length ; i ++ ) padded [ i ] = outputs [ i ] ;
return padded ;
}
2026-03-30 16:05:45 +02:00
// ---- MLP Resize ----
// Returns the target output count for a given mode
function outputCountForMode ( mode ) {
if ( mode === 'visual' ) return N _VISUAL _OUTPUTS ;
if ( mode === 'synth' ) return N _SYNTH _OUTPUTS ;
if ( mode === 'midi-cc' ) return midiCCMap . length ;
2026-04-03 21:36:24 +02:00
if ( mode === 'audio-canvas' ) return audioCanvas ? audioCanvas . getOutputCount ( ) : 12 ;
2026-03-30 16:05:45 +02:00
return N _SYNTH _OUTPUTS ;
}
2026-04-03 19:01:10 +02:00
/ * *
* Total MLP output count : engine params + EOC params when in Shared mode .
* In all other NISPS modes ( bypass / linked / independent ) or when no EOC chain
* exists , only the engine ' s own param count is included .
*
* @ returns { number }
* /
function totalOutputCount ( ) {
const engineParams = activeEngine ? . paramCount ? ? N _SYNTH _OUTPUTS ;
const eocParams = ( eocChain ? . nispsMode === 'shared' ) ? ( eocChain ? . paramCount ? ? 0 ) : 0 ;
return engineParams + eocParams ;
}
2026-03-30 16:05:45 +02:00
/ * *
* Recreate IML instances with a new output count .
2026-04-03 18:27:54 +02:00
* Joystick IML uses warm - start weight transfer to preserve learned mappings .
* Training examples are always cleared ( dataset is JS - side and output - count - specific ) .
2026-03-30 16:05:45 +02:00
* /
2026-04-03 18:49:07 +02:00
// ---- EOC audio graph wiring ----
/ * *
* Wire the EOC chain into the audio graph after the C15 AudioContext is created .
* Safe to call multiple times — guarded by _eocInited flag .
*
* Call this immediately after any c15 . start ( ) / activeEngine . init ( ) that brings
* the AudioContext into existence .
* /
async function _startEocChain ( ) {
if ( _eocInited || ! eocChain ) return ;
2026-04-03 18:56:35 +02:00
// C15Adapter exposes _bridge.audioContext; Faust engines expose _audioCtx
const audioCtx = activeEngine ? . _bridge ? . audioContext ? ? activeEngine ? . _audioCtx ;
2026-04-03 18:49:07 +02:00
if ( ! audioCtx ) {
console . warn ( '[EOC] _startEocChain: no AudioContext yet — skipping' ) ;
return ;
}
await eocChain . init ( audioCtx ) ;
_eocInited = true ;
// Disconnect the limiter from destination (it auto-connects there in C15Bridge.start())
const outputNode = activeEngine . getOutputNode ( ) ;
try { outputNode . disconnect ( ) ; } catch ( _ ) { /* already disconnected */ }
eocChain . connect ( outputNode , audioCtx . destination ) ;
console . log ( '[EOC] chain wired into audio graph' ) ;
}
2026-04-03 20:01:14 +02:00
/ * *
* Lazily initialise ShapeSeq subsystem .
* Guarded by _shapeSeqInited — safe to call multiple times .
* Requires audio to be running ( AudioContext must exist ) .
* /
async function _ensureShapeSeqInit ( ) {
if ( _shapeSeqInited || ! shapeSeqEnabled ) return ;
// Import modules lazily
if ( ! ShapeSeqEngine ) {
( { ShapeSeqEngine } = await import ( './shapeseq/sequencer.js' ) ) ;
( { StepVisualizer } = await import ( './shapeseq/step-viz.js' ) ) ;
( { ChainBuilderUI } = await import ( './shapeseq/chain-ui.js' ) ) ;
( { EventBus , getDefaultBus } = await import ( './shapeseq/event-bus.js' ) ) ;
}
const audioCtx = activeEngine ? . _bridge ? . audioContext ? ? activeEngine ? . _audioCtx ;
if ( ! audioCtx ) {
console . warn ( '[ShapeSeq] _ensureShapeSeqInit: no AudioContext yet — skipping' ) ;
return ;
}
shapeSeqBus = getDefaultBus ( audioCtx ) ;
// activeEngine (C15Adapter) exposes noteOn/noteOff directly — use it as the c15Bridge
shapeSeqEngine = new ShapeSeqEngine ( {
audioContext : audioCtx ,
eventBus : shapeSeqBus ,
c15Bridge : activeEngine ,
} ) ;
await shapeSeqEngine . init ( ) ;
_shapeSeqInited = true ;
// Show the ShapeSeq container
const container = document . getElementById ( 'shapeseq-container' ) ;
if ( container && outputMode === 'synth' ) {
container . style . display = 'block' ;
}
console . log ( '[ShapeSeq] engine initialised' ) ;
}
/ * *
* Destroy the ShapeSeq engine ( e . g . on engine switch ) .
* /
function _destroyShapeSeq ( ) {
if ( shapeSeqEngine ) {
shapeSeqEngine . destroy ( ) ;
shapeSeqEngine = null ;
}
shapeSeqViz = null ;
shapeSeqChainUI = null ;
shapeSeqBus = null ;
_shapeSeqInited = false ;
const container = document . getElementById ( 'shapeseq-container' ) ;
if ( container ) container . style . display = 'none' ;
console . log ( '[ShapeSeq] engine destroyed' ) ;
}
2026-04-03 18:30:26 +02:00
// ---- Engine switching (stub for meml-phg UI) ----
/ * *
* Hot - swap the active synth engine .
* Reinitialises the arpeggiator and resizes the MLP to match the new engine ' s
* param count . The new engine must already be initialised ( init ( ) called ) .
*
* @ param { import ( './synth/engine-interface.js' ) . SynthEngine } engine
* /
async function setActiveEngine ( engine ) {
2026-04-03 20:01:14 +02:00
// Destroy ShapeSeq on engine switch — it holds a reference to the old engine
if ( shapeSeqEnabled && _shapeSeqInited ) {
_destroyShapeSeq ( ) ;
}
2026-04-03 18:30:26 +02:00
activeEngine = engine ;
c15 = engine ; // keep alias in sync
arpeggiator . setEngine ( engine ) ;
2026-04-03 19:20:29 +02:00
if ( midiInput ) midiInput . setEngine ( engine ) ;
2026-04-03 18:34:14 +02:00
const btn = document . getElementById ( 'synth-mode-btn' ) ;
if ( btn ) btn . textContent = engine . displayName ;
2026-04-03 19:30:20 +02:00
const synthDockBtn = document . querySelector ( '[data-drawer="synth"]' ) ;
if ( synthDockBtn ) synthDockBtn . title = ` Synth: ${ engine . displayName } ` ;
2026-04-03 18:49:07 +02:00
EngineSwitcher . setActive ( engine . id ) ;
EngineSwitcher . setLoading ( engine . id , false ) ;
2026-04-03 19:01:10 +02:00
await resizeMLP ( totalOutputCount ( ) ) ;
2026-04-03 18:49:07 +02:00
2026-04-03 18:55:06 +02:00
// Reload MIDI CC map for the new engine (scoped storage key)
reloadMidiCCMap ( ) ;
2026-04-03 19:01:10 +02:00
// Rebuild heatmap cells — include EOC params when in shared mode
if ( eocChain ? . nispsMode === 'shared' ) {
const combinedMeta = [ ... ( engine . paramMeta ? ? [ ] ) , ... eocChain . paramMeta ] ;
rebuildHeatmap ( combinedMeta ) ;
} else {
rebuildHeatmap ( engine . paramMeta ) ;
}
document . getElementById ( 'heatmap-cells' ) ? . parentElement
? . classList . toggle ( 'shared-mode' , eocChain ? . nispsMode === 'shared' ) ;
2026-04-03 18:55:06 +02:00
2026-04-03 18:49:07 +02:00
// Rewire EOC chain to the new engine's output node
if ( eocChain && _eocInited ) {
const outputNode = engine . getOutputNode ( ) ;
if ( outputNode ) {
try { outputNode . disconnect ( ) ; } catch ( _ ) { /* not yet connected */ }
2026-04-03 18:56:35 +02:00
// C15Adapter exposes _bridge.audioContext; Faust engines expose _audioCtx
const rewireCtx = engine . _bridge ? . audioContext ? ? engine . _audioCtx ;
if ( rewireCtx ) {
eocChain . connect ( outputNode , rewireCtx . destination ) ;
}
2026-04-03 18:49:07 +02:00
}
}
2026-04-03 19:26:12 +02:00
2026-04-03 19:41:05 +02:00
// Rebuild paramToSection and SynthVisualizer for the new engine
if ( engine . id === 'shaper-feedback' ) {
restoreC15ParamToSection ( ) ;
// Restore C15 section map in SynthVisualizer
if ( synthVisualizer ) {
synthVisualizer . sectionMap = [ ] ;
let idx = 0 ;
for ( const sec of SYNTH _SECTIONS ) {
for ( let i = 0 ; i < sec . count ; i ++ , idx ++ ) {
synthVisualizer . sectionMap . push ( sec ) ;
}
}
while ( synthVisualizer . sectionMap . length < N _SYNTH _OUTPUTS ) {
synthVisualizer . sectionMap . push ( { name : 'Other' , count : 1 , color : '#666666' } ) ;
}
synthVisualizer . params = new Array ( N _SYNTH _OUTPUTS ) . fill ( 0.5 ) ;
synthVisualizer . displayParams = new Array ( N _SYNTH _OUTPUTS ) . fill ( 0.5 ) ;
}
} else if ( engine . paramMeta ? . length > 0 ) {
rebuildParamToSection ( engine . paramMeta ) ;
2026-04-11 08:31:19 +02:00
rebuildNonC15Sections ( engine . paramMeta ) ;
2026-04-03 19:41:05 +02:00
if ( synthVisualizer ) {
synthVisualizer . rebuild ( engine . paramMeta ) ;
}
}
2026-04-03 19:26:12 +02:00
// Build engine-specific param overrides (null for C15, flat array for Faust)
if ( engine . id === 'shaper-feedback' ) {
engineParamOverrides = null ;
} else {
buildEngineParamOverrides ( ) ;
}
// Rebuild the preset selector for the new engine
rebuildPresetSelector ( ) ;
// Clear active preset — it belongs to the previous engine
activeSynthPresetId = null ;
feat(playground): add Modular audio mode with shared mod pool
New "Modular" engine in a-immersive with three hot-swappable Faust
sub-engines (subtractive/additive/fm) sharing a common modulation pool:
16 ADSR slots + 32 LFO slots (single-knob sine->tri->square->saw
wavemorph) routed through a 48-source x 10-destination matrix per
engine. Per-connection scalar amounts in [-1, 1], summed at each
destination. Default MLP output count is 512 (32 mod-source params +
480 matrix cells); model reinits on sub-engine swap, count change, or
engine-param exposure toggle.
Faust layer:
- mod-pool.lib: shared ADSR/LFO/source-bus library
- gen-modular-dsp.py: byte-reproducible generator (source of truth)
- modular-subtractive: faithful Minimoog (3 osc, ladder filter, no envs)
- modular-additive: 64-partial, spectral shape + formants, no envs/LFOs
- modular-fm: 4-op matrix + self-feedback, no envs
- All three share d08=amp, d09=pan conventions
- MODULAR_DESTINATIONS.md: authoritative destination table
JS layer:
- ModularEngine: self-contained SynthEngine with getState/setState,
setSubEngine, setModSourceCount, setExposeEngineParam
- modular-ui: drawer with sub-engine toggle, ADSR/LFO count steppers,
per-slot enable switches, matrix grid editor (tap-cycle, long-press
precise, right-click menu, negative amounts), preset overlay
- modular-presets: 6 named presets (Slow pad, Plucky bass, Crystal,
DX bell, Morphing drone + default)
- a-app.js: Modular mode registered, paramMeta:change -> resizeMLP,
modular DSP state persisted under modularDspState, window.__nisps
debug hooks for programmatic control
Tests: tests/e2e/modular-mode.spec.js (11 Playwright tests, all passing
including DSP state survives reload, sub-engine swap keeps paramCount,
preset apply verification).
Also fixes a pre-existing build.sh bug where the -e flag caused faust
to overwrite .wasm outputs with expanded DSP source text, leaving
additive/fm-matrix/eoc-* committed as invalid WebAssembly. Rebuilt all
affected engines with the corrected script. Added an early-message
buffer to faust-worklet-processor.js so setParam calls arriving before
wasm instantiation are queued rather than dropped (needed when the user
configures modular state before clicking Start Audio).
2026-04-11 07:34:41 +02:00
// Modular UI show/hide (Phase C). The UI is created once during init()
// and stays in the DOM; we toggle its visibility via the dock icon.
if ( modularUI ) {
if ( engine . id === 'modular' ) modularUI . show ( ) ;
else modularUI . hide ( ) ;
}
2026-04-03 18:30:26 +02:00
}
2026-03-30 16:05:45 +02:00
async function resizeMLP ( newOutputCount ) {
if ( newOutputCount === N _OUTPUTS ) return ;
N _OUTPUTS = newOutputCount ;
2026-04-03 18:27:54 +02:00
// Extract joystick weights before destroying (for warm-start transfer)
const joySnapshot = imlJoy ? imlJoy . extractWeights ( ) : null ;
2026-04-02 21:35:38 +02:00
// Destroy old IML instances (free WASM memory)
if ( imlJoy ) imlJoy . destroy ( ) ;
if ( imlHand ) imlHand . destroy ( ) ;
2026-04-03 18:27:54 +02:00
// Joystick IML: warm-start from previous weights when possible
if ( joySnapshot ) {
imlJoy = await WasmIML . createWithWarmStart ( joySnapshot , N _OUTPUTS , 1000 , 1.0 , 0.00001 ) ;
} else {
imlJoy = await WasmIML . create ( N _JOY _INPUTS , N _OUTPUTS , [ 32 , 48 , 64 ] , 1000 , 1.0 , 0.00001 ) ;
imlJoy . randomiseWeights ( spreadLevel ) ;
}
2026-03-30 16:05:45 +02:00
imlJoy . setLogger ( msg => console . log ( '[NISPS:joy]' , msg ) ) ;
2026-04-03 18:27:54 +02:00
// Hand IML: fresh init (warm-start for 14-input networks is a future concern)
2026-03-30 16:05:45 +02:00
imlHand = await WasmIML . create ( N _HAND _INPUTS , N _OUTPUTS , [ 48 , 48 , 64 ] , 1000 , 1.0 , 0.00001 ) ;
imlHand . setLogger ( msg => console . log ( '[NISPS:hand]' , msg ) ) ;
2026-04-03 18:27:54 +02:00
imlHand . randomiseWeights ( spreadLevel ) ;
2026-03-30 16:05:45 +02:00
iml = ( inputMode === 'joystick' ) ? imlJoy : imlHand ;
// Reset dependent state
rawParamValues = new Array ( N _OUTPUTS ) . fill ( 0.5 ) ;
2026-04-03 18:30:26 +02:00
_lastSentParams = new Float32Array ( N _OUTPUTS ) ;
2026-03-30 16:05:45 +02:00
// Re-run inference
iml . setInput ( 0 , joyX ) ;
iml . setInput ( 1 , joyY ) ;
iml . process ( ) ;
2026-04-03 18:27:54 +02:00
console . log ( ` [NISPS] MLP resized to ${ N _OUTPUTS } outputs (joystick IML warm-started) ` ) ;
2026-03-30 16:05:45 +02:00
}
2026-03-21 23:57:58 +01:00
// ---- Init ----
2026-03-23 22:23:07 +01:00
async function init ( ) {
2026-03-21 23:57:58 +01:00
// Parse ?tame URL param
const urlParams = new URLSearchParams ( window . location . search ) ;
2026-03-22 22:59:08 +01:00
// tame is handled at groupOverrides init time, no longer needed here
2026-03-22 01:10:26 +01:00
spreadLevel = parseFloat ( urlParams . get ( 'spread' ) ? ? '0.6' ) ;
if ( isNaN ( spreadLevel ) ) spreadLevel = 0.6 ;
2026-03-22 00:36:00 +01:00
spreadLevel = Math . max ( 0 , Math . min ( 1 , spreadLevel ) ) ;
2026-03-21 23:57:58 +01:00
feat(playground): add MediaPipe hand tracking input with dev panel
Hand tracking via webcam as alternative to joystick input:
- Right hand tracks 14 derived features (palm XY, finger curls, spread,
roll, pitch, pinch) through a separate 14-input MLP
- Left hand gesture recognition (1 finger = thumbs up, 2 = thumbs down)
with 400ms debounce hold
- Split-zone PIP display (no camera feed, skeleton only) with dashed
divider and cross-zone dimming
- Dual IML architecture: independent imlJoy (2 inputs) and imlHand
(14 inputs) with pointer swap, preserving training data per mode
- Dev panel (?devmode=true): draggable/collapsible floating panel with
sliders for MediaPipe confidence thresholds, smoothing, gesture hold
time, world landmarks toggle, and live feature bar monitor
- Visual presets always route to joystick IML (prevents dimension mismatch)
- Race condition guard on input mode switching
2026-03-23 22:28:58 +01:00
// Dual IML instances — joystick (2 inputs) and hand tracking (14 inputs)
imlJoy = await WasmIML . create ( N _JOY _INPUTS , N _OUTPUTS , [ 32 , 48 , 64 ] , 1000 , 1.0 , 0.00001 ) ;
imlJoy . setLogger ( msg => console . log ( '[NISPS:joy]' , msg ) ) ;
imlHand = await WasmIML . create ( N _HAND _INPUTS , N _OUTPUTS , [ 48 , 48 , 64 ] , 1000 , 1.0 , 0.00001 ) ;
imlHand . setLogger ( msg => console . log ( '[NISPS:hand]' , msg ) ) ;
iml = imlJoy ; // default to joystick
2026-03-21 23:57:58 +01:00
// Canvas + Visualizer
$canvas = document . getElementById ( 'vis-canvas' ) ;
visualizer = new FlowFieldVisualizer ( $canvas ) ;
// Synth visualizer
$synthVisCanvas = document . getElementById ( 'synth-vis-canvas' ) ;
synthVisualizer = new SynthVisualizer ( $synthVisCanvas ) ;
2026-04-03 18:30:26 +02:00
// Synth — use C15Adapter to satisfy the SynthEngine interface
activeEngine = new C15Adapter ( ) ;
c15 = activeEngine ; // alias: existing code that references c15 continues to work
activeEngine . onStatusChange = msg => {
2026-03-21 23:57:58 +01:00
const el = document . getElementById ( 'synth-status' ) ;
if ( el ) el . textContent = msg ;
} ;
2026-04-03 18:30:26 +02:00
arpeggiator = new Arpeggiator ( activeEngine ) ;
2026-04-03 19:20:29 +02:00
midiInput = new MIDIInput ( activeEngine ) ;
2026-03-22 10:01:50 +01:00
initMIDIControls ( ) ;
2026-03-21 23:57:58 +01:00
2026-03-30 16:05:45 +02:00
// MIDI Output
midiOutput = new MIDIOutput ( ) ;
initMIDICCControls ( ) ;
2026-03-25 10:55:58 +01:00
2026-03-21 23:57:58 +01:00
// DOM refs
$heatmapCells = document . getElementById ( 'heatmap-cells' ) ;
$heatmapTooltip = document . getElementById ( 'heatmap-tooltip' ) ;
$joystickContainer = document . getElementById ( 'joystick-container' ) ;
$joyMap = document . getElementById ( 'joy-map' ) ;
$joyMapCtx = $joyMap . getContext ( '2d' ) ;
$noiseRing = document . getElementById ( 'noise-ring' ) ;
$followBadge = document . getElementById ( 'follow-badge' ) ;
$rlButtons = document . getElementById ( 'rl-buttons' ) ;
$btnThumbsUp = document . getElementById ( 'btn-thumbsup' ) ;
$btnThumbsDown = document . getElementById ( 'btn-thumbsdown' ) ;
$statusText = document . getElementById ( 'status-text' ) ;
$lossCanvas = document . getElementById ( 'loss-canvas' ) ;
$lossCtx = $lossCanvas . getContext ( '2d' ) ;
2026-03-30 16:05:45 +02:00
$engineParams = document . getElementById ( 'engine-params' ) ;
2026-03-21 23:57:58 +01:00
$followPill = document . getElementById ( 'follow-pill' ) ;
2026-04-03 18:55:06 +02:00
// Build heatmap cells (engine-aware: uses activeEngine.paramMeta for synth mode)
rebuildHeatmap ( activeEngine . paramMeta ) ;
2026-03-21 23:57:58 +01:00
// Build raw param sliders
2026-03-30 16:05:45 +02:00
buildEngineParams ( ) ;
2026-03-21 23:57:58 +01:00
2026-04-03 18:49:07 +02:00
// Engine switcher — prepended above the tuning sliders in the Engine drawer
const ENGINES = [
{
id : 'shaper-feedback' ,
displayName : 'C15 Shaper-Feedback' ,
paramCount : 126 ,
description : 'Phase-aligned waveshapers with feedback mixer. Complex harmonic textures.' ,
} ,
{
id : 'additive' ,
displayName : 'Additive' ,
paramCount : 48 ,
description : 'Spectral envelope additive synthesis. 64 harmonics shaped by ML.' ,
} ,
{
id : 'fm' ,
displayName : 'FM Matrix' ,
paramCount : 55 ,
description : '4-operator FM with continuous routing matrix. Algorithm emerges from exploration.' ,
} ,
feat(playground): add Modular audio mode with shared mod pool
New "Modular" engine in a-immersive with three hot-swappable Faust
sub-engines (subtractive/additive/fm) sharing a common modulation pool:
16 ADSR slots + 32 LFO slots (single-knob sine->tri->square->saw
wavemorph) routed through a 48-source x 10-destination matrix per
engine. Per-connection scalar amounts in [-1, 1], summed at each
destination. Default MLP output count is 512 (32 mod-source params +
480 matrix cells); model reinits on sub-engine swap, count change, or
engine-param exposure toggle.
Faust layer:
- mod-pool.lib: shared ADSR/LFO/source-bus library
- gen-modular-dsp.py: byte-reproducible generator (source of truth)
- modular-subtractive: faithful Minimoog (3 osc, ladder filter, no envs)
- modular-additive: 64-partial, spectral shape + formants, no envs/LFOs
- modular-fm: 4-op matrix + self-feedback, no envs
- All three share d08=amp, d09=pan conventions
- MODULAR_DESTINATIONS.md: authoritative destination table
JS layer:
- ModularEngine: self-contained SynthEngine with getState/setState,
setSubEngine, setModSourceCount, setExposeEngineParam
- modular-ui: drawer with sub-engine toggle, ADSR/LFO count steppers,
per-slot enable switches, matrix grid editor (tap-cycle, long-press
precise, right-click menu, negative amounts), preset overlay
- modular-presets: 6 named presets (Slow pad, Plucky bass, Crystal,
DX bell, Morphing drone + default)
- a-app.js: Modular mode registered, paramMeta:change -> resizeMLP,
modular DSP state persisted under modularDspState, window.__nisps
debug hooks for programmatic control
Tests: tests/e2e/modular-mode.spec.js (11 Playwright tests, all passing
including DSP state survives reload, sub-engine swap keeps paramCount,
preset apply verification).
Also fixes a pre-existing build.sh bug where the -e flag caused faust
to overwrite .wasm outputs with expanded DSP source text, leaving
additive/fm-matrix/eoc-* committed as invalid WebAssembly. Rebuilt all
affected engines with the corrected script. Added an early-message
buffer to faust-worklet-processor.js so setParam calls arriving before
wasm instantiation are queued rather than dropped (needed when the user
configures modular state before clicking Start Audio).
2026-04-11 07:34:41 +02:00
{
id : 'modular' ,
displayName : 'Modular' ,
fix(modular): restore matrix in paramMeta; amp floor via positive-only mod_amp
b290144 made matrix cells opt-in to prevent joystick-silences-voice,
but that broke modular-ui.updateLive(): the matrix DOM stopped
reflecting live MLP outputs because matrixIndexCache was empty when
buildMatrixIndex() walked paramMeta. This was the same regression
6072fe8 had previously fixed.
Fix it structurally at the DSP layer instead: amp_val now computes
as `clamp(base_amp + max(0, mod_amp)) * level * vel_gain`, so matrix
d08_amp cells can only boost the amp floor — never cut it. base_amp
defaults to 1.0 (always audible), and presets that want classic
envelope-gated voices (slow pad, plucky bass, crystal, morphing
drone) drop base_amp to 0 and layer a positive ADSR→amp route on top.
With the DSP guard in place, all 480 matrix cells can safely live
in paramMeta again, and updateLive() gets its live visual feedback
back. Revert the opt-in gate in _rebuildParamMeta and the 32-param
test counts, and add a regression test asserting that every matrix
destination has 48 cells in paramMeta — that's what updateLive needs.
2026-04-11 09:39:58 +02:00
paramCount : 512 ,
description : 'Shared mod pool (4 ADSRs + 8 LFOs) routed through a matrix into a swappable voice. Starts with a 3-osc subtractive sub-engine.' ,
feat(playground): add Modular audio mode with shared mod pool
New "Modular" engine in a-immersive with three hot-swappable Faust
sub-engines (subtractive/additive/fm) sharing a common modulation pool:
16 ADSR slots + 32 LFO slots (single-knob sine->tri->square->saw
wavemorph) routed through a 48-source x 10-destination matrix per
engine. Per-connection scalar amounts in [-1, 1], summed at each
destination. Default MLP output count is 512 (32 mod-source params +
480 matrix cells); model reinits on sub-engine swap, count change, or
engine-param exposure toggle.
Faust layer:
- mod-pool.lib: shared ADSR/LFO/source-bus library
- gen-modular-dsp.py: byte-reproducible generator (source of truth)
- modular-subtractive: faithful Minimoog (3 osc, ladder filter, no envs)
- modular-additive: 64-partial, spectral shape + formants, no envs/LFOs
- modular-fm: 4-op matrix + self-feedback, no envs
- All three share d08=amp, d09=pan conventions
- MODULAR_DESTINATIONS.md: authoritative destination table
JS layer:
- ModularEngine: self-contained SynthEngine with getState/setState,
setSubEngine, setModSourceCount, setExposeEngineParam
- modular-ui: drawer with sub-engine toggle, ADSR/LFO count steppers,
per-slot enable switches, matrix grid editor (tap-cycle, long-press
precise, right-click menu, negative amounts), preset overlay
- modular-presets: 6 named presets (Slow pad, Plucky bass, Crystal,
DX bell, Morphing drone + default)
- a-app.js: Modular mode registered, paramMeta:change -> resizeMLP,
modular DSP state persisted under modularDspState, window.__nisps
debug hooks for programmatic control
Tests: tests/e2e/modular-mode.spec.js (11 Playwright tests, all passing
including DSP state survives reload, sub-engine swap keeps paramCount,
preset apply verification).
Also fixes a pre-existing build.sh bug where the -e flag caused faust
to overwrite .wasm outputs with expanded DSP source text, leaving
additive/fm-matrix/eoc-* committed as invalid WebAssembly. Rebuilt all
affected engines with the corrected script. Added an early-message
buffer to faust-worklet-processor.js so setParam calls arriving before
wasm instantiation are queued rather than dropped (needed when the user
configures modular state before clicking Start Audio).
2026-04-11 07:34:41 +02:00
} ,
2026-04-03 18:49:07 +02:00
] ;
2026-04-03 19:30:20 +02:00
const engineSwitcherEl = document . getElementById ( 'synth-engine-switcher' ) ;
2026-04-03 18:49:07 +02:00
if ( engineSwitcherEl ) {
EngineSwitcher . init ( engineSwitcherEl , ENGINES , async ( engineId ) => {
2026-04-03 18:56:35 +02:00
if ( engineId === activeEngine ? . id ) return ;
EngineSwitcher . setLoading ( engineId , true ) ;
try {
let newEngine ;
if ( engineId === 'shaper-feedback' ) {
// Switch back to C15 — reuse the existing instance
newEngine = c15 instanceof C15Adapter ? c15 : new C15Adapter ( ) ;
} else if ( engineId === 'additive' ) {
newEngine = new AdditiveEngine ( ) ;
} else if ( engineId === 'fm' ) {
newEngine = new FMEngine ( ) ;
feat(playground): add Modular audio mode with shared mod pool
New "Modular" engine in a-immersive with three hot-swappable Faust
sub-engines (subtractive/additive/fm) sharing a common modulation pool:
16 ADSR slots + 32 LFO slots (single-knob sine->tri->square->saw
wavemorph) routed through a 48-source x 10-destination matrix per
engine. Per-connection scalar amounts in [-1, 1], summed at each
destination. Default MLP output count is 512 (32 mod-source params +
480 matrix cells); model reinits on sub-engine swap, count change, or
engine-param exposure toggle.
Faust layer:
- mod-pool.lib: shared ADSR/LFO/source-bus library
- gen-modular-dsp.py: byte-reproducible generator (source of truth)
- modular-subtractive: faithful Minimoog (3 osc, ladder filter, no envs)
- modular-additive: 64-partial, spectral shape + formants, no envs/LFOs
- modular-fm: 4-op matrix + self-feedback, no envs
- All three share d08=amp, d09=pan conventions
- MODULAR_DESTINATIONS.md: authoritative destination table
JS layer:
- ModularEngine: self-contained SynthEngine with getState/setState,
setSubEngine, setModSourceCount, setExposeEngineParam
- modular-ui: drawer with sub-engine toggle, ADSR/LFO count steppers,
per-slot enable switches, matrix grid editor (tap-cycle, long-press
precise, right-click menu, negative amounts), preset overlay
- modular-presets: 6 named presets (Slow pad, Plucky bass, Crystal,
DX bell, Morphing drone + default)
- a-app.js: Modular mode registered, paramMeta:change -> resizeMLP,
modular DSP state persisted under modularDspState, window.__nisps
debug hooks for programmatic control
Tests: tests/e2e/modular-mode.spec.js (11 Playwright tests, all passing
including DSP state survives reload, sub-engine swap keeps paramCount,
preset apply verification).
Also fixes a pre-existing build.sh bug where the -e flag caused faust
to overwrite .wasm outputs with expanded DSP source text, leaving
additive/fm-matrix/eoc-* committed as invalid WebAssembly. Rebuilt all
affected engines with the corrected script. Added an early-message
buffer to faust-worklet-processor.js so setParam calls arriving before
wasm instantiation are queued rather than dropped (needed when the user
configures modular state before clicking Start Audio).
2026-04-11 07:34:41 +02:00
} else if ( engineId === 'modular' ) {
newEngine = new ModularEngine ( ) ;
// When the modular engine's paramMeta changes (sub-engine swap or
// expose/unexpose toggles from Phase C UI), resize the MLP and
// rebuild downstream state that depends on paramCount.
newEngine . on ( 'paramMeta:change' , async ( ) => {
if ( activeEngine !== newEngine ) return ;
await resizeMLP ( totalOutputCount ( ) ) ;
if ( eocChain ? . nispsMode === 'shared' ) {
const combinedMeta = [ ... ( newEngine . paramMeta ? ? [ ] ) , ... eocChain . paramMeta ] ;
rebuildHeatmap ( combinedMeta ) ;
} else {
rebuildHeatmap ( newEngine . paramMeta ) ;
}
if ( newEngine . paramMeta ? . length > 0 ) {
rebuildParamToSection ( newEngine . paramMeta ) ;
2026-04-11 08:31:19 +02:00
rebuildNonC15Sections ( newEngine . paramMeta ) ;
feat(playground): add Modular audio mode with shared mod pool
New "Modular" engine in a-immersive with three hot-swappable Faust
sub-engines (subtractive/additive/fm) sharing a common modulation pool:
16 ADSR slots + 32 LFO slots (single-knob sine->tri->square->saw
wavemorph) routed through a 48-source x 10-destination matrix per
engine. Per-connection scalar amounts in [-1, 1], summed at each
destination. Default MLP output count is 512 (32 mod-source params +
480 matrix cells); model reinits on sub-engine swap, count change, or
engine-param exposure toggle.
Faust layer:
- mod-pool.lib: shared ADSR/LFO/source-bus library
- gen-modular-dsp.py: byte-reproducible generator (source of truth)
- modular-subtractive: faithful Minimoog (3 osc, ladder filter, no envs)
- modular-additive: 64-partial, spectral shape + formants, no envs/LFOs
- modular-fm: 4-op matrix + self-feedback, no envs
- All three share d08=amp, d09=pan conventions
- MODULAR_DESTINATIONS.md: authoritative destination table
JS layer:
- ModularEngine: self-contained SynthEngine with getState/setState,
setSubEngine, setModSourceCount, setExposeEngineParam
- modular-ui: drawer with sub-engine toggle, ADSR/LFO count steppers,
per-slot enable switches, matrix grid editor (tap-cycle, long-press
precise, right-click menu, negative amounts), preset overlay
- modular-presets: 6 named presets (Slow pad, Plucky bass, Crystal,
DX bell, Morphing drone + default)
- a-app.js: Modular mode registered, paramMeta:change -> resizeMLP,
modular DSP state persisted under modularDspState, window.__nisps
debug hooks for programmatic control
Tests: tests/e2e/modular-mode.spec.js (11 Playwright tests, all passing
including DSP state survives reload, sub-engine swap keeps paramCount,
preset apply verification).
Also fixes a pre-existing build.sh bug where the -e flag caused faust
to overwrite .wasm outputs with expanded DSP source text, leaving
additive/fm-matrix/eoc-* committed as invalid WebAssembly. Rebuilt all
affected engines with the corrected script. Added an early-message
buffer to faust-worklet-processor.js so setParam calls arriving before
wasm instantiation are queued rather than dropped (needed when the user
configures modular state before clicking Start Audio).
2026-04-11 07:34:41 +02:00
if ( synthVisualizer ) synthVisualizer . rebuild ( newEngine . paramMeta ) ;
}
buildEngineParamOverrides ( ) ;
} ) ;
2026-04-03 18:56:35 +02:00
} else {
showToast ( ` Unknown engine: ${ engineId } ` ) ;
return ;
}
if ( newEngine !== activeEngine ) {
2026-04-03 20:01:14 +02:00
// Stop arpeggiator/ShapeSeq before switching
if ( shapeSeqEnabled && shapeSeqEngine ) shapeSeqEngine . stop ( ) ;
2026-04-03 18:56:35 +02:00
if ( arpeggiator ) arpeggiator . stop ( ) ;
2026-04-03 19:41:05 +02:00
// Ensure paramMeta is available even if audio isn't started yet.
// This prevents MLP resizing to 0 outputs for Faust engines.
if ( newEngine . loadParamMeta ) {
await newEngine . loadParamMeta ( ) ;
}
2026-04-03 18:56:35 +02:00
// If audio is already running, init the new engine now
const audioCtxForInit = activeEngine ? . _audioCtx
? ? activeEngine ? . _bridge ? . audioContext
? ? null ;
if ( audioCtxForInit && ! newEngine . _running ) {
await newEngine . init ( audioCtxForInit ) ;
}
feat(playground): add Modular audio mode with shared mod pool
New "Modular" engine in a-immersive with three hot-swappable Faust
sub-engines (subtractive/additive/fm) sharing a common modulation pool:
16 ADSR slots + 32 LFO slots (single-knob sine->tri->square->saw
wavemorph) routed through a 48-source x 10-destination matrix per
engine. Per-connection scalar amounts in [-1, 1], summed at each
destination. Default MLP output count is 512 (32 mod-source params +
480 matrix cells); model reinits on sub-engine swap, count change, or
engine-param exposure toggle.
Faust layer:
- mod-pool.lib: shared ADSR/LFO/source-bus library
- gen-modular-dsp.py: byte-reproducible generator (source of truth)
- modular-subtractive: faithful Minimoog (3 osc, ladder filter, no envs)
- modular-additive: 64-partial, spectral shape + formants, no envs/LFOs
- modular-fm: 4-op matrix + self-feedback, no envs
- All three share d08=amp, d09=pan conventions
- MODULAR_DESTINATIONS.md: authoritative destination table
JS layer:
- ModularEngine: self-contained SynthEngine with getState/setState,
setSubEngine, setModSourceCount, setExposeEngineParam
- modular-ui: drawer with sub-engine toggle, ADSR/LFO count steppers,
per-slot enable switches, matrix grid editor (tap-cycle, long-press
precise, right-click menu, negative amounts), preset overlay
- modular-presets: 6 named presets (Slow pad, Plucky bass, Crystal,
DX bell, Morphing drone + default)
- a-app.js: Modular mode registered, paramMeta:change -> resizeMLP,
modular DSP state persisted under modularDspState, window.__nisps
debug hooks for programmatic control
Tests: tests/e2e/modular-mode.spec.js (11 Playwright tests, all passing
including DSP state survives reload, sub-engine swap keeps paramCount,
preset apply verification).
Also fixes a pre-existing build.sh bug where the -e flag caused faust
to overwrite .wasm outputs with expanded DSP source text, leaving
additive/fm-matrix/eoc-* committed as invalid WebAssembly. Rebuilt all
affected engines with the corrected script. Added an early-message
buffer to faust-worklet-processor.js so setParam calls arriving before
wasm instantiation are queued rather than dropped (needed when the user
configures modular state before clicking Start Audio).
2026-04-11 07:34:41 +02:00
// Phase E — apply any pending modular DSP snapshot from loadState().
// Runs BEFORE setActiveEngine so paramMeta is correct when the MLP
// resizes below. Note: Phase C's UI state (counts, sub-engine,
// exposed params, enables) is applied separately by modular-ui.js
// on its first refresh(); that runs AFTER setActiveEngine via
// modularUI.show(), so the load ordering is:
// 1. setState() here — restores raw DSP values + sub-engine
// 2. modular-ui refresh() — replays Phase C's UI state
// Step 2 may call setModSourceCount() which re-emits
// paramMeta:change; that's fine since the raw values are already
// in _lastRawByLabel.
if ( engineId === 'modular' && _pendingModularDspState &&
typeof newEngine . setState === 'function' ) {
try {
await newEngine . setState ( _pendingModularDspState ) ;
} catch ( err ) {
console . warn ( '[NISPS] modular setState on engine switch failed:' , err ) ;
}
_pendingModularDspState = null ;
}
2026-04-03 18:56:35 +02:00
await setActiveEngine ( newEngine ) ;
}
} catch ( err ) {
console . error ( '[EngineSwitcher] Failed to switch engine:' , err ) ;
showToast ( ` Failed to load ${ engineId } : ${ err . message } ` ) ;
} finally {
EngineSwitcher . setLoading ( engineId , false ) ;
2026-04-03 18:49:07 +02:00
}
} , {
hasTrainingData : ( ) => ( imlJoy ? . exampleCount ? ? 0 ) > 0 || ( imlHand ? . exampleCount ? ? 0 ) > 0 ,
} ) ;
EngineSwitcher . setActive ( activeEngine . id ) ;
const engineDockBtn = document . querySelector ( '[data-drawer="params"]' ) ;
if ( engineDockBtn ) engineDockBtn . title = ` Engine: ${ activeEngine . displayName } ` ;
}
2026-03-21 23:57:58 +01:00
// Wire events
wireJoystick ( ) ;
2026-03-30 16:05:45 +02:00
wireDock ( ) ;
feat(playground): add Modular audio mode with shared mod pool
New "Modular" engine in a-immersive with three hot-swappable Faust
sub-engines (subtractive/additive/fm) sharing a common modulation pool:
16 ADSR slots + 32 LFO slots (single-knob sine->tri->square->saw
wavemorph) routed through a 48-source x 10-destination matrix per
engine. Per-connection scalar amounts in [-1, 1], summed at each
destination. Default MLP output count is 512 (32 mod-source params +
480 matrix cells); model reinits on sub-engine swap, count change, or
engine-param exposure toggle.
Faust layer:
- mod-pool.lib: shared ADSR/LFO/source-bus library
- gen-modular-dsp.py: byte-reproducible generator (source of truth)
- modular-subtractive: faithful Minimoog (3 osc, ladder filter, no envs)
- modular-additive: 64-partial, spectral shape + formants, no envs/LFOs
- modular-fm: 4-op matrix + self-feedback, no envs
- All three share d08=amp, d09=pan conventions
- MODULAR_DESTINATIONS.md: authoritative destination table
JS layer:
- ModularEngine: self-contained SynthEngine with getState/setState,
setSubEngine, setModSourceCount, setExposeEngineParam
- modular-ui: drawer with sub-engine toggle, ADSR/LFO count steppers,
per-slot enable switches, matrix grid editor (tap-cycle, long-press
precise, right-click menu, negative amounts), preset overlay
- modular-presets: 6 named presets (Slow pad, Plucky bass, Crystal,
DX bell, Morphing drone + default)
- a-app.js: Modular mode registered, paramMeta:change -> resizeMLP,
modular DSP state persisted under modularDspState, window.__nisps
debug hooks for programmatic control
Tests: tests/e2e/modular-mode.spec.js (11 Playwright tests, all passing
including DSP state survives reload, sub-engine swap keeps paramCount,
preset apply verification).
Also fixes a pre-existing build.sh bug where the -e flag caused faust
to overwrite .wasm outputs with expanded DSP source text, leaving
additive/fm-matrix/eoc-* committed as invalid WebAssembly. Rebuilt all
affected engines with the corrected script. Added an early-message
buffer to faust-worklet-processor.js so setParam calls arriving before
wasm instantiation are queued rather than dropped (needed when the user
configures modular state before clicking Start Audio).
2026-04-11 07:34:41 +02:00
// Phase C: Modular UI. Mount once here (after dock + drawer-stack exist).
// The UI is hidden until the modular engine becomes active.
try {
modularUI = initModularUI ( {
getEngine : ( ) => activeEngine ,
// Phase E: when modular UI mutates its state (preset applied,
// enables toggled, etc.) trigger the top-level save so the DSP
// snapshot is re-persisted alongside Phase C's UI-state key.
onStateChange : ( ) => saveState ( ) ,
} ) ;
if ( activeEngine ? . id === 'modular' ) modularUI . show ( ) ;
} catch ( err ) {
console . error ( '[NISPS] Failed to init modular UI:' , err ) ;
}
2026-03-21 23:57:58 +01:00
wireControls ( ) ;
wireSynthControls ( ) ;
wireGamepad ( ) ;
wireKeyboard ( ) ;
feat(playground): add MediaPipe hand tracking input with dev panel
Hand tracking via webcam as alternative to joystick input:
- Right hand tracks 14 derived features (palm XY, finger curls, spread,
roll, pitch, pinch) through a separate 14-input MLP
- Left hand gesture recognition (1 finger = thumbs up, 2 = thumbs down)
with 400ms debounce hold
- Split-zone PIP display (no camera feed, skeleton only) with dashed
divider and cross-zone dimming
- Dual IML architecture: independent imlJoy (2 inputs) and imlHand
(14 inputs) with pointer swap, preserving training data per mode
- Dev panel (?devmode=true): draggable/collapsible floating panel with
sliders for MediaPipe confidence thresholds, smoothing, gesture hold
time, world landmarks toggle, and live feature bar monitor
- Visual presets always route to joystick IML (prevents dimension mismatch)
- Race condition guard on input mode switching
2026-03-23 22:28:58 +01:00
wireInputToggle ( ) ;
createDevPanel ( ( ) => handTracker ) ;
2026-03-21 23:57:58 +01:00
wireQuickPlayControls ( ) ;
2026-03-22 20:57:21 +01:00
wireGroupDrawer ( ) ;
2026-03-30 16:05:45 +02:00
wireParamPopup ( ) ;
2026-03-22 21:41:49 +01:00
wireHelp ( ) ;
2026-03-21 23:57:58 +01:00
// Resize
window . addEventListener ( 'resize' , onResize ) ;
onResize ( ) ;
// Restore saved state (if any)
2026-03-30 16:05:45 +02:00
await loadState ( ) ;
2026-03-21 23:57:58 +01:00
2026-03-22 23:50:56 +01:00
// Wire synth preset selector
wireSynthPresets ( ) ;
2026-04-03 19:26:12 +02:00
// Check URL ?preset param (overrides localStorage) — search all engines
2026-03-22 23:50:56 +01:00
const urlPreset = urlParams . get ( 'preset' ) ;
2026-04-03 19:26:12 +02:00
const allPresets = [ ... SYNTH _PRESETS , ... ADDITIVE _PRESETS , ... FM _PRESETS ] ;
if ( urlPreset && allPresets . some ( p => p . id === urlPreset ) ) {
2026-03-22 23:50:56 +01:00
applyPreset ( urlPreset ) ;
} else if ( activeSynthPresetId ) {
// Restored from localStorage — sync the dropdown
const $ps = document . getElementById ( 'synth-preset-select' ) ;
if ( $ps ) $ps . value = activeSynthPresetId ;
}
2026-03-30 16:05:45 +02:00
// Initial inference
iml . setInput ( 0 , joyX ) ;
iml . setInput ( 1 , joyY ) ;
iml . process ( ) ;
routeOutputs ( iml . getOutputs ( ) ) ;
updateHeatmap ( iml . getOutputs ( ) ) ;
2026-03-21 23:57:58 +01:00
updateStatus ( ) ;
drawJoyMap ( ) ;
drawLossPlot ( ) ;
// Auto-save every 10 seconds
setInterval ( saveState , 10000 ) ;
2026-04-03 18:42:59 +02:00
// EOC Effects Chain — initialise chain and wire drawer UI
2026-04-03 18:49:07 +02:00
eocChain = new EOCChain ( ) ;
2026-04-03 18:42:59 +02:00
const eocDrawerBody = document . getElementById ( 'eoc-drawer-body' ) ;
if ( eocDrawerBody ) {
EOCChainUI . init ( eocChain , eocDrawerBody ) ;
}
2026-04-03 19:21:59 +02:00
// EOC Independent mode joystick
const eocJoyCanvas = document . getElementById ( 'eoc-joy-map' ) ;
if ( eocJoyCanvas ) {
eocJoystick = new EOCJoystick ( eocJoyCanvas ) ;
eocJoystick . onChange = ( x , y ) => {
if ( ! imlEoc || eocChain ? . nispsMode !== 'independent' ) return ;
imlEoc . setInput ( 0 , x ) ;
imlEoc . setInput ( 1 , y ) ;
imlEoc . process ( ) ;
const eocOutputs = imlEoc . getOutputs ( ) ;
for ( let i = 0 ; i < eocOutputs . length ; i ++ ) {
eocChain . setParam ( i , eocOutputs [ i ] ) ;
}
} ;
}
2026-04-03 19:16:54 +02:00
// Handle EOC structural changes — Shared mode resizes MLP; Linked mode manages second IML
2026-04-03 19:01:10 +02:00
window . addEventListener ( 'eoc:change' , async ( e ) => {
const reason = e . detail ? . reason ;
console . log ( '[EOC] chain changed, reason:' , reason , 'paramCount:' , eocChain . paramCount , 'nispsMode:' , eocChain . nispsMode ) ;
2026-04-03 19:03:12 +02:00
if ( reason === 'nispsMode-changed' ) {
2026-04-03 19:21:59 +02:00
// Linked/Independent mode lifecycle — both need a second IML
if ( eocChain . nispsMode === 'linked' || eocChain . nispsMode === 'independent' ) {
2026-04-03 19:03:12 +02:00
await initEocIML ( ) ;
} else {
destroyEocIML ( ) ;
eocTrainingTarget = 'synth' ;
}
2026-04-03 19:21:59 +02:00
// Show/hide EOC joystick (only visible in independent mode)
const eocJoyContainer = document . getElementById ( 'eoc-joy-container' ) ;
if ( eocJoyContainer ) {
eocJoyContainer . classList . toggle ( 'hidden' , eocChain . nispsMode !== 'independent' ) ;
}
2026-04-03 19:16:54 +02:00
}
// Shared mode: resize MLP and rebuild heatmap on mode switch or module change
2026-04-03 19:01:10 +02:00
if ( reason === 'nispsMode-changed' || reason ? . startsWith ( 'module' ) ) {
const newTotal = totalOutputCount ( ) ;
if ( newTotal !== N _OUTPUTS ) {
await resizeMLP ( newTotal ) ;
}
if ( eocChain . nispsMode === 'shared' ) {
const combinedMeta = [
... ( activeEngine ? . paramMeta ? ? [ ] ) ,
... eocChain . paramMeta ,
] ;
rebuildHeatmap ( combinedMeta ) ;
} else {
rebuildHeatmap ( activeEngine ? . paramMeta ? ? [ ] ) ;
}
document . getElementById ( 'heatmap-cells' ) ? . parentElement
? . classList . toggle ( 'shared-mode' , eocChain . nispsMode === 'shared' ) ;
2026-04-03 19:16:54 +02:00
2026-04-03 19:21:59 +02:00
// Linked/Independent mode: recreate EOC IML when modules change
if ( ( eocChain . nispsMode === 'linked' || eocChain . nispsMode === 'independent' ) && imlEoc ) {
2026-04-03 19:16:54 +02:00
await initEocIML ( ) ;
}
2026-04-03 19:01:10 +02:00
}
2026-04-03 19:20:52 +02:00
// Show/hide EOC RL buttons and Synth label based on linked mode
const isLinked = eocChain . nispsMode === 'linked' ;
const eocRlContainer = document . getElementById ( 'eoc-rl-buttons' ) ;
if ( eocRlContainer ) eocRlContainer . classList . toggle ( 'hidden' , ! isLinked ) ;
const rlLabel = document . getElementById ( 'rl-label' ) ;
if ( rlLabel ) rlLabel . classList . toggle ( 'hidden' , ! isLinked ) ;
2026-04-03 18:49:07 +02:00
} ) ;
feat(tests): add Playwright e2e suite + debug probe for a-immersive
- Add window.__nisps debug probe (gated on ?debug=1) exposing iml state,
getOutputs/getLoss/getWeights/getExampleCount, and action triggers
(thumbsUp/thumbsDown/train/randomise/clearExamples/saveState)
- Fix WasmIML bug: this.dataset was a plain object; import Dataset and
use new Dataset(100) so computeWeights() is available for training
- Fix WasmIML.addExample/clearDataset to use Dataset API methods
- 44 Playwright e2e tests across 4 spec files:
- ml-engine.spec.js: WASM inference bounds, training loss, thumbs
up/down behavior, async training, example capture semantics
- ui-interactions.spec.js: drawer open/close, mode switching,
heatmap bar counts, preset chips, keyboard shortcuts (1/2/Z)
- input-pipeline.spec.js: input→output variation, clamping, joystick
drag, post-training output bounds across the full input space
- persistence.spec.js: URL params (?preset, ?spread), localStorage
round-trip, saveState probe
2026-04-03 17:38:04 +02:00
// Debug probe — exposed on window when ?debug=1 is in the URL.
// Used by Playwright e2e tests. Zero footprint in production.
if ( new URLSearchParams ( window . location . search ) . has ( 'debug' ) ) {
window . _ _nisps = {
get iml ( ) { return iml ; } ,
get imlJoy ( ) { return imlJoy ; } ,
get imlHand ( ) { return imlHand ; } ,
getOutputs : ( ) => [ ... iml . getOutputs ( ) ] ,
getLoss : ( ) => iml . lastLoss ,
getWeights : ( ) => iml . _getFlatWeights ( ) ,
getExampleCount : ( ) => iml . exampleCount ,
setInputs : ( x , y ) => {
iml . setInput ( 0 , x ) ;
iml . setInput ( 1 , y ) ;
iml . process ( ) ;
const outputs = iml . getOutputs ( ) ;
routeOutputs ( outputs ) ;
updateHeatmap ( outputs ) ;
} ,
thumbsUp : ( ) => onThumbsUp ( ) ,
thumbsDown : ( ) => onThumbsDown ( ) ,
train : ( ) => {
const loss = trainModel ( ) ;
const outputs = iml . getOutputs ( ) ;
routeOutputs ( outputs ) ;
updateHeatmap ( outputs ) ;
syncRawParamsFromOutputs ( outputs ) ;
updateStatus ( ) ;
drawLossPlot ( ) ;
return loss ;
} ,
trainAsync : ( ) => new Promise ( resolve => trainModelAsync ( resolve ) ) ,
randomise : ( ) => {
iml . randomiseWeights ( spreadLevel ) ;
const outputs = iml . getOutputs ( ) ;
routeOutputs ( outputs ) ;
updateHeatmap ( outputs ) ;
syncRawParamsFromOutputs ( outputs ) ;
updateStatus ( ) ;
} ,
clearExamples : ( ) => {
iml . clearDataset ( ) ;
updateStatus ( ) ;
} ,
saveState : ( ) => saveState ( ) ,
2026-04-03 18:17:59 +02:00
evalLoss : ( ) => iml . evalLoss ( ) ,
inferBatch : ( points ) => iml . inferBatch ( points ) ,
getLayerStats : ( ) => iml . getLayerStats ( ) ,
2026-04-03 18:42:59 +02:00
eocChain ,
feat(playground): add Modular audio mode with shared mod pool
New "Modular" engine in a-immersive with three hot-swappable Faust
sub-engines (subtractive/additive/fm) sharing a common modulation pool:
16 ADSR slots + 32 LFO slots (single-knob sine->tri->square->saw
wavemorph) routed through a 48-source x 10-destination matrix per
engine. Per-connection scalar amounts in [-1, 1], summed at each
destination. Default MLP output count is 512 (32 mod-source params +
480 matrix cells); model reinits on sub-engine swap, count change, or
engine-param exposure toggle.
Faust layer:
- mod-pool.lib: shared ADSR/LFO/source-bus library
- gen-modular-dsp.py: byte-reproducible generator (source of truth)
- modular-subtractive: faithful Minimoog (3 osc, ladder filter, no envs)
- modular-additive: 64-partial, spectral shape + formants, no envs/LFOs
- modular-fm: 4-op matrix + self-feedback, no envs
- All three share d08=amp, d09=pan conventions
- MODULAR_DESTINATIONS.md: authoritative destination table
JS layer:
- ModularEngine: self-contained SynthEngine with getState/setState,
setSubEngine, setModSourceCount, setExposeEngineParam
- modular-ui: drawer with sub-engine toggle, ADSR/LFO count steppers,
per-slot enable switches, matrix grid editor (tap-cycle, long-press
precise, right-click menu, negative amounts), preset overlay
- modular-presets: 6 named presets (Slow pad, Plucky bass, Crystal,
DX bell, Morphing drone + default)
- a-app.js: Modular mode registered, paramMeta:change -> resizeMLP,
modular DSP state persisted under modularDspState, window.__nisps
debug hooks for programmatic control
Tests: tests/e2e/modular-mode.spec.js (11 Playwright tests, all passing
including DSP state survives reload, sub-engine swap keeps paramCount,
preset apply verification).
Also fixes a pre-existing build.sh bug where the -e flag caused faust
to overwrite .wasm outputs with expanded DSP source text, leaving
additive/fm-matrix/eoc-* committed as invalid WebAssembly. Rebuilt all
affected engines with the corrected script. Added an early-message
buffer to faust-worklet-processor.js so setParam calls arriving before
wasm instantiation are queued rather than dropped (needed when the user
configures modular state before clicking Start Audio).
2026-04-11 07:34:41 +02:00
// ---- Phase E — modular engine hooks ----
get activeEngine ( ) { return activeEngine ; } ,
get activeEngineId ( ) { return activeEngine ? . id ? ? null ; } ,
get paramCount ( ) { return activeEngine ? . paramCount ? ? 0 ; } ,
/** Returns the modular engine's full DSP snapshot, or null if not active. */
getModularState : ( ) => {
if ( activeEngine ? . id !== 'modular' ||
typeof activeEngine . getState !== 'function' ) return null ;
return activeEngine . getState ( ) ;
} ,
/** Restores a modular DSP snapshot. Returns a Promise. */
setModularState : async ( s ) => {
if ( activeEngine ? . id !== 'modular' ||
typeof activeEngine . setState !== 'function' ) return false ;
await activeEngine . setState ( s ) ;
return true ;
} ,
/** Swap the active sub-engine (subtractive/additive/fm). */
setModularSubEngine : async ( id ) => {
if ( activeEngine ? . id !== 'modular' ||
typeof activeEngine . setSubEngine !== 'function' ) return false ;
await activeEngine . setSubEngine ( id ) ;
return true ;
} ,
/** Apply a named modular preset. Returns a Promise<boolean>. */
applyModularPreset : async ( presetId ) => {
if ( activeEngine ? . id !== 'modular' ) return false ;
const p = findModularPreset ( presetId ) ;
if ( ! p ) return false ;
await applyModularPreset ( activeEngine , p ) ;
return true ;
} ,
/** Change modular ADSR/LFO counts. */
setModularSourceCount : ( adsr , lfo ) => {
if ( activeEngine ? . id !== 'modular' ||
typeof activeEngine . setModSourceCount !== 'function' ) return false ;
activeEngine . setModSourceCount ( adsr , lfo ) ;
return true ;
} ,
/** List of known modular preset ids. */
listModularPresets : ( ) => MODULAR _PRESETS . map ( p => ( { id : p . id , name : p . name } ) ) ,
feat(tests): add Playwright e2e suite + debug probe for a-immersive
- Add window.__nisps debug probe (gated on ?debug=1) exposing iml state,
getOutputs/getLoss/getWeights/getExampleCount, and action triggers
(thumbsUp/thumbsDown/train/randomise/clearExamples/saveState)
- Fix WasmIML bug: this.dataset was a plain object; import Dataset and
use new Dataset(100) so computeWeights() is available for training
- Fix WasmIML.addExample/clearDataset to use Dataset API methods
- 44 Playwright e2e tests across 4 spec files:
- ml-engine.spec.js: WASM inference bounds, training loss, thumbs
up/down behavior, async training, example capture semantics
- ui-interactions.spec.js: drawer open/close, mode switching,
heatmap bar counts, preset chips, keyboard shortcuts (1/2/Z)
- input-pipeline.spec.js: input→output variation, clamping, joystick
drag, post-training output bounds across the full input space
- persistence.spec.js: URL params (?preset, ?spread), localStorage
round-trip, saveState probe
2026-04-03 17:38:04 +02:00
} ;
}
2026-04-03 19:03:12 +02:00
// EOC Linked-mode probe — exposed unconditionally so the training-target UI
// (meml-0r0) can wire buttons without requiring ?debug=1.
window . _ _nispsEoc = {
get trainingTarget ( ) { return eocTrainingTarget ; } ,
setTrainingTarget ( t ) { eocTrainingTarget = t ; } ,
get imlEoc ( ) { return imlEoc ; } ,
} ;
2026-03-21 23:57:58 +01:00
// Start animation
requestAnimationFrame ( animate ) ;
}
2026-04-03 18:55:06 +02:00
// ---- Heatmap helpers ----
/ * *
* Derive a stable HSL color string from a group name .
* Uses a simple djb2 - style hash to map any group string to a consistent hue .
* /
function _colorFromGroup ( group ) {
let hash = 5381 ;
for ( let i = 0 ; i < group . length ; i ++ ) {
hash = ( ( hash << 5 ) + hash ) + group . charCodeAt ( i ) ;
hash |= 0 ; // force 32-bit int
}
const hue = ( ( hash >>> 0 ) % 360 ) ;
return ` hsl( ${ hue } , 70%, 55%) ` ;
}
/ * *
* Rebuild heatmap cells ( and reset per - param state arrays ) from a paramMeta array .
* Call this when the active engine switches to update the heatmap for the new param layout .
* @ param { Array < { id : string , name : string , group : string } > } paramMeta
* /
function rebuildHeatmap ( paramMeta ) {
// Resize state arrays to match the new engine's param count
rawParamValues = new Array ( paramMeta . length ) . fill ( 0.5 ) ;
_lastSentParams = new Float32Array ( paramMeta . length ) ;
buildHeatmap ( ) ;
}
2026-03-21 23:57:58 +01:00
// ---- Heatmap (bar chart style) ----
function buildHeatmap ( ) {
$heatmapCells . innerHTML = '' ;
const isSynth = outputMode === 'synth' ;
2026-03-30 16:05:45 +02:00
const isMidiCC = outputMode === 'midi-cc' ;
2026-04-02 23:12:18 +02:00
const isAudioCanvas = outputMode === 'audio-canvas' ;
2026-04-03 18:55:06 +02:00
// For synth mode, derive names and colors from the active engine's paramMeta.
// For all other modes, use the static arrays as before.
let count , names , colors ;
if ( isSynth && activeEngine ? . paramMeta ) {
const meta = activeEngine . paramMeta ;
count = meta . length ;
// C15 has curated per-param colors from SYNTH_PARAM_COLORS; other engines derive from group.
const useCuratedColors = activeEngine . id === 'shaper-feedback' && SYNTH _PARAM _COLORS . length === meta . length ;
names = meta . map ( p => p . name ) ;
colors = useCuratedColors ? SYNTH _PARAM _COLORS : meta . map ( p => _colorFromGroup ( p . group ) ) ;
} else {
2026-04-03 21:36:24 +02:00
if ( isAudioCanvas && audioCanvas ) {
count = audioCanvas . getOutputCount ( ) ;
names = audioCanvas . getAudioParamNames ( ) ;
colors = audioCanvas . getAudioParamColors ( ) ;
_ensureAudioCanvasOverrides ( count ) ;
} else if ( isMidiCC ) {
count = midiCCMap . length ;
names = midiCCMap . map ( p => p . name ) ;
colors = MIDI _CC _PARAM _COLORS ;
} else {
count = N _VISUAL _OUTPUTS ;
names = VISUAL _PARAM _NAMES ;
colors = VISUAL _PARAM _COLORS ;
}
2026-04-03 18:55:06 +02:00
}
2026-03-21 23:57:58 +01:00
for ( let i = 0 ; i < count ; i ++ ) {
const cell = document . createElement ( 'div' ) ;
cell . className = 'heatmap-cell' ;
cell . dataset . index = i ;
const bar = document . createElement ( 'div' ) ;
bar . className = 'heatmap-cell-bar' ;
bar . style . background = colors [ i ] ;
bar . style . width = '30%' ; // default
bar . style . height = '100%' ;
cell . appendChild ( bar ) ;
2026-03-30 16:05:45 +02:00
// Tooltip on hover (quick info) + click to open popup
2026-03-21 23:57:58 +01:00
cell . addEventListener ( 'pointerenter' , ( e ) => {
2026-03-30 16:05:45 +02:00
if ( cell . _dragging ) return ;
2026-03-21 23:57:58 +01:00
const outputs = iml . getOutputs ( ) ;
2026-03-30 16:05:45 +02:00
$heatmapTooltip . textContent = ` ${ names [ i ] } : ${ outputs [ i ] . toFixed ( 3 ) } ▾ ` ;
2026-03-21 23:57:58 +01:00
$heatmapTooltip . classList . add ( 'visible' ) ;
const rect = cell . getBoundingClientRect ( ) ;
$heatmapTooltip . style . left = ` ${ rect . left } px ` ;
} ) ;
cell . addEventListener ( 'pointerleave' , ( ) => {
2026-03-30 16:05:45 +02:00
if ( ! cell . _dragging ) $heatmapTooltip . classList . remove ( 'visible' ) ;
// Delay hide popup so user can move to it
if ( activePopupParam === i ) {
popupHideTimer = setTimeout ( ( ) => hideParamPopup ( ) , 300 ) ;
}
} ) ;
// Click opens popup; drag sets parameter value
cell . addEventListener ( 'pointerdown' , ( e ) => {
e . preventDefault ( ) ;
cell . _downX = e . clientX ;
cell . _downY = e . clientY ;
cell . _didDrag = false ;
cell . _dragging = true ;
cell . classList . add ( 'dragging' ) ;
cell . setPointerCapture ( e . pointerId ) ;
} ) ;
cell . addEventListener ( 'pointermove' , ( e ) => {
if ( ! cell . _dragging ) return ;
const dx = Math . abs ( e . clientX - cell . _downX ) ;
const dy = Math . abs ( e . clientY - cell . _downY ) ;
if ( dx > 3 || dy > 3 ) cell . _didDrag = true ;
if ( cell . _didDrag ) setHeatmapValue ( i , e , cell ) ;
} ) ;
cell . addEventListener ( 'pointerup' , ( ) => {
cell . _dragging = false ;
cell . classList . remove ( 'dragging' ) ;
if ( ! cell . _didDrag ) {
// Click — toggle popup
clearTimeout ( popupHideTimer ) ;
if ( activePopupParam === i ) {
hideParamPopup ( ) ;
} else {
showParamPopup ( i , cell ) ;
}
}
} ) ;
cell . addEventListener ( 'pointercancel' , ( ) => {
cell . _dragging = false ;
cell . classList . remove ( 'dragging' ) ;
2026-03-21 23:57:58 +01:00
} ) ;
$heatmapCells . appendChild ( cell ) ;
}
}
2026-03-30 16:05:45 +02:00
function setHeatmapValue ( paramIndex , e , cell ) {
const rect = cell . getBoundingClientRect ( ) ;
const x = Math . max ( 0 , Math . min ( 1 , ( e . clientX - rect . left ) / rect . width ) ) ;
// If frozen, dragging updates the fixed value directly
const ov = getParamOverride ( paramIndex ) ;
if ( ov && ov . frozen ) {
ov . fixedValue = x ;
}
rawParamValues [ paramIndex ] = x ;
routeOutputs ( rawParamValues ) ;
updateHeatmap ( rawParamValues ) ;
syncRawParamsFromOutputs ( rawParamValues ) ;
// Update tooltip
const isSynth = outputMode === 'synth' ;
const isMidiCC = outputMode === 'midi-cc' ;
2026-04-02 23:12:18 +02:00
const isAudioCanvas = outputMode === 'audio-canvas' ;
2026-04-03 18:55:06 +02:00
let paramName ;
if ( isSynth && activeEngine ? . paramMeta ? . [ paramIndex ] ) {
paramName = activeEngine . paramMeta [ paramIndex ] . name ;
} else if ( isMidiCC ) {
paramName = midiCCMap [ paramIndex ] ? . name ? ? ` p ${ paramIndex } ` ;
2026-04-03 21:36:24 +02:00
} else if ( isAudioCanvas && audioCanvas ) {
const acNames = audioCanvas . getAudioParamNames ( ) ;
paramName = acNames [ paramIndex ] ? ? ` p ${ paramIndex } ` ;
2026-04-03 18:55:06 +02:00
} else {
paramName = VISUAL _PARAM _NAMES [ paramIndex ] ? ? ` p ${ paramIndex } ` ;
}
$heatmapTooltip . textContent = ` ${ paramName } : ${ x . toFixed ( 3 ) } ` ;
2026-03-30 16:05:45 +02:00
$heatmapTooltip . classList . add ( 'visible' ) ;
$heatmapTooltip . style . left = ` ${ rect . left } px ` ;
}
2026-03-21 23:57:58 +01:00
function updateHeatmap ( outputs ) {
const cells = $heatmapCells . children ;
for ( let i = 0 ; i < cells . length && i < outputs . length ; i ++ ) {
const bar = cells [ i ] . querySelector ( '.heatmap-cell-bar' ) ;
if ( bar ) {
const pct = Math . max ( 2 , Math . round ( outputs [ i ] * 100 ) ) ;
bar . style . width = pct + '%' ;
}
2026-03-30 16:05:45 +02:00
// Dim frozen/muted cells
const ov = getParamOverride ( i ) ;
if ( ov ) {
cells [ i ] . classList . toggle ( 'heatmap-cell-frozen' , ! ! ov . frozen ) ;
}
2026-03-21 23:57:58 +01:00
}
}
2026-03-30 16:05:45 +02:00
/** Get the override object for a heatmap param (works in both visual & synth mode) */
function getParamOverride ( paramIndex ) {
if ( outputMode === 'synth' ) {
2026-04-03 20:01:14 +02:00
// Non-C15 engines use the flat engineParamOverrides array
if ( engineParamOverrides && paramIndex < engineParamOverrides . length ) {
const p = engineParamOverrides [ paramIndex ] ;
return {
get min ( ) { return p . min ; } , set min ( v ) { p . min = v ; } ,
get max ( ) { return p . max ; } , set max ( v ) { p . max = v ; } ,
get curve ( ) { return p . curve ; } , set curve ( v ) { p . curve = v ; } ,
get frozen ( ) { return p . muted ; } , set frozen ( v ) { p . muted = v ; } ,
get fixedValue ( ) { return p . fixedValue ; } , set fixedValue ( v ) { p . fixedValue = v ; } ,
} ;
}
// C15 path: use groupOverrides + paramToSection
2026-03-30 16:05:45 +02:00
const mapping = paramToSection [ paramIndex ] ;
if ( ! mapping ) return null ;
2026-04-03 20:01:14 +02:00
const group = groupOverrides [ mapping . si ] ;
if ( ! group ) return null ;
const p = group . params [ mapping . li ] ;
if ( ! p ) return null ;
2026-03-30 16:05:45 +02:00
// Expose as { min, max, curve, frozen, fixedValue } — map 'muted' to 'frozen'
return {
get min ( ) { return p . min ; } , set min ( v ) { p . min = v ; } ,
get max ( ) { return p . max ; } , set max ( v ) { p . max = v ; } ,
get curve ( ) { return p . curve ; } , set curve ( v ) { p . curve = v ; } ,
get frozen ( ) { return p . muted ; } , set frozen ( v ) { p . muted = v ; } ,
get fixedValue ( ) { return p . fixedValue ; } , set fixedValue ( v ) { p . fixedValue = v ; } ,
} ;
} else if ( outputMode === 'midi-cc' ) {
return midiCCOverrides [ paramIndex ] || null ;
2026-04-02 23:12:18 +02:00
} else if ( outputMode === 'audio-canvas' ) {
return audioCanvasOverrides [ paramIndex ] || null ;
2026-03-30 16:05:45 +02:00
} else {
return visualOverrides [ paramIndex ] || null ;
}
}
// ---- Param Popup (hover menu on heatmap cells) ----
function wireParamPopup ( ) {
$paramPopup = document . createElement ( 'div' ) ;
$paramPopup . className = 'param-popup' ;
$paramPopup . innerHTML = '<div class="pp-header"></div><div class="pp-body"></div>' ;
document . body . appendChild ( $paramPopup ) ;
$paramPopup . addEventListener ( 'pointerenter' , ( ) => {
clearTimeout ( popupHideTimer ) ;
} ) ;
$paramPopup . addEventListener ( 'pointerleave' , ( ) => {
popupHideTimer = setTimeout ( ( ) => hideParamPopup ( ) , 300 ) ;
} ) ;
}
function showParamPopup ( paramIndex , cell ) {
if ( activePopupParam === paramIndex ) return ;
activePopupParam = paramIndex ;
const isSynth = outputMode === 'synth' ;
const isMidiCC = outputMode === 'midi-cc' ;
2026-04-02 23:12:18 +02:00
const isAudioCanvas = outputMode === 'audio-canvas' ;
2026-04-03 18:55:06 +02:00
let name , color ;
if ( isSynth && activeEngine ? . paramMeta ? . [ paramIndex ] ) {
const pm = activeEngine . paramMeta [ paramIndex ] ;
name = pm . name ;
// Use curated C15 colors when available, else derive from group.
const useCuratedColors = activeEngine . id === 'shaper-feedback' && SYNTH _PARAM _COLORS . length > paramIndex ;
color = useCuratedColors ? SYNTH _PARAM _COLORS [ paramIndex ] : _colorFromGroup ( pm . group ) ;
} else if ( isMidiCC ) {
name = midiCCMap [ paramIndex ] ? . name ? ? ` p ${ paramIndex } ` ;
color = MIDI _CC _PARAM _COLORS [ paramIndex ] ? ? '#888' ;
2026-04-03 21:36:24 +02:00
} else if ( isAudioCanvas && audioCanvas ) {
const acNames = audioCanvas . getAudioParamNames ( ) ;
const acColors = audioCanvas . getAudioParamColors ( ) ;
name = acNames [ paramIndex ] ? ? ` p ${ paramIndex } ` ;
color = acColors [ paramIndex ] ? ? '#888' ;
2026-04-03 18:55:06 +02:00
} else {
name = VISUAL _PARAM _NAMES [ paramIndex ] ? ? ` p ${ paramIndex } ` ;
color = VISUAL _PARAM _COLORS [ paramIndex ] ? ? '#888' ;
}
2026-03-30 16:05:45 +02:00
const ov = getParamOverride ( paramIndex ) ;
if ( ! ov ) return ;
const header = $paramPopup . querySelector ( '.pp-header' ) ;
header . textContent = name ;
header . style . color = color ;
const body = $paramPopup . querySelector ( '.pp-body' ) ;
body . innerHTML = '' ;
// -- MIDI CC specific: name, CC#, channel editors --
if ( isMidiCC ) {
const ccParam = midiCCMap [ paramIndex ] ;
// Editable name
const nameRow = document . createElement ( 'div' ) ;
nameRow . className = 'pp-row' ;
const nameLabel = document . createElement ( 'span' ) ;
nameLabel . className = 'pp-label' ;
nameLabel . textContent = 'Name' ;
const nameInput = document . createElement ( 'input' ) ;
nameInput . type = 'text' ;
nameInput . value = ccParam . name ;
nameInput . className = 'pp-text-input' ;
nameInput . addEventListener ( 'change' , ( ) => {
ccParam . name = nameInput . value ;
header . textContent = nameInput . value ;
buildHeatmap ( ) ;
updateHeatmap ( iml . getOutputs ( ) ) ;
2026-04-03 18:55:06 +02:00
saveCCMap ( midiCCMap , midiCCStorageKey ( ) ) ;
feat(playground): implement Phase 1 control surface — compound axes, input pipeline, enhanced joy-map
Three new standalone ES modules + integration into a-app.js:
- input-pipeline.js: 5-stage processing (deadzone → zoom → curve → smoothing →
momentum-as-zoom), 3 anchor modes, zoom-at-zero freeze, per-axis overrides
- control-surface.js: compound axes (Boldness/Memory/Precision) with interpolation
tables, offset-based override resolution (trim-pot model), 6 built-in presets
- control-surface-ui.js: floating bar axis sliders, gear icon settings drawer with
Input/Training/Exploration/Output sections, log-scale sliders, override dots
- joy-map-enhanced.js: zoom minimap with adaptive grid (4×4→32×32), vanishing trail
with Catmull-Rom spline + tap-to-return, dual concentric noise rings, frozen overlay
Integration fixes from fresh-eyes review:
- getCurrentInputs()/setCurrentInputs() use cached pipeline coords (not raw)
- CSS noise ring hidden when canvas version active (no doubling)
- Input mode switch re-runs through pipeline
- Control surface state persisted to localStorage
Implements full Phase 1 of SPEC-controls.md plus bonus items from later phases
(zoom-aware feedback, control presets with override resolution, input curve/deadzone/
smoothing/momentum all wired).
2026-03-26 09:24:24 +01:00
} ) ;
2026-03-30 16:05:45 +02:00
nameRow . appendChild ( nameLabel ) ;
nameRow . appendChild ( nameInput ) ;
body . appendChild ( nameRow ) ;
// CC number
const ccRow = document . createElement ( 'div' ) ;
ccRow . className = 'pp-row' ;
const ccLabel = document . createElement ( 'span' ) ;
ccLabel . className = 'pp-label' ;
ccLabel . textContent = 'CC#' ;
const ccInput = document . createElement ( 'input' ) ;
ccInput . type = 'number' ;
ccInput . min = '0' ;
ccInput . max = '127' ;
ccInput . value = ccParam . cc ;
ccInput . className = 'pp-num-input' ;
ccInput . addEventListener ( 'change' , ( ) => {
ccParam . cc = parseInt ( ccInput . value ) || 0 ;
// Auto-name if name matches old CC name
if ( CC _NAMES [ ccParam . cc ] ) {
ccParam . name = CC _NAMES [ ccParam . cc ] ;
nameInput . value = ccParam . name ;
header . textContent = ccParam . name ;
buildHeatmap ( ) ;
updateHeatmap ( iml . getOutputs ( ) ) ;
}
2026-04-03 18:55:06 +02:00
saveCCMap ( midiCCMap , midiCCStorageKey ( ) ) ;
2026-03-30 16:05:45 +02:00
} ) ;
ccRow . appendChild ( ccLabel ) ;
ccRow . appendChild ( ccInput ) ;
body . appendChild ( ccRow ) ;
// Channel
const chRow = document . createElement ( 'div' ) ;
chRow . className = 'pp-row' ;
const chLabel = document . createElement ( 'span' ) ;
chLabel . className = 'pp-label' ;
chLabel . textContent = 'Ch' ;
const chInput = document . createElement ( 'input' ) ;
chInput . type = 'number' ;
chInput . min = '1' ;
chInput . max = '16' ;
chInput . value = ccParam . channel ;
chInput . className = 'pp-num-input' ;
chInput . addEventListener ( 'change' , ( ) => {
ccParam . channel = Math . max ( 1 , Math . min ( 16 , parseInt ( chInput . value ) || 1 ) ) ;
chInput . value = ccParam . channel ;
2026-04-03 18:55:06 +02:00
saveCCMap ( midiCCMap , midiCCStorageKey ( ) ) ;
2026-03-30 16:05:45 +02:00
} ) ;
chRow . appendChild ( chLabel ) ;
chRow . appendChild ( chInput ) ;
body . appendChild ( chRow ) ;
}
// -- Curve row --
const curveRow = document . createElement ( 'div' ) ;
curveRow . className = 'pp-row' ;
const curveLabel = document . createElement ( 'span' ) ;
curveLabel . className = 'pp-label' ;
curveLabel . textContent = 'Curve' ;
const curveCanvas = document . createElement ( 'canvas' ) ;
curveCanvas . className = 'pp-curve-canvas' ;
curveCanvas . width = 36 ;
curveCanvas . height = 36 ;
const curveVal = document . createElement ( 'span' ) ;
curveVal . className = 'pp-val' ;
curveVal . textContent = ov . curve . toFixed ( 2 ) ;
function redrawCurve ( ) {
_drawCurveOnCanvas ( curveCanvas , ov . curve , color ) ;
feat(playground): implement Phase 1 control surface — compound axes, input pipeline, enhanced joy-map
Three new standalone ES modules + integration into a-app.js:
- input-pipeline.js: 5-stage processing (deadzone → zoom → curve → smoothing →
momentum-as-zoom), 3 anchor modes, zoom-at-zero freeze, per-axis overrides
- control-surface.js: compound axes (Boldness/Memory/Precision) with interpolation
tables, offset-based override resolution (trim-pot model), 6 built-in presets
- control-surface-ui.js: floating bar axis sliders, gear icon settings drawer with
Input/Training/Exploration/Output sections, log-scale sliders, override dots
- joy-map-enhanced.js: zoom minimap with adaptive grid (4×4→32×32), vanishing trail
with Catmull-Rom spline + tap-to-return, dual concentric noise rings, frozen overlay
Integration fixes from fresh-eyes review:
- getCurrentInputs()/setCurrentInputs() use cached pipeline coords (not raw)
- CSS noise ring hidden when canvas version active (no doubling)
- Input mode switch re-runs through pipeline
- Control surface state persisted to localStorage
Implements full Phase 1 of SPEC-controls.md plus bonus items from later phases
(zoom-aware feedback, control presets with override resolution, input curve/deadzone/
smoothing/momentum all wired).
2026-03-26 09:24:24 +01:00
}
2026-03-30 16:05:45 +02:00
_wireCurveDrag ( curveCanvas , ( ) => ov . curve , ( v ) => {
ov . curve = v ;
curveVal . textContent = v . toFixed ( 2 ) ;
redrawCurve ( ) ;
routeOutputs ( iml . getOutputs ( ) ) ;
} ) ;
redrawCurve ( ) ;
curveRow . appendChild ( curveLabel ) ;
curveRow . appendChild ( curveCanvas ) ;
curveRow . appendChild ( curveVal ) ;
body . appendChild ( curveRow ) ;
// -- Min/Max row --
const rangeRow = document . createElement ( 'div' ) ;
rangeRow . className = 'pp-row' ;
const rangeLabel = document . createElement ( 'span' ) ;
rangeLabel . className = 'pp-label' ;
rangeLabel . textContent = 'Range' ;
const rangeWrap = document . createElement ( 'div' ) ;
rangeWrap . className = 'pp-range-wrap' ;
const rangeFill = document . createElement ( 'div' ) ;
rangeFill . className = 'gd-range-fill' ;
const minSlider = document . createElement ( 'input' ) ;
minSlider . type = 'range' ; minSlider . min = '0' ; minSlider . max = '1' ; minSlider . step = '0.01' ;
minSlider . value = ov . min ;
minSlider . className = 'gd-range-input gd-range-min' ;
const maxSlider = document . createElement ( 'input' ) ;
maxSlider . type = 'range' ; maxSlider . min = '0' ; maxSlider . max = '1' ; maxSlider . step = '0.01' ;
maxSlider . value = ov . max ;
maxSlider . className = 'gd-range-input gd-range-max' ;
const rangeValSpan = document . createElement ( 'span' ) ;
rangeValSpan . className = 'pp-val' ;
function updateRangeFill ( ) {
rangeFill . style . left = ` ${ ov . min * 100 } % ` ;
rangeFill . style . width = ` ${ ( ov . max - ov . min ) * 100 } % ` ;
rangeValSpan . textContent = ` ${ ov . min . toFixed ( 2 ) } – ${ ov . max . toFixed ( 2 ) } ` ;
}
updateRangeFill ( ) ;
minSlider . addEventListener ( 'input' , ( ) => {
ov . min = parseFloat ( minSlider . value ) ;
if ( ov . min > ov . max ) { ov . max = ov . min ; maxSlider . value = ov . max ; }
updateRangeFill ( ) ;
routeOutputs ( iml . getOutputs ( ) ) ;
} ) ;
maxSlider . addEventListener ( 'input' , ( ) => {
ov . max = parseFloat ( maxSlider . value ) ;
if ( ov . max < ov . min ) { ov . min = ov . max ; minSlider . value = ov . min ; }
updateRangeFill ( ) ;
routeOutputs ( iml . getOutputs ( ) ) ;
} ) ;
rangeWrap . appendChild ( rangeFill ) ;
rangeWrap . appendChild ( minSlider ) ;
rangeWrap . appendChild ( maxSlider ) ;
rangeRow . appendChild ( rangeLabel ) ;
rangeRow . appendChild ( rangeWrap ) ;
rangeRow . appendChild ( rangeValSpan ) ;
body . appendChild ( rangeRow ) ;
// -- Freeze row --
const freezeRow = document . createElement ( 'div' ) ;
freezeRow . className = 'pp-row pp-freeze-row' ;
const freezeBtn = document . createElement ( 'button' ) ;
freezeBtn . className = 'pp-freeze-btn' + ( ov . frozen ? ' frozen' : '' ) ;
freezeBtn . textContent = ov . frozen ? 'Frozen' : 'Freeze' ;
freezeBtn . title = ov . frozen ? 'Unfreeze — re-enable NISPS control' : 'Freeze — lock value, disable NISPS' ;
const valSlider = document . createElement ( 'input' ) ;
valSlider . type = 'range' ; valSlider . min = '0' ; valSlider . max = '1' ; valSlider . step = '0.01' ;
valSlider . value = ov . fixedValue ;
valSlider . className = 'pp-val-slider' ;
if ( ! ov . frozen ) valSlider . style . display = 'none' ;
const valDisplay = document . createElement ( 'span' ) ;
valDisplay . className = 'pp-val' ;
valDisplay . textContent = ov . frozen ? ov . fixedValue . toFixed ( 2 ) : '' ;
freezeBtn . addEventListener ( 'click' , ( ) => {
ov . frozen = ! ov . frozen ;
freezeBtn . classList . toggle ( 'frozen' , ov . frozen ) ;
freezeBtn . textContent = ov . frozen ? 'Frozen' : 'Freeze' ;
freezeBtn . title = ov . frozen ? 'Unfreeze — re-enable NISPS control' : 'Freeze — lock value, disable NISPS' ;
valSlider . style . display = ov . frozen ? '' : 'none' ;
valDisplay . textContent = ov . frozen ? ov . fixedValue . toFixed ( 2 ) : '' ;
// When freezing, capture the current output value
if ( ov . frozen ) {
const outputs = iml . getOutputs ( ) ;
ov . fixedValue = outputs [ paramIndex ] ;
valSlider . value = ov . fixedValue ;
valDisplay . textContent = ov . fixedValue . toFixed ( 2 ) ;
}
routeOutputs ( iml . getOutputs ( ) ) ;
} ) ;
valSlider . addEventListener ( 'input' , ( ) => {
ov . fixedValue = parseFloat ( valSlider . value ) ;
valDisplay . textContent = ov . fixedValue . toFixed ( 2 ) ;
routeOutputs ( iml . getOutputs ( ) ) ;
} ) ;
freezeRow . appendChild ( freezeBtn ) ;
freezeRow . appendChild ( valSlider ) ;
freezeRow . appendChild ( valDisplay ) ;
body . appendChild ( freezeRow ) ;
// Position below the heatmap cell
const cellRect = cell . getBoundingClientRect ( ) ;
const popupWidth = 260 ;
let left = cellRect . left + cellRect . width / 2 - popupWidth / 2 ;
left = Math . max ( 4 , Math . min ( left , window . innerWidth - popupWidth - 4 ) ) ;
$paramPopup . style . left = ` ${ left } px ` ;
$paramPopup . style . top = ` ${ cellRect . bottom + 4 } px ` ;
$paramPopup . classList . add ( 'visible' ) ;
}
function hideParamPopup ( ) {
activePopupParam = - 1 ;
if ( $paramPopup ) $paramPopup . classList . remove ( 'visible' ) ;
}
// ---- Joy Map (merged joystick + minimap) ----
function drawJoyMap ( ) {
2026-03-21 23:57:58 +01:00
const canvas = $joyMap ;
const ctx = $joyMapCtx ;
const w = canvas . width ;
const h = canvas . height ;
const cx = w / 2 ;
const cy = h / 2 ;
const r = w / 2 - 2 ;
// Clear with circle clip
ctx . save ( ) ;
ctx . beginPath ( ) ;
ctx . arc ( cx , cy , r , 0 , Math . PI * 2 ) ;
ctx . clip ( ) ;
// Background
ctx . fillStyle = 'rgba(13, 13, 13, 0.7)' ;
ctx . fillRect ( 0 , 0 , w , h ) ;
// Grid lines
ctx . strokeStyle = 'rgba(255, 255, 255, 0.06)' ;
ctx . lineWidth = 0.5 ;
for ( let frac = 0.25 ; frac < 1 ; frac += 0.25 ) {
ctx . beginPath ( ) ;
ctx . moveTo ( frac * w , 0 ) ; ctx . lineTo ( frac * w , h ) ;
ctx . moveTo ( 0 , frac * h ) ; ctx . lineTo ( w , frac * h ) ;
ctx . stroke ( ) ;
}
// Ring border
ctx . strokeStyle = 'rgba(255, 255, 255, 0.12)' ;
ctx . lineWidth = 2 ;
ctx . beginPath ( ) ;
ctx . arc ( cx , cy , r , 0 , Math . PI * 2 ) ;
ctx . stroke ( ) ;
// Training example dots
const features = iml . dataset . features ;
const labels = iml . dataset . labels ;
for ( let i = 0 ; i < features . length ; i ++ ) {
const fx = features [ i ] [ 0 ] * w ;
const fy = ( 1 - features [ i ] [ 1 ] ) * h ;
let hue = 0 ;
if ( labels [ i ] ) {
hue = ( labels [ i ] [ 3 ] || 0 ) * 360 ;
}
ctx . fillStyle = ` hsla( ${ hue } , 80%, 60%, 0.85) ` ;
ctx . beginPath ( ) ;
ctx . arc ( fx , fy , 3 , 0 , Math . PI * 2 ) ;
ctx . fill ( ) ;
}
// Current position knob (accent dot with glow)
const px = joyX * w ;
const py = ( 1 - joyY ) * h ;
// Glow
2026-03-22 00:36:00 +01:00
ctx . shadowColor = 'rgba(255, 106, 0, 0.6)' ;
2026-03-21 23:57:58 +01:00
ctx . shadowBlur = 12 ;
2026-03-22 00:36:00 +01:00
ctx . fillStyle = 'rgba(255, 106, 0, 0.9)' ;
2026-03-21 23:57:58 +01:00
ctx . beginPath ( ) ;
ctx . arc ( px , py , 8 , 0 , Math . PI * 2 ) ;
ctx . fill ( ) ;
// Inner bright dot
ctx . shadowBlur = 0 ;
ctx . fillStyle = 'rgba(255, 255, 255, 0.8)' ;
ctx . beginPath ( ) ;
ctx . arc ( px , py , 3 , 0 , Math . PI * 2 ) ;
ctx . fill ( ) ;
// Crosshair
2026-03-22 00:36:00 +01:00
ctx . strokeStyle = 'rgba(255, 106, 0, 0.2)' ;
2026-03-21 23:57:58 +01:00
ctx . lineWidth = 0.5 ;
ctx . shadowBlur = 0 ;
ctx . beginPath ( ) ;
ctx . moveTo ( px , 0 ) ; ctx . lineTo ( px , h ) ;
ctx . moveTo ( 0 , py ) ; ctx . lineTo ( w , py ) ;
ctx . stroke ( ) ;
ctx . restore ( ) ;
}
// ---- Joystick ----
function wireJoystick ( ) {
const canvas = $joyMap ;
let startX , startY , startJX , startJY ;
let lastDoubleTap = 0 ;
function onStart ( e ) {
e . preventDefault ( ) ;
const touch = e . touches ? e . touches [ 0 ] : e ;
// Double-tap detection for follow mode
const now = Date . now ( ) ;
if ( now - lastDoubleTap < 350 ) {
toggleFollowMode ( ) ;
lastDoubleTap = 0 ;
return ;
}
lastDoubleTap = now ;
// Snap to tap position within the circle
const rect = canvas . getBoundingClientRect ( ) ;
const size = rect . width ;
const dx = touch . clientX - rect . left - size / 2 ;
const dy = touch . clientY - rect . top - size / 2 ;
const maxR = size / 2 - 8 ;
joyX = Math . max ( 0 , Math . min ( 1 , 0.5 + ( dx / maxR ) * 0.5 ) ) ;
joyY = Math . max ( 0 , Math . min ( 1 , 0.5 - ( dy / maxR ) * 0.5 ) ) ;
drawJoyMap ( ) ;
onJoystickMove ( ) ;
joyDragging = true ;
startX = touch . clientX ;
startY = touch . clientY ;
startJX = joyX ;
startJY = joyY ;
}
function onMove ( e ) {
if ( ! joyDragging && ! joyFollowMode ) return ;
e . preventDefault ( ) ;
const touch = e . touches ? e . touches [ 0 ] : e ;
if ( joyFollowMode ) {
joyX = Math . max ( 0 , Math . min ( 1 , touch . clientX / window . innerWidth ) ) ;
joyY = Math . max ( 0 , Math . min ( 1 , 1 - touch . clientY / window . innerHeight ) ) ;
} else if ( joyDragging ) {
const dx = touch . clientX - startX ;
const dy = touch . clientY - startY ;
const size = canvas . getBoundingClientRect ( ) . width ;
const maxR = size / 2 - 8 ;
const scale = 1 / maxR ;
joyX = Math . max ( 0 , Math . min ( 1 , startJX + dx * scale * 0.5 ) ) ;
joyY = Math . max ( 0 , Math . min ( 1 , startJY - dy * scale * 0.5 ) ) ;
}
drawJoyMap ( ) ;
onJoystickMove ( ) ;
}
function onEnd ( ) {
joyDragging = false ;
}
canvas . addEventListener ( 'pointerdown' , onStart ) ;
window . addEventListener ( 'pointermove' , onMove ) ;
window . addEventListener ( 'pointerup' , onEnd ) ;
window . addEventListener ( 'pointercancel' , onEnd ) ;
// Follow mode: track pointer on main canvas when active
$canvas . addEventListener ( 'pointermove' , ( e ) => {
if ( ! joyFollowMode ) return ;
joyX = Math . max ( 0 , Math . min ( 1 , e . clientX / window . innerWidth ) ) ;
joyY = Math . max ( 0 , Math . min ( 1 , 1 - e . clientY / window . innerHeight ) ) ;
drawJoyMap ( ) ;
onJoystickMove ( ) ;
} ) ;
}
function toggleFollowMode ( ) {
joyFollowMode = ! joyFollowMode ;
console . log ( '[Joystick] Follow mode:' , joyFollowMode ) ;
updateFollowUI ( ) ;
}
function updateFollowUI ( ) {
if ( joyFollowMode ) {
$followBadge . classList . remove ( 'hidden' ) ;
$joystickContainer . classList . add ( 'follow-active' ) ;
$followPill . classList . add ( 'active' ) ;
} else {
$followBadge . classList . add ( 'hidden' ) ;
$joystickContainer . classList . remove ( 'follow-active' ) ;
$followPill . classList . remove ( 'active' ) ;
}
}
function onJoystickMove ( ) {
feat(playground): add MediaPipe hand tracking input with dev panel
Hand tracking via webcam as alternative to joystick input:
- Right hand tracks 14 derived features (palm XY, finger curls, spread,
roll, pitch, pinch) through a separate 14-input MLP
- Left hand gesture recognition (1 finger = thumbs up, 2 = thumbs down)
with 400ms debounce hold
- Split-zone PIP display (no camera feed, skeleton only) with dashed
divider and cross-zone dimming
- Dual IML architecture: independent imlJoy (2 inputs) and imlHand
(14 inputs) with pointer swap, preserving training data per mode
- Dev panel (?devmode=true): draggable/collapsible floating panel with
sliders for MediaPipe confidence thresholds, smoothing, gesture hold
time, world landmarks toggle, and live feature bar monitor
- Visual presets always route to joystick IML (prevents dimension mismatch)
- Race condition guard on input mode switching
2026-03-23 22:28:58 +01:00
if ( inputMode !== 'joystick' ) return ;
2026-03-30 16:05:45 +02:00
iml . setInput ( 0 , joyX ) ;
iml . setInput ( 1 , joyY ) ;
2026-03-21 23:57:58 +01:00
iml . process ( ) ;
const outputs = iml . getOutputs ( ) ;
routeOutputs ( outputs ) ;
updateHeatmap ( outputs ) ;
2026-03-22 10:01:50 +01:00
syncRawParamsFromOutputs ( outputs ) ;
2026-03-21 23:57:58 +01:00
2026-04-03 19:03:12 +02:00
// EOC Linked mode: run EOC IML with same joystick inputs
if ( imlEoc && eocChain ? . nispsMode === 'linked' ) {
imlEoc . setInput ( 0 , joyX ) ;
imlEoc . setInput ( 1 , joyY ) ;
imlEoc . process ( ) ;
const eocOutputs = imlEoc . getOutputs ( ) ;
for ( let i = 0 ; i < eocOutputs . length ; i ++ ) {
eocChain . setParam ( i , eocOutputs [ i ] ) ;
}
}
2026-04-03 20:01:14 +02:00
// ShapeSeq: route joystick inputs to the sequence engine
if ( shapeSeqEnabled && shapeSeqEngine && shapeSeqEngine . isPlaying ) {
if ( shapeSeqEngine . mlpMode . mode === 'unified' ) {
// Unified mode: timbre MLP already ran — pass its outputs to the sequencer
shapeSeqEngine . setSequenceOutputsFromTimbre ( outputs ) ;
} else {
// Dual mode: sequence engine runs its own MLP
shapeSeqEngine . setSequenceInputs ( [ joyX , joyY ] ) ;
}
}
2026-03-30 16:05:45 +02:00
// Trail
2026-03-21 23:57:58 +01:00
joyTrail . push ( { x : joyX , y : joyY , t : Date . now ( ) } ) ;
if ( joyTrail . length > 30 ) joyTrail . shift ( ) ;
}
// ---- Output routing ----
2026-03-24 00:42:37 +01:00
// Synth param throttling: only send values that changed beyond a dead zone.
// Prevents ring buffer flooding (126 params × 30fps = 3780 msg/s > 512 capacity).
2026-04-03 18:30:26 +02:00
let _lastSentParams = new Float32Array ( N _OUTPUTS ) ;
2026-03-24 00:42:37 +01:00
const PARAM _DEAD _ZONE = 0.002 ; // ~0.2% change threshold
let _lastParamSendTime = 0 ;
const PARAM _SEND _INTERVAL = 50 ; // max ~20fps for synth param updates
2026-03-21 23:57:58 +01:00
function routeOutputs ( outputs ) {
if ( outputMode === 'synth' ) {
fix(modular): restore matrix in paramMeta + bypass MLP when untrained
Previous fix pulled matrix cells out of paramMeta to avoid the
default-patch-clobbering silence issue. Side effect: paramCount dropped
from 512 to 32, the heatmap strip and synth visualizer shrank
dramatically, and moving the joystick no longer animated matrix cells
(the user's two most recent complaints).
Better approach: put matrix cells back in paramMeta (512 outputs), but
when iml.exampleCount === 0 substitute the normalised default-patch
vector for the raw MLP output in routeOutputs. The engine sees the
default patch exactly, audio works, and the user still sees 512 cells
in the heatmap / matrix grid. As soon as they capture their first
training example the MLP resumes driving everything normally.
ModularEngine.getDefaultNormalizedOutputs() returns the normalised
default value per paramMeta entry, reading from _lastRawByLabel first
(so user edits via click or _setRawByLabel propagate) and falling back
to the walk-entry init field.
modular-ui setCell now prefers engine.setParam over _setRawByLabel when
the cell is in paramMeta, so writes flow through the normal tracking
path and are visible to getDefaultNormalizedOutputs on the next tick.
Verified in headless Chromium: switching to modular, cold start, reading
engine._lastRawByLabel after routeOutputs ticks shows ampRaw=1,
attackRaw=0.01, enableRaw=1 — the default patch is preserved. paramCount
is 512 again. 0 console errors.
2026-04-11 08:50:59 +02:00
// Modular cold-start: before the user has captured any training
// examples, the MLP output is arbitrary — an untrained sigmoid
// network with spread=0.6 on 512 outputs tends toward ~0, which
// denormalises matrix cells to raw=0 and silences the default
// patch's ADSR1→amp routing. Ignore the MLP entirely and drive the
// engine with the default-normalised vector. Once exampleCount > 0
// the MLP has a target to hit and resumes driving normally.
let shiftedOutputs = outputs ;
if ( activeEngine ? . id === 'modular' &&
( iml ? . exampleCount ? ? 0 ) === 0 &&
typeof activeEngine . getDefaultNormalizedOutputs === 'function' ) {
const defaults = activeEngine . getDefaultNormalizedOutputs ( ) ;
if ( defaults && defaults . length === outputs . length ) {
shiftedOutputs = defaults ;
}
}
const overridden = new Array ( shiftedOutputs . length ) ;
for ( let i = 0 ; i < shiftedOutputs . length ; i ++ ) {
overridden [ i ] = applyGroupOverrides ( shiftedOutputs [ i ] , i ) ;
2026-03-22 20:57:21 +01:00
}
2026-03-24 00:42:37 +01:00
// Visualizer always gets every frame (it's local, no buffer)
2026-03-22 20:57:21 +01:00
synthVisualizer . setParams ( overridden ) ;
2026-03-24 00:42:37 +01:00
2026-04-11 08:08:29 +02:00
// Modular matrix UI: mirror live MLP outputs into visible matrix cells
// (the UI throttles its own DOM writes internally)
if ( modularUI && activeEngine ? . id === 'modular' ) {
modularUI . updateLive ( overridden ) ;
}
2026-04-03 18:30:26 +02:00
// Engine param updates: throttle + dead-zone filter
2026-04-11 08:31:19 +02:00
if ( activeEngine && activeEngine . running ) {
2026-03-24 00:42:37 +01:00
const now = performance . now ( ) ;
if ( now - _lastParamSendTime >= PARAM _SEND _INTERVAL ) {
_lastParamSendTime = now ;
2026-04-03 19:01:10 +02:00
const engineParamCount = activeEngine . paramCount ;
2026-04-03 18:56:35 +02:00
for ( let i = 0 ; i < overridden . length && i < N _OUTPUTS ; i ++ ) {
2026-03-24 00:42:37 +01:00
const v = overridden [ i ] ;
if ( Math . abs ( v - _lastSentParams [ i ] ) > PARAM _DEAD _ZONE ) {
2026-04-03 19:01:10 +02:00
// Only send engine params (indices before engineParamCount) to the engine
if ( i < engineParamCount ) {
activeEngine . setParam ( i , v ) ;
}
2026-03-24 00:42:37 +01:00
_lastSentParams [ i ] = v ;
}
}
2026-03-21 23:57:58 +01:00
}
}
2026-04-03 19:01:10 +02:00
// In Shared mode, route outputs beyond engine params to the EOC chain
if ( eocChain ? . nispsMode === 'shared' ) {
const engineParamCount = activeEngine ? . paramCount ? ? 0 ;
const eocParamCount = eocChain . paramCount ;
for ( let i = 0 ; i < eocParamCount ; i ++ ) {
const outputIndex = engineParamCount + i ;
if ( outputIndex < outputs . length ) {
eocChain . setParam ( i , outputs [ outputIndex ] ) ;
}
}
}
2026-03-30 16:05:45 +02:00
} else if ( outputMode === 'midi-cc' ) {
// MIDI CC output — route through overrides then send CC messages
if ( midiOutput && midiOutput . enabled && midiOutput . activeOutput ) {
const messages = [ ] ;
for ( let i = 0 ; i < midiCCMap . length && i < outputs . length ; i ++ ) {
const ccParam = midiCCMap [ i ] ;
const ov = midiCCOverrides [ i ] ;
if ( ! ov || ov . frozen ) {
if ( ov ) messages . push ( { channel : ccParam . channel , cc : ccParam . cc , value : Math . round ( ov . fixedValue * 127 ) } ) ;
continue ;
}
const v = applyGroupOverride ( outputs [ i ] , ov . curve , ov . min , ov . max ) ;
messages . push ( { channel : ccParam . channel , cc : ccParam . cc , value : Math . round ( v * 127 ) } ) ;
}
midiOutput . sendBatch ( messages ) ;
}
2026-04-02 23:12:18 +02:00
} else if ( outputMode === 'audio-canvas' ) {
if ( audioCanvas ) audioCanvas . setOutputs ( outputs ) ;
2026-03-21 23:57:58 +01:00
} else {
2026-03-30 16:05:45 +02:00
const vis = new Array ( N _VISUAL _OUTPUTS ) ;
for ( let i = 0 ; i < N _VISUAL _OUTPUTS ; i ++ ) {
const vo = visualOverrides [ i ] ;
if ( vo . frozen ) {
vis [ i ] = vo . fixedValue ;
} else {
vis [ i ] = applyGroupOverride ( outputs [ i ] , vo . curve , vo . min , vo . max ) ;
}
}
visualizer . setParams ( vis ) ;
2026-03-25 10:55:58 +01:00
}
2026-03-21 23:57:58 +01:00
}
// ---- Bottom sheet ----
2026-03-30 16:05:45 +02:00
// ---- Dock & Drawer system ----
function wireDock ( ) {
const dock = document . getElementById ( 'dock' ) ;
// Toggle drawers on dock icon click
dock . addEventListener ( 'click' , ( e ) => {
const icon = e . target . closest ( '.dock-icon' ) ;
if ( ! icon ) return ;
const drawerId = icon . dataset . drawer ;
// Special case: help opens modal instead of drawer
if ( drawerId === 'help' ) {
document . getElementById ( 'help-overlay' ) . classList . remove ( 'hidden' ) ;
return ;
}
const drawer = document . getElementById ( ` drawer- ${ drawerId } ` ) ;
if ( ! drawer ) return ;
const isOpen = ! drawer . classList . contains ( 'hidden' ) ;
if ( isOpen ) {
drawer . classList . add ( 'hidden' ) ;
icon . classList . remove ( 'active' ) ;
2026-03-21 23:57:58 +01:00
} else {
2026-03-30 16:05:45 +02:00
drawer . classList . remove ( 'hidden' ) ;
icon . classList . add ( 'active' ) ;
2026-03-21 23:57:58 +01:00
}
} ) ;
2026-03-30 16:05:45 +02:00
// Close buttons inside drawers
document . querySelectorAll ( '.drawer-close' ) . forEach ( btn => {
btn . addEventListener ( 'click' , ( ) => {
const drawerId = btn . dataset . drawer ;
const drawer = document . getElementById ( ` drawer- ${ drawerId } ` ) ;
if ( drawer ) drawer . classList . add ( 'hidden' ) ;
const icon = dock . querySelector ( ` .dock-icon[data-drawer=" ${ drawerId } "] ` ) ;
if ( icon ) icon . classList . remove ( 'active' ) ;
} ) ;
} ) ;
2026-03-21 23:57:58 +01:00
// Follow pill toggle
$followPill . addEventListener ( 'click' , ( ) => {
toggleFollowMode ( ) ;
} ) ;
}
// ---- Controls wiring ----
function wireControls ( ) {
2026-03-30 16:05:45 +02:00
// Output mode toggle (in mode drawer)
2026-03-22 10:07:38 +01:00
document . querySelectorAll ( '#output-toggle-float .pill-opt' ) . forEach ( btn => {
2026-03-22 01:10:26 +01:00
btn . addEventListener ( 'click' , ( ) => {
syncOutputToggles ( btn . dataset . mode ) ;
2026-03-21 23:57:58 +01:00
setOutputMode ( btn . dataset . mode ) ;
} ) ;
} ) ;
2026-03-30 16:05:45 +02:00
// Action buttons (in training drawer)
2026-03-21 23:57:58 +01:00
document . getElementById ( 'btn-add-example' ) . addEventListener ( 'click' , onAddExample ) ;
document . getElementById ( 'btn-train' ) . addEventListener ( 'click' , onTrain ) ;
document . getElementById ( 'btn-clear' ) . addEventListener ( 'click' , onClear ) ;
2026-03-22 10:01:50 +01:00
document . getElementById ( 'btn-clear-examples' ) . addEventListener ( 'click' , onClearExamples ) ;
2026-03-21 23:57:58 +01:00
document . getElementById ( 'btn-randomize' ) . addEventListener ( 'click' , onRandomize ) ;
// RL buttons
$btnThumbsUp . addEventListener ( 'click' , onThumbsUp ) ;
$btnThumbsDown . addEventListener ( 'click' , onThumbsDown ) ;
2026-04-03 19:20:52 +02:00
// EOC RL buttons (Linked mode — target EOC IML directly)
const eocRlPlus = document . getElementById ( 'eoc-rl-plus' ) ;
const eocRlMinus = document . getElementById ( 'eoc-rl-minus' ) ;
if ( eocRlPlus && eocRlMinus ) {
eocRlPlus . addEventListener ( 'click' , ( ) => {
if ( ! imlEoc ) return ;
const prevTarget = eocTrainingTarget ;
eocTrainingTarget = 'eoc' ;
onThumbsUp ( ) ;
eocTrainingTarget = prevTarget ;
flash ( 'eoc-rl-plus' ) ;
} ) ;
eocRlMinus . addEventListener ( 'click' , ( ) => {
if ( ! imlEoc ) return ;
const prevTarget = eocTrainingTarget ;
eocTrainingTarget = 'eoc' ;
onThumbsDown ( ) ;
eocTrainingTarget = prevTarget ;
flash ( 'eoc-rl-minus' ) ;
} ) ;
}
2026-03-30 16:05:45 +02:00
// Undo button
document . getElementById ( 'btn-undo' ) . addEventListener ( 'click' , onUndo ) ;
2026-03-21 23:57:58 +01:00
// Presets
document . querySelectorAll ( '.preset-chip' ) . forEach ( chip => {
chip . addEventListener ( 'click' , ( ) => loadPreset ( chip . dataset . preset ) ) ;
} ) ;
2026-03-22 10:01:50 +01:00
// Initial noise ring update
updateNoiseRing ( ) ;
2026-03-21 23:57:58 +01:00
}
feat(playground): add MediaPipe hand tracking input with dev panel
Hand tracking via webcam as alternative to joystick input:
- Right hand tracks 14 derived features (palm XY, finger curls, spread,
roll, pitch, pinch) through a separate 14-input MLP
- Left hand gesture recognition (1 finger = thumbs up, 2 = thumbs down)
with 400ms debounce hold
- Split-zone PIP display (no camera feed, skeleton only) with dashed
divider and cross-zone dimming
- Dual IML architecture: independent imlJoy (2 inputs) and imlHand
(14 inputs) with pointer swap, preserving training data per mode
- Dev panel (?devmode=true): draggable/collapsible floating panel with
sliders for MediaPipe confidence thresholds, smoothing, gesture hold
time, world landmarks toggle, and live feature bar monitor
- Visual presets always route to joystick IML (prevents dimension mismatch)
- Race condition guard on input mode switching
2026-03-23 22:28:58 +01:00
// ---- Input mode (joystick / hands) ----
function wireInputToggle ( ) {
document . querySelectorAll ( '#input-toggle .pill-opt' ) . forEach ( btn => {
btn . addEventListener ( 'click' , ( ) => {
const mode = btn . dataset . input ;
if ( mode === inputMode ) return ;
setInputMode ( mode ) ;
syncInputToggle ( mode ) ;
} ) ;
} ) ;
}
function syncInputToggle ( mode ) {
document . querySelectorAll ( '#input-toggle .pill-opt' ) . forEach ( b =>
b . classList . toggle ( 'active' , b . dataset . input === mode )
) ;
}
let _inputModeSwitching = false ;
async function setInputMode ( mode ) {
if ( _inputModeSwitching ) return ;
_inputModeSwitching = true ;
try {
await _setInputModeInner ( mode ) ;
} finally {
_inputModeSwitching = false ;
}
}
async function _setInputModeInner ( mode ) {
inputMode = mode ;
const $pip = document . getElementById ( 'hand-pip' ) ;
if ( mode === 'hands' ) {
iml = imlHand ;
$joystickContainer . style . display = 'none' ;
$pip . classList . remove ( 'hidden' ) ;
if ( ! handTracker ) {
const $status = document . getElementById ( 'hand-status' ) ;
$status . textContent = 'Loading model...' ;
handTracker = new HandTracker ( {
videoElement : document . getElementById ( 'hand-video' ) ,
overlayCanvas : document . getElementById ( 'hand-overlay' ) ,
onTrackingInput : onHandInput ,
onGesture : onHandGesture ,
onConnectionChange : ( active ) => {
console . log ( '[HandTracker] active:' , active ) ;
} ,
} ) ;
try {
await handTracker . start ( ) ;
$status . textContent = 'Tracking' ;
$status . classList . add ( 'tracking' ) ;
} catch ( e ) {
$status . textContent = 'Camera error' ;
console . error ( '[HandTracker]' , e ) ;
setInputMode ( 'joystick' ) ;
syncInputToggle ( 'joystick' ) ;
return ;
}
} else {
await handTracker . start ( ) ;
document . getElementById ( 'hand-status' ) . textContent = 'Tracking' ;
document . getElementById ( 'hand-status' ) . classList . add ( 'tracking' ) ;
}
iml . process ( ) ;
routeOutputs ( iml . getOutputs ( ) ) ;
updateHeatmap ( iml . getOutputs ( ) ) ;
} else {
iml = imlJoy ;
$joystickContainer . style . display = '' ;
$pip . classList . add ( 'hidden' ) ;
if ( handTracker ) {
handTracker . stop ( ) ;
document . getElementById ( 'hand-status' ) . classList . remove ( 'tracking' ) ;
}
2026-03-30 16:05:45 +02:00
iml . setInput ( 0 , joyX ) ;
iml . setInput ( 1 , joyY ) ;
iml . process ( ) ;
routeOutputs ( iml . getOutputs ( ) ) ;
updateHeatmap ( iml . getOutputs ( ) ) ;
feat(playground): add MediaPipe hand tracking input with dev panel
Hand tracking via webcam as alternative to joystick input:
- Right hand tracks 14 derived features (palm XY, finger curls, spread,
roll, pitch, pinch) through a separate 14-input MLP
- Left hand gesture recognition (1 finger = thumbs up, 2 = thumbs down)
with 400ms debounce hold
- Split-zone PIP display (no camera feed, skeleton only) with dashed
divider and cross-zone dimming
- Dual IML architecture: independent imlJoy (2 inputs) and imlHand
(14 inputs) with pointer swap, preserving training data per mode
- Dev panel (?devmode=true): draggable/collapsible floating panel with
sliders for MediaPipe confidence thresholds, smoothing, gesture hold
time, world landmarks toggle, and live feature bar monitor
- Visual presets always route to joystick IML (prevents dimension mismatch)
- Race condition guard on input mode switching
2026-03-23 22:28:58 +01:00
}
updateStatus ( ) ;
drawJoyMap ( ) ;
}
function getCurrentInputs ( ) {
if ( inputMode === 'hands' && handTracker ) {
return [ ... handTracker . features ] ;
}
2026-03-30 16:05:45 +02:00
return [ joyX , joyY ] ;
feat(playground): add MediaPipe hand tracking input with dev panel
Hand tracking via webcam as alternative to joystick input:
- Right hand tracks 14 derived features (palm XY, finger curls, spread,
roll, pitch, pinch) through a separate 14-input MLP
- Left hand gesture recognition (1 finger = thumbs up, 2 = thumbs down)
with 400ms debounce hold
- Split-zone PIP display (no camera feed, skeleton only) with dashed
divider and cross-zone dimming
- Dual IML architecture: independent imlJoy (2 inputs) and imlHand
(14 inputs) with pointer swap, preserving training data per mode
- Dev panel (?devmode=true): draggable/collapsible floating panel with
sliders for MediaPipe confidence thresholds, smoothing, gesture hold
time, world landmarks toggle, and live feature bar monitor
- Visual presets always route to joystick IML (prevents dimension mismatch)
- Race condition guard on input mode switching
2026-03-23 22:28:58 +01:00
}
function setCurrentInputs ( ) {
if ( inputMode === 'hands' && handTracker ) {
const f = handTracker . features ;
for ( let i = 0 ; i < f . length ; i ++ ) iml . setInput ( i , f [ i ] ) ;
} else {
2026-03-30 16:05:45 +02:00
iml . setInput ( 0 , joyX ) ;
iml . setInput ( 1 , joyY ) ;
feat(playground): add MediaPipe hand tracking input with dev panel
Hand tracking via webcam as alternative to joystick input:
- Right hand tracks 14 derived features (palm XY, finger curls, spread,
roll, pitch, pinch) through a separate 14-input MLP
- Left hand gesture recognition (1 finger = thumbs up, 2 = thumbs down)
with 400ms debounce hold
- Split-zone PIP display (no camera feed, skeleton only) with dashed
divider and cross-zone dimming
- Dual IML architecture: independent imlJoy (2 inputs) and imlHand
(14 inputs) with pointer swap, preserving training data per mode
- Dev panel (?devmode=true): draggable/collapsible floating panel with
sliders for MediaPipe confidence thresholds, smoothing, gesture hold
time, world landmarks toggle, and live feature bar monitor
- Visual presets always route to joystick IML (prevents dimension mismatch)
- Race condition guard on input mode switching
2026-03-23 22:28:58 +01:00
}
}
function onHandInput ( features ) {
if ( inputMode !== 'hands' ) return ;
for ( let i = 0 ; i < features . length ; i ++ ) {
iml . setInput ( i , features [ i ] ) ;
}
iml . process ( ) ;
const outputs = iml . getOutputs ( ) ;
routeOutputs ( outputs ) ;
updateHeatmap ( outputs ) ;
syncRawParamsFromOutputs ( outputs ) ;
updateGestureIndicator ( ) ;
}
function onHandGesture ( gesture ) {
if ( gesture === 'thumbsup' ) {
onThumbsUp ( ) ;
} else if ( gesture === 'thumbsdown' ) {
onThumbsDown ( ) ;
}
}
function updateGestureIndicator ( ) {
if ( ! handTracker ) return ;
const $indicator = document . getElementById ( 'gesture-indicator' ) ;
const $label = document . getElementById ( 'gesture-label' ) ;
const $progress = $indicator . querySelector ( '.gesture-ring-progress' ) ;
if ( handTracker . gestureCandidate && handTracker . gestureProgress > 0 ) {
$indicator . classList . add ( 'active' ) ;
const circumference = 2 * Math . PI * 16 ;
const offset = circumference * ( 1 - handTracker . gestureProgress ) ;
$progress . style . strokeDashoffset = offset ;
$label . textContent = handTracker . gestureCandidate === 'thumbsup' ? '+' : '\u2212' ;
} else {
$indicator . classList . remove ( 'active' ) ;
}
const $status = document . getElementById ( 'hand-status' ) ;
if ( handTracker . active ) {
if ( handTracker . trackingRight && handTracker . trackingLeft ) {
$status . textContent = 'Both hands' ;
} else if ( handTracker . trackingRight ) {
$status . textContent = 'Tracking' ;
} else {
$status . textContent = 'No hand' ;
}
$status . classList . toggle ( 'tracking' , handTracker . trackingRight ) ;
}
}
2026-03-22 01:10:26 +01:00
function syncOutputToggles ( mode ) {
2026-03-22 10:07:38 +01:00
document . querySelectorAll ( '#output-toggle-float .pill-opt' ) . forEach ( b => b . classList . toggle ( 'active' , b . dataset . mode === mode ) ) ;
2026-03-22 01:10:26 +01:00
}
2026-03-30 16:05:45 +02:00
async function setOutputMode ( mode , { skipConfirm = false } = { } ) {
2026-04-07 01:01:37 +02:00
const heatmapStrip = document . getElementById ( 'heatmap-strip' ) ;
const synthQuickControls = document . getElementById ( 'synth-quick-controls' ) ;
const midiCCQuickControls = document . getElementById ( 'midi-cc-quick-controls' ) ;
const audioCanvasWrap = document . getElementById ( 'audio-canvas-wrap' ) ;
// Lazily create AudioCanvas BEFORE computing targetOutputs so that
// outputCountForMode returns the real count (not the 12-output fallback).
if ( mode === 'audio-canvas' && ! audioCanvas ) {
audioCanvas = new AudioCanvas ( audioCanvasWrap ) ;
_ensureAudioCanvasOverrides ( audioCanvas . getOutputCount ( ) ) ;
// Listen for dynamic output count changes (loop count changes)
audioCanvasWrap . addEventListener ( 'ac:outputcount-changed' , async ( e ) => {
if ( outputMode !== 'audio-canvas' ) return ;
const newCount = e . detail . count ;
_ensureAudioCanvasOverrides ( newCount ) ;
if ( newCount !== N _OUTPUTS ) {
await resizeMLP ( newCount ) ;
}
buildHeatmap ( ) ;
updateHeatmap ( iml . getOutputs ( ) ) ;
} ) ;
}
2026-03-30 16:05:45 +02:00
const targetOutputs = outputCountForMode ( mode ) ;
const needsResize = targetOutputs !== N _OUTPUTS ;
// Warn about weight reset if resizing (skip during state restore)
if ( needsResize && ! skipConfirm && iml . dataset . features . length > 0 ) {
2026-04-03 18:27:54 +02:00
if ( ! confirm ( ` Switching to ${ mode } mode requires ${ targetOutputs } outputs (currently ${ N _OUTPUTS } ). This will reset training examples. Network weights will be partially preserved. Continue? ` ) ) {
2026-03-30 16:05:45 +02:00
syncOutputToggles ( outputMode ) ; // revert pill UI
return ;
}
}
2026-03-21 23:57:58 +01:00
outputMode = mode ;
2026-03-22 20:57:21 +01:00
hideGroupDrawer ( ) ;
2026-03-30 16:05:45 +02:00
hideParamPopup ( ) ;
// Resize MLP if needed
if ( needsResize ) {
await resizeMLP ( targetOutputs ) ;
}
2026-03-21 23:57:58 +01:00
buildHeatmap ( ) ;
updateHeatmap ( iml . getOutputs ( ) ) ;
2026-04-02 23:12:18 +02:00
// Hide/show audio-canvas wrap
if ( audioCanvasWrap ) audioCanvasWrap . style . display = mode === 'audio-canvas' ? 'block' : 'none' ;
2026-03-21 23:57:58 +01:00
2026-04-03 20:01:14 +02:00
// Show/hide ShapeSeq container (only visible in synth mode when shapeseq=1)
const shapeSeqContainer = document . getElementById ( 'shapeseq-container' ) ;
if ( shapeSeqContainer ) {
shapeSeqContainer . style . display = ( shapeSeqEnabled && mode === 'synth' ) ? 'block' : 'none' ;
}
2026-03-21 23:57:58 +01:00
if ( mode === 'synth' ) {
$canvas . classList . add ( 'hidden-canvas' ) ;
$synthVisCanvas . classList . add ( 'active' ) ;
heatmapStrip . classList . add ( 'hidden' ) ;
synthQuickControls . classList . remove ( 'hidden' ) ;
2026-03-30 16:05:45 +02:00
if ( midiCCQuickControls ) midiCCQuickControls . classList . add ( 'hidden' ) ;
2026-03-22 10:01:50 +01:00
synthVisualizer . enableInteraction ( true ) ;
2026-03-22 20:57:21 +01:00
const qp = document . getElementById ( 'quick-play' ) ;
if ( qp ) qp . classList . toggle ( 'audio-needs-init' , ! ( c15 && c15 . running ) ) ;
2026-03-30 16:05:45 +02:00
} else if ( mode === 'midi-cc' ) {
$canvas . classList . add ( 'hidden-canvas' ) ;
$synthVisCanvas . classList . remove ( 'active' ) ;
heatmapStrip . classList . remove ( 'hidden' ) ;
synthQuickControls . classList . add ( 'hidden' ) ;
if ( midiCCQuickControls ) midiCCQuickControls . classList . remove ( 'hidden' ) ;
synthVisualizer . enableInteraction ( false ) ;
2026-04-02 23:12:18 +02:00
} else if ( mode === 'audio-canvas' ) {
$canvas . classList . add ( 'hidden-canvas' ) ;
$synthVisCanvas . classList . remove ( 'active' ) ;
heatmapStrip . classList . remove ( 'hidden' ) ;
synthQuickControls . classList . add ( 'hidden' ) ;
if ( midiCCQuickControls ) midiCCQuickControls . classList . add ( 'hidden' ) ;
synthVisualizer . enableInteraction ( false ) ;
2026-03-21 23:57:58 +01:00
} else {
$canvas . classList . remove ( 'hidden-canvas' ) ;
$synthVisCanvas . classList . remove ( 'active' ) ;
heatmapStrip . classList . remove ( 'hidden' ) ;
synthQuickControls . classList . add ( 'hidden' ) ;
2026-03-30 16:05:45 +02:00
if ( midiCCQuickControls ) midiCCQuickControls . classList . add ( 'hidden' ) ;
2026-03-21 23:57:58 +01:00
synthVisualizer . enableInteraction ( false ) ;
}
routeOutputs ( iml . getOutputs ( ) ) ;
2026-03-30 16:05:45 +02:00
buildEngineParams ( ) ;
2026-03-21 23:57:58 +01:00
syncRawParamsFromOutputs ( iml . getOutputs ( ) ) ;
}
function updateNoiseRing ( ) {
if ( noiseLevel > 0.15 ) {
$noiseRing . className = 'noise-ring active high' ;
2026-03-22 10:01:50 +01:00
} else if ( noiseLevel > 0.01 ) {
2026-03-21 23:57:58 +01:00
$noiseRing . className = 'noise-ring active' ;
} else {
$noiseRing . className = 'noise-ring' ;
}
}
// ---- Examples mode ----
function onAddExample ( ) {
feat(playground): add MediaPipe hand tracking input with dev panel
Hand tracking via webcam as alternative to joystick input:
- Right hand tracks 14 derived features (palm XY, finger curls, spread,
roll, pitch, pinch) through a separate 14-input MLP
- Left hand gesture recognition (1 finger = thumbs up, 2 = thumbs down)
with 400ms debounce hold
- Split-zone PIP display (no camera feed, skeleton only) with dashed
divider and cross-zone dimming
- Dual IML architecture: independent imlJoy (2 inputs) and imlHand
(14 inputs) with pointer swap, preserving training data per mode
- Dev panel (?devmode=true): draggable/collapsible floating panel with
sliders for MediaPipe confidence thresholds, smoothing, gesture hold
time, world landmarks toggle, and live feature bar monitor
- Visual presets always route to joystick IML (prevents dimension mismatch)
- Race condition guard on input mode switching
2026-03-23 22:28:58 +01:00
const inputs = getCurrentInputs ( ) ;
2026-03-21 23:57:58 +01:00
const outputs = [ ... rawParamValues ] ;
iml . addExample ( inputs , outputs ) ;
updateStatus ( ) ;
drawJoyMap ( ) ;
flash ( 'btn-add-example' ) ;
}
function onTrain ( ) {
2026-03-23 22:23:07 +01:00
if ( iml . isTraining ) return ;
flash ( 'btn-train' ) ;
trainModelAsync ( ) ;
2026-03-21 23:57:58 +01:00
}
function onRandomize ( ) {
2026-03-22 00:36:00 +01:00
iml . randomiseWeights ( spreadLevel ) ;
feat(playground): add MediaPipe hand tracking input with dev panel
Hand tracking via webcam as alternative to joystick input:
- Right hand tracks 14 derived features (palm XY, finger curls, spread,
roll, pitch, pinch) through a separate 14-input MLP
- Left hand gesture recognition (1 finger = thumbs up, 2 = thumbs down)
with 400ms debounce hold
- Split-zone PIP display (no camera feed, skeleton only) with dashed
divider and cross-zone dimming
- Dual IML architecture: independent imlJoy (2 inputs) and imlHand
(14 inputs) with pointer swap, preserving training data per mode
- Dev panel (?devmode=true): draggable/collapsible floating panel with
sliders for MediaPipe confidence thresholds, smoothing, gesture hold
time, world landmarks toggle, and live feature bar monitor
- Visual presets always route to joystick IML (prevents dimension mismatch)
- Race condition guard on input mode switching
2026-03-23 22:28:58 +01:00
setCurrentInputs ( ) ;
2026-03-22 01:10:26 +01:00
iml . process ( ) ;
2026-03-21 23:57:58 +01:00
const outputs = iml . getOutputs ( ) ;
routeOutputs ( outputs ) ;
updateHeatmap ( outputs ) ;
syncRawParamsFromOutputs ( outputs ) ;
noiseLevel = 0.05 ;
updateStatus ( ) ;
}
2026-03-22 10:01:50 +01:00
function onClearExamples ( ) {
iml . clearDataset ( ) ;
updateStatus ( ) ;
drawJoyMap ( ) ;
}
2026-03-21 23:57:58 +01:00
function onClear ( ) {
iml . clearDataset ( ) ;
iml . lossHistory = [ ] ;
iml . bestLoss = null ;
iml . totalTrainingIterations = 0 ;
noiseLevel = 0.05 ;
clearState ( ) ;
updateStatus ( ) ;
drawLossPlot ( ) ;
drawJoyMap ( ) ;
}
2026-03-30 16:05:45 +02:00
// ---- Undo ----
function pushUndoSnapshot ( ) {
const snapshot = {
weights : iml . _getFlatWeights ? iml . _getFlatWeights ( ) : ( iml . mlp ? iml . mlp . getWeights ( ) : null ) ,
noiseLevel ,
exampleCount : iml . exampleCount ,
} ;
if ( snapshot . weights ) {
undoStack . push ( snapshot ) ;
if ( undoStack . length > MAX _UNDO ) undoStack . shift ( ) ;
updateUndoButton ( ) ;
}
}
function onUndo ( ) {
if ( undoStack . length === 0 ) return ;
const snapshot = undoStack . pop ( ) ;
if ( iml . _setFlatWeights && Array . isArray ( snapshot . weights ) ) {
iml . _setFlatWeights ( snapshot . weights ) ;
} else if ( iml . mlp && snapshot . weights ) {
iml . mlp . setWeights ( snapshot . weights ) ;
}
noiseLevel = snapshot . noiseLevel ;
iml . process ( ) ;
const outputs = iml . getOutputs ( ) ;
routeOutputs ( outputs ) ;
updateHeatmap ( outputs ) ;
syncRawParamsFromOutputs ( outputs ) ;
updateStatus ( ) ;
updateNoiseRing ( ) ;
updateUndoButton ( ) ;
flash ( 'btn-undo' ) ;
}
function updateUndoButton ( ) {
const btn = document . getElementById ( 'btn-undo' ) ;
if ( btn ) btn . classList . toggle ( 'has-undo' , undoStack . length > 0 ) ;
}
2026-03-21 23:57:58 +01:00
// ---- RL mode ----
2026-04-03 19:03:12 +02:00
/ * *
* Return the IML instance that RL feedback should target .
2026-04-03 19:21:59 +02:00
* In Linked / Independent mode the user may direct feedback to the EOC IML instead .
2026-04-03 19:03:12 +02:00
* /
function _rlTarget ( ) {
2026-04-03 19:21:59 +02:00
const mode = eocChain ? . nispsMode ;
if ( ( mode === 'linked' || mode === 'independent' ) && eocTrainingTarget === 'eoc' && imlEoc ) {
2026-04-03 19:03:12 +02:00
return imlEoc ;
}
return iml ;
}
2026-03-21 23:57:58 +01:00
function onThumbsUp ( ) {
2026-04-03 19:03:12 +02:00
const target = _rlTarget ( ) ;
if ( ! target || target . isTraining ) return ;
2026-03-23 22:23:07 +01:00
2026-03-30 16:05:45 +02:00
pushUndoSnapshot ( ) ;
feat(playground): add MediaPipe hand tracking input with dev panel
Hand tracking via webcam as alternative to joystick input:
- Right hand tracks 14 derived features (palm XY, finger curls, spread,
roll, pitch, pinch) through a separate 14-input MLP
- Left hand gesture recognition (1 finger = thumbs up, 2 = thumbs down)
with 400ms debounce hold
- Split-zone PIP display (no camera feed, skeleton only) with dashed
divider and cross-zone dimming
- Dual IML architecture: independent imlJoy (2 inputs) and imlHand
(14 inputs) with pointer swap, preserving training data per mode
- Dev panel (?devmode=true): draggable/collapsible floating panel with
sliders for MediaPipe confidence thresholds, smoothing, gesture hold
time, world landmarks toggle, and live feature bar monitor
- Visual presets always route to joystick IML (prevents dimension mismatch)
- Race condition guard on input mode switching
2026-03-23 22:28:58 +01:00
const inputs = getCurrentInputs ( ) ;
2026-04-02 21:35:38 +02:00
const outputs = [ ... rawParamValues ] ;
2026-04-03 19:03:12 +02:00
target . addExample ( inputs , outputs ) ;
2026-03-21 23:57:58 +01:00
2026-03-30 16:05:45 +02:00
noiseLevel *= rlExplorationDecay ;
noiseLevel = Math . max ( noiseLevel , 0.005 ) ;
2026-03-21 23:57:58 +01:00
flash ( 'btn-thumbsup' ) ;
2026-03-23 22:23:07 +01:00
updateNoiseRing ( ) ;
2026-04-03 19:03:12 +02:00
// Only run async training on the target; synth IML uses trainModelAsync, EOC IML trains directly
if ( target === iml ) {
trainModelAsync ( ) ;
} else {
target . trainAsync ( ( { loss } ) => {
updateStatus ( ) ;
} ) ;
}
2026-03-21 23:57:58 +01:00
}
function onThumbsDown ( ) {
2026-04-03 19:03:12 +02:00
const target = _rlTarget ( ) ;
if ( ! target ) return ;
2026-03-30 16:05:45 +02:00
pushUndoSnapshot ( ) ;
const noiseCap = 0.3 * ( 1 - spreadLevel ) + 0.05 * spreadLevel ;
noiseLevel = Math . min ( noiseLevel * 1.5 , noiseCap ) ;
2026-03-21 23:57:58 +01:00
2026-04-03 19:03:12 +02:00
target . moveWeights ( noiseLevel , spreadLevel ) ;
2026-03-21 23:57:58 +01:00
2026-04-03 19:03:12 +02:00
// Re-route outputs from whichever IML was affected
if ( target === iml ) {
const outputs = iml . getOutputs ( ) ;
routeOutputs ( outputs ) ;
updateHeatmap ( outputs ) ;
syncRawParamsFromOutputs ( outputs ) ;
2026-04-03 19:21:59 +02:00
} else if ( imlEoc && ( eocChain ? . nispsMode === 'linked' || eocChain ? . nispsMode === 'independent' ) ) {
2026-04-03 19:03:12 +02:00
imlEoc . process ( ) ;
const eocOutputs = imlEoc . getOutputs ( ) ;
for ( let i = 0 ; i < eocOutputs . length ; i ++ ) {
eocChain . setParam ( i , eocOutputs [ i ] ) ;
}
}
2026-03-21 23:57:58 +01:00
updateStatus ( ) ;
updateNoiseRing ( ) ;
flash ( 'btn-thumbsdown' ) ;
}
// ---- Training ----
2026-03-23 22:23:07 +01:00
// Sync — used for preset loading and state restore
2026-03-21 23:57:58 +01:00
function trainModel ( ) {
2026-03-23 22:23:07 +01:00
return iml . train ( ) ;
}
// Async — used for interactive training (thumbs-up, train button)
function trainModelAsync ( onDone ) {
iml . trainAsync ( ( { loss , outputs } ) => {
routeOutputs ( outputs ) ;
updateHeatmap ( outputs ) ;
syncRawParamsFromOutputs ( outputs ) ;
updateStatus ( ) ;
drawLossPlot ( ) ;
drawJoyMap ( ) ;
if ( onDone ) onDone ( ) ;
2026-03-21 23:57:58 +01:00
} ) ;
}
// ---- Presets ----
function loadPreset ( name ) {
const preset = PRESETS [ name ] ;
if ( ! preset ) return ;
feat(playground): add MediaPipe hand tracking input with dev panel
Hand tracking via webcam as alternative to joystick input:
- Right hand tracks 14 derived features (palm XY, finger curls, spread,
roll, pitch, pinch) through a separate 14-input MLP
- Left hand gesture recognition (1 finger = thumbs up, 2 = thumbs down)
with 400ms debounce hold
- Split-zone PIP display (no camera feed, skeleton only) with dashed
divider and cross-zone dimming
- Dual IML architecture: independent imlJoy (2 inputs) and imlHand
(14 inputs) with pointer swap, preserving training data per mode
- Dev panel (?devmode=true): draggable/collapsible floating panel with
sliders for MediaPipe confidence thresholds, smoothing, gesture hold
time, world landmarks toggle, and live feature bar monitor
- Visual presets always route to joystick IML (prevents dimension mismatch)
- Race condition guard on input mode switching
2026-03-23 22:28:58 +01:00
// Visual presets have 2-element inputs — always apply to joystick IML
imlJoy . clearDataset ( ) ;
2026-03-21 23:57:58 +01:00
for ( const ex of preset ) {
feat(playground): add MediaPipe hand tracking input with dev panel
Hand tracking via webcam as alternative to joystick input:
- Right hand tracks 14 derived features (palm XY, finger curls, spread,
roll, pitch, pinch) through a separate 14-input MLP
- Left hand gesture recognition (1 finger = thumbs up, 2 = thumbs down)
with 400ms debounce hold
- Split-zone PIP display (no camera feed, skeleton only) with dashed
divider and cross-zone dimming
- Dual IML architecture: independent imlJoy (2 inputs) and imlHand
(14 inputs) with pointer swap, preserving training data per mode
- Dev panel (?devmode=true): draggable/collapsible floating panel with
sliders for MediaPipe confidence thresholds, smoothing, gesture hold
time, world landmarks toggle, and live feature bar monitor
- Visual presets always route to joystick IML (prevents dimension mismatch)
- Race condition guard on input mode switching
2026-03-23 22:28:58 +01:00
imlJoy . addExample ( ex . input , padPresetOutputs ( ex . output ) ) ;
2026-03-21 23:57:58 +01:00
}
feat(playground): add MediaPipe hand tracking input with dev panel
Hand tracking via webcam as alternative to joystick input:
- Right hand tracks 14 derived features (palm XY, finger curls, spread,
roll, pitch, pinch) through a separate 14-input MLP
- Left hand gesture recognition (1 finger = thumbs up, 2 = thumbs down)
with 400ms debounce hold
- Split-zone PIP display (no camera feed, skeleton only) with dashed
divider and cross-zone dimming
- Dual IML architecture: independent imlJoy (2 inputs) and imlHand
(14 inputs) with pointer swap, preserving training data per mode
- Dev panel (?devmode=true): draggable/collapsible floating panel with
sliders for MediaPipe confidence thresholds, smoothing, gesture hold
time, world landmarks toggle, and live feature bar monitor
- Visual presets always route to joystick IML (prevents dimension mismatch)
- Race condition guard on input mode switching
2026-03-23 22:28:58 +01:00
// Temporarily point iml to imlJoy for training, then restore
const prevIml = iml ;
iml = imlJoy ;
2026-03-21 23:57:58 +01:00
const loss = trainModel ( ) ;
feat(playground): add MediaPipe hand tracking input with dev panel
Hand tracking via webcam as alternative to joystick input:
- Right hand tracks 14 derived features (palm XY, finger curls, spread,
roll, pitch, pinch) through a separate 14-input MLP
- Left hand gesture recognition (1 finger = thumbs up, 2 = thumbs down)
with 400ms debounce hold
- Split-zone PIP display (no camera feed, skeleton only) with dashed
divider and cross-zone dimming
- Dual IML architecture: independent imlJoy (2 inputs) and imlHand
(14 inputs) with pointer swap, preserving training data per mode
- Dev panel (?devmode=true): draggable/collapsible floating panel with
sliders for MediaPipe confidence thresholds, smoothing, gesture hold
time, world landmarks toggle, and live feature bar monitor
- Visual presets always route to joystick IML (prevents dimension mismatch)
- Race condition guard on input mode switching
2026-03-23 22:28:58 +01:00
iml = prevIml ;
// Show results from joystick IML
imlJoy . setInput ( 0 , joyX ) ;
imlJoy . setInput ( 1 , joyY ) ;
imlJoy . process ( ) ;
const outputs = imlJoy . getOutputs ( ) ;
2026-03-21 23:57:58 +01:00
routeOutputs ( outputs ) ;
updateHeatmap ( outputs ) ;
syncRawParamsFromOutputs ( outputs ) ;
updateStatus ( ) ;
drawLossPlot ( ) ;
drawJoyMap ( ) ;
}
// ---- Status ----
function updateStatus ( ) {
const count = iml . exampleCount ;
const loss = iml . lastLoss ;
let text = ` ${ count } example ${ count !== 1 ? 's' : '' } ` ;
if ( loss !== null ) {
text += ` \u 00b7 loss ${ loss . toFixed ( 5 ) } ` ;
} else {
text += ' \u00b7 untrained' ;
}
2026-03-22 10:01:50 +01:00
text += ` \u 00b7 noise ${ noiseLevel . toFixed ( 3 ) } ` ;
2026-03-21 23:57:58 +01:00
$statusText . textContent = text ;
2026-03-30 16:05:45 +02:00
syncEngineParams ( ) ;
2026-03-21 23:57:58 +01:00
}
// ---- Loss plot ----
function drawLossPlot ( ) {
const ctx = $lossCtx ;
const w = $lossCanvas . width ;
const h = $lossCanvas . height ;
const history = iml . lossHistory ;
ctx . fillStyle = 'rgba(0, 0, 0, 0.3)' ;
ctx . fillRect ( 0 , 0 , w , h ) ;
if ( history . length < 2 ) return ;
const maxLoss = Math . max ( ... history . slice ( - 200 ) ) || 1 ;
const points = history . slice ( - 200 ) ;
2026-03-22 00:36:00 +01:00
ctx . strokeStyle = 'rgba(255, 106, 0, 0.6)' ;
2026-03-21 23:57:58 +01:00
ctx . lineWidth = 1.5 ;
ctx . beginPath ( ) ;
for ( let i = 0 ; i < points . length ; i ++ ) {
const x = ( i / ( points . length - 1 ) ) * w ;
const y = h - ( points [ i ] / maxLoss ) * h * 0.9 ;
if ( i === 0 ) ctx . moveTo ( x , y ) ;
else ctx . lineTo ( x , y ) ;
}
ctx . stroke ( ) ;
}
// ---- Raw param sliders ----
2026-03-30 16:05:45 +02:00
// ---- Engine Parameters (NISPS tuning) ----
function buildEngineParams ( ) {
2026-04-03 18:49:07 +02:00
if ( ! $engineParams ) return ;
// Use a sub-container so the EngineSwitcher section (prepended) is not clobbered.
let container = $engineParams . querySelector ( '.engine-tuning' ) ;
if ( ! container ) {
container = document . createElement ( 'div' ) ;
container . className = 'engine-tuning' ;
$engineParams . appendChild ( container ) ;
}
2026-03-30 16:05:45 +02:00
container . innerHTML = '' ;
const params = [
{
label : 'Spread' , key : 'spread' ,
get : ( ) => spreadLevel , set : ( v ) => { spreadLevel = v ; } ,
min : 0 , max : 1 , step : 0.01 ,
desc : 'Weight init scale & RL noise regime (0=polarised, 1=centered)' ,
} ,
{
label : 'Noise' , key : 'noise' ,
get : ( ) => noiseLevel , set : ( v ) => { noiseLevel = v ; updateNoiseRing ( ) ; } ,
min : 0 , max : 0.5 , step : 0.001 ,
desc : 'Current RL exploration noise level' ,
} ,
{
label : 'RL Decay' , key : 'rlDecay' ,
get : ( ) => rlExplorationDecay , set : ( v ) => { rlExplorationDecay = v ; } ,
min : 0.8 , max : 1.0 , step : 0.005 ,
desc : 'Noise decay per thumbs-up (lower = faster convergence)' ,
} ,
{
label : 'Learn Rate' , key : 'lr' ,
get : ( ) => iml . learningRate , set : ( v ) => { iml . learningRate = v ; } ,
min : 0.01 , max : 5.0 , step : 0.01 ,
desc : 'MLP training learning rate' ,
} ,
{
label : 'Max Iters' , key : 'maxIter' ,
get : ( ) => iml . maxIterations , set : ( v ) => { iml . maxIterations = Math . round ( v ) ; } ,
min : 50 , max : 5000 , step : 50 ,
desc : 'Max training iterations per train() call' ,
} ,
{
label : 'Convergence' , key : 'conv' ,
get : ( ) => iml . convergenceThreshold , set : ( v ) => { iml . convergenceThreshold = v ; } ,
min : 0.000001 , max : 0.01 , step : 0.000001 ,
desc : 'Stop training when loss drops below this' ,
} ,
] ;
2026-03-21 23:57:58 +01:00
2026-03-30 16:05:45 +02:00
for ( const p of params ) {
2026-03-21 23:57:58 +01:00
const row = document . createElement ( 'div' ) ;
2026-03-30 16:05:45 +02:00
row . className = 'engine-param' ;
2026-03-21 23:57:58 +01:00
const label = document . createElement ( 'span' ) ;
2026-03-30 16:05:45 +02:00
label . className = 'engine-param-label' ;
label . textContent = p . label ;
label . title = p . desc ;
2026-03-21 23:57:58 +01:00
const slider = document . createElement ( 'input' ) ;
slider . type = 'range' ;
2026-03-30 16:05:45 +02:00
slider . min = String ( p . min ) ;
slider . max = String ( p . max ) ;
slider . step = String ( p . step ) ;
slider . value = p . get ( ) ;
2026-03-21 23:57:58 +01:00
const val = document . createElement ( 'span' ) ;
2026-03-30 16:05:45 +02:00
val . className = 'engine-param-val' ;
val . textContent = formatEngineVal ( p . get ( ) , p . step ) ;
2026-03-21 23:57:58 +01:00
slider . addEventListener ( 'input' , ( ) => {
2026-03-30 16:05:45 +02:00
const v = parseFloat ( slider . value ) ;
p . set ( v ) ;
val . textContent = formatEngineVal ( v , p . step ) ;
2026-03-21 23:57:58 +01:00
} ) ;
row . appendChild ( label ) ;
row . appendChild ( slider ) ;
row . appendChild ( val ) ;
2026-03-30 16:05:45 +02:00
container . appendChild ( row ) ;
// Store ref for live updates
row . _paramDef = p ;
row . _slider = slider ;
row . _val = val ;
2026-03-21 23:57:58 +01:00
}
}
2026-03-30 16:05:45 +02:00
function formatEngineVal ( v , step ) {
if ( step < 0.0001 ) return v . toExponential ( 1 ) ;
if ( step < 0.01 ) return v . toFixed ( 3 ) ;
if ( step < 1 ) return v . toFixed ( 2 ) ;
return String ( Math . round ( v ) ) ;
}
function syncEngineParams ( ) {
if ( ! $engineParams ) return ;
const rows = $engineParams . querySelectorAll ( '.engine-param' ) ;
rows . forEach ( row => {
const p = row . _paramDef ;
if ( ! p ) return ;
const v = p . get ( ) ;
row . _slider . value = v ;
row . _val . textContent = formatEngineVal ( v , parseFloat ( row . _slider . step ) ) ;
2026-03-21 23:57:58 +01:00
} ) ;
}
2026-03-30 16:05:45 +02:00
function syncRawParamsFromOutputs ( outputs ) {
rawParamValues = [ ... outputs ] ;
feat(shapeseq): integrate ShapeSeq into synth mode, replace arpeggiator
- Remove Arpeggiator import and all arp references from a-app.js
- Add ShapeSeq imports (ShapeSeqEngine, StepVisualizer, ChainBuilderUI,
getDefaultBus) and lazy initialization on first audio start
- In synth mode: show ShapeSeq container (circular viz + chain builder),
hide flow-field particles. Visual mode unchanged.
- Route joystick or hand tracking inputs to shapeSeq.setSequenceInputs()
each frame; render step viz at 60fps
- Add HTML: #shapeseq-container with #shapeseq-viz canvas and
#shapeseq-chain wrapper, sequencer transport controls (play/stop,
tempo, step count)
- Add CSS: full-screen flex layout, viz takes upper 45%, chain builder
scrollable below, hidden by default
- Quick play button wires to ShapeSeq start/stop instead of arp
Closes meml-6ry. This completes ShapeSeq Phase 1.
2026-03-24 01:17:32 +01:00
}
2026-03-21 23:57:58 +01:00
// ---- Synth controls ----
function wireSynthControls ( ) {
const startBtn = document . getElementById ( 'synth-start' ) ;
const volumeSlider = document . getElementById ( 'synth-volume' ) ;
2026-03-24 10:54:13 +01:00
const arpToggle = document . getElementById ( 'arp-toggle' ) ;
const arpProgression = document . getElementById ( 'arp-progression' ) ;
const arpTempo = document . getElementById ( 'arp-tempo' ) ;
const arpOctaves = document . getElementById ( 'arp-octaves' ) ;
const arpOffset = document . getElementById ( 'arp-offset' ) ;
2026-03-30 16:05:45 +02:00
if ( ! startBtn || ! volumeSlider || ! arpToggle ) return ;
2026-03-21 23:57:58 +01:00
startBtn . addEventListener ( 'click' , async ( ) => {
2026-03-22 20:57:21 +01:00
const quickPlay = document . getElementById ( 'quick-play' ) ;
2026-03-21 23:57:58 +01:00
if ( c15 . running ) {
2026-04-03 20:01:14 +02:00
if ( shapeSeqEnabled && shapeSeqEngine ) shapeSeqEngine . stop ( ) ;
2026-03-24 10:54:13 +01:00
arpeggiator . stop ( ) ;
arpToggle . textContent = 'Play' ;
2026-03-21 23:57:58 +01:00
await c15 . stop ( ) ;
startBtn . textContent = 'Start Audio' ;
2026-03-22 20:57:21 +01:00
if ( quickPlay ) quickPlay . classList . add ( 'audio-needs-init' ) ;
2026-03-21 23:57:58 +01:00
} else {
2026-04-03 20:01:14 +02:00
await activeEngine . init ( ) ;
2026-04-03 18:49:07 +02:00
await _startEocChain ( ) ;
2026-04-03 20:01:14 +02:00
if ( shapeSeqEnabled ) await _ensureShapeSeqInit ( ) ;
2026-03-21 23:57:58 +01:00
startBtn . textContent = 'Stop Audio' ;
2026-03-22 20:57:21 +01:00
if ( quickPlay ) quickPlay . classList . remove ( 'audio-needs-init' ) ;
2026-03-21 23:57:58 +01:00
routeOutputs ( iml . getOutputs ( ) ) ;
}
} ) ;
volumeSlider . addEventListener ( 'input' , ( e ) => {
c15 . setMasterVolume ( parseFloat ( e . target . value ) ) ;
2026-03-22 22:59:08 +01:00
const quickVol = document . getElementById ( 'quick-vol' ) ;
if ( quickVol ) quickVol . value = e . target . value ;
2026-03-21 23:57:58 +01:00
} ) ;
2026-03-24 10:54:13 +01:00
arpToggle . addEventListener ( 'click' , ( ) => {
if ( ! c15 . running ) return ;
2026-04-03 20:01:14 +02:00
const isPlaying = shapeSeqEnabled
? ( shapeSeqEngine && shapeSeqEngine . isPlaying )
: arpeggiator . playing ;
if ( isPlaying ) {
if ( shapeSeqEnabled && shapeSeqEngine ) shapeSeqEngine . stop ( ) ;
else arpeggiator . stop ( ) ;
2026-03-24 10:54:13 +01:00
arpToggle . textContent = 'Play' ;
} else {
2026-04-03 20:01:14 +02:00
if ( shapeSeqEnabled && shapeSeqEngine ) shapeSeqEngine . start ( ) ;
else arpeggiator . start ( ) ;
2026-03-24 10:54:13 +01:00
arpToggle . textContent = 'Stop' ;
}
} ) ;
2026-03-21 23:57:58 +01:00
2026-03-24 10:54:13 +01:00
arpProgression . addEventListener ( 'change' , ( e ) => {
arpeggiator . progression = e . target . value ;
} ) ;
feat(shapeseq): integrate ShapeSeq into synth mode, replace arpeggiator
- Remove Arpeggiator import and all arp references from a-app.js
- Add ShapeSeq imports (ShapeSeqEngine, StepVisualizer, ChainBuilderUI,
getDefaultBus) and lazy initialization on first audio start
- In synth mode: show ShapeSeq container (circular viz + chain builder),
hide flow-field particles. Visual mode unchanged.
- Route joystick or hand tracking inputs to shapeSeq.setSequenceInputs()
each frame; render step viz at 60fps
- Add HTML: #shapeseq-container with #shapeseq-viz canvas and
#shapeseq-chain wrapper, sequencer transport controls (play/stop,
tempo, step count)
- Add CSS: full-screen flex layout, viz takes upper 45%, chain builder
scrollable below, hidden by default
- Quick play button wires to ShapeSeq start/stop instead of arp
Closes meml-6ry. This completes ShapeSeq Phase 1.
2026-03-24 01:17:32 +01:00
2026-03-24 10:54:13 +01:00
arpTempo . addEventListener ( 'input' , ( e ) => {
const val = parseInt ( e . target . value ) ;
arpeggiator . bpm = val ;
document . getElementById ( 'tempo-val' ) . textContent = val ;
const quickBpm = document . getElementById ( 'quick-bpm' ) ;
const quickBpmVal = document . getElementById ( 'quick-bpm-val' ) ;
if ( quickBpm ) quickBpm . value = val ;
if ( quickBpmVal ) quickBpmVal . textContent = val ;
} ) ;
2026-03-21 23:57:58 +01:00
2026-03-24 10:54:13 +01:00
arpOctaves . addEventListener ( 'input' , ( e ) => {
const val = parseInt ( e . target . value ) ;
arpeggiator . octaves = val ;
document . getElementById ( 'octaves-val' ) . textContent = val ;
} ) ;
2026-03-21 23:57:58 +01:00
2026-03-24 10:54:13 +01:00
arpOffset . addEventListener ( 'input' , ( e ) => {
const val = parseInt ( e . target . value ) ;
arpeggiator . octaveOffset = val ;
document . getElementById ( 'offset-val' ) . textContent = val ;
} ) ;
2026-03-21 23:57:58 +01:00
}
2026-03-22 10:01:50 +01:00
// ---- MIDI Input ----
async function initMIDIControls ( ) {
const row = document . getElementById ( 'midi-row' ) ;
const statusRow = document . getElementById ( 'midi-status-row' ) ;
const toggle = document . getElementById ( 'midi-toggle' ) ;
const select = document . getElementById ( 'midi-select' ) ;
const statusEl = document . getElementById ( 'midi-status' ) ;
if ( ! row || ! toggle || ! select ) return ;
2026-03-21 23:57:58 +01:00
2026-03-22 10:01:50 +01:00
const available = await midiInput . init ( ) ;
if ( ! available ) return ;
2026-03-21 23:57:58 +01:00
2026-03-22 10:01:50 +01:00
row . style . display = '' ;
2026-03-21 23:57:58 +01:00
2026-03-22 10:01:50 +01:00
function populateInputs ( ) {
const inputs = midiInput . getInputs ( ) ;
select . innerHTML = '<option value="">All Inputs</option>' +
inputs . map ( i => ` <option value=" ${ i . id } "> ${ i . name } </option> ` ) . join ( '' ) ;
}
2026-03-21 23:57:58 +01:00
2026-03-22 10:01:50 +01:00
populateInputs ( ) ;
midiInput . onInputsChange = ( ) => populateInputs ( ) ;
2026-03-21 23:57:58 +01:00
2026-03-22 10:01:50 +01:00
midiInput . onStatusChange = ( msg ) => {
if ( statusEl ) {
statusEl . textContent = msg ;
statusRow . style . display = '' ;
}
} ;
2026-03-21 23:57:58 +01:00
2026-03-22 10:01:50 +01:00
midiInput . onCC = ( cc , value ) => {
if ( cc === 1 ) { joyX = value ; onJoystickMove ( ) ; drawJoyMap ( ) ; }
if ( cc === 2 ) { joyY = value ; onJoystickMove ( ) ; drawJoyMap ( ) ; }
} ;
2026-03-21 23:57:58 +01:00
2026-03-22 10:01:50 +01:00
toggle . addEventListener ( 'click' , ( ) => {
if ( ! c15 . running ) return ;
midiInput . toggle ( ) ;
toggle . textContent = midiInput . enabled ? 'Disable' : 'Enable' ;
toggle . classList . toggle ( 'playing' , midiInput . enabled ) ;
} ) ;
2026-03-21 23:57:58 +01:00
2026-03-22 10:01:50 +01:00
select . addEventListener ( 'change' , ( e ) => {
midiInput . selectInput ( e . target . value || null ) ;
} ) ;
}
2026-03-21 23:57:58 +01:00
2026-04-02 21:35:38 +02:00
// ---- MIDI CC Preset Application ----
async function applyMidiCCPreset ( preset ) {
// Replace the CC map with preset params
midiCCMap . length = 0 ;
midiCCMap . push ( ... preset . params . map ( p => ( { ... p } ) ) ) ;
// Rebuild overrides
midiCCOverrides . length = 0 ;
midiCCOverrides . push ( ... preset . params . map ( p => ( {
min : p . min ? ? 0 , max : p . max ? ? 1 , curve : p . curve ? ? 0.5 ,
frozen : p . muted ? ? false , fixedValue : p . fixedValue ? ? 0.5 ,
} ) ) ) ;
_generateCCColors ( midiCCMap . length ) ;
2026-04-03 18:55:06 +02:00
saveCCMap ( midiCCMap , midiCCStorageKey ( ) ) ;
2026-04-02 21:35:38 +02:00
// Resize MLP if in midi-cc mode
if ( outputMode === 'midi-cc' ) {
await resizeMLP ( midiCCMap . length ) ;
buildHeatmap ( ) ;
updateHeatmap ( iml . getOutputs ( ) ) ;
}
// Update UI
const countEl = document . getElementById ( 'midi-cc-count' ) ;
if ( countEl ) countEl . textContent = midiCCMap . length ;
buildMidiCCParamList ( ) ;
console . log ( ` [NISPS] Applied MIDI CC preset: ${ preset . name } ( ${ preset . params . length } params) ` ) ;
}
2026-03-30 16:05:45 +02:00
// ---- MIDI CC Controls ----
async function initMIDICCControls ( ) {
const enableBtn = document . getElementById ( 'midi-cc-enable-btn' ) ;
const outputSelect = document . getElementById ( 'midi-cc-output-select' ) ;
const statusEl = document . getElementById ( 'midi-cc-status' ) ;
const countEl = document . getElementById ( 'midi-cc-count' ) ;
const addBtn = document . getElementById ( 'midi-cc-add' ) ;
const removeBtn = document . getElementById ( 'midi-cc-remove' ) ;
const importBtn = document . getElementById ( 'midi-cc-import' ) ;
const exportBtn = document . getElementById ( 'midi-cc-export' ) ;
2026-04-02 21:35:38 +02:00
const presetSelect = document . getElementById ( 'midi-cc-preset-select' ) ;
const fileImportBtn = document . getElementById ( 'midi-cc-file-import' ) ;
2026-03-30 16:05:45 +02:00
if ( ! enableBtn || ! outputSelect ) return ;
const available = await midiOutput . init ( ) ;
function populateOutputs ( ) {
const outputs = midiOutput . getOutputs ( ) ;
outputSelect . innerHTML = '<option value="">No device</option>' +
outputs . map ( o => ` <option value=" ${ o . id } "> ${ o . name } </option> ` ) . join ( '' ) ;
}
if ( available ) {
populateOutputs ( ) ;
midiOutput . onOutputsChange = ( ) => populateOutputs ( ) ;
}
midiOutput . onStatusChange = ( msg ) => {
if ( statusEl ) statusEl . textContent = msg ;
} ;
enableBtn . addEventListener ( 'click' , ( ) => {
midiOutput . toggle ( ) ;
enableBtn . classList . toggle ( 'playing' , midiOutput . enabled ) ;
} ) ;
outputSelect . addEventListener ( 'change' , ( e ) => {
midiOutput . selectOutput ( e . target . value || null ) ;
if ( e . target . value ) midiOutput . enable ( ) ;
enableBtn . classList . toggle ( 'playing' , midiOutput . enabled ) ;
} ) ;
// Update count display
function updateCountDisplay ( ) {
if ( countEl ) countEl . textContent = midiCCMap . length ;
}
updateCountDisplay ( ) ;
// Add CC param
if ( addBtn ) {
addBtn . addEventListener ( 'click' , async ( ) => {
midiCCMap . push ( createCCParam ( 74 , 1 ) ) ;
midiCCOverrides . push ( { min : 0 , max : 1 , curve : 0.5 , frozen : false , fixedValue : 0.5 } ) ;
_generateCCColors ( midiCCMap . length ) ;
2026-04-03 18:55:06 +02:00
saveCCMap ( midiCCMap , midiCCStorageKey ( ) ) ;
2026-03-30 16:05:45 +02:00
updateCountDisplay ( ) ;
// Resize MLP if we're in midi-cc mode
if ( outputMode === 'midi-cc' ) {
await resizeMLP ( midiCCMap . length ) ;
buildHeatmap ( ) ;
updateHeatmap ( iml . getOutputs ( ) ) ;
}
} ) ;
}
// Remove last CC param
if ( removeBtn ) {
removeBtn . addEventListener ( 'click' , async ( ) => {
if ( midiCCMap . length <= 1 ) return ;
midiCCMap . pop ( ) ;
midiCCOverrides . pop ( ) ;
_generateCCColors ( midiCCMap . length ) ;
2026-04-03 18:55:06 +02:00
saveCCMap ( midiCCMap , midiCCStorageKey ( ) ) ;
2026-03-30 16:05:45 +02:00
updateCountDisplay ( ) ;
if ( outputMode === 'midi-cc' ) {
await resizeMLP ( midiCCMap . length ) ;
buildHeatmap ( ) ;
updateHeatmap ( iml . getOutputs ( ) ) ;
}
} ) ;
}
// Export
if ( exportBtn ) {
exportBtn . addEventListener ( 'click' , ( ) => {
const json = exportCCMap ( midiCCMap ) ;
navigator . clipboard . writeText ( json ) . then (
( ) => { if ( statusEl ) statusEl . textContent = 'Copied to clipboard' ; } ,
( ) => {
// Fallback: prompt
prompt ( 'Copy this JSON:' , json ) ;
}
) ;
} ) ;
}
// Import
if ( importBtn ) {
importBtn . addEventListener ( 'click' , async ( ) => {
const json = prompt ( 'Paste MIDI CC map JSON:' ) ;
if ( ! json ) return ;
const parsed = importCCMap ( json ) ;
if ( ! parsed ) {
alert ( 'Invalid MIDI CC map JSON' ) ;
return ;
}
midiCCMap . length = 0 ;
midiCCMap . push ( ... parsed ) ;
midiCCOverrides . length = 0 ;
midiCCOverrides . push ( ... parsed . map ( p => ( {
min : p . min ? ? 0 , max : p . max ? ? 1 , curve : p . curve ? ? 0.5 ,
frozen : p . muted ? ? false , fixedValue : p . fixedValue ? ? 0.5 ,
} ) ) ) ;
_generateCCColors ( midiCCMap . length ) ;
2026-04-03 18:55:06 +02:00
saveCCMap ( midiCCMap , midiCCStorageKey ( ) ) ;
2026-03-30 16:05:45 +02:00
updateCountDisplay ( ) ;
if ( outputMode === 'midi-cc' ) {
await resizeMLP ( midiCCMap . length ) ;
buildHeatmap ( ) ;
updateHeatmap ( iml . getOutputs ( ) ) ;
}
} ) ;
}
2026-04-02 21:35:38 +02:00
// Quick preset selector (in quick controls bar)
const quickPresetSelect = document . getElementById ( 'midi-cc-quick-preset' ) ;
if ( quickPresetSelect ) {
quickPresetSelect . innerHTML = '<option value="">Manual</option>' ;
for ( const p of listMidiPresets ( ) ) {
const opt = document . createElement ( 'option' ) ;
opt . value = p . id ;
opt . textContent = p . name ;
quickPresetSelect . appendChild ( opt ) ;
}
quickPresetSelect . addEventListener ( 'change' , async ( ) => {
const id = quickPresetSelect . value ;
if ( ! id ) return ;
try {
const preset = await loadMidiPreset ( id ) ;
await applyMidiCCPreset ( preset ) ;
updateCountDisplay ( ) ;
if ( statusEl ) statusEl . textContent = ` Loaded: ${ preset . name } ` ;
// Sync drawer preset selector
if ( presetSelect ) presetSelect . value = id ;
} catch ( err ) {
console . error ( '[MIDI CC] Failed to load preset:' , err ) ;
}
} ) ;
}
// Preset selector (in drawer)
if ( presetSelect ) {
// Populate with built-in presets
presetSelect . innerHTML = '<option value="">Manual</option>' ;
for ( const p of listMidiPresets ( ) ) {
const opt = document . createElement ( 'option' ) ;
opt . value = p . id ;
opt . textContent = p . name ;
presetSelect . appendChild ( opt ) ;
}
presetSelect . addEventListener ( 'change' , async ( ) => {
const id = presetSelect . value ;
if ( ! id ) return ; // "Manual" selected — keep current config
try {
const preset = await loadMidiPreset ( id ) ;
await applyMidiCCPreset ( preset ) ;
updateCountDisplay ( ) ;
if ( statusEl ) statusEl . textContent = ` Loaded: ${ preset . name } ` ;
// Sync quick preset selector
if ( quickPresetSelect ) quickPresetSelect . value = id ;
} catch ( err ) {
console . error ( '[MIDI CC] Failed to load preset:' , err ) ;
if ( statusEl ) statusEl . textContent = ` Error: ${ err . message } ` ;
}
} ) ;
}
// File import (JSON file from disk)
if ( fileImportBtn ) {
// Create hidden file input
const fileInput = document . createElement ( 'input' ) ;
fileInput . type = 'file' ;
fileInput . accept = '.json' ;
fileInput . style . display = 'none' ;
document . body . appendChild ( fileInput ) ;
fileImportBtn . addEventListener ( 'click' , ( ) => fileInput . click ( ) ) ;
fileInput . addEventListener ( 'change' , async ( ) => {
const file = fileInput . files ? . [ 0 ] ;
if ( ! file ) return ;
const preset = await loadMidiPresetFromFile ( file ) ;
if ( ! preset ) {
alert ( 'Invalid MIDI CC preset file' ) ;
return ;
}
await applyMidiCCPreset ( preset ) ;
updateCountDisplay ( ) ;
if ( statusEl ) statusEl . textContent = ` Loaded: ${ preset . name || file . name } ` ;
// Reset file input so the same file can be re-selected
fileInput . value = '' ;
} ) ;
}
2026-03-30 16:05:45 +02:00
// Build the per-param editor in the drawer
buildMidiCCParamList ( ) ;
}
function buildMidiCCParamList ( ) {
const container = document . getElementById ( 'midi-cc-param-list' ) ;
if ( ! container ) return ;
container . innerHTML = '' ;
for ( let i = 0 ; i < midiCCMap . length ; i ++ ) {
const p = midiCCMap [ i ] ;
const row = document . createElement ( 'div' ) ;
row . className = 'synth-row midi-cc-param-row' ;
row . style . borderLeft = ` 3px solid ${ MIDI _CC _PARAM _COLORS [ i ] || '#888' } ` ;
const nameSpan = document . createElement ( 'span' ) ;
nameSpan . className = 'midi-cc-param-name' ;
nameSpan . textContent = ` ${ p . name } ` ;
nameSpan . style . flex = '1' ;
nameSpan . style . fontSize = '0.75rem' ;
const ccSpan = document . createElement ( 'span' ) ;
ccSpan . style . opacity = '0.6' ;
ccSpan . style . fontSize = '0.65rem' ;
ccSpan . textContent = ` CC ${ p . cc } Ch ${ p . channel } ` ;
row . appendChild ( nameSpan ) ;
row . appendChild ( ccSpan ) ;
container . appendChild ( row ) ;
}
}
2026-03-22 10:01:50 +01:00
// ---- Gamepad ----
function wireGamepad ( ) {
gamepad = new GamepadInput ( {
invertY : true ,
onMove : ( x , y ) => {
joyX = x ;
joyY = y ;
drawJoyMap ( ) ;
onJoystickMove ( ) ;
} ,
onButton : ( btn ) => {
if ( btn === 'lb' ) onThumbsDown ( ) ;
2026-03-22 21:40:15 +01:00
if ( btn === 'rb' ) onThumbsUp ( ) ;
if ( btn === 'a' ) onTrain ( ) ;
if ( btn === 'x' ) onRandomize ( ) ;
if ( btn === 'b' ) onClearExamples ( ) ;
2026-03-22 10:01:50 +01:00
} ,
onConnectionChange : ( connected ) => {
const el = document . getElementById ( 'gamepad-status' ) ;
2026-03-22 21:40:15 +01:00
if ( el ) el . textContent = connected ? 'Gamepad connected' : 'Press any button to activate gamepad' ;
2026-03-22 10:01:50 +01:00
} ,
} ) ;
2026-03-22 21:40:15 +01:00
// Show hint immediately if no gamepad detected yet
if ( ! gamepad . connected ) {
const el = document . getElementById ( 'gamepad-status' ) ;
if ( el ) el . textContent = 'Press any gamepad button to connect' ;
}
2026-03-21 23:57:58 +01:00
}
// ---- Keyboard ----
function wireKeyboard ( ) {
window . addEventListener ( 'keydown' , ( e ) => {
if ( e . repeat ) return ;
2026-03-22 10:52:30 +01:00
// Don't intercept keys when an input/select has focus
const tag = document . activeElement ? . tagName ;
if ( tag === 'INPUT' || tag === 'SELECT' || tag === 'TEXTAREA' ) return ;
2026-03-22 10:01:50 +01:00
if ( e . key === '1' || e . code === 'Numpad1' ) {
e . preventDefault ( ) ;
onThumbsDown ( ) ;
} else if ( e . key === '2' || e . code === 'Numpad2' ) {
e . preventDefault ( ) ;
onThumbsUp ( ) ;
2026-04-03 19:20:52 +02:00
} else if ( e . key === '3' || e . code === 'Numpad3' ) {
e . preventDefault ( ) ;
if ( imlEoc && eocChain ? . nispsMode === 'linked' ) {
const prevTarget = eocTrainingTarget ;
eocTrainingTarget = 'eoc' ;
onThumbsDown ( ) ;
eocTrainingTarget = prevTarget ;
flash ( 'eoc-rl-minus' ) ;
}
} else if ( e . key === '4' || e . code === 'Numpad4' ) {
e . preventDefault ( ) ;
if ( imlEoc && eocChain ? . nispsMode === 'linked' ) {
const prevTarget = eocTrainingTarget ;
eocTrainingTarget = 'eoc' ;
onThumbsUp ( ) ;
eocTrainingTarget = prevTarget ;
flash ( 'eoc-rl-plus' ) ;
}
2026-03-30 16:05:45 +02:00
} else if ( e . key === 'z' || e . key === 'Z' ) {
e . preventDefault ( ) ;
onUndo ( ) ;
2026-03-21 23:57:58 +01:00
}
} ) ;
}
// ---- Quick play controls ----
2026-03-22 23:50:56 +01:00
// ---- Synth Preset Selector ----
2026-04-03 19:26:12 +02:00
/ * *
* Rebuild the < select > dropdown options for the current engine ' s presets .
* Called on engine switch and at init time .
* /
function rebuildPresetSelector ( ) {
const $select = document . getElementById ( 'synth-preset-select' ) ;
if ( ! $select ) return ;
const engineId = activeEngine ? . id ? ? 'shaper-feedback' ;
const presets = getPresetsForEngine ( engineId ) ;
const tiers = getPresetTiersForEngine ( engineId ) ;
// Clear existing options
$select . innerHTML = '' ;
// Manual option
const manualOpt = document . createElement ( 'option' ) ;
manualOpt . value = '' ;
manualOpt . textContent = 'Manual' ;
$select . appendChild ( manualOpt ) ;
// Group by tier
for ( const tier of tiers ) {
const group = document . createElement ( 'optgroup' ) ;
group . label = tier . label ;
for ( const preset of presets . filter ( p => p . tier === tier . tier ) ) {
const opt = document . createElement ( 'option' ) ;
opt . value = preset . id ;
opt . textContent = preset . name ;
group . appendChild ( opt ) ;
}
if ( group . children . length > 0 ) {
$select . appendChild ( group ) ;
}
}
// Sync current selection
if ( activeSynthPresetId && presets . some ( p => p . id === activeSynthPresetId ) ) {
$select . value = activeSynthPresetId ;
} else {
$select . value = '' ;
}
}
2026-03-22 23:50:56 +01:00
function wireSynthPresets ( ) {
const $select = document . getElementById ( 'synth-preset-select' ) ;
if ( ! $select ) return ;
2026-04-03 19:26:12 +02:00
// Build initial options for the active engine
rebuildPresetSelector ( ) ;
2026-03-22 23:50:56 +01:00
$select . addEventListener ( 'change' , ( ) => {
const val = $select . value ;
if ( val === '' ) {
// "Manual" selected — clear preset tracking but don't change overrides
activeSynthPresetId = null ;
saveState ( ) ;
} else {
applyPreset ( val ) ;
}
} ) ;
}
2026-03-21 23:57:58 +01:00
function wireQuickPlayControls ( ) {
const quickPlay = document . getElementById ( 'quick-play' ) ;
const quickPlayIcon = document . getElementById ( 'quick-play-icon' ) ;
const quickVol = document . getElementById ( 'quick-vol' ) ;
const quickBpm = document . getElementById ( 'quick-bpm' ) ;
const quickBpmVal = document . getElementById ( 'quick-bpm-val' ) ;
2026-03-30 16:05:45 +02:00
if ( ! quickPlay || ! quickPlayIcon || ! quickVol || ! quickBpm ) return ;
2026-03-21 23:57:58 +01:00
const playIconSVG = '<path d="M4 2l10 6-10 6z"/>' ;
const pauseIconSVG = '<rect x="3" y="2" width="4" height="12"/><rect x="9" y="2" width="4" height="12"/>' ;
function updatePlayIcon ( ) {
const isPlaying = c15 && c15 . running ;
quickPlayIcon . innerHTML = isPlaying ? pauseIconSVG : playIconSVG ;
quickPlay . classList . toggle ( 'playing' , isPlaying ) ;
2026-03-22 20:57:21 +01:00
quickPlay . classList . toggle ( 'audio-needs-init' , ! isPlaying ) ;
2026-03-21 23:57:58 +01:00
}
quickPlay . addEventListener ( 'click' , async ( ) => {
if ( c15 . running ) {
2026-04-03 20:01:14 +02:00
if ( shapeSeqEnabled && shapeSeqEngine ) shapeSeqEngine . stop ( ) ;
else arpeggiator . stop ( ) ;
2026-03-21 23:57:58 +01:00
await c15 . stop ( ) ;
// Also update the bottom sheet controls
const startBtn = document . getElementById ( 'synth-start' ) ;
2026-03-24 10:54:13 +01:00
const arpToggle = document . getElementById ( 'arp-toggle' ) ;
2026-03-21 23:57:58 +01:00
if ( startBtn ) startBtn . textContent = 'Start Audio' ;
2026-03-24 10:54:13 +01:00
if ( arpToggle ) arpToggle . textContent = 'Play' ;
2026-03-21 23:57:58 +01:00
} else {
2026-04-03 20:01:14 +02:00
await activeEngine . init ( ) ;
2026-04-03 18:49:07 +02:00
await _startEocChain ( ) ;
2026-04-03 20:01:14 +02:00
if ( shapeSeqEnabled ) {
await _ensureShapeSeqInit ( ) ;
if ( shapeSeqEngine ) shapeSeqEngine . start ( ) ;
} else {
arpeggiator . start ( ) ;
}
2026-03-21 23:57:58 +01:00
routeOutputs ( iml . getOutputs ( ) ) ;
// Also update the bottom sheet controls
const startBtn = document . getElementById ( 'synth-start' ) ;
2026-03-24 10:54:13 +01:00
const arpToggle = document . getElementById ( 'arp-toggle' ) ;
2026-03-21 23:57:58 +01:00
if ( startBtn ) startBtn . textContent = 'Stop Audio' ;
2026-03-24 10:54:13 +01:00
if ( arpToggle ) arpToggle . textContent = 'Stop' ;
2026-03-21 23:57:58 +01:00
}
updatePlayIcon ( ) ;
} ) ;
quickVol . addEventListener ( 'input' , ( e ) => {
c15 . setMasterVolume ( parseFloat ( e . target . value ) ) ;
// Sync with bottom sheet volume slider
const sheetVol = document . getElementById ( 'synth-volume' ) ;
if ( sheetVol ) sheetVol . value = e . target . value ;
} ) ;
quickBpm . addEventListener ( 'input' , ( e ) => {
const val = parseInt ( e . target . value ) ;
2026-03-24 10:54:13 +01:00
arpeggiator . bpm = val ;
2026-03-21 23:57:58 +01:00
quickBpmVal . textContent = val ;
// Sync with bottom sheet tempo slider
2026-03-24 10:54:13 +01:00
const sheetTempo = document . getElementById ( 'arp-tempo' ) ;
2026-03-21 23:57:58 +01:00
const sheetTempoVal = document . getElementById ( 'tempo-val' ) ;
if ( sheetTempo ) sheetTempo . value = val ;
if ( sheetTempoVal ) sheetTempoVal . textContent = val ;
} ) ;
}
2026-03-22 20:57:21 +01:00
// ---- Group Override Drawer ----
let $groupDrawer = null ;
let activeDrawerSection = - 1 ;
let drawerHideTimer = null ;
2026-03-30 16:05:45 +02:00
// ---- Heatmap param popup ----
let $paramPopup = null ;
let activePopupParam = - 1 ;
let popupHideTimer = null ;
2026-03-22 20:57:21 +01:00
function wireGroupDrawer ( ) {
// Create the drawer DOM element once
$groupDrawer = document . createElement ( 'div' ) ;
$groupDrawer . className = 'group-drawer' ;
$groupDrawer . innerHTML = '<div class="group-drawer-header"></div><div class="group-drawer-body"></div>' ;
document . body . appendChild ( $groupDrawer ) ;
// Keep drawer open while hovering over it
$groupDrawer . addEventListener ( 'pointerenter' , ( ) => {
clearTimeout ( drawerHideTimer ) ;
} ) ;
$groupDrawer . addEventListener ( 'pointerleave' , ( ) => {
drawerHideTimer = setTimeout ( ( ) => hideGroupDrawer ( ) , 300 ) ;
} ) ;
// Detect hover over section labels on the synth vis canvas
$synthVisCanvas . addEventListener ( 'pointermove' , ( e ) => {
if ( outputMode !== 'synth' ) return ;
const region = synthVisualizer . hitTestSection ( e . clientX , e . clientY ) ;
if ( region ) {
clearTimeout ( drawerHideTimer ) ;
if ( activeDrawerSection !== region . index ) {
showGroupDrawer ( region ) ;
}
} else {
// Leaving section label area, delay hide
if ( activeDrawerSection >= 0 ) {
drawerHideTimer = setTimeout ( ( ) => hideGroupDrawer ( ) , 300 ) ;
}
}
} ) ;
// Also handle click for mobile
$synthVisCanvas . addEventListener ( 'pointerdown' , ( e ) => {
if ( outputMode !== 'synth' ) return ;
const region = synthVisualizer . hitTestSection ( e . clientX , e . clientY ) ;
if ( region ) {
e . preventDefault ( ) ;
e . stopPropagation ( ) ;
clearTimeout ( drawerHideTimer ) ;
if ( activeDrawerSection === region . index ) {
hideGroupDrawer ( ) ;
} else {
showGroupDrawer ( region ) ;
}
}
} , true ) ; // capture phase so it fires before bar interaction
}
2026-04-11 08:31:19 +02:00
/ * *
* Uniform view of a "section" ( aka param group ) for the active engine .
* C15 gets the static SYNTH _SECTIONS + groupOverrides + SYNTH _PARAM _MAP view ;
* non - C15 gets a view derived from paramMeta + engineParamOverrides +
* nonC15Sections / nonC15GroupCurves .
*
* Returns { name , color , count , startIndex , getCurve , setCurve ,
* getParamName , getParamOverride } or null if the section doesn ' t
* exist for the active engine .
* /
function getSectionView ( sectionIndex ) {
if ( activeEngine ? . id === 'shaper-feedback' ) {
const sec = SYNTH _SECTIONS [ sectionIndex ] ;
const ov = groupOverrides [ sectionIndex ] ;
if ( ! sec || ! ov ) return null ;
let start = 0 ;
for ( let i = 0 ; i < sectionIndex ; i ++ ) start += SYNTH _SECTIONS [ i ] . count ;
return {
name : sec . name ,
color : sec . color ,
count : sec . count ,
startIndex : start ,
getCurve : ( ) => ov . curve ,
setCurve : ( v ) => { ov . curve = v ; } ,
getParamName : ( li ) => SYNTH _PARAM _MAP [ start + li ] ? . label ? ? ` p ${ start + li } ` ,
getParamOverride : ( li ) => ov . params [ li ] ,
} ;
}
// Non-C15 engines
const sec = nonC15Sections [ sectionIndex ] ;
if ( ! sec || ! engineParamOverrides ) return null ;
const start = sec . startIndex ;
return {
name : sec . name ,
color : sec . color ,
count : sec . count ,
startIndex : start ,
getCurve : ( ) => nonC15GroupCurves [ sectionIndex ] ? ? 0.5 ,
setCurve : ( v ) => {
nonC15GroupCurves [ sectionIndex ] = v ;
_nonC15GroupCurveMemory . set ( sec . name , v ) ;
} ,
getParamName : ( li ) => activeEngine ? . paramMeta ? . [ start + li ] ? . name ? ? ` p ${ start + li } ` ,
getParamOverride : ( li ) => engineParamOverrides [ start + li ] ,
} ;
}
2026-03-22 20:57:21 +01:00
function showGroupDrawer ( region ) {
2026-04-11 08:31:19 +02:00
const view = getSectionView ( region . index ) ;
if ( ! view ) return ;
2026-03-22 20:57:21 +01:00
activeDrawerSection = region . index ;
// Header with section name
const header = $groupDrawer . querySelector ( '.group-drawer-header' ) ;
2026-04-11 08:31:19 +02:00
header . textContent = view . name ;
header . style . color = view . color ;
2026-03-22 20:57:21 +01:00
// Body: group curve + per-param rows
const body = $groupDrawer . querySelector ( '.group-drawer-body' ) ;
body . innerHTML = '' ;
// -- Group master curve (draggable canvas) --
const curveRow = document . createElement ( 'div' ) ;
curveRow . className = 'gd-curve-row' ;
const curveLabel = document . createElement ( 'span' ) ;
curveLabel . className = 'gd-label' ;
curveLabel . textContent = 'Group' ;
const curveCanvas = document . createElement ( 'canvas' ) ;
curveCanvas . className = 'gd-curve-canvas' ;
curveCanvas . width = 48 ;
curveCanvas . height = 48 ;
const curveVal = document . createElement ( 'span' ) ;
curveVal . className = 'gd-val' ;
2026-04-11 08:31:19 +02:00
curveVal . textContent = view . getCurve ( ) . toFixed ( 2 ) ;
2026-03-22 20:57:21 +01:00
function drawGroupCurvePreview ( ) {
2026-04-11 08:31:19 +02:00
_drawCurveOnCanvas ( curveCanvas , view . getCurve ( ) , view . color ) ;
2026-03-22 20:57:21 +01:00
}
// Vertical drag on group curve — applies relative delta to all param curves
{
let dragging = false , startY = 0 , startGroupCurve = 0 , startParamCurves = [ ] ;
curveCanvas . style . cursor = 'ns-resize' ;
curveCanvas . style . touchAction = 'none' ;
curveCanvas . addEventListener ( 'pointerdown' , ( e ) => {
e . preventDefault ( ) ; e . stopPropagation ( ) ;
dragging = true ;
startY = e . clientY ;
2026-04-11 08:31:19 +02:00
startGroupCurve = view . getCurve ( ) ;
startParamCurves = [ ] ;
for ( let i = 0 ; i < view . count ; i ++ ) {
startParamCurves . push ( view . getParamOverride ( i ) ? . curve ? ? 0.5 ) ;
}
2026-03-22 20:57:21 +01:00
curveCanvas . setPointerCapture ( e . pointerId ) ;
} ) ;
curveCanvas . addEventListener ( 'pointermove' , ( e ) => {
if ( ! dragging ) return ;
e . preventDefault ( ) ;
const dy = e . clientY - startY ;
const delta = dy / 80 ;
const newGroup = Math . max ( 0 , Math . min ( 1 , startGroupCurve + delta ) ) ;
2026-04-11 08:31:19 +02:00
view . setCurve ( newGroup ) ;
2026-03-22 20:57:21 +01:00
curveVal . textContent = newGroup . toFixed ( 2 ) ;
// Apply same delta to each param, preserving relative offsets
2026-04-11 08:31:19 +02:00
for ( let i = 0 ; i < view . count ; i ++ ) {
const pov = view . getParamOverride ( i ) ;
if ( pov ) pov . curve = Math . max ( 0 , Math . min ( 1 , startParamCurves [ i ] + delta ) ) ;
2026-03-22 20:57:21 +01:00
}
drawGroupCurvePreview ( ) ;
body . querySelectorAll ( '.gd-param-curve-canvas' ) . forEach ( c => {
if ( c . _redraw ) c . _redraw ( ) ;
} ) ;
routeOutputs ( iml . getOutputs ( ) ) ;
} ) ;
curveCanvas . addEventListener ( 'pointerup' , ( ) => { dragging = false ; } ) ;
curveCanvas . addEventListener ( 'pointercancel' , ( ) => { dragging = false ; } ) ;
}
curveRow . appendChild ( curveLabel ) ;
curveRow . appendChild ( curveCanvas ) ;
curveRow . appendChild ( curveVal ) ;
body . appendChild ( curveRow ) ;
drawGroupCurvePreview ( ) ;
// -- Per-param rows --
2026-04-11 08:31:19 +02:00
for ( let li = 0 ; li < view . count ; li ++ ) {
const pov = view . getParamOverride ( li ) ;
if ( ! pov ) continue ;
const paramName = view . getParamName ( li ) ;
2026-03-22 20:57:21 +01:00
const row = document . createElement ( 'div' ) ;
row . className = 'gd-param-row' ;
if ( pov . muted ) row . classList . add ( 'gd-muted' ) ;
// Name
const nameSpan = document . createElement ( 'span' ) ;
nameSpan . className = 'gd-param-name' ;
2026-04-11 08:31:19 +02:00
nameSpan . textContent = paramName ;
2026-03-22 20:57:21 +01:00
// Per-param curve canvas (vertically draggable, no slider)
const pCurveCanvas = document . createElement ( 'canvas' ) ;
pCurveCanvas . className = 'gd-param-curve-canvas' ;
pCurveCanvas . width = 28 ;
pCurveCanvas . height = 28 ;
2026-04-11 08:31:19 +02:00
pCurveCanvas . _redraw = ( ) => _drawCurveOnCanvas ( pCurveCanvas , pov . curve , view . color ) ;
2026-03-22 20:57:21 +01:00
_wireCurveDrag ( pCurveCanvas , ( ) => pov . curve , ( v ) => {
pov . curve = v ;
pCurveCanvas . _redraw ( ) ;
routeOutputs ( iml . getOutputs ( ) ) ;
} ) ;
pCurveCanvas . _redraw ( ) ;
// Dual-range slider (min/max as two overlapping range inputs)
const rangeWrap = document . createElement ( 'div' ) ;
rangeWrap . className = 'gd-range-wrap' ;
const rangeFill = document . createElement ( 'div' ) ;
rangeFill . className = 'gd-range-fill' ;
const minSlider = document . createElement ( 'input' ) ;
minSlider . type = 'range' ; minSlider . min = '0' ; minSlider . max = '1' ; minSlider . step = '0.01' ;
minSlider . value = pov . min ;
minSlider . className = 'gd-range-input gd-range-min' ;
const maxSlider = document . createElement ( 'input' ) ;
maxSlider . type = 'range' ; maxSlider . min = '0' ; maxSlider . max = '1' ; maxSlider . step = '0.01' ;
maxSlider . value = pov . max ;
maxSlider . className = 'gd-range-input gd-range-max' ;
// Value slider (shown when muted)
const valSlider = document . createElement ( 'input' ) ;
valSlider . type = 'range' ; valSlider . min = '0' ; valSlider . max = '1' ; valSlider . step = '0.01' ;
valSlider . value = pov . fixedValue ;
valSlider . className = 'gd-val-slider' ;
function updateRangeFill ( ) {
rangeFill . style . left = ` ${ pov . min * 100 } % ` ;
rangeFill . style . width = ` ${ ( pov . max - pov . min ) * 100 } % ` ;
}
updateRangeFill ( ) ;
minSlider . addEventListener ( 'input' , ( ) => {
pov . min = parseFloat ( minSlider . value ) ;
if ( pov . min > pov . max ) { pov . max = pov . min ; maxSlider . value = pov . max ; }
updateRangeFill ( ) ;
routeOutputs ( iml . getOutputs ( ) ) ;
} ) ;
maxSlider . addEventListener ( 'input' , ( ) => {
pov . max = parseFloat ( maxSlider . value ) ;
if ( pov . max < pov . min ) { pov . min = pov . max ; minSlider . value = pov . min ; }
updateRangeFill ( ) ;
routeOutputs ( iml . getOutputs ( ) ) ;
} ) ;
valSlider . addEventListener ( 'input' , ( ) => {
pov . fixedValue = parseFloat ( valSlider . value ) ;
routeOutputs ( iml . getOutputs ( ) ) ;
} ) ;
rangeWrap . appendChild ( rangeFill ) ;
rangeWrap . appendChild ( minSlider ) ;
rangeWrap . appendChild ( maxSlider ) ;
// Mute toggle
const muteBtn = document . createElement ( 'button' ) ;
muteBtn . className = 'gd-mute-btn' + ( pov . muted ? ' muted' : '' ) ;
muteBtn . textContent = pov . muted ? 'M' : 'M' ;
muteBtn . title = pov . muted ? 'Unmute (re-enable NISPS control)' : 'Mute (remove from NISPS)' ;
muteBtn . addEventListener ( 'click' , ( ) => {
pov . muted = ! pov . muted ;
muteBtn . classList . toggle ( 'muted' , pov . muted ) ;
muteBtn . title = pov . muted ? 'Unmute (re-enable NISPS control)' : 'Mute (remove from NISPS)' ;
row . classList . toggle ( 'gd-muted' , pov . muted ) ;
routeOutputs ( iml . getOutputs ( ) ) ;
} ) ;
row . appendChild ( nameSpan ) ;
row . appendChild ( pCurveCanvas ) ;
row . appendChild ( rangeWrap ) ;
row . appendChild ( valSlider ) ;
row . appendChild ( muteBtn ) ;
body . appendChild ( row ) ;
}
// Position the drawer below the section label
const canvasRect = $synthVisCanvas . getBoundingClientRect ( ) ;
const centerX = ( region . left + region . right ) / 2 + canvasRect . left ;
const topY = region . bottom + canvasRect . top + 4 ;
const drawerWidth = 320 ;
let left = centerX - drawerWidth / 2 ;
left = Math . max ( 4 , Math . min ( left , window . innerWidth - drawerWidth - 4 ) ) ;
$groupDrawer . style . left = ` ${ left } px ` ;
$groupDrawer . style . top = ` ${ topY } px ` ;
$groupDrawer . classList . add ( 'visible' ) ;
}
/** Draw a curve preview on a canvas element */
function _drawCurveOnCanvas ( canvas , curveFactor , color ) {
const ctx = canvas . getContext ( '2d' ) ;
const w = canvas . width ;
const h = canvas . height ;
ctx . clearRect ( 0 , 0 , w , h ) ;
ctx . fillStyle = 'rgba(255,255,255,0.03)' ;
ctx . fillRect ( 0 , 0 , w , h ) ;
// Linear reference
ctx . strokeStyle = 'rgba(255,255,255,0.1)' ;
ctx . lineWidth = 1 ;
ctx . beginPath ( ) ;
ctx . moveTo ( 0 , h ) ;
ctx . lineTo ( w , 0 ) ;
ctx . stroke ( ) ;
// Curve
ctx . strokeStyle = color ;
ctx . lineWidth = Math . min ( 2 , w / 16 ) ;
ctx . beginPath ( ) ;
const steps = 30 ;
for ( let i = 0 ; i <= steps ; i ++ ) {
const t = i / steps ;
const v = applyCurve ( t , curveFactor ) ;
const px = t * w ;
const py = ( 1 - v ) * h ;
if ( i === 0 ) ctx . moveTo ( px , py ) ;
else ctx . lineTo ( px , py ) ;
}
ctx . stroke ( ) ;
}
/** Wire vertical drag on a canvas to control a curve factor */
function _wireCurveDrag ( canvas , getter , setter ) {
let dragging = false ;
let startY = 0 ;
let startVal = 0 ;
canvas . style . cursor = 'ns-resize' ;
canvas . style . touchAction = 'none' ;
canvas . addEventListener ( 'pointerdown' , ( e ) => {
e . preventDefault ( ) ;
e . stopPropagation ( ) ;
dragging = true ;
startY = e . clientY ;
startVal = getter ( ) ;
canvas . setPointerCapture ( e . pointerId ) ;
} ) ;
canvas . addEventListener ( 'pointermove' , ( e ) => {
if ( ! dragging ) return ;
e . preventDefault ( ) ;
// Drag down = more exponential (higher curve), drag up = more logarithmic (lower curve)
const dy = e . clientY - startY ;
const newVal = Math . max ( 0 , Math . min ( 1 , startVal + dy / 80 ) ) ;
setter ( newVal ) ;
} ) ;
canvas . addEventListener ( 'pointerup' , ( ) => { dragging = false ; } ) ;
canvas . addEventListener ( 'pointercancel' , ( ) => { dragging = false ; } ) ;
}
function hideGroupDrawer ( ) {
activeDrawerSection = - 1 ;
if ( $groupDrawer ) $groupDrawer . classList . remove ( 'visible' ) ;
}
2026-03-21 23:57:58 +01:00
// ---- Resize ----
function onResize ( ) {
2026-03-22 20:57:21 +01:00
hideGroupDrawer ( ) ;
2026-03-30 16:05:45 +02:00
hideParamPopup ( ) ;
2026-03-21 23:57:58 +01:00
visualizer . resize ( ) ;
visualizer . initParticles ( ) ;
synthVisualizer . resize ( ) ;
// Resize joy-map canvas to match container
const size = $joystickContainer . offsetWidth ;
if ( size > 0 ) {
$joyMap . width = size ;
$joyMap . height = size ;
drawJoyMap ( ) ;
}
}
2026-03-22 21:41:49 +01:00
// ---- Help modal ----
function wireHelp ( ) {
const overlay = document . getElementById ( 'help-overlay' ) ;
const btnClose = document . getElementById ( 'help-close' ) ;
const btnGotIt = document . getElementById ( 'help-got-it' ) ;
function hide ( ) { overlay . classList . add ( 'hidden' ) ; localStorage . setItem ( 'nisps-help-seen' , '1' ) ; }
2026-03-30 16:05:45 +02:00
// Help is opened via dock icon (handled in wireDock)
2026-03-22 21:41:49 +01:00
btnClose . addEventListener ( 'click' , hide ) ;
btnGotIt . addEventListener ( 'click' , hide ) ;
overlay . addEventListener ( 'click' , ( e ) => { if ( e . target === overlay ) hide ( ) ; } ) ;
// Show on first visit
2026-03-30 16:05:45 +02:00
if ( ! localStorage . getItem ( 'nisps-help-seen' ) ) overlay . classList . remove ( 'hidden' ) ;
2026-03-22 21:41:49 +01:00
}
2026-03-21 23:57:58 +01:00
// ---- Animation ----
function animate ( ) {
2026-03-22 10:01:50 +01:00
if ( gamepad ) gamepad . poll ( ) ;
2026-03-21 23:57:58 +01:00
if ( outputMode === 'synth' ) {
synthVisualizer . draw ( ) ;
2026-03-30 16:05:45 +02:00
} else if ( outputMode === 'visual' ) {
2026-03-21 23:57:58 +01:00
visualizer . draw ( ) ;
}
requestAnimationFrame ( animate ) ;
}
// ---- Utility ----
function flash ( id ) {
const el = document . getElementById ( id ) ;
if ( ! el ) return ;
el . classList . add ( 'flash' ) ;
setTimeout ( ( ) => el . classList . remove ( 'flash' ) , 250 ) ;
}
2026-04-03 19:21:59 +02:00
// ---- EOC Linked/Independent-mode IML helpers ----
2026-04-03 19:03:12 +02:00
/ * *
2026-04-03 19:21:59 +02:00
* Create ( or recreate ) the EOC IML instance for Linked / Independent mode .
2026-04-03 19:03:12 +02:00
* Uses a smaller network than the synth IML since EOC params are more independent .
* /
async function initEocIML ( ) {
if ( imlEoc ) { imlEoc . destroy ( ) ; imlEoc = null ; }
if ( ! eocChain || eocChain . paramCount === 0 ) return ;
imlEoc = await WasmIML . create (
N _JOY _INPUTS ,
eocChain . paramCount ,
[ 16 , 32 ] , // smaller network — EOC params are more independent
1000 , 1.0 , 0.00001 ,
) ;
imlEoc . randomiseWeights ( spreadLevel ) ;
2026-04-03 19:21:59 +02:00
console . log ( ` [EOC] IML created ( ${ eocChain . nispsMode } ) — ${ eocChain . paramCount } outputs ` ) ;
2026-04-03 19:03:12 +02:00
}
2026-04-03 19:21:59 +02:00
/** Destroy the EOC IML instance (e.g. when leaving Linked/Independent mode). */
2026-04-03 19:03:12 +02:00
function destroyEocIML ( ) {
if ( imlEoc ) {
imlEoc . destroy ( ) ;
imlEoc = null ;
console . log ( '[EOC] Linked IML destroyed' ) ;
}
}
2026-03-21 23:57:58 +01:00
// ---- Persistence ----
function saveState ( ) {
try {
const state = {
feat(playground): add MediaPipe hand tracking input with dev panel
Hand tracking via webcam as alternative to joystick input:
- Right hand tracks 14 derived features (palm XY, finger curls, spread,
roll, pitch, pinch) through a separate 14-input MLP
- Left hand gesture recognition (1 finger = thumbs up, 2 = thumbs down)
with 400ms debounce hold
- Split-zone PIP display (no camera feed, skeleton only) with dashed
divider and cross-zone dimming
- Dual IML architecture: independent imlJoy (2 inputs) and imlHand
(14 inputs) with pointer swap, preserving training data per mode
- Dev panel (?devmode=true): draggable/collapsible floating panel with
sliders for MediaPipe confidence thresholds, smoothing, gesture hold
time, world landmarks toggle, and live feature bar monitor
- Visual presets always route to joystick IML (prevents dimension mismatch)
- Race condition guard on input mode switching
2026-03-23 22:28:58 +01:00
features : imlJoy . dataset . features ,
labels : imlJoy . dataset . labels ,
handFeatures : imlHand . dataset . features ,
handLabels : imlHand . dataset . labels ,
2026-03-21 23:57:58 +01:00
noiseLevel ,
outputMode ,
feat(playground): add MediaPipe hand tracking input with dev panel
Hand tracking via webcam as alternative to joystick input:
- Right hand tracks 14 derived features (palm XY, finger curls, spread,
roll, pitch, pinch) through a separate 14-input MLP
- Left hand gesture recognition (1 finger = thumbs up, 2 = thumbs down)
with 400ms debounce hold
- Split-zone PIP display (no camera feed, skeleton only) with dashed
divider and cross-zone dimming
- Dual IML architecture: independent imlJoy (2 inputs) and imlHand
(14 inputs) with pointer swap, preserving training data per mode
- Dev panel (?devmode=true): draggable/collapsible floating panel with
sliders for MediaPipe confidence thresholds, smoothing, gesture hold
time, world landmarks toggle, and live feature bar monitor
- Visual presets always route to joystick IML (prevents dimension mismatch)
- Race condition guard on input mode switching
2026-03-23 22:28:58 +01:00
inputMode ,
2026-03-21 23:57:58 +01:00
joyX ,
joyY ,
2026-03-22 20:57:21 +01:00
groupOverrides ,
2026-03-30 16:05:45 +02:00
visualOverrides ,
midiCCOverrides ,
2026-04-02 23:12:18 +02:00
audioCanvasState : audioCanvas ? audioCanvas . getState ( ) : null ,
2026-03-22 23:50:56 +01:00
synthPresetId : activeSynthPresetId ,
2026-04-03 18:49:07 +02:00
engineId : activeEngine ? activeEngine . id : 'shaper-feedback' ,
// EOC state
eocModules : eocChain ? eocChain . modules . map ( m => ( {
id : m . id ,
enabled : m . enabled ,
params : m . paramMeta . map ( ( _ , i ) => m . getCurrentParamValue ( i ) ) ,
} ) ) : [ ] ,
eocNispsMode : eocChain ? eocChain . nispsMode : 'bypass' ,
2026-04-03 19:03:12 +02:00
eocTrainingTarget ,
feat(playground): add Modular audio mode with shared mod pool
New "Modular" engine in a-immersive with three hot-swappable Faust
sub-engines (subtractive/additive/fm) sharing a common modulation pool:
16 ADSR slots + 32 LFO slots (single-knob sine->tri->square->saw
wavemorph) routed through a 48-source x 10-destination matrix per
engine. Per-connection scalar amounts in [-1, 1], summed at each
destination. Default MLP output count is 512 (32 mod-source params +
480 matrix cells); model reinits on sub-engine swap, count change, or
engine-param exposure toggle.
Faust layer:
- mod-pool.lib: shared ADSR/LFO/source-bus library
- gen-modular-dsp.py: byte-reproducible generator (source of truth)
- modular-subtractive: faithful Minimoog (3 osc, ladder filter, no envs)
- modular-additive: 64-partial, spectral shape + formants, no envs/LFOs
- modular-fm: 4-op matrix + self-feedback, no envs
- All three share d08=amp, d09=pan conventions
- MODULAR_DESTINATIONS.md: authoritative destination table
JS layer:
- ModularEngine: self-contained SynthEngine with getState/setState,
setSubEngine, setModSourceCount, setExposeEngineParam
- modular-ui: drawer with sub-engine toggle, ADSR/LFO count steppers,
per-slot enable switches, matrix grid editor (tap-cycle, long-press
precise, right-click menu, negative amounts), preset overlay
- modular-presets: 6 named presets (Slow pad, Plucky bass, Crystal,
DX bell, Morphing drone + default)
- a-app.js: Modular mode registered, paramMeta:change -> resizeMLP,
modular DSP state persisted under modularDspState, window.__nisps
debug hooks for programmatic control
Tests: tests/e2e/modular-mode.spec.js (11 Playwright tests, all passing
including DSP state survives reload, sub-engine swap keeps paramCount,
preset apply verification).
Also fixes a pre-existing build.sh bug where the -e flag caused faust
to overwrite .wasm outputs with expanded DSP source text, leaving
additive/fm-matrix/eoc-* committed as invalid WebAssembly. Rebuilt all
affected engines with the corrected script. Added an early-message
buffer to faust-worklet-processor.js so setParam calls arriving before
wasm instantiation are queued rather than dropped (needed when the user
configures modular state before clicking Start Audio).
2026-04-11 07:34:41 +02:00
// Phase E — full modular DSP snapshot. Only stored when the active
// engine is the modular engine. Contains raw per-label values; the
// UI-facing state (sub-engine choice, counts, enables, exposed params)
// lives in a separate localStorage key 'nisps-modular-state' owned by
// modular-ui.js. On load we apply the UI state first (sub-engine swap,
// counts) and then this dsp dict overwrites any default values.
modularDspState : ( activeEngine && activeEngine . id === 'modular' &&
typeof activeEngine . getState === 'function' )
? ( ( ) => {
const s = activeEngine . getState ( ) ;
// Strip fields already owned by Phase C's UI state key to keep a
// single source of truth per field.
return { version : s . version , dsp : s . dsp } ;
} ) ( )
: null ,
2026-03-21 23:57:58 +01:00
} ;
localStorage . setItem ( STORAGE _KEY , JSON . stringify ( state ) ) ;
} catch ( e ) {
console . warn ( '[NISPS] Failed to save state:' , e ) ;
}
}
2026-03-30 16:05:45 +02:00
async function loadState ( ) {
2026-03-21 23:57:58 +01:00
try {
const raw = localStorage . getItem ( STORAGE _KEY ) ;
if ( ! raw ) return ;
const state = JSON . parse ( raw ) ;
feat(playground): add MediaPipe hand tracking input with dev panel
Hand tracking via webcam as alternative to joystick input:
- Right hand tracks 14 derived features (palm XY, finger curls, spread,
roll, pitch, pinch) through a separate 14-input MLP
- Left hand gesture recognition (1 finger = thumbs up, 2 = thumbs down)
with 400ms debounce hold
- Split-zone PIP display (no camera feed, skeleton only) with dashed
divider and cross-zone dimming
- Dual IML architecture: independent imlJoy (2 inputs) and imlHand
(14 inputs) with pointer swap, preserving training data per mode
- Dev panel (?devmode=true): draggable/collapsible floating panel with
sliders for MediaPipe confidence thresholds, smoothing, gesture hold
time, world landmarks toggle, and live feature bar monitor
- Visual presets always route to joystick IML (prevents dimension mismatch)
- Race condition guard on input mode switching
2026-03-23 22:28:58 +01:00
// Restore joystick IML training data (pad old 20-element labels to N_OUTPUTS)
2026-03-21 23:57:58 +01:00
if ( state . features && state . labels && state . features . length > 0 ) {
for ( let i = 0 ; i < state . features . length ; i ++ ) {
feat(playground): add MediaPipe hand tracking input with dev panel
Hand tracking via webcam as alternative to joystick input:
- Right hand tracks 14 derived features (palm XY, finger curls, spread,
roll, pitch, pinch) through a separate 14-input MLP
- Left hand gesture recognition (1 finger = thumbs up, 2 = thumbs down)
with 400ms debounce hold
- Split-zone PIP display (no camera feed, skeleton only) with dashed
divider and cross-zone dimming
- Dual IML architecture: independent imlJoy (2 inputs) and imlHand
(14 inputs) with pointer swap, preserving training data per mode
- Dev panel (?devmode=true): draggable/collapsible floating panel with
sliders for MediaPipe confidence thresholds, smoothing, gesture hold
time, world landmarks toggle, and live feature bar monitor
- Visual presets always route to joystick IML (prevents dimension mismatch)
- Race condition guard on input mode switching
2026-03-23 22:28:58 +01:00
imlJoy . addExample ( state . features [ i ] , padPresetOutputs ( state . labels [ i ] ) ) ;
2026-03-21 23:57:58 +01:00
}
feat(playground): add MediaPipe hand tracking input with dev panel
Hand tracking via webcam as alternative to joystick input:
- Right hand tracks 14 derived features (palm XY, finger curls, spread,
roll, pitch, pinch) through a separate 14-input MLP
- Left hand gesture recognition (1 finger = thumbs up, 2 = thumbs down)
with 400ms debounce hold
- Split-zone PIP display (no camera feed, skeleton only) with dashed
divider and cross-zone dimming
- Dual IML architecture: independent imlJoy (2 inputs) and imlHand
(14 inputs) with pointer swap, preserving training data per mode
- Dev panel (?devmode=true): draggable/collapsible floating panel with
sliders for MediaPipe confidence thresholds, smoothing, gesture hold
time, world landmarks toggle, and live feature bar monitor
- Visual presets always route to joystick IML (prevents dimension mismatch)
- Race condition guard on input mode switching
2026-03-23 22:28:58 +01:00
const prevIml = iml ;
iml = imlJoy ;
2026-03-21 23:57:58 +01:00
trainModel ( ) ;
feat(playground): add MediaPipe hand tracking input with dev panel
Hand tracking via webcam as alternative to joystick input:
- Right hand tracks 14 derived features (palm XY, finger curls, spread,
roll, pitch, pinch) through a separate 14-input MLP
- Left hand gesture recognition (1 finger = thumbs up, 2 = thumbs down)
with 400ms debounce hold
- Split-zone PIP display (no camera feed, skeleton only) with dashed
divider and cross-zone dimming
- Dual IML architecture: independent imlJoy (2 inputs) and imlHand
(14 inputs) with pointer swap, preserving training data per mode
- Dev panel (?devmode=true): draggable/collapsible floating panel with
sliders for MediaPipe confidence thresholds, smoothing, gesture hold
time, world landmarks toggle, and live feature bar monitor
- Visual presets always route to joystick IML (prevents dimension mismatch)
- Race condition guard on input mode switching
2026-03-23 22:28:58 +01:00
iml = prevIml ;
}
// Restore hand IML training data
if ( state . handFeatures && state . handLabels && state . handFeatures . length > 0 ) {
for ( let i = 0 ; i < state . handFeatures . length ; i ++ ) {
imlHand . addExample ( state . handFeatures [ i ] , padPresetOutputs ( state . handLabels [ i ] ) ) ;
}
const prevIml = iml ;
iml = imlHand ;
trainModel ( ) ;
iml = prevIml ;
2026-03-21 23:57:58 +01:00
}
feat(playground): add MediaPipe hand tracking input with dev panel
Hand tracking via webcam as alternative to joystick input:
- Right hand tracks 14 derived features (palm XY, finger curls, spread,
roll, pitch, pinch) through a separate 14-input MLP
- Left hand gesture recognition (1 finger = thumbs up, 2 = thumbs down)
with 400ms debounce hold
- Split-zone PIP display (no camera feed, skeleton only) with dashed
divider and cross-zone dimming
- Dual IML architecture: independent imlJoy (2 inputs) and imlHand
(14 inputs) with pointer swap, preserving training data per mode
- Dev panel (?devmode=true): draggable/collapsible floating panel with
sliders for MediaPipe confidence thresholds, smoothing, gesture hold
time, world landmarks toggle, and live feature bar monitor
- Visual presets always route to joystick IML (prevents dimension mismatch)
- Race condition guard on input mode switching
2026-03-23 22:28:58 +01:00
routeOutputs ( iml . getOutputs ( ) ) ;
2026-03-21 23:57:58 +01:00
if ( typeof state . noiseLevel === 'number' ) noiseLevel = state . noiseLevel ;
if ( typeof state . joyX === 'number' ) joyX = state . joyX ;
if ( typeof state . joyY === 'number' ) joyY = state . joyY ;
2026-03-22 20:57:21 +01:00
// Restore group overrides
if ( state . groupOverrides && Array . isArray ( state . groupOverrides ) ) {
for ( let si = 0 ; si < state . groupOverrides . length && si < groupOverrides . length ; si ++ ) {
const saved = state . groupOverrides [ si ] ;
if ( typeof saved . curve === 'number' ) groupOverrides [ si ] . curve = saved . curve ;
if ( Array . isArray ( saved . params ) ) {
for ( let li = 0 ; li < saved . params . length && li < groupOverrides [ si ] . params . length ; li ++ ) {
const sp = saved . params [ li ] , gp = groupOverrides [ si ] . params [ li ] ;
if ( typeof sp . min === 'number' ) gp . min = sp . min ;
if ( typeof sp . max === 'number' ) gp . max = sp . max ;
if ( typeof sp . curve === 'number' ) gp . curve = sp . curve ;
if ( typeof sp . muted === 'boolean' ) gp . muted = sp . muted ;
if ( typeof sp . fixedValue === 'number' ) gp . fixedValue = sp . fixedValue ;
}
}
}
}
2026-03-30 16:05:45 +02:00
// Restore visual overrides
if ( Array . isArray ( state . visualOverrides ) ) {
for ( let i = 0 ; i < state . visualOverrides . length && i < visualOverrides . length ; i ++ ) {
const sv = state . visualOverrides [ i ] , vo = visualOverrides [ i ] ;
if ( typeof sv . min === 'number' ) vo . min = sv . min ;
if ( typeof sv . max === 'number' ) vo . max = sv . max ;
if ( typeof sv . curve === 'number' ) vo . curve = sv . curve ;
if ( typeof sv . frozen === 'boolean' ) vo . frozen = sv . frozen ;
if ( typeof sv . fixedValue === 'number' ) vo . fixedValue = sv . fixedValue ;
}
}
// Restore MIDI CC overrides
if ( Array . isArray ( state . midiCCOverrides ) ) {
for ( let i = 0 ; i < state . midiCCOverrides . length && i < midiCCOverrides . length ; i ++ ) {
const sv = state . midiCCOverrides [ i ] , ov = midiCCOverrides [ i ] ;
if ( typeof sv . min === 'number' ) ov . min = sv . min ;
if ( typeof sv . max === 'number' ) ov . max = sv . max ;
if ( typeof sv . curve === 'number' ) ov . curve = sv . curve ;
if ( typeof sv . frozen === 'boolean' ) ov . frozen = sv . frozen ;
if ( typeof sv . fixedValue === 'number' ) ov . fixedValue = sv . fixedValue ;
}
}
2026-03-22 23:50:56 +01:00
// Restore synth preset id (just track it, don't re-apply — groupOverrides already restored above)
if ( typeof state . synthPresetId === 'string' ) {
activeSynthPresetId = state . synthPresetId ;
}
2026-04-02 23:12:18 +02:00
// Restore audio canvas state (pan/zoom/submode)
if ( state . audioCanvasState && audioCanvas ) {
audioCanvas . setState ( state . audioCanvasState ) ;
}
2026-03-21 23:57:58 +01:00
// Restore output mode
if ( state . outputMode && state . outputMode !== outputMode ) {
2026-03-30 16:05:45 +02:00
await setOutputMode ( state . outputMode , { skipConfirm : true } ) ;
2026-03-22 01:10:26 +01:00
syncOutputToggles ( outputMode ) ;
2026-03-21 23:57:58 +01:00
}
feat(playground): add MediaPipe hand tracking input with dev panel
Hand tracking via webcam as alternative to joystick input:
- Right hand tracks 14 derived features (palm XY, finger curls, spread,
roll, pitch, pinch) through a separate 14-input MLP
- Left hand gesture recognition (1 finger = thumbs up, 2 = thumbs down)
with 400ms debounce hold
- Split-zone PIP display (no camera feed, skeleton only) with dashed
divider and cross-zone dimming
- Dual IML architecture: independent imlJoy (2 inputs) and imlHand
(14 inputs) with pointer swap, preserving training data per mode
- Dev panel (?devmode=true): draggable/collapsible floating panel with
sliders for MediaPipe confidence thresholds, smoothing, gesture hold
time, world landmarks toggle, and live feature bar monitor
- Visual presets always route to joystick IML (prevents dimension mismatch)
- Race condition guard on input mode switching
2026-03-23 22:28:58 +01:00
// Note: don't auto-restore inputMode='hands' — requires camera permission
2026-04-03 18:49:07 +02:00
2026-04-03 18:56:35 +02:00
// Restore engine selection — sync switcher highlight; deferred engine init
// happens on first audio-start click (Faust engines require a running AudioContext).
if ( typeof state . engineId === 'string' ) {
2026-04-03 18:49:07 +02:00
EngineSwitcher . setActive ( activeEngine ? activeEngine . id : 'shaper-feedback' ) ;
}
feat(playground): add Modular audio mode with shared mod pool
New "Modular" engine in a-immersive with three hot-swappable Faust
sub-engines (subtractive/additive/fm) sharing a common modulation pool:
16 ADSR slots + 32 LFO slots (single-knob sine->tri->square->saw
wavemorph) routed through a 48-source x 10-destination matrix per
engine. Per-connection scalar amounts in [-1, 1], summed at each
destination. Default MLP output count is 512 (32 mod-source params +
480 matrix cells); model reinits on sub-engine swap, count change, or
engine-param exposure toggle.
Faust layer:
- mod-pool.lib: shared ADSR/LFO/source-bus library
- gen-modular-dsp.py: byte-reproducible generator (source of truth)
- modular-subtractive: faithful Minimoog (3 osc, ladder filter, no envs)
- modular-additive: 64-partial, spectral shape + formants, no envs/LFOs
- modular-fm: 4-op matrix + self-feedback, no envs
- All three share d08=amp, d09=pan conventions
- MODULAR_DESTINATIONS.md: authoritative destination table
JS layer:
- ModularEngine: self-contained SynthEngine with getState/setState,
setSubEngine, setModSourceCount, setExposeEngineParam
- modular-ui: drawer with sub-engine toggle, ADSR/LFO count steppers,
per-slot enable switches, matrix grid editor (tap-cycle, long-press
precise, right-click menu, negative amounts), preset overlay
- modular-presets: 6 named presets (Slow pad, Plucky bass, Crystal,
DX bell, Morphing drone + default)
- a-app.js: Modular mode registered, paramMeta:change -> resizeMLP,
modular DSP state persisted under modularDspState, window.__nisps
debug hooks for programmatic control
Tests: tests/e2e/modular-mode.spec.js (11 Playwright tests, all passing
including DSP state survives reload, sub-engine swap keeps paramCount,
preset apply verification).
Also fixes a pre-existing build.sh bug where the -e flag caused faust
to overwrite .wasm outputs with expanded DSP source text, leaving
additive/fm-matrix/eoc-* committed as invalid WebAssembly. Rebuilt all
affected engines with the corrected script. Added an early-message
buffer to faust-worklet-processor.js so setParam calls arriving before
wasm instantiation are queued rather than dropped (needed when the user
configures modular state before clicking Start Audio).
2026-04-11 07:34:41 +02:00
// Phase E — stash modular DSP state so it can be applied once the
// ModularEngine instance actually exists. Applied either right now (if
// the active engine is already modular) or later from the engine-switch
// handler when the user clicks the Modular card.
if ( state . modularDspState && typeof state . modularDspState === 'object' ) {
_pendingModularDspState = state . modularDspState ;
if ( activeEngine ? . id === 'modular' && typeof activeEngine . setState === 'function' ) {
try {
await activeEngine . setState ( _pendingModularDspState ) ;
_pendingModularDspState = null ;
} catch ( err ) {
console . warn ( '[NISPS] modular setState on load failed:' , err ) ;
}
}
}
2026-04-03 18:49:07 +02:00
// Restore EOC chain state (modules, enabled flags, param values, nispsMode)
if ( eocChain && Array . isArray ( state . eocModules ) && state . eocModules . length > 0 ) {
for ( const saved of state . eocModules ) {
// Only add if not already in the chain
if ( ! eocChain . getModule ( saved . id ) ) {
try {
eocChain . addModule ( moduleFactory ( saved . id ) ) ;
} catch ( e ) {
console . warn ( ` [EOC] Could not restore module ' ${ saved . id } ': ` , e . message ) ;
continue ;
}
}
const mod = eocChain . getModule ( saved . id ) ;
if ( mod ) {
mod . enabled = saved . enabled ? ? true ;
if ( Array . isArray ( saved . params ) ) {
saved . params . forEach ( ( v , i ) => mod . setParam ( i , v ) ) ;
}
}
}
}
if ( eocChain && typeof state . eocNispsMode === 'string' ) {
try { eocChain . nispsMode = state . eocNispsMode ; } catch ( _ ) { /* invalid mode in old save */ }
}
2026-04-03 19:03:12 +02:00
// Restore EOC training target
if ( state . eocTrainingTarget === 'synth' || state . eocTrainingTarget === 'eoc' ) {
eocTrainingTarget = state . eocTrainingTarget ;
}
feat(playground): add MediaPipe hand tracking input with dev panel
Hand tracking via webcam as alternative to joystick input:
- Right hand tracks 14 derived features (palm XY, finger curls, spread,
roll, pitch, pinch) through a separate 14-input MLP
- Left hand gesture recognition (1 finger = thumbs up, 2 = thumbs down)
with 400ms debounce hold
- Split-zone PIP display (no camera feed, skeleton only) with dashed
divider and cross-zone dimming
- Dual IML architecture: independent imlJoy (2 inputs) and imlHand
(14 inputs) with pointer swap, preserving training data per mode
- Dev panel (?devmode=true): draggable/collapsible floating panel with
sliders for MediaPipe confidence thresholds, smoothing, gesture hold
time, world landmarks toggle, and live feature bar monitor
- Visual presets always route to joystick IML (prevents dimension mismatch)
- Race condition guard on input mode switching
2026-03-23 22:28:58 +01:00
console . log ( ` [NISPS] Restored ${ state . features ? . length || 0 } joy examples, ${ state . handFeatures ? . length || 0 } hand examples from storage ` ) ;
2026-03-21 23:57:58 +01:00
} catch ( e ) {
console . warn ( '[NISPS] Failed to load state:' , e ) ;
}
}
2026-04-03 18:49:07 +02:00
function showToast ( message , durationMs = 2500 ) {
let toast = document . getElementById ( 'nisps-toast' ) ;
if ( ! toast ) {
toast = document . createElement ( 'div' ) ;
toast . id = 'nisps-toast' ;
toast . className = 'nisps-toast' ;
document . body . appendChild ( toast ) ;
}
toast . textContent = message ;
toast . classList . add ( 'visible' ) ;
clearTimeout ( toast . _hideTimer ) ;
toast . _hideTimer = setTimeout ( ( ) => toast . classList . remove ( 'visible' ) , durationMs ) ;
}
2026-03-21 23:57:58 +01:00
function clearState ( ) {
try {
localStorage . removeItem ( STORAGE _KEY ) ;
} catch ( e ) {
console . warn ( '[NISPS] Failed to clear state:' , e ) ;
}
}
// ---- Start ----
document . addEventListener ( 'DOMContentLoaded' , init ) ;