fix(manifold): smooth particle rendering

This commit is contained in:
monkey-w1n5t0n 2026-07-25 17:11:24 +02:00
parent a1cd26ff68
commit 856642a962
14 changed files with 427 additions and 114 deletions

21
MAP.md
View file

@ -39,16 +39,19 @@ anchor + locked decisions) and the `docs/specs/*-spec.md` set.
framework-neutral: `wasm-iml.ts` (rewired off Solid stores onto an injected `EngineSink`), `engine-host.ts` + framework-neutral: `wasm-iml.ts` (rewired off Solid stores onto an injected `EngineSink`), `engine-host.ts` +
`worklet/nisps-processor.ts` (audio), thin WASM wrappers over the core pipelines + curve catalog (the TS `worklet/nisps-processor.ts` (audio), thin WASM wrappers over the core pipelines + curve catalog (the TS
`input-pipeline`/`output-pipeline`/`curves` implementations died at P4), `wasm-worker.ts`, `input-pipeline`/`output-pipeline`/`curves` implementations died at P4), `wasm-worker.ts`,
`spine.ts` (the reactive spine BELOW React — `setInput` derives processed→ml→routed eagerly off-render), `spine.ts` (the reactive spine BELOW React — `setInput` derives processed→ml→routed eagerly off-render;
structural/training state and throttled live-output notifications are separate channels),
`engine-api.ts` (`EngineApi` façade incl. live architecture/weight/example metrics and `feedback.*` wrappers over the `nisps_ml_feedback_*` C ABI), `engine-api.ts` (`EngineApi` façade incl. live architecture/weight/example metrics and `feedback.*` wrappers over the `nisps_ml_feedback_*` C ABI),
`EngineProvider.tsx`/`useEngine.ts` (React binding via `useSyncExternalStore` version counter). nisps.js is `EngineProvider.tsx`/`useEngine.ts` (React bindings via `useSyncExternalStore`: state version plus an
opt-in 30 Hz output version for DOM consumers; canvases read live buffers directly). nisps.js is
loaded via fetch+indirect-eval (Emscripten MODULARIZE glue has no ES exports), base-aware via `document.baseURI` loaded via fetch+indirect-eval (Emscripten MODULARIZE glue has no ES exports), base-aware via `document.baseURI`
for the `/next` sub-path. for the `/next` sub-path.
- `manifold/src/primitives/` — the 7 design primitives as typed React (Badge, Button, PillToggle, Slider, Switch, VirtualJoystick, XYPad). Five unused ones were deleted in the 2026-07 sweep (L22). - `manifold/src/primitives/` — the 7 design primitives as typed React (Badge, Button, PillToggle, Slider, Switch, VirtualJoystick, XYPad). Five unused ones were deleted in the 2026-07 sweep (L22).
- `manifold/src/console/` — the convertible Console: `ConsoleApp`, `CompositeStage` (single-divider convertible - `manifold/src/console/` — the convertible Console: `ConsoleApp`, `CompositeStage` (single-divider convertible
with snap/magnetism/minimap-demotion), `OutputStage`/`SandwichStage`/`ParticleStage`/`Manifold` (canvas, with snap/magnetism/minimap-demotion), `OutputStage`/`SandwichStage`/`ParticleStage`/`Manifold` (canvas,
rect↔circular + feedback markers; ParticleStage has interactive cursor-labelled heatmap sliders, an adjustable rect↔circular + feedback markers; ParticleStage batches its 400 particles into 32 colour paths per frame,
joystick, and double-click whole-screen follow-mouse input), `Dock` (top Mode selector + 5 vertically-centred drawers), `Drawers` has interactive cursor-labelled heatmap sliders, an adjustable circular Manifold pad with feedback markers and cursor trail,
and double-click whole-screen follow-mouse input), `Dock` (top Mode selector + 5 vertically-centred drawers), `Drawers`
(Learning/Inputs/Outputs/Settings/Help; Learning includes the live model-architecture inspector and (Learning/Inputs/Outputs/Settings/Help; Learning includes the live model-architecture inspector and
Outputs owns the remaining scroll height), `TrainingHealth` (real per-iteration loss curve from Outputs owns the remaining scroll height), `TrainingHealth` (real per-iteration loss curve from
`nisps_ml_loss_history` + per-layer weight health from `nisps_ml_get_layer_stats`; rendered only at `nisps_ml_loss_history` + per-layer weight health from `nisps_ml_get_layer_stats`; rendered only at
@ -73,10 +76,10 @@ anchor + locked decisions) and the `docs/specs/*-spec.md` set.
- `manifold/src/inputs/` — modular INPUT layer feeding the ML head. The Inputs dock picks ONE exclusive mode - `manifold/src/inputs/` — modular INPUT layer feeding the ML head. The Inputs dock picks ONE exclusive mode
(`InputMode` = `internal` | `gamepad` | `midi`; Internal/XY-pad is default). `input-layer.ts` owns a single rAF (`InputMode` = `internal` | `gamepad` | `midi`; Internal/XY-pad is default). `input-layer.ts` owns a single rAF
loop composing the active source's axes → **one dedicated engine input slot per axis, 1:1, no blending** → one loop composing the active source's axes → **one dedicated engine input slot per axis, 1:1, no blending** → one
`setInputs`, plus an `onReducedInput` callback the manifold tracks. The WASM net is over-provisioned to a `setInputs`, plus an `onReducedInput` callback the manifold tracks. Normal Manifold models default to a
32-input head (`MAX_AXES`, `nisps/wasm/bindings.cpp`); unused slots are zero-padded and a zero input is inert, runtime-shaped 2-input head; the Inputs dock can select 4 inputs, and active sources can grow the head when
so idle sources cannot perturb the net. Mean-blending was removed deliberately — it diluted every source and they need more slots. Mean-blending was removed deliberately — it diluted every source and biased the net
biased the net toward idle sources' resting values. Active-axis edits follow the persistent I/O policy: toward idle sources' resting values. Active-axis edits follow the persistent I/O policy:
keep-capacity permutes stable identities in place until more slots are required; exact-I/O reconstructs keep-capacity permutes stable identities in place until more slots are required; exact-I/O reconstructs
to the active count. Surviving weights and (under adapt policy) examples are identity-remapped; feedback to the active count. Surviving weights and (under adapt policy) examples are identity-remapped; feedback
scratch state resets. Sources: `xy-pad-source` (push-driven), scratch state resets. Sources: `xy-pad-source` (push-driven),
@ -178,7 +181,7 @@ includes; no `nisps-core`.
- C++ identifiers: `PascalCase` types, `snake_case` functions/variables, `kPascalCase` constexpr. JSON keys `snake_case`. TS types `PascalCase`, components `PascalCase.tsx`, modules `kebab-case.ts`. - C++ identifiers: `PascalCase` types, `snake_case` functions/variables, `kPascalCase` constexpr. JSON keys `snake_case`. TS types `PascalCase`, components `PascalCase.tsx`, modules `kebab-case.ts`.
- `Curve` enum lives in `nisps/core/math.hpp` (lowercase: `linear/exp/log/square/sqrt/sigmoid/cubic`, plus the parameterised `centered_power` free function); generated mode headers re-export via `using Curve = ::nisps::Curve;`. Since P4 there is NO TS mirror — the browser samples the WASM catalog (`nisps_curve_apply(+batch)`). - `Curve` enum lives in `nisps/core/math.hpp` (lowercase: `linear/exp/log/square/sqrt/sigmoid/cubic`, plus the parameterised `centered_power` free function); generated mode headers re-export via `using Curve = ::nisps::Curve;`. Since P4 there is NO TS mirror — the browser samples the WASM catalog (`nisps_curve_apply(+batch)`).
- Modes are TSX components composed of primitives; mode parameter contracts are JSON schemas with codegen → C++ **and** TS types (`MF_MODES` derives params/ml-config from the generated schemas since P5; labels/ordering stay a manifold overlay). **No declarative JSON UI.** - Modes are TSX components composed of primitives; mode parameter contracts are JSON schemas with codegen → C++ **and** TS types (`MF_MODES` derives params/ml-config from the generated schemas since P5; labels/ordering stay a manifold overlay). **No declarative JSON UI.**
- WASM and firmware share the same C++; the browser MLP is runtime-shaped (`MLPCore<DynamicStorage>`, since P2): `nisps_ml_create` honours `(input, output, hidden[3])` with non-positive/null args defaulting to `32→[10,14,18]→126`; `nisps_ml_reshape` warm-starts a new shape. Firmware keeps compile-time `MLP<...>` (zero heap). Per-mode dims are schema-real on both targets since P5.3 (the browser reshapes on mode switch). - WASM and firmware share the same C++; the browser MLP is runtime-shaped (`MLPCore<DynamicStorage>`, since P2): `nisps_ml_create` honours `(input, output, hidden[3])` with non-positive/null args defaulting to `32→[10,14,18]→126`; `nisps_ml_reshape` warm-starts a new shape. Firmware keeps compile-time `MLP<...>` (zero heap). Manifold normal modes use a 2-input UI default and can opt into 4 inputs; audio-analysis keeps its schema-defined feature arity. Firmware mode shapes remain schema-real.
- Cross-platform parity: `scripts/parity-check.sh` enforces native vs WASM agreement within 1e-5. - Cross-platform parity: `scripts/parity-check.sh` enforces native vs WASM agreement within 1e-5.
## Gotchas ## Gotchas

View file

@ -70,12 +70,13 @@ single `backend.send` at the action's tail, off-render. Contract rules:
call in the derivation path. call in the derivation path.
- **No per-frame allocation.** Buffers are reused (`routedBuf` threaded through the output - **No per-frame allocation.** Buffers are reused (`routedBuf` threaded through the output
pipeline and handed to the backend). pipeline and handed to the backend).
- **Weights mutate only through engine actions** (train / feedback / reshape), each of which bumps - **Weights mutate only through engine actions** (train / feedback / reshape). Structural/training
the spine's version counter — there is no write path that can update audio without notifying state bumps `version`; every inference advances a separate `outputVersion`. Both expose explicit
the UI. subscriptions, so audio propagation never depends on React scheduling.
- **React subscribes via `useSyncExternalStore(subscribe, version)`** — the version counter, not - **React state and live output are separate channels.** Structural/training consumers use
the arrays; canvases read the live `Float32Array`s imperatively in rAF and never re-render per `useSyncExternalStore(subscribe, version)`. DOM output consumers opt into the throttled 30 Hz
frame. `subscribeOutputs/outputVersion` channel. Canvases read and shape the live `Float32Array`s
imperatively in rAF and never re-render per frame.
### 2.2 The live-feedback guarantee and its e2e assertion ### 2.2 The live-feedback guarantee and its e2e assertion

View file

@ -66,13 +66,13 @@ bun run test:e2e # Playwright smoke (needs `bun run build` first; runs again
│ ConsoleApp = spine of the UI: holds state, picks a Stage, │ │ ConsoleApp = spine of the UI: holds state, picks a Stage, │
│ renders the Dock. "Convertible" = swappable Stages. │ │ renders the Dock. "Convertible" = swappable Stages. │
└───────────────┬─────────────────────────────────────────────────-┘ └───────────────┬─────────────────────────────────────────────────-┘
│ reads engine.version (useSyncExternalStore), reads buffers imperatively │ reads state/output versions (useSyncExternalStore), canvases read buffers imperatively
│ writes via engine.setInput / setParam / feedback.* │ writes via engine.setInput / setParam / feedback.*
┌───────────────▼─────────────────────────────────────────────────┐ ┌───────────────▼─────────────────────────────────────────────────┐
│ Engine layer (NO React) src/engine/ src/inputs/ src/feedback/ src/backends/ │ Engine layer (NO React) src/engine/ src/inputs/ src/feedback/ src/backends/
│ Spine = reactive store below React. Per-frame ML inference is │ │ Spine = reactive store below React. Per-frame ML inference is │
│ eager + synchronous, OFF the render cycle. React only watches │ │ eager + synchronous, OFF the render cycle. React watches state
a monotonic version counter. changes plus an opt-in 30 Hz output channel for DOM surfaces.
└───────────────┬─────────────────────────────────────────────────┘ └───────────────┬─────────────────────────────────────────────────┘
│ C ABI │ C ABI
┌───────────────▼─────────────────────────────────────────────────┐ ┌───────────────▼─────────────────────────────────────────────────┐
@ -109,7 +109,7 @@ Manifold ships a single "composite" altitude. Selection is now a plain three-way
|---|---|---|---| |---|---|---|---|
| CompositeStage | `CompositeStage.tsx` | **default / hero** | Draggable split-ratio; magnet-snaps to 0.14/0.33/0.5/0.66/0.86; collapses a side to a corner minimap at extremes. | | CompositeStage | `CompositeStage.tsx` | **default / hero** | Draggable split-ratio; magnet-snaps to 0.14/0.33/0.5/0.66/0.86; collapses a side to a corner minimap at extremes. |
| SandwichStage | `SandwichStage.tsx` | `sandwich===true` (wins over the others) | Three-pane layout: `Manifold` input surface left, 3D parameter-landscape centre (input → MLP heatmap grid → outputs, drag to orbit), compact `OutputStage` right. | | SandwichStage | `SandwichStage.tsx` | `sandwich===true` (wins over the others) | Three-pane layout: `Manifold` input surface left, 3D parameter-landscape centre (input → MLP heatmap grid → outputs, drag to orbit), compact `OutputStage` right. |
| ParticleStage | `ParticleStage.tsx` | `outputMode==='particles'` | Flow-field visualiser (`flow-field.ts`, 400-particle Canvas2D port) + interactive output heatmap sliders with cursor tooltips + a larger, explicitly adjustable/repositionable joystick; double-click anywhere in the stage enters whole-screen follow-mouse mode. | | ParticleStage | `ParticleStage.tsx` | `outputMode==='particles'` | Flow-field visualiser (`flow-field.ts`, 400 particles batched into 32 contiguous colour paths per frame) + interactive output heatmap sliders with cursor tooltips + a larger, explicitly adjustable/repositionable circular Manifold pad (feedback markers + cursor trail); double-click anywhere in the stage enters whole-screen follow-mouse mode. |
`Manifold.tsx` and `OutputStage.tsx` are no longer top-level stages — they are panes composed by `Manifold.tsx` and `OutputStage.tsx` are no longer top-level stages — they are panes composed by
CompositeStage/SandwichStage. `Manifold.tsx` is the full-bleed 2D input surface (canvas trail + pins CompositeStage/SandwichStage. `Manifold.tsx` is the full-bleed 2D input surface (canvas trail + pins
@ -208,12 +208,15 @@ a setting → `--r-*` tokens.
- `engine-api.ts`**`EngineApi`, the framework-neutral facade** everything in the UI talks to: - `engine-api.ts`**`EngineApi`, the framework-neutral facade** everything in the UI talks to:
`setInput/setInputs`, `getOutputs/routedOutput`, training (`addExample/train/trainAsync/evalLoss`), `setInput/setInputs`, `getOutputs/routedOutput`, training (`addExample/train/trainAsync/evalLoss`),
weights (`getWeights/setWeights/process/randomise`), telemetry weights (`getWeights/setWeights/process/randomise`), telemetry
(`architecture/weightCount/exampleCount/lossHistory/getLayerStats`), `subscribe/version/on`, plus nested `.feedback` and `.audio` (`architecture/weightCount/exampleCount/lossHistory/getLayerStats`), structural `subscribe/version`,
throttled `subscribeOutputs/outputVersion`, and `on`, plus nested `.feedback` and `.audio`
facades. **`lossHistory()` reads SPINE STATE, not the MLP handle** — an async train runs on the facades. **`lossHistory()` reads SPINE STATE, not the MLP handle** — an async train runs on the
worker's mirror net, so the main handle's own history is empty for those runs; both paths worker's mirror net, so the main handle's own history is empty for those runs; both paths
publish to the spine. publish to the spine.
- `EngineProvider.tsx` / `useEngine.ts` — the **only** React coupling. `useEngine()` returns the API - `EngineProvider.tsx` / `useEngine.ts` — the **only** React coupling. `useEngine()` returns the API
(null until WASM ready); `useEngineVersion()` = `useSyncExternalStore(subscribe, version)`. (null until WASM ready); `useEngineVersion()` watches structural/training state, while
`useEngineOutputVersion()` opts DOM consumers into the throttled 30 Hz output channel. Particle canvases
use neither output subscription nor React state for live values; they shape the reused buffer in rAF.
- `engine-host.ts` — main-thread audio wiring: AudioContext (user-gesture gated), fetch `nisps.wasm`, - `engine-host.ts` — main-thread audio wiring: AudioContext (user-gesture gated), fetch `nisps.wasm`,
register + feed the worklet. register + feed the worklet.
- `wasm-iml.ts` (**~750 lines**) — the ML interface to `nisps.wasm`: one MLP handle, dataset, heap - `wasm-iml.ts` (**~750 lines**) — the ML interface to `nisps.wasm`: one MLP handle, dataset, heap
@ -243,16 +246,16 @@ a setting → `--r-*` tokens.
### Inputs — `src/inputs/` ### Inputs — `src/inputs/`
- `input-layer.ts` — composition hub. One rAF loop polls sources, pulls all axes into a vector, - `input-layer.ts` — composition hub. One rAF loop polls sources, pulls all axes into a vector,
forwards N→engine. **`MAX_AXES = 32`** (WASM net over-provisioned to 32 inputs). **Dedicated forwards N→engine. **`MAX_AXES = 32`** is the source-vector safety cap, not the model shape.
dimensions, NO mean-blending** — each active axis drives its own engine slot 1:1; unused slots **Dedicated dimensions, NO mean-blending** — each active axis drives its own engine slot 1:1;
zero-padded (inert). Changing the layout uses the same persisted identity-aware I/O policy as output the ConsoleApp reshapes the runtime model when the selected 2/4-input capacity or active layout
cards. requires it. Changing the layout uses the same persisted identity-aware I/O policy as output cards.
- `base-source.ts` + sources: `xy-pad-source.ts` (push, 2 axes), `gamepad-source.ts` (single=2 / - `base-source.ts` + sources: `xy-pad-source.ts` (push, 2 axes), `gamepad-source.ts` (single=2 /
double=4 axes, deadzone 0.08), `midi-input-source.ts` (Web MIDI, batch CC-learn, multi-port). double=4 axes, deadzone 0.08), `midi-input-source.ts` (Web MIDI, batch CC-learn, multi-port).
- `useInputLayer.ts` — React binding; manages exclusive input mode + gamepad stick mode + MIDI - `useInputLayer.ts` — React binding; manages exclusive input mode + gamepad stick mode + MIDI
device/learn map; exposes `pushPad`, `sources`, `channelLayout`, etc. device/learn map; exposes `pushPad`, `sources`, `channelLayout`, etc.
- **I/O migration (P2.3, live):** the net is **runtime-shaped**. It boots at the default - **I/O migration (P2.3, live):** the net is **runtime-shaped**. Normal Manifold modes boot at
over-provisioned 32-input head (zero-padding preserved), and `EngineApi.reshape({ inputSize, … })` 2 inputs; the Inputs dock can select 4, and active layouts can grow the model further. `EngineApi.reshape({ inputSize, … })`
`WasmIML.reshape`. `engine/io-reshape.ts` owns the identity map: **Keep capacity** (default) `WasmIML.reshape`. `engine/io-reshape.ts` owns the identity map: **Keep capacity** (default)
permutes weights/examples in place and reconstructs only when active I/O outgrows the net; permutes weights/examples in place and reconstructs only when active I/O outgrows the net;
**Exact I/O** reconstructs whenever active arity changes. Existing examples either adapt by **Exact I/O** reconstructs whenever active arity changes. Existing examples either adapt by
@ -262,14 +265,15 @@ a setting → `--r-*` tokens.
(`wasm-worker.ts`) carries the current dims in its train message and re-creates its mirror net to (`wasm-worker.ts`) carries the current dims in its train message and re-creates its mirror net to
match. The raw debug `window.__nisps.reshape(nIn)` remains a low-level reconstruct-and-clear call. match. The raw debug `window.__nisps.reshape(nIn)` remains a low-level reconstruct-and-clear call.
- **Per-mode net dims (P5.3):** switching INSTRUMENT mode reshapes the net to that mode's schema - **Per-mode net dims (P5.3):** switching INSTRUMENT mode reshapes the net to that mode's schema
`ml` config (`MFMode.ml` — input/hidden/output + legacy spread) via a `ConsoleApp` effect keyed on hidden/output config plus Manifold's effective input arity via a `ConsoleApp` effect keyed on
`[engine, modeId]`. No confirm modal (switching instrument is deliberate). The effect depends on `engine`, so on boot `[engine, modeId, modelInputSize]`. No confirm modal (switching instrument is deliberate). Normal
it fires once WASM is ready and lands the boot mode's dims (**paf_synth → 4→[10,10,14]→33**, weights modes boot at **2 inputs** (paf_synth → 2→[10,10,14]→33, weights 787); the Inputs dock's **4 inputs**
809 — NOT the 32→126 default). The reshape-offer effect reads the engine's CURRENT `inputSize` option is retained across mode switches. Audio-analysis keeps its schema-defined input feature count.
The reshape-offer effect reads the engine's CURRENT `inputSize`
live, so a mode switch that changes arity doesn't spuriously prompt (its baseline tracks axis live, so a mode switch that changes arity doesn't spuriously prompt (its baseline tracks axis
COUNT, unchanged by a pure dim change). Non-schema modes restore `DEFAULT_MODE_ML` (32→126). COUNT, unchanged by a pure dim change). Manifold-only modes use the same 2/4 input selector.
Debug seam for tests: under `?debug=1` ConsoleApp installs `window.__mf` Debug seam for tests: under `?debug=1` ConsoleApp installs `window.__mf`
(`setMode`/`getModeId`/`paramCount`/`modeIds`) — the UI-level analogue of `__nisps`, since no (`setMode`/`getModeId`/`paramCount`/`modeIds`/`getModelInputSize`/`setModelInputSize`) — the UI-level analogue of `__nisps`, since no
in-UI instrument picker exists yet (`ctx.modes`/`setModeId` are plumbed but unrendered). in-UI instrument picker exists yet (`ctx.modes`/`setModeId` are plumbed but unrendered).
Manifold passes `spread=0` for boot, mode-switch reshapes, direct re-rolls, explore-and-place Manifold passes `spread=0` for boot, mode-switch reshapes, direct re-rolls, explore-and-place
scratchpad rolls, and VCV-forwarded randomise gestures by default. Settings → Experimental scratchpad rolls, and VCV-forwarded randomise gestures by default. Settings → Experimental

View file

@ -6,8 +6,9 @@
* - The pseudo-inference (`MF_infer`, sin/cos) is GONE. The `values` every * - The pseudo-inference (`MF_infer`, sin/cos) is GONE. The `values` every
* consumer reads now come from `engine.getOutputs()`, mapped onto the mode's * consumer reads now come from `engine.getOutputs()`, mapped onto the mode's
* params by `shapeValues` (status/min/max/curve applied here). Pad/joystick * params by `shapeValues` (status/min/max/curve applied here). Pad/joystick
* motion drives `engine.setInput(x,y)`; we subscribe to engine changes via * motion drives `engine.setInput(x,y)`; structural state and throttled DOM
* `useEngineVersion` and re-derive `values` imperatively on render. * outputs have separate subscriptions, while ParticleStage shapes the live
* reused buffer directly in rAF.
* - Verdicts wire to the engine: commit feedback.thumbsUp(); perturb * - Verdicts wire to the engine: commit feedback.thumbsUp(); perturb
* feedback.thumbsDown(); reroll randomise(); each followed by process(). * feedback.thumbsDown(); reroll randomise(); each followed by process().
* - The default feedback mode is "Explore and place" the shared C++ core's * - The default feedback mode is "Explore and place" the shared C++ core's
@ -31,6 +32,7 @@ import { useEffect, useMemo, useRef, useState } from 'react';
import type { CSSProperties } from 'react'; import type { CSSProperties } from 'react';
import { import {
useEngine, useEngine,
useEngineOutputVersion,
useEngineVersion, useEngineVersion,
ExplorationController, ExplorationController,
DEFAULT_GEOMETRIC_FEEDBACK_CONFIG, DEFAULT_GEOMETRIC_FEEDBACK_CONFIG,
@ -51,6 +53,7 @@ import type {
DrawerKey, DrawerKey,
FeedbackMarker, FeedbackMarker,
FeedbackModeUI, FeedbackModeUI,
ManifoldInputSize,
OutputMode, OutputMode,
Pin, Pin,
SoloMode, SoloMode,
@ -82,6 +85,8 @@ export interface MfDebugHook {
setOutputMode: (mode: OutputMode) => void; setOutputMode: (mode: OutputMode) => void;
setMidiCcCount: (count: number) => void; setMidiCcCount: (count: number) => void;
displayOutputCount: () => number; displayOutputCount: () => number;
getModelInputSize: () => ManifoldInputSize;
setModelInputSize: (size: ManifoldInputSize) => void;
} }
declare global { declare global {
@ -106,11 +111,12 @@ function pillBtn(color: string): CSSProperties {
export function ConsoleApp() { export function ConsoleApp() {
const engine = useEngine(); const engine = useEngine();
const version = useEngineVersion(engine); const stateVersion = useEngineVersion(engine);
const { settings } = useSettings(); const { settings } = useSettings();
const [modeId, setModeId] = useState('paf_synth'); const [modeId, setModeId] = useState('paf_synth');
const mode = MF_MODES.find((m) => m.id === modeId) ?? MF_MODES[0]; const mode = MF_MODES.find((m) => m.id === modeId) ?? MF_MODES[0];
const [modelInputSize, setModelInputSizeState] = useState<ManifoldInputSize>(2);
const [params, setParams] = useState<MFParam[]>(() => mode.params.map((p) => ({ ...p }))); const [params, setParams] = useState<MFParam[]>(() => mode.params.map((p) => ({ ...p })));
const [paramsModeId, setParamsModeId] = useState(mode.id); const [paramsModeId, setParamsModeId] = useState(mode.id);
const [pos, setPos] = useState<[number, number]>([0.5, 0.5]); const [pos, setPos] = useState<[number, number]>([0.5, 0.5]);
@ -156,6 +162,16 @@ export function ConsoleApp() {
// Active output MODE (TOP dock selector) — default Particle System. The dock // Active output MODE (TOP dock selector) — default Particle System. The dock
// backend + audio backend derive from this. // backend + audio backend derive from this.
const [outputMode, setOutputModeState] = useState<OutputMode>(DEFAULT_OUTPUT_MODE); const [outputMode, setOutputModeState] = useState<OutputMode>(DEFAULT_OUTPUT_MODE);
// Canvas stages read the live buffer imperatively. DOM-heavy stages opt into
// the separate 30 Hz output channel instead of making the whole console a
// per-inference subscriber.
const outputVersion = useEngineOutputVersion(
engine,
outputMode !== 'particles' ||
sandwich ||
params.some((param) => param.manualOverride),
);
const version = stateVersion + outputVersion;
const outputBackend: BackendId = outputModeDescriptor(outputMode).backend; const outputBackend: BackendId = outputModeDescriptor(outputMode).backend;
// Per-backend transport settings (backends-spec §2.3/§2.4). Persisted via the // Per-backend transport settings (backends-spec §2.3/§2.4). Persisted via the
// named-preset system; these are the live working values. // named-preset system; these are the live working values.
@ -283,19 +299,17 @@ export function ConsoleApp() {
if (engine) engine.audio.setBackend(modeEngineId(modeId) as Parameters<typeof engine.audio.setBackend>[0]); if (engine) engine.audio.setBackend(modeEngineId(modeId) as Parameters<typeof engine.audio.setBackend>[0]);
}, [engine, modeId]); }, [engine, modeId]);
// Per-mode net dims (one-core-engine P5.3). On mode switch — and once WASM is // Per-mode net dims (one-core-engine P5.3). Normal Manifold modes expose a
// ready on boot (this effect depends on `engine`, so it fires when the engine // deliberately smaller 2-input working shape for now, while the Inputs dock
// transitions null→ready with the boot mode) — reshape the runtime-shaped MLP // can opt into the schema's usual 4-input shape. Audio-analysis is different:
// to the active mode's schema `ml` config (warm-started; the C-side dataset + // its schema input arity describes the feature vector and must stay intact.
// feedback reset, which the transient reset below also clears). NO confirm
// modal: switching instrument is already a deliberate act. The P2.3 axis-count
// modal stays for input-LAYOUT changes only (see the reshape-offer effect).
useEffect(() => { useEffect(() => {
if (!engine) return; if (!engine) return;
const { inputSize, outputSize, hidden } = mode.ml; const inputSize = mode.input === 'audio_in' ? mode.ml.inputSize : modelInputSize;
const { outputSize, hidden } = mode.ml;
engine.reshape({ inputSize, outputSize, hidden }, randomisationSpread); engine.reshape({ inputSize, outputSize, hidden }, randomisationSpread);
// eslint-disable-next-line react-hooks/exhaustive-deps // eslint-disable-next-line react-hooks/exhaustive-deps
}, [engine, modeId]); }, [engine, modeId, modelInputSize]);
// reset transient state on mode switch // reset transient state on mode switch
useEffect(() => { useEffect(() => {
@ -344,8 +358,13 @@ export function ConsoleApp() {
return; return;
} }
const activeMap: DimensionMap = keys.map((key) => previous.get(key) ?? null); const activeMap: DimensionMap = keys.map((key) => previous.get(key) ?? null);
// Keep the explicit dock choice as a minimum capacity. Active sources may
// still require more (for example, a double-stick gamepad), but a source
// with fewer axes must not silently shrink a deliberate 4-D model.
const minimumInputSize = mode.input === 'audio_in' ? mode.ml.inputSize : modelInputSize;
const requiredSize = Math.max(minimumInputSize, keys.length);
const target = const target =
resizeTarget(keys.length, inSize, settings.networkResizePolicy) ?? inSize; resizeTarget(requiredSize, inSize, settings.networkResizePolicy) ?? inSize;
const inputMap = completeDimensionMap(activeMap, target, inSize); const inputMap = completeDimensionMap(activeMap, target, inSize);
const needsMigration = const needsMigration =
target !== inSize || inputMap.some((oldIndex, newIndex) => oldIndex !== newIndex); target !== inSize || inputMap.some((oldIndex, newIndex) => oldIndex !== newIndex);
@ -369,6 +388,9 @@ export function ConsoleApp() {
engine, engine,
inputLayoutKey, inputLayoutKey,
inputs.engineInputSize, inputs.engineInputSize,
mode.input,
mode.ml.inputSize,
modelInputSize,
randomisationSpread, randomisationSpread,
settings.networkResizePolicy, settings.networkResizePolicy,
settings.exampleResizePolicy, settings.exampleResizePolicy,
@ -393,11 +415,13 @@ export function ConsoleApp() {
setOutputMode, setOutputMode,
setMidiCcCount, setMidiCcCount,
displayOutputCount: () => displayOutputCount, displayOutputCount: () => displayOutputCount,
getModelInputSize: () => modelInputSize,
setModelInputSize: (size) => setModelInputSizeState(size === 4 ? 4 : 2),
}; };
return () => { return () => {
if (window.__mf) delete window.__mf; if (window.__mf) delete window.__mf;
}; };
}, [modeId, params, displayOutputCount]); }, [modeId, params, displayOutputCount, modelInputSize]);
// Drive a pad/joystick/manifold move through the input layer's XY-pad source, // Drive a pad/joystick/manifold move through the input layer's XY-pad source,
// then mirror the raw position into React state for readouts. The layer's loop // then mirror the raw position into React state for readouts. The layer's loop
@ -644,7 +668,10 @@ export function ConsoleApp() {
} }
setAddingExample(false); setAddingExample(false);
// Snapshot the current input → current (shaped) output as a training example. // Snapshot the current input → current (shaped) output as a training example.
engine?.addExample([pos[0], pos[1]], Array.from(values)); engine?.addExample(
[pos[0], pos[1]],
shapeValues(params, engine?.getOutputs() ?? null),
);
setExamples((e) => e + 1); setExamples((e) => e + 1);
train(); train();
}; };
@ -979,6 +1006,8 @@ export function ConsoleApp() {
cvDisconnect, cvDisconnect,
setParams: (next: MFParam[]) => setParams(next), setParams: (next: MFParam[]) => setParams(next),
inputs, inputs,
modelInputSize,
setModelInputSize: (size: ManifoldInputSize) => setModelInputSizeState(size),
spread, spread,
setSpread, setSpread,
xavierSpreadEnabled: settings.xavierSpreadEnabled, xavierSpreadEnabled: settings.xavierSpreadEnabled,
@ -1096,6 +1125,7 @@ export function ConsoleApp() {
params={displayedParams} params={displayedParams}
values={displayedValues} values={displayedValues}
onChange={setParam} onChange={setParam}
markers={markers}
/> />
) : ( ) : (
<CompositeStage <CompositeStage

View file

@ -21,8 +21,9 @@
import { useEffect, useRef, useState } from 'react'; import { useEffect, useRef, useState } from 'react';
import type { MouseEvent as ReactMouseEvent, PointerEvent as ReactPointerEvent } from 'react'; import type { MouseEvent as ReactMouseEvent, PointerEvent as ReactPointerEvent } from 'react';
import { useEngine } from '../engine'; import { useEngine } from '../engine';
import { VirtualJoystick } from '../primitives/VirtualJoystick'; import { shapeValuesInto, type MFParam } from './model';
import type { MFParam } from './model'; import { Manifold } from './Manifold';
import type { FeedbackMarker } from './types';
import { import {
FlowFieldVisualizer, FlowFieldVisualizer,
N_VISUAL_OUTPUTS, N_VISUAL_OUTPUTS,
@ -36,16 +37,19 @@ export interface ParticleStageProps {
params: MFParam[]; params: MFParam[];
values: number[]; values: number[];
onChange: (i: number, patch: Partial<MFParam>) => void; onChange: (i: number, patch: Partial<MFParam>) => void;
markers?: FeedbackMarker[];
} }
export function ParticleStage({ pos, onMove, params, values, onChange }: ParticleStageProps) { export function ParticleStage({ pos, onMove, params, values, onChange, markers = [] }: ParticleStageProps) {
const engine = useEngine(); const engine = useEngine();
const canvasRef = useRef<HTMLCanvasElement>(null); const canvasRef = useRef<HTMLCanvasElement>(null);
const vizRef = useRef<FlowFieldVisualizer | null>(null); const vizRef = useRef<FlowFieldVisualizer | null>(null);
const barsRef = useRef<(HTMLDivElement | null)[]>([]); const barsRef = useRef<(HTMLDivElement | null)[]>([]);
const tooltipRef = useRef<HTMLDivElement>(null); const tooltipRef = useRef<HTMLDivElement>(null);
const outputsRef = useRef<Float32Array | null>(null); const outputsRef = useRef<Float32Array | null>(null);
const valuesRef = useRef(values); const displayOutputsRef = useRef<ArrayLike<number>>(values);
const shapedOutputsRef = useRef(new Float32Array(N_VISUAL_OUTPUTS));
const paramsRef = useRef(params);
const hoverRef = useRef<number | null>(null); const hoverRef = useRef<number | null>(null);
const [hover, setHover] = useState<number | null>(null); const [hover, setHover] = useState<number | null>(null);
const [tooltipPos, setTooltipPos] = useState({ x: 12, y: 28 }); const [tooltipPos, setTooltipPos] = useState({ x: 12, y: 28 });
@ -73,7 +77,7 @@ export function ParticleStage({ pos, onMove, params, values, onChange }: Particl
startX: 0, startX: 0,
el: null, el: null,
}); });
valuesRef.current = values; paramsRef.current = params;
onMoveRef.current = onMove; onMoveRef.current = onMove;
const clamp = (n: number, lo: number, hi: number) => Math.max(lo, Math.min(hi, n)); const clamp = (n: number, lo: number, hi: number) => Math.max(lo, Math.min(hi, n));
@ -125,7 +129,7 @@ export function ParticleStage({ pos, onMove, params, values, onChange }: Particl
x: Math.min(e.clientX + 12, Math.max(12, window.innerWidth - 190)), x: Math.min(e.clientX + 12, Math.max(12, window.innerWidth - 190)),
y: Math.min(e.clientY + 12, Math.max(28, window.innerHeight - 32)), y: Math.min(e.clientY + 12, Math.max(28, window.innerHeight - 32)),
}); });
const output = valuesRef.current[i] ?? outputsRef.current?.[i] ?? 0; const output = displayOutputsRef.current[i] ?? outputsRef.current?.[i] ?? 0;
if (tooltipRef.current) tooltipRef.current.textContent = `${VISUAL_PARAM_NAMES[i]}: ${output.toFixed(3)}`; if (tooltipRef.current) tooltipRef.current.textContent = `${VISUAL_PARAM_NAMES[i]}: ${output.toFixed(3)}`;
}; };
@ -194,7 +198,11 @@ export function ParticleStage({ pos, onMove, params, values, onChange }: Particl
const outputs = engine?.getOutputs(); const outputs = engine?.getOutputs();
if (outputs) { if (outputs) {
outputsRef.current = outputs; outputsRef.current = outputs;
const displayOutputs = valuesRef.current.length >= N_VISUAL_OUTPUTS ? valuesRef.current : outputs; const displayOutputs =
paramsRef.current.length >= N_VISUAL_OUTPUTS
? shapeValuesInto(paramsRef.current, outputs, shapedOutputsRef.current)
: outputs;
displayOutputsRef.current = displayOutputs;
viz.setParams(displayOutputs); viz.setParams(displayOutputs);
// Drive the heatmap bar widths imperatively (cheap; no React churn). // Drive the heatmap bar widths imperatively (cheap; no React churn).
for (let i = 0; i < N_VISUAL_OUTPUTS; i++) { for (let i = 0; i < N_VISUAL_OUTPUTS; i++) {
@ -394,7 +402,10 @@ export function ParticleStage({ pos, onMove, params, values, onChange }: Particl
</div> </div>
)} )}
{/* Adjustable circular pad — explicit edit handles keep repositioning/resizing deliberate. */} {/* Adjustable circular pad reuse Manifold's visual surface so Particle
mode shows the same feedback marks and cursor trail as every other
input presentation. The outer stage owns follow-mouse, therefore the
embedded Manifold's double-click mode is disabled. */}
<div <div
style={{ style={{
position: 'absolute', position: 'absolute',
@ -406,14 +417,21 @@ export function ParticleStage({ pos, onMove, params, values, onChange }: Particl
pointerEvents: 'none', pointerEvents: 'none',
}} }}
> >
<VirtualJoystick <div
size={padSize} role="application"
position={pos} aria-label="particle input pad"
tabIndex={padEdit || followMouse ? -1 : 0}
style={{ position: 'absolute', inset: 0, pointerEvents: 'auto' }}
>
<Manifold
pos={pos}
onMove={onMove} onMove={onMove}
disabled={padEdit || followMouse} markers={markers}
ariaLabel="particle input pad" variant="circular"
style={{ pointerEvents: padEdit || followMouse ? 'none' : 'auto' }} frozen={padEdit || followMouse}
followMouseEnabled={false}
/> />
</div>
<button <button
type="button" type="button"
aria-label={padEdit ? 'Finish adjusting joystick' : 'Adjust joystick size and position'} aria-label={padEdit ? 'Finish adjusting joystick' : 'Adjust joystick size and position'}

View file

@ -69,6 +69,15 @@ function noise2D(x: number, y: number): number {
const TWO_PI = Math.PI * 2; const TWO_PI = Math.PI * 2;
export const N_FLOW_PARTICLES = 400;
/**
* Contiguous id bands used to batch Canvas2D paths. Thirty-two bands keep the
* original hue sweep visually continuous (at most 3.75° between bands at the
* maximum 120° spread) while replacing 400 dynamic colour parses + fills with
* 32 per frame.
*/
export const FLOW_COLOR_BUCKETS = 32;
/** /**
* The 20 visual output params, in output order (p0..p19). Names are verbatim * The 20 visual output params, in output order (p0..p19). Names are verbatim
* from the a-immersive original (`VISUAL_PARAM_NAMES`, a-app.js:46). * from the a-immersive original (`VISUAL_PARAM_NAMES`, a-app.js:46).
@ -157,7 +166,11 @@ export class FlowFieldVisualizer {
private canvas: HTMLCanvasElement; private canvas: HTMLCanvasElement;
private ctx: CanvasRenderingContext2D; private ctx: CanvasRenderingContext2D;
private particles: Particle[] = []; private particles: Particle[] = [];
private numParticles = 400; private colorBuckets: Particle[][] = Array.from(
{ length: FLOW_COLOR_BUCKETS },
() => [],
);
private numParticles = N_FLOW_PARTICLES;
private time = 0; private time = 0;
private width = 0; private width = 0;
private height = 0; private height = 0;
@ -210,8 +223,15 @@ export class FlowFieldVisualizer {
private initParticles(): void { private initParticles(): void {
this.particles = []; this.particles = [];
for (const bucket of this.colorBuckets) bucket.length = 0;
for (let i = 0; i < this.numParticles; i++) { for (let i = 0; i < this.numParticles; i++) {
this.particles.push(this.makeParticle(i)); const particle = this.makeParticle(i);
this.particles.push(particle);
const bucket = Math.min(
FLOW_COLOR_BUCKETS - 1,
Math.floor((i * FLOW_COLOR_BUCKETS) / this.numParticles),
);
this.colorBuckets[bucket].push(particle);
} }
this.ctx.fillStyle = '#0d0d0d'; this.ctx.fillStyle = '#0d0d0d';
this.ctx.fillRect(0, 0, this.width, this.height); this.ctx.fillRect(0, 0, this.width, this.height);
@ -317,10 +337,14 @@ export class FlowFieldVisualizer {
ctx.fillStyle = `rgba(13, 13, 13, ${params.fadeRate})`; ctx.fillStyle = `rgba(13, 13, 13, ${params.fadeRate})`;
ctx.fillRect(0, 0, width, height); ctx.fillRect(0, 0, width, height);
for (const p of this.particles) {
const cx = width * 0.5; const cx = width * 0.5;
const cy = height * 0.5; const cy = height * 0.5;
const repulsorRadius = Math.min(width, height) * 0.28;
// Advance the simulation without touching Canvas state. Rendering is
// batched by colour below, avoiding one dynamic colour parse + fill call
// per particle.
for (const p of this.particles) {
// Sample flow field // Sample flow field
const nx = p.x * params.scale; const nx = p.x * params.scale;
const ny = p.y * params.scale; const ny = p.y * params.scale;
@ -378,7 +402,6 @@ export class FlowFieldVisualizer {
nextY -= nyCenter * dispersionForce; nextY -= nyCenter * dispersionForce;
// Orbiting repulsor points carve dynamic voids and bursts. // Orbiting repulsor points carve dynamic voids and bursts.
const repulsorRadius = Math.min(width, height) * 0.28;
for (let r = 0; r < params.repulsorCount; r++) { for (let r = 0; r < params.repulsorCount; r++) {
const phase = this.time * params.repulsorOrbitRate + (r / 4) * TWO_PI; const phase = this.time * params.repulsorOrbitRate + (r / 4) * TWO_PI;
const wobble = 0.6 + 0.15 * r; const wobble = 0.6 + 0.15 * r;
@ -404,14 +427,28 @@ export class FlowFieldVisualizer {
p.age += 1; p.age += 1;
if (p.age >= p.life) this.respawnParticle(p); if (p.age >= p.life) this.respawnParticle(p);
}
// Colour based on particle id + hue params // Preserve the original id-ordered hue/lightness sweep with one
const hue = (params.hueBase + (p.id / this.numParticles) * params.hueSpread) % 360; // representative colour per contiguous band. Each band becomes one path
const lightness = 50 + Math.sin(p.id * 0.1 + this.time) * 15; // and one fill instead of every particle parsing its own HSL string.
for (const bucket of this.colorBuckets) {
if (bucket.length === 0) continue;
const representative = bucket[Math.floor(bucket.length * 0.5)];
const hue =
(params.hueBase +
(representative.id / this.numParticles) * params.hueSpread) %
360;
const lightness =
50 + Math.sin(representative.id * 0.1 + this.time) * 15;
ctx.fillStyle = `hsl(${hue}, 75%, ${lightness}%)`; ctx.fillStyle = `hsl(${hue}, 75%, ${lightness}%)`;
ctx.beginPath(); ctx.beginPath();
for (const p of bucket) {
// Start an independent sub-path so adjacent circles never acquire a
// connecting edge when the whole colour band is filled at once.
ctx.moveTo(p.x + params.particleSize, p.y);
ctx.arc(p.x, p.y, params.particleSize, 0, TWO_PI); ctx.arc(p.x, p.y, params.particleSize, 0, TWO_PI);
}
ctx.fill(); ctx.fill();
} }
} }

View file

@ -379,17 +379,32 @@ export { applyCurve };
* onto its own N outputs; the `i < engineOut.length` guard keeps it safe during * onto its own N outputs; the `i < engineOut.length` guard keeps it safe during
* the async reshape window. * the async reshape window.
*/ */
export function shapeValues(params: MFParam[], engineOut: Float32Array | null): number[] { export function shapeValuesInto<T extends { length: number; [index: number]: number }>(
return params.map((p, i) => { params: MFParam[],
if (p.status === 'off') return 0; engineOut: Float32Array | null,
out: T,
): T {
const count = Math.min(params.length, out.length);
for (let i = 0; i < count; i++) {
const p = params[i];
if (p.status === 'off') {
out[i] = 0;
continue;
}
if (p.status === 'fixed') { if (p.status === 'fixed') {
return p.min + Math.max(0, Math.min(1, p.val ?? 0.5)) * (p.max - p.min); out[i] = p.min + Math.max(0, Math.min(1, p.val ?? 0.5)) * (p.max - p.min);
continue;
} }
const engineIndex = p.engineIndex ?? i; const engineIndex = p.engineIndex ?? i;
const raw = engineOut && engineIndex < engineOut.length ? engineOut[engineIndex] : 0.5; const raw = engineOut && engineIndex < engineOut.length ? engineOut[engineIndex] : 0.5;
const v = p.min + applyCurve(raw, p.curve) * (p.max - p.min); const v = p.min + applyCurve(raw, p.curve) * (p.max - p.min);
return Math.max(0, Math.min(1, v)); out[i] = Math.max(0, Math.min(1, v));
}); }
return out;
}
export function shapeValues(params: MFParam[], engineOut: Float32Array | null): number[] {
return shapeValuesInto(params, engineOut, new Array<number>(params.length));
} }
/** Create a backend-agnostic output card after every schema output is active. */ /** Create a backend-agnostic output card after every schema output is active. */

View file

@ -10,8 +10,9 @@
* The engine imports NO React. The only React in `engine/` is the * The engine imports NO React. The only React in `engine/` is the
* EngineProvider/useEngine binding layer (separate files). * EngineProvider/useEngine binding layer (separate files).
* *
* `subscribe(cb)` + `version()` are the `useSyncExternalStore` contract: React * `subscribe(cb)` + `version()` expose structural/training state changes.
* re-reads on a version bump but consumers read the live Float32Array * `subscribeOutputs(cb)` + `outputVersion()` expose a throttled live-output
* channel for DOM consumers. Canvas consumers read the live Float32Array
* imperatively via `getOutputs()` / `routedOutput()`. * imperatively via `getOutputs()` / `routedOutput()`.
*/ */
@ -410,6 +411,16 @@ export class EngineApi {
return this.spine.version(); return this.spine.version();
} }
/** Subscribe to throttled live-output changes for non-canvas UI consumers. */
subscribeOutputs(cb: () => void): () => void {
return this.spine.subscribeOutputs(cb);
}
/** Latest live-output revision; increments on every inference. */
outputVersion(): number {
return this.spine.outputVersion();
}
/** Subscribe to a named engine event (`ml.*`, `feedback.*`, …). */ /** Subscribe to a named engine event (`ml.*`, `feedback.*`, …). */
on(event: string, fn: (payload?: unknown) => void): () => void { on(event: string, fn: (payload?: unknown) => void): () => void {
return this.spine.on(event, fn); return this.spine.on(event, fn);

View file

@ -55,7 +55,12 @@ export type {
export { EngineProvider, EngineContext } from './EngineProvider'; export { EngineProvider, EngineContext } from './EngineProvider';
export type { EngineProviderProps } from './EngineProvider'; export type { EngineProviderProps } from './EngineProvider';
export { useEngine, useEngineOrThrow, useEngineVersion } from './useEngine'; export {
useEngine,
useEngineOrThrow,
useEngineVersion,
useEngineOutputVersion,
} from './useEngine';
// Pipeline config types + defaults. The PROCESSING lives in the C++/WASM core // Pipeline config types + defaults. The PROCESSING lives in the C++/WASM core
// (one-core-engine P4); configure via EngineApi.setInputConfig / setOutputConfig. // (one-core-engine P4); configure via EngineApi.setInputConfig / setOutputConfig.

View file

@ -10,9 +10,10 @@
* all C++/WASM chains since one-core-engine P4) and fires the single * all C++/WASM chains since one-core-engine P4) and fires the single
* `backend.send` at the action TAIL, off React's render cycle. * `backend.send` at the action TAIL, off React's render cycle.
* *
* React subscribes via `useSyncExternalStore(subscribe, version)` the version * Structural/training state uses `subscribe/version`; opt-in DOM output
* counter, NOT the array and reads the live `Float32Array` imperatively (so * consumers use the throttled `subscribeOutputs/outputVersion` channel.
* canvases never re-render per frame). * Canvases read the live `Float32Array` imperatively and never re-render per
* frame.
* *
* Buffers are reused (no per-frame allocation): `routedBuf` is a single * Buffers are reused (no per-frame allocation): `routedBuf` is a single
* Float32Array threaded through the output pipeline and handed to the backend. * Float32Array threaded through the output pipeline and handed to the backend.
@ -69,7 +70,11 @@ export class Spine implements EngineSink {
}; };
private listeners = new Set<() => void>(); private listeners = new Set<() => void>();
private outputListeners = new Set<() => void>();
private eventListeners = new Map<string, Set<(payload?: unknown) => void>>(); private eventListeners = new Map<string, Set<(payload?: unknown) => void>>();
private outputVersion_ = 0;
private lastOutputNotifyMs = -Infinity;
private readonly outputNotifyIntervalMs = 1000 / 30;
// Engine handles wired in via `attach`. // Engine handles wired in via `attach`.
private iml: WasmIML | null = null; private iml: WasmIML | null = null;
@ -235,7 +240,9 @@ export class Spine implements EngineSink {
* the 2-D manifold / XY-pad path delegates to {@link setInputs}. * the 2-D manifold / XY-pad path delegates to {@link setInputs}.
*/ */
setInput(x: number, y: number): Float32Array | null { setInput(x: number, y: number): Float32Array | null {
return this.setInputs([x, y]); this.rawInput[0] = x;
this.rawInput[1] = y;
return this.setInputs(this.rawInput);
} }
/** /**
@ -282,7 +289,7 @@ export class Spine implements EngineSink {
// 3. ml (inference into the reused buffer; no alloc). // 3. ml (inference into the reused buffer; no alloc).
iml.processInto(this.mlBuf); iml.processInto(this.mlBuf);
this.liveOutputs.set(this.mlBuf.subarray(0, this.liveOutputs.length)); this.liveOutputs.set(this.mlBuf);
// 4. routed (output chain, in place on the reused routedBuf → C++-side state). // 4. routed (output chain, in place on the reused routedBuf → C++-side state).
if (!this.routedBuf || this.routedBuf.length !== this.mlBuf.length) { if (!this.routedBuf || this.routedBuf.length !== this.mlBuf.length) {
@ -298,7 +305,7 @@ export class Spine implements EngineSink {
// 5. single backend.send at the tail (off React render). // 5. single backend.send at the tail (off React render).
if (this.backendSend && this.routedBuf) this.backendSend(this.routedBuf); if (this.backendSend && this.routedBuf) this.backendSend(this.routedBuf);
this.bump_(); this.publishOutputs_();
return this.routedBuf; return this.routedBuf;
} }
@ -342,6 +349,18 @@ export class Spine implements EngineSink {
version = (): number => this.state_.version; version = (): number => this.state_.version;
/**
* Live-output subscription kept separate from structural/training state.
* Notifications are capped at 30 Hz for DOM consumers; canvas consumers read
* the reused output buffers directly in their own rAF loops.
*/
subscribeOutputs = (cb: () => void): (() => void) => {
this.outputListeners.add(cb);
return () => { this.outputListeners.delete(cb); };
};
outputVersion = (): number => this.outputVersion_;
getState(): Readonly<SpineState> { getState(): Readonly<SpineState> {
return this.state_; return this.state_;
} }
@ -357,4 +376,14 @@ export class Spine implements EngineSink {
this.state_ = { ...this.state_, version: this.state_.version + 1 }; this.state_ = { ...this.state_, version: this.state_.version + 1 };
for (const fn of this.listeners) fn(); for (const fn of this.listeners) fn();
} }
private publishOutputs_(): void {
this.outputVersion_++;
if (this.outputListeners.size === 0) return;
const now =
typeof performance !== 'undefined' ? performance.now() : Date.now();
if (now - this.lastOutputNotifyMs < this.outputNotifyIntervalMs) return;
this.lastOutputNotifyMs = now;
for (const fn of this.outputListeners) fn();
}
} }

View file

@ -3,12 +3,9 @@
* *
* `useEngine()` returns the EngineApi from context (or null before it loads). * `useEngine()` returns the EngineApi from context (or null before it loads).
* *
* `useEngineVersion()` subscribes to engine state changes via * `useEngineVersion()` subscribes only to structural/training state.
* `useSyncExternalStore(engine.subscribe, engine.version)`. It returns the * `useEngineOutputVersion()` is the opt-in, throttled live-output channel for
* VERSION COUNTER (a number), NOT the output array so a component re-renders * DOM consumers. Canvas consumers read live buffers imperatively instead.
* when engine state changes but reads the live `Float32Array` imperatively
* (`engine.getOutputs()` / `engine.routedOutput()`) inside a rAF loop or on
* render. This keeps per-frame audio inference off React's render cycle.
*/ */
import { useContext, useSyncExternalStore } from 'react'; import { useContext, useSyncExternalStore } from 'react';
@ -41,3 +38,18 @@ export function useEngineVersion(engine: EngineApi | null): number {
() => 0, () => 0,
); );
} }
/**
* Subscribe to the engine's throttled live-output channel. Disabled consumers
* neither subscribe nor re-render and report version 0.
*/
export function useEngineOutputVersion(
engine: EngineApi | null,
enabled = true,
): number {
return useSyncExternalStore(
(cb) => (engine && enabled ? engine.subscribeOutputs(cb) : () => {}),
() => (engine && enabled ? engine.outputVersion() : 0),
() => 0,
);
}

View file

@ -5,7 +5,7 @@
* Sits ABOVE the engine: it owns a single rAF loop that, each frame, * Sits ABOVE the engine: it owns a single rAF loop that, each frame,
* 1. polls poll-based sources (gamepad buttons actions), * 1. polls poll-based sources (gamepad buttons actions),
* 2. pulls every active source's axes into the shared `vector` (pull-based), * 2. pulls every active source's axes into the shared `vector` (pull-based),
* 3. blends/maps the N-dim vector down to the engine's input arity, and * 3. maps the N-dim vector onto the engine's input arity, and
* 4. fires engine.setInputs(...) exactly once. * 4. fires engine.setInputs(...) exactly once.
* *
* The XY pad remains push-driven via the existing onMove handler when it's the * The XY pad remains push-driven via the existing onMove handler when it's the
@ -14,14 +14,12 @@
* composable and the channel layout coherent. * composable and the channel layout coherent.
* *
* Dedicated dimensions (no blending) * Dedicated dimensions (no blending)
* The WASM net is over-provisioned to a 32-input head (= MAX_AXES; see * Each active axis drives its OWN engine input slot 1:1 a double-stick
* nisps/wasm/bindings.cpp). Each active axis drives its OWN engine input slot * gamepad is 4 genuine dims, and a learned MIDI surface is N genuine dims.
* 1:1 a double-stick gamepad is 4 genuine dims, a learned MIDI surface is N * `compose()` forwards the active axes and truncates only when a transient or
* genuine dims. `compose()` simply forwards the active axes; the engine * deliberately smaller model cannot accept all of them. We do NOT mean-blend
* zero-pads the remaining slots and a zero input is inert (0 × weight = 0), so * (the previous behaviour) that diluted every source and biased the net
* unused dimensions never perturb the net. We do NOT mean-blend (the previous * toward idle sources' resting values.
* behaviour) that diluted every source and biased the net toward idle
* sources' resting values.
* *
* Changing the ACTIVE layout runs through ConsoleApp's persistent I/O policy: * Changing the ACTIVE layout runs through ConsoleApp's persistent I/O policy:
* keep-capacity remaps stable dimensions in place until capacity is exceeded; * keep-capacity remaps stable dimensions in place until capacity is exceeded;
@ -42,6 +40,7 @@ export class InputLayer {
private sources: InputSource[] = []; private sources: InputSource[] = [];
private engine: InputEngineSink | null = null; private engine: InputEngineSink | null = null;
private vector = new Float32Array(MAX_AXES); private vector = new Float32Array(MAX_AXES);
private reduced: number[] = [];
private running = false; private running = false;
private rafId: number | null = null; private rafId: number | null = null;
private actionListeners = new Set<(a: InputAction) => void>(); private actionListeners = new Set<(a: InputAction) => void>();
@ -136,7 +135,7 @@ export class InputLayer {
} }
if (n === 0) return; if (n === 0) return;
// 3. reduce to the engine's input arity (see file header — blend, not fake). // 3. reduce to the engine's current input arity (dedicated, not blended).
const reduced = this.compose(n, engine.architecture.inputSize); const reduced = this.compose(n, engine.architecture.inputSize);
// 4. one engine write. // 4. one engine write.
@ -157,17 +156,14 @@ export class InputLayer {
* the engine zero-pads the slots beyond `count` and a zero input is inert * the engine zero-pads the slots beyond `count` and a zero input is inert
* (0 × weight = 0), so unused dimensions never perturb the net. * (0 × weight = 0), so unused dimensions never perturb the net.
* *
* The net's input arity is over-provisioned (32, = MAX_AXES), so `inputSize` * The ConsoleApp owns the reshape policy and grows the model when needed.
* is effectively always n; the `min` only guards a transient where more * The `min` guards the short transition without blending or reassigning axes.
* axes are active than the net can take. We deliberately do NOT mean-blend
* (the old behaviour) that diluted every source and biased the net toward
* idle sources' resting values.
*/ */
private compose(n: number, inputSize: number): number[] { private compose(n: number, inputSize: number): number[] {
const count = Math.min(n, inputSize); const count = Math.min(n, inputSize);
const out = new Array<number>(count); this.reduced.length = count;
for (let i = 0; i < count; i++) out[i] = this.vector[i]; for (let i = 0; i < count; i++) this.reduced[i] = this.vector[i];
return out; return this.reduced;
} }
// ---- actions + layout fan-out ------------------------------------------- // ---- actions + layout fan-out -------------------------------------------

View file

@ -0,0 +1,65 @@
import { expect, test } from 'bun:test';
import { Spine } from '../src/engine/spine';
import type { WasmIML } from '../src/engine/wasm-iml';
import { InputLayer } from '../src/inputs/input-layer';
import { XYPadSource } from '../src/inputs/xy-pad-source';
function makeFakeIml(outputSize = 20): WasmIML {
return {
architecture: {
inputSize: 2,
hidden: [0, 0, 0] as [number, number, number],
outputSize,
numLayers: 3,
},
setInputConfig: () => {},
setOutputConfig: () => {},
setOutputFreezeMask: () => {},
resetInput: () => {},
resetOutput: () => {},
processInput: (x: number, y: number) => ({ x, y, frozen: false }),
setInput: () => {},
processInto: (buf: Float32Array) => buf.fill(0.25),
processOutput: () => {},
} as unknown as WasmIML;
}
test('live inference publishes through the output channel without notifying state subscribers', () => {
const spine = new Spine();
spine.attach(makeFakeIml(), null);
spine.setState({ outputSize: 20 });
let stateNotifications = 0;
let outputNotifications = 0;
spine.subscribe(() => stateNotifications++);
spine.subscribeOutputs(() => outputNotifications++);
spine.setInputs([0.2, 0.8]);
spine.setInputs([0.3, 0.7]);
expect(stateNotifications).toBe(0);
expect(outputNotifications).toBe(1);
expect(spine.outputVersion()).toBe(2);
expect(Array.from(spine.outputs())).toEqual(new Array(20).fill(0.25));
});
test('InputLayer reuses its reduced input vector across animation frames', () => {
const layer = new InputLayer();
const source = new XYPadSource();
const writes: ReadonlyArray<number>[] = [];
layer.attach({
architecture: { inputSize: 2 },
setInputs: (values) => writes.push(values),
});
layer.setSources([source]);
source.pushAxes(0.2, 0.8);
layer.frame();
source.pushAxes(0.3, 0.7);
layer.frame();
expect(writes).toHaveLength(2);
expect(writes[0]).toBe(writes[1]);
expect(Array.from(writes[1])).toEqual(
Array.from(new Float32Array([0.3, 0.7])),
);
});

View file

@ -0,0 +1,87 @@
import { expect, test } from 'bun:test';
import {
FLOW_COLOR_BUCKETS,
FlowFieldVisualizer,
N_FLOW_PARTICLES,
} from '../src/console/flow-field';
interface FakeContext {
fillStyles: string[];
fillCount: number;
arcCount: number;
fillStyle: string;
setTransform(): void;
scale(): void;
fillRect(): void;
beginPath(): void;
moveTo(): void;
arc(): void;
fill(): void;
}
function makeContext(): FakeContext {
const ctx = {
fillStyles: [] as string[],
fillCount: 0,
arcCount: 0,
setTransform() {},
scale() {},
fillRect() {},
beginPath() {},
moveTo() {},
arc() {
ctx.arcCount++;
},
fill() {
ctx.fillCount++;
},
} as FakeContext;
Object.defineProperty(ctx, 'fillStyle', {
set(value: string) {
ctx.fillStyles.push(value);
},
get() {
return ctx.fillStyles.at(-1) ?? '';
},
});
return ctx;
}
test('particle draw batches dynamic colours instead of parsing one HSL string per particle', () => {
const previousWindow = globalThis.window;
Object.defineProperty(globalThis, 'window', {
value: { devicePixelRatio: 1 },
configurable: true,
});
try {
const ctx = makeContext();
const canvas = {
getContext: () => ctx,
getBoundingClientRect: () => ({ width: 640, height: 480 }),
width: 0,
height: 0,
} as unknown as HTMLCanvasElement;
const visualizer = new FlowFieldVisualizer(canvas);
ctx.fillStyles.length = 0;
ctx.fillCount = 0;
ctx.arcCount = 0;
visualizer.draw();
expect(ctx.arcCount).toBe(N_FLOW_PARTICLES);
expect(ctx.fillCount).toBe(FLOW_COLOR_BUCKETS);
expect(ctx.fillStyles.filter((value) => value.startsWith('hsl('))).toHaveLength(
FLOW_COLOR_BUCKETS,
);
} finally {
if (previousWindow === undefined) {
delete (globalThis as { window?: Window }).window;
} else {
Object.defineProperty(globalThis, 'window', {
value: previousWindow,
configurable: true,
});
}
}
});