fix(manifold): smooth particle rendering
This commit is contained in:
parent
a1cd26ff68
commit
856642a962
14 changed files with 427 additions and 114 deletions
21
MAP.md
21
MAP.md
|
|
@ -39,16 +39,19 @@ anchor + locked decisions) and the `docs/specs/*-spec.md` set.
|
|||
framework-neutral: `wasm-iml.ts` (rewired off Solid stores onto an injected `EngineSink`), `engine-host.ts` +
|
||||
`worklet/nisps-processor.ts` (audio), thin WASM wrappers over the core pipelines + curve catalog (the TS
|
||||
`input-pipeline`/`output-pipeline`/`curves` implementations died at P4), `wasm-worker.ts`,
|
||||
`spine.ts` (the reactive spine BELOW React — `setInput` derives processed→ml→routed eagerly off-render),
|
||||
`spine.ts` (the reactive spine BELOW React — `setInput` derives processed→ml→routed eagerly off-render;
|
||||
structural/training state and throttled live-output notifications are separate channels),
|
||||
`engine-api.ts` (`EngineApi` façade incl. live architecture/weight/example metrics and `feedback.*` wrappers over the `nisps_ml_feedback_*` C ABI),
|
||||
`EngineProvider.tsx`/`useEngine.ts` (React binding via `useSyncExternalStore` version counter). nisps.js is
|
||||
`EngineProvider.tsx`/`useEngine.ts` (React bindings via `useSyncExternalStore`: state version plus an
|
||||
opt-in 30 Hz output version for DOM consumers; canvases read live buffers directly). nisps.js is
|
||||
loaded via fetch+indirect-eval (Emscripten MODULARIZE glue has no ES exports), base-aware via `document.baseURI`
|
||||
for the `/next` sub-path.
|
||||
- `manifold/src/primitives/` — the 7 design primitives as typed React (Badge, Button, PillToggle, Slider, Switch, VirtualJoystick, XYPad). Five unused ones were deleted in the 2026-07 sweep (L22).
|
||||
- `manifold/src/console/` — the convertible Console: `ConsoleApp`, `CompositeStage` (single-divider convertible
|
||||
with snap/magnetism/minimap-demotion), `OutputStage`/`SandwichStage`/`ParticleStage`/`Manifold` (canvas,
|
||||
rect↔circular + feedback markers; ParticleStage has interactive cursor-labelled heatmap sliders, an adjustable
|
||||
joystick, and double-click whole-screen follow-mouse input), `Dock` (top Mode selector + 5 vertically-centred drawers), `Drawers`
|
||||
rect↔circular + feedback markers; ParticleStage batches its 400 particles into 32 colour paths per frame,
|
||||
has interactive cursor-labelled heatmap sliders, an adjustable circular Manifold pad with feedback markers and cursor trail,
|
||||
and double-click whole-screen follow-mouse input), `Dock` (top Mode selector + 5 vertically-centred drawers), `Drawers`
|
||||
(Learning/Inputs/Outputs/Settings/Help; Learning includes the live model-architecture inspector and
|
||||
Outputs owns the remaining scroll height), `TrainingHealth` (real per-iteration loss curve from
|
||||
`nisps_ml_loss_history` + per-layer weight health from `nisps_ml_get_layer_stats`; rendered only at
|
||||
|
|
@ -73,10 +76,10 @@ anchor + locked decisions) and the `docs/specs/*-spec.md` set.
|
|||
- `manifold/src/inputs/` — modular INPUT layer feeding the ML head. The Inputs dock picks ONE exclusive mode
|
||||
(`InputMode` = `internal` | `gamepad` | `midi`; Internal/XY-pad is default). `input-layer.ts` owns a single rAF
|
||||
loop composing the active source's axes → **one dedicated engine input slot per axis, 1:1, no blending** → one
|
||||
`setInputs`, plus an `onReducedInput` callback the manifold tracks. The WASM net is over-provisioned to a
|
||||
32-input head (`MAX_AXES`, `nisps/wasm/bindings.cpp`); unused slots are zero-padded and a zero input is inert,
|
||||
so idle sources cannot perturb the net. Mean-blending was removed deliberately — it diluted every source and
|
||||
biased the net toward idle sources' resting values. Active-axis edits follow the persistent I/O policy:
|
||||
`setInputs`, plus an `onReducedInput` callback the manifold tracks. Normal Manifold models default to a
|
||||
runtime-shaped 2-input head; the Inputs dock can select 4 inputs, and active sources can grow the head when
|
||||
they need more slots. Mean-blending was removed deliberately — it diluted every source and biased the net
|
||||
toward idle sources' resting values. Active-axis edits follow the persistent I/O policy:
|
||||
keep-capacity permutes stable identities in place until more slots are required; exact-I/O reconstructs
|
||||
to the active count. Surviving weights and (under adapt policy) examples are identity-remapped; feedback
|
||||
scratch state resets. Sources: `xy-pad-source` (push-driven),
|
||||
|
|
@ -178,7 +181,7 @@ includes; no `nisps-core`.
|
|||
- C++ identifiers: `PascalCase` types, `snake_case` functions/variables, `kPascalCase` constexpr. JSON keys `snake_case`. TS types `PascalCase`, components `PascalCase.tsx`, modules `kebab-case.ts`.
|
||||
- `Curve` enum lives in `nisps/core/math.hpp` (lowercase: `linear/exp/log/square/sqrt/sigmoid/cubic`, plus the parameterised `centered_power` free function); generated mode headers re-export via `using Curve = ::nisps::Curve;`. Since P4 there is NO TS mirror — the browser samples the WASM catalog (`nisps_curve_apply(+batch)`).
|
||||
- Modes are TSX components composed of primitives; mode parameter contracts are JSON schemas with codegen → C++ **and** TS types (`MF_MODES` derives params/ml-config from the generated schemas since P5; labels/ordering stay a manifold overlay). **No declarative JSON UI.**
|
||||
- WASM and firmware share the same C++; the browser MLP is runtime-shaped (`MLPCore<DynamicStorage>`, since P2): `nisps_ml_create` honours `(input, output, hidden[3])` with non-positive/null args defaulting to `32→[10,14,18]→126`; `nisps_ml_reshape` warm-starts a new shape. Firmware keeps compile-time `MLP<...>` (zero heap). Per-mode dims are schema-real on both targets since P5.3 (the browser reshapes on mode switch).
|
||||
- WASM and firmware share the same C++; the browser MLP is runtime-shaped (`MLPCore<DynamicStorage>`, since P2): `nisps_ml_create` honours `(input, output, hidden[3])` with non-positive/null args defaulting to `32→[10,14,18]→126`; `nisps_ml_reshape` warm-starts a new shape. Firmware keeps compile-time `MLP<...>` (zero heap). Manifold normal modes use a 2-input UI default and can opt into 4 inputs; audio-analysis keeps its schema-defined feature arity. Firmware mode shapes remain schema-real.
|
||||
- Cross-platform parity: `scripts/parity-check.sh` enforces native vs WASM agreement within 1e-5.
|
||||
|
||||
## Gotchas
|
||||
|
|
|
|||
|
|
@ -70,12 +70,13 @@ single `backend.send` at the action's tail, off-render. Contract rules:
|
|||
call in the derivation path.
|
||||
- **No per-frame allocation.** Buffers are reused (`routedBuf` threaded through the output
|
||||
pipeline and handed to the backend).
|
||||
- **Weights mutate only through engine actions** (train / feedback / reshape), each of which bumps
|
||||
the spine's version counter — there is no write path that can update audio without notifying
|
||||
the UI.
|
||||
- **React subscribes via `useSyncExternalStore(subscribe, version)`** — the version counter, not
|
||||
the arrays; canvases read the live `Float32Array`s imperatively in rAF and never re-render per
|
||||
frame.
|
||||
- **Weights mutate only through engine actions** (train / feedback / reshape). Structural/training
|
||||
state bumps `version`; every inference advances a separate `outputVersion`. Both expose explicit
|
||||
subscriptions, so audio propagation never depends on React scheduling.
|
||||
- **React state and live output are separate channels.** Structural/training consumers use
|
||||
`useSyncExternalStore(subscribe, version)`. DOM output consumers opt into the throttled 30 Hz
|
||||
`subscribeOutputs/outputVersion` channel. Canvases read and shape the live `Float32Array`s
|
||||
imperatively in rAF and never re-render per frame.
|
||||
|
||||
### 2.2 The live-feedback guarantee and its e2e assertion
|
||||
|
||||
|
|
|
|||
|
|
@ -66,13 +66,13 @@ bun run test:e2e # Playwright smoke (needs `bun run build` first; runs again
|
|||
│ ConsoleApp = spine of the UI: holds state, picks a Stage, │
|
||||
│ renders the Dock. "Convertible" = swappable Stages. │
|
||||
└───────────────┬─────────────────────────────────────────────────-┘
|
||||
│ reads engine.version (useSyncExternalStore), reads buffers imperatively
|
||||
│ reads state/output versions (useSyncExternalStore), canvases read buffers imperatively
|
||||
│ writes via engine.setInput / setParam / feedback.*
|
||||
┌───────────────▼─────────────────────────────────────────────────┐
|
||||
│ Engine layer (NO React) src/engine/ src/inputs/ src/feedback/ src/backends/
|
||||
│ Spine = reactive store below React. Per-frame ML inference is │
|
||||
│ eager + synchronous, OFF the render cycle. React only watches │
|
||||
│ a monotonic version counter. │
|
||||
│ eager + synchronous, OFF the render cycle. React watches state │
|
||||
│ changes plus an opt-in 30 Hz output channel for DOM surfaces. │
|
||||
└───────────────┬─────────────────────────────────────────────────┘
|
||||
│ C ABI
|
||||
┌───────────────▼─────────────────────────────────────────────────┐
|
||||
|
|
@ -109,7 +109,7 @@ Manifold ships a single "composite" altitude. Selection is now a plain three-way
|
|||
|---|---|---|---|
|
||||
| CompositeStage | `CompositeStage.tsx` | **default / hero** | Draggable split-ratio; magnet-snaps to 0.14/0.33/0.5/0.66/0.86; collapses a side to a corner minimap at extremes. |
|
||||
| SandwichStage | `SandwichStage.tsx` | `sandwich===true` (wins over the others) | Three-pane layout: `Manifold` input surface left, 3D parameter-landscape centre (input → MLP heatmap grid → outputs, drag to orbit), compact `OutputStage` right. |
|
||||
| ParticleStage | `ParticleStage.tsx` | `outputMode==='particles'` | Flow-field visualiser (`flow-field.ts`, 400-particle Canvas2D port) + interactive output heatmap sliders with cursor tooltips + a larger, explicitly adjustable/repositionable joystick; double-click anywhere in the stage enters whole-screen follow-mouse mode. |
|
||||
| ParticleStage | `ParticleStage.tsx` | `outputMode==='particles'` | Flow-field visualiser (`flow-field.ts`, 400 particles batched into 32 contiguous colour paths per frame) + interactive output heatmap sliders with cursor tooltips + a larger, explicitly adjustable/repositionable circular Manifold pad (feedback markers + cursor trail); double-click anywhere in the stage enters whole-screen follow-mouse mode. |
|
||||
|
||||
`Manifold.tsx` and `OutputStage.tsx` are no longer top-level stages — they are panes composed by
|
||||
CompositeStage/SandwichStage. `Manifold.tsx` is the full-bleed 2D input surface (canvas trail + pins
|
||||
|
|
@ -208,12 +208,15 @@ a setting → `--r-*` tokens.
|
|||
- `engine-api.ts` — **`EngineApi`, the framework-neutral facade** everything in the UI talks to:
|
||||
`setInput/setInputs`, `getOutputs/routedOutput`, training (`addExample/train/trainAsync/evalLoss`),
|
||||
weights (`getWeights/setWeights/process/randomise`), telemetry
|
||||
(`architecture/weightCount/exampleCount/lossHistory/getLayerStats`), `subscribe/version/on`, plus nested `.feedback` and `.audio`
|
||||
(`architecture/weightCount/exampleCount/lossHistory/getLayerStats`), structural `subscribe/version`,
|
||||
throttled `subscribeOutputs/outputVersion`, and `on`, plus nested `.feedback` and `.audio`
|
||||
facades. **`lossHistory()` reads SPINE STATE, not the MLP handle** — an async train runs on the
|
||||
worker's mirror net, so the main handle's own history is empty for those runs; both paths
|
||||
publish to the spine.
|
||||
- `EngineProvider.tsx` / `useEngine.ts` — the **only** React coupling. `useEngine()` returns the API
|
||||
(null until WASM ready); `useEngineVersion()` = `useSyncExternalStore(subscribe, version)`.
|
||||
(null until WASM ready); `useEngineVersion()` watches structural/training state, while
|
||||
`useEngineOutputVersion()` opts DOM consumers into the throttled 30 Hz output channel. Particle canvases
|
||||
use neither output subscription nor React state for live values; they shape the reused buffer in rAF.
|
||||
- `engine-host.ts` — main-thread audio wiring: AudioContext (user-gesture gated), fetch `nisps.wasm`,
|
||||
register + feed the worklet.
|
||||
- `wasm-iml.ts` (**~750 lines**) — the ML interface to `nisps.wasm`: one MLP handle, dataset, heap
|
||||
|
|
@ -243,16 +246,16 @@ a setting → `--r-*` tokens.
|
|||
|
||||
### Inputs — `src/inputs/`
|
||||
- `input-layer.ts` — composition hub. One rAF loop polls sources, pulls all axes into a vector,
|
||||
forwards N→engine. **`MAX_AXES = 32`** (WASM net over-provisioned to 32 inputs). **Dedicated
|
||||
dimensions, NO mean-blending** — each active axis drives its own engine slot 1:1; unused slots
|
||||
zero-padded (inert). Changing the layout uses the same persisted identity-aware I/O policy as output
|
||||
cards.
|
||||
forwards N→engine. **`MAX_AXES = 32`** is the source-vector safety cap, not the model shape.
|
||||
**Dedicated dimensions, NO mean-blending** — each active axis drives its own engine slot 1:1;
|
||||
the ConsoleApp reshapes the runtime model when the selected 2/4-input capacity or active layout
|
||||
requires it. Changing the layout uses the same persisted identity-aware I/O policy as output cards.
|
||||
- `base-source.ts` + sources: `xy-pad-source.ts` (push, 2 axes), `gamepad-source.ts` (single=2 /
|
||||
double=4 axes, deadzone 0.08), `midi-input-source.ts` (Web MIDI, batch CC-learn, multi-port).
|
||||
- `useInputLayer.ts` — React binding; manages exclusive input mode + gamepad stick mode + MIDI
|
||||
device/learn map; exposes `pushPad`, `sources`, `channelLayout`, etc.
|
||||
- **I/O migration (P2.3, live):** the net is **runtime-shaped**. It boots at the default
|
||||
over-provisioned 32-input head (zero-padding preserved), and `EngineApi.reshape({ inputSize, … })`
|
||||
- **I/O migration (P2.3, live):** the net is **runtime-shaped**. Normal Manifold modes boot at
|
||||
2 inputs; the Inputs dock can select 4, and active layouts can grow the model further. `EngineApi.reshape({ inputSize, … })`
|
||||
→ `WasmIML.reshape`. `engine/io-reshape.ts` owns the identity map: **Keep capacity** (default)
|
||||
permutes weights/examples in place and reconstructs only when active I/O outgrows the net;
|
||||
**Exact I/O** reconstructs whenever active arity changes. Existing examples either adapt by
|
||||
|
|
@ -262,14 +265,15 @@ a setting → `--r-*` tokens.
|
|||
(`wasm-worker.ts`) carries the current dims in its train message and re-creates its mirror net to
|
||||
match. The raw debug `window.__nisps.reshape(nIn)` remains a low-level reconstruct-and-clear call.
|
||||
- **Per-mode net dims (P5.3):** switching INSTRUMENT mode reshapes the net to that mode's schema
|
||||
`ml` config (`MFMode.ml` — input/hidden/output + legacy spread) via a `ConsoleApp` effect keyed on
|
||||
`[engine, modeId]`. No confirm modal (switching instrument is deliberate). The effect depends on `engine`, so on boot
|
||||
it fires once WASM is ready and lands the boot mode's dims (**paf_synth → 4→[10,10,14]→33**, weights
|
||||
809 — NOT the 32→126 default). The reshape-offer effect reads the engine's CURRENT `inputSize`
|
||||
hidden/output config plus Manifold's effective input arity via a `ConsoleApp` effect keyed on
|
||||
`[engine, modeId, modelInputSize]`. No confirm modal (switching instrument is deliberate). Normal
|
||||
modes boot at **2 inputs** (paf_synth → 2→[10,10,14]→33, weights 787); the Inputs dock's **4 inputs**
|
||||
option is retained across mode switches. Audio-analysis keeps its schema-defined input feature count.
|
||||
The reshape-offer effect reads the engine's CURRENT `inputSize`
|
||||
live, so a mode switch that changes arity doesn't spuriously prompt (its baseline tracks axis
|
||||
COUNT, unchanged by a pure dim change). Non-schema modes restore `DEFAULT_MODE_ML` (32→126).
|
||||
COUNT, unchanged by a pure dim change). Manifold-only modes use the same 2/4 input selector.
|
||||
Debug seam for tests: under `?debug=1` ConsoleApp installs `window.__mf`
|
||||
(`setMode`/`getModeId`/`paramCount`/`modeIds`) — the UI-level analogue of `__nisps`, since no
|
||||
(`setMode`/`getModeId`/`paramCount`/`modeIds`/`getModelInputSize`/`setModelInputSize`) — the UI-level analogue of `__nisps`, since no
|
||||
in-UI instrument picker exists yet (`ctx.modes`/`setModeId` are plumbed but unrendered).
|
||||
Manifold passes `spread=0` for boot, mode-switch reshapes, direct re-rolls, explore-and-place
|
||||
scratchpad rolls, and VCV-forwarded randomise gestures by default. Settings → Experimental
|
||||
|
|
|
|||
|
|
@ -6,8 +6,9 @@
|
|||
* - The pseudo-inference (`MF_infer`, sin/cos) is GONE. The `values` every
|
||||
* consumer reads now come from `engine.getOutputs()`, mapped onto the mode's
|
||||
* params by `shapeValues` (status/min/max/curve applied here). Pad/joystick
|
||||
* motion drives `engine.setInput(x,y)`; we subscribe to engine changes via
|
||||
* `useEngineVersion` and re-derive `values` imperatively on render.
|
||||
* motion drives `engine.setInput(x,y)`; structural state and throttled DOM
|
||||
* outputs have separate subscriptions, while ParticleStage shapes the live
|
||||
* reused buffer directly in rAF.
|
||||
* - Verdicts wire to the engine: commit → feedback.thumbsUp(); perturb →
|
||||
* feedback.thumbsDown(); reroll → randomise(); each followed by process().
|
||||
* - The default feedback mode is "Explore and place" → the shared C++ core's
|
||||
|
|
@ -31,6 +32,7 @@ import { useEffect, useMemo, useRef, useState } from 'react';
|
|||
import type { CSSProperties } from 'react';
|
||||
import {
|
||||
useEngine,
|
||||
useEngineOutputVersion,
|
||||
useEngineVersion,
|
||||
ExplorationController,
|
||||
DEFAULT_GEOMETRIC_FEEDBACK_CONFIG,
|
||||
|
|
@ -51,6 +53,7 @@ import type {
|
|||
DrawerKey,
|
||||
FeedbackMarker,
|
||||
FeedbackModeUI,
|
||||
ManifoldInputSize,
|
||||
OutputMode,
|
||||
Pin,
|
||||
SoloMode,
|
||||
|
|
@ -82,6 +85,8 @@ export interface MfDebugHook {
|
|||
setOutputMode: (mode: OutputMode) => void;
|
||||
setMidiCcCount: (count: number) => void;
|
||||
displayOutputCount: () => number;
|
||||
getModelInputSize: () => ManifoldInputSize;
|
||||
setModelInputSize: (size: ManifoldInputSize) => void;
|
||||
}
|
||||
|
||||
declare global {
|
||||
|
|
@ -106,11 +111,12 @@ function pillBtn(color: string): CSSProperties {
|
|||
|
||||
export function ConsoleApp() {
|
||||
const engine = useEngine();
|
||||
const version = useEngineVersion(engine);
|
||||
const stateVersion = useEngineVersion(engine);
|
||||
const { settings } = useSettings();
|
||||
|
||||
const [modeId, setModeId] = useState('paf_synth');
|
||||
const mode = MF_MODES.find((m) => m.id === modeId) ?? MF_MODES[0];
|
||||
const [modelInputSize, setModelInputSizeState] = useState<ManifoldInputSize>(2);
|
||||
const [params, setParams] = useState<MFParam[]>(() => mode.params.map((p) => ({ ...p })));
|
||||
const [paramsModeId, setParamsModeId] = useState(mode.id);
|
||||
const [pos, setPos] = useState<[number, number]>([0.5, 0.5]);
|
||||
|
|
@ -156,6 +162,16 @@ export function ConsoleApp() {
|
|||
// Active output MODE (TOP dock selector) — default Particle System. The dock
|
||||
// backend + audio backend derive from this.
|
||||
const [outputMode, setOutputModeState] = useState<OutputMode>(DEFAULT_OUTPUT_MODE);
|
||||
// Canvas stages read the live buffer imperatively. DOM-heavy stages opt into
|
||||
// the separate 30 Hz output channel instead of making the whole console a
|
||||
// per-inference subscriber.
|
||||
const outputVersion = useEngineOutputVersion(
|
||||
engine,
|
||||
outputMode !== 'particles' ||
|
||||
sandwich ||
|
||||
params.some((param) => param.manualOverride),
|
||||
);
|
||||
const version = stateVersion + outputVersion;
|
||||
const outputBackend: BackendId = outputModeDescriptor(outputMode).backend;
|
||||
// Per-backend transport settings (backends-spec §2.3/§2.4). Persisted via the
|
||||
// named-preset system; these are the live working values.
|
||||
|
|
@ -283,19 +299,17 @@ export function ConsoleApp() {
|
|||
if (engine) engine.audio.setBackend(modeEngineId(modeId) as Parameters<typeof engine.audio.setBackend>[0]);
|
||||
}, [engine, modeId]);
|
||||
|
||||
// Per-mode net dims (one-core-engine P5.3). On mode switch — and once WASM is
|
||||
// ready on boot (this effect depends on `engine`, so it fires when the engine
|
||||
// transitions null→ready with the boot mode) — reshape the runtime-shaped MLP
|
||||
// to the active mode's schema `ml` config (warm-started; the C-side dataset +
|
||||
// feedback reset, which the transient reset below also clears). NO confirm
|
||||
// modal: switching instrument is already a deliberate act. The P2.3 axis-count
|
||||
// modal stays for input-LAYOUT changes only (see the reshape-offer effect).
|
||||
// Per-mode net dims (one-core-engine P5.3). Normal Manifold modes expose a
|
||||
// deliberately smaller 2-input working shape for now, while the Inputs dock
|
||||
// can opt into the schema's usual 4-input shape. Audio-analysis is different:
|
||||
// its schema input arity describes the feature vector and must stay intact.
|
||||
useEffect(() => {
|
||||
if (!engine) return;
|
||||
const { inputSize, outputSize, hidden } = mode.ml;
|
||||
const inputSize = mode.input === 'audio_in' ? mode.ml.inputSize : modelInputSize;
|
||||
const { outputSize, hidden } = mode.ml;
|
||||
engine.reshape({ inputSize, outputSize, hidden }, randomisationSpread);
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [engine, modeId]);
|
||||
}, [engine, modeId, modelInputSize]);
|
||||
|
||||
// reset transient state on mode switch
|
||||
useEffect(() => {
|
||||
|
|
@ -344,8 +358,13 @@ export function ConsoleApp() {
|
|||
return;
|
||||
}
|
||||
const activeMap: DimensionMap = keys.map((key) => previous.get(key) ?? null);
|
||||
// Keep the explicit dock choice as a minimum capacity. Active sources may
|
||||
// still require more (for example, a double-stick gamepad), but a source
|
||||
// with fewer axes must not silently shrink a deliberate 4-D model.
|
||||
const minimumInputSize = mode.input === 'audio_in' ? mode.ml.inputSize : modelInputSize;
|
||||
const requiredSize = Math.max(minimumInputSize, keys.length);
|
||||
const target =
|
||||
resizeTarget(keys.length, inSize, settings.networkResizePolicy) ?? inSize;
|
||||
resizeTarget(requiredSize, inSize, settings.networkResizePolicy) ?? inSize;
|
||||
const inputMap = completeDimensionMap(activeMap, target, inSize);
|
||||
const needsMigration =
|
||||
target !== inSize || inputMap.some((oldIndex, newIndex) => oldIndex !== newIndex);
|
||||
|
|
@ -369,6 +388,9 @@ export function ConsoleApp() {
|
|||
engine,
|
||||
inputLayoutKey,
|
||||
inputs.engineInputSize,
|
||||
mode.input,
|
||||
mode.ml.inputSize,
|
||||
modelInputSize,
|
||||
randomisationSpread,
|
||||
settings.networkResizePolicy,
|
||||
settings.exampleResizePolicy,
|
||||
|
|
@ -393,11 +415,13 @@ export function ConsoleApp() {
|
|||
setOutputMode,
|
||||
setMidiCcCount,
|
||||
displayOutputCount: () => displayOutputCount,
|
||||
getModelInputSize: () => modelInputSize,
|
||||
setModelInputSize: (size) => setModelInputSizeState(size === 4 ? 4 : 2),
|
||||
};
|
||||
return () => {
|
||||
if (window.__mf) delete window.__mf;
|
||||
};
|
||||
}, [modeId, params, displayOutputCount]);
|
||||
}, [modeId, params, displayOutputCount, modelInputSize]);
|
||||
|
||||
// Drive a pad/joystick/manifold move through the input layer's XY-pad source,
|
||||
// then mirror the raw position into React state for readouts. The layer's loop
|
||||
|
|
@ -644,7 +668,10 @@ export function ConsoleApp() {
|
|||
}
|
||||
setAddingExample(false);
|
||||
// Snapshot the current input → current (shaped) output as a training example.
|
||||
engine?.addExample([pos[0], pos[1]], Array.from(values));
|
||||
engine?.addExample(
|
||||
[pos[0], pos[1]],
|
||||
shapeValues(params, engine?.getOutputs() ?? null),
|
||||
);
|
||||
setExamples((e) => e + 1);
|
||||
train();
|
||||
};
|
||||
|
|
@ -979,6 +1006,8 @@ export function ConsoleApp() {
|
|||
cvDisconnect,
|
||||
setParams: (next: MFParam[]) => setParams(next),
|
||||
inputs,
|
||||
modelInputSize,
|
||||
setModelInputSize: (size: ManifoldInputSize) => setModelInputSizeState(size),
|
||||
spread,
|
||||
setSpread,
|
||||
xavierSpreadEnabled: settings.xavierSpreadEnabled,
|
||||
|
|
@ -1096,6 +1125,7 @@ export function ConsoleApp() {
|
|||
params={displayedParams}
|
||||
values={displayedValues}
|
||||
onChange={setParam}
|
||||
markers={markers}
|
||||
/>
|
||||
) : (
|
||||
<CompositeStage
|
||||
|
|
|
|||
|
|
@ -21,8 +21,9 @@
|
|||
import { useEffect, useRef, useState } from 'react';
|
||||
import type { MouseEvent as ReactMouseEvent, PointerEvent as ReactPointerEvent } from 'react';
|
||||
import { useEngine } from '../engine';
|
||||
import { VirtualJoystick } from '../primitives/VirtualJoystick';
|
||||
import type { MFParam } from './model';
|
||||
import { shapeValuesInto, type MFParam } from './model';
|
||||
import { Manifold } from './Manifold';
|
||||
import type { FeedbackMarker } from './types';
|
||||
import {
|
||||
FlowFieldVisualizer,
|
||||
N_VISUAL_OUTPUTS,
|
||||
|
|
@ -36,16 +37,19 @@ export interface ParticleStageProps {
|
|||
params: MFParam[];
|
||||
values: number[];
|
||||
onChange: (i: number, patch: Partial<MFParam>) => void;
|
||||
markers?: FeedbackMarker[];
|
||||
}
|
||||
|
||||
export function ParticleStage({ pos, onMove, params, values, onChange }: ParticleStageProps) {
|
||||
export function ParticleStage({ pos, onMove, params, values, onChange, markers = [] }: ParticleStageProps) {
|
||||
const engine = useEngine();
|
||||
const canvasRef = useRef<HTMLCanvasElement>(null);
|
||||
const vizRef = useRef<FlowFieldVisualizer | null>(null);
|
||||
const barsRef = useRef<(HTMLDivElement | null)[]>([]);
|
||||
const tooltipRef = useRef<HTMLDivElement>(null);
|
||||
const outputsRef = useRef<Float32Array | null>(null);
|
||||
const valuesRef = useRef(values);
|
||||
const displayOutputsRef = useRef<ArrayLike<number>>(values);
|
||||
const shapedOutputsRef = useRef(new Float32Array(N_VISUAL_OUTPUTS));
|
||||
const paramsRef = useRef(params);
|
||||
const hoverRef = useRef<number | null>(null);
|
||||
const [hover, setHover] = useState<number | null>(null);
|
||||
const [tooltipPos, setTooltipPos] = useState({ x: 12, y: 28 });
|
||||
|
|
@ -73,7 +77,7 @@ export function ParticleStage({ pos, onMove, params, values, onChange }: Particl
|
|||
startX: 0,
|
||||
el: null,
|
||||
});
|
||||
valuesRef.current = values;
|
||||
paramsRef.current = params;
|
||||
onMoveRef.current = onMove;
|
||||
|
||||
const clamp = (n: number, lo: number, hi: number) => Math.max(lo, Math.min(hi, n));
|
||||
|
|
@ -125,7 +129,7 @@ export function ParticleStage({ pos, onMove, params, values, onChange }: Particl
|
|||
x: Math.min(e.clientX + 12, Math.max(12, window.innerWidth - 190)),
|
||||
y: Math.min(e.clientY + 12, Math.max(28, window.innerHeight - 32)),
|
||||
});
|
||||
const output = valuesRef.current[i] ?? outputsRef.current?.[i] ?? 0;
|
||||
const output = displayOutputsRef.current[i] ?? outputsRef.current?.[i] ?? 0;
|
||||
if (tooltipRef.current) tooltipRef.current.textContent = `${VISUAL_PARAM_NAMES[i]}: ${output.toFixed(3)}`;
|
||||
};
|
||||
|
||||
|
|
@ -194,7 +198,11 @@ export function ParticleStage({ pos, onMove, params, values, onChange }: Particl
|
|||
const outputs = engine?.getOutputs();
|
||||
if (outputs) {
|
||||
outputsRef.current = outputs;
|
||||
const displayOutputs = valuesRef.current.length >= N_VISUAL_OUTPUTS ? valuesRef.current : outputs;
|
||||
const displayOutputs =
|
||||
paramsRef.current.length >= N_VISUAL_OUTPUTS
|
||||
? shapeValuesInto(paramsRef.current, outputs, shapedOutputsRef.current)
|
||||
: outputs;
|
||||
displayOutputsRef.current = displayOutputs;
|
||||
viz.setParams(displayOutputs);
|
||||
// Drive the heatmap bar widths imperatively (cheap; no React churn).
|
||||
for (let i = 0; i < N_VISUAL_OUTPUTS; i++) {
|
||||
|
|
@ -394,7 +402,10 @@ export function ParticleStage({ pos, onMove, params, values, onChange }: Particl
|
|||
</div>
|
||||
)}
|
||||
|
||||
{/* Adjustable circular pad — explicit edit handles keep repositioning/resizing deliberate. */}
|
||||
{/* Adjustable circular pad — reuse Manifold's visual surface so Particle
|
||||
mode shows the same feedback marks and cursor trail as every other
|
||||
input presentation. The outer stage owns follow-mouse, therefore the
|
||||
embedded Manifold's double-click mode is disabled. */}
|
||||
<div
|
||||
style={{
|
||||
position: 'absolute',
|
||||
|
|
@ -406,14 +417,21 @@ export function ParticleStage({ pos, onMove, params, values, onChange }: Particl
|
|||
pointerEvents: 'none',
|
||||
}}
|
||||
>
|
||||
<VirtualJoystick
|
||||
size={padSize}
|
||||
position={pos}
|
||||
onMove={onMove}
|
||||
disabled={padEdit || followMouse}
|
||||
ariaLabel="particle input pad"
|
||||
style={{ pointerEvents: padEdit || followMouse ? 'none' : 'auto' }}
|
||||
/>
|
||||
<div
|
||||
role="application"
|
||||
aria-label="particle input pad"
|
||||
tabIndex={padEdit || followMouse ? -1 : 0}
|
||||
style={{ position: 'absolute', inset: 0, pointerEvents: 'auto' }}
|
||||
>
|
||||
<Manifold
|
||||
pos={pos}
|
||||
onMove={onMove}
|
||||
markers={markers}
|
||||
variant="circular"
|
||||
frozen={padEdit || followMouse}
|
||||
followMouseEnabled={false}
|
||||
/>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
aria-label={padEdit ? 'Finish adjusting joystick' : 'Adjust joystick size and position'}
|
||||
|
|
|
|||
|
|
@ -69,6 +69,15 @@ function noise2D(x: number, y: number): number {
|
|||
|
||||
const TWO_PI = Math.PI * 2;
|
||||
|
||||
export const N_FLOW_PARTICLES = 400;
|
||||
/**
|
||||
* Contiguous id bands used to batch Canvas2D paths. Thirty-two bands keep the
|
||||
* original hue sweep visually continuous (at most 3.75° between bands at the
|
||||
* maximum 120° spread) while replacing 400 dynamic colour parses + fills with
|
||||
* 32 per frame.
|
||||
*/
|
||||
export const FLOW_COLOR_BUCKETS = 32;
|
||||
|
||||
/**
|
||||
* The 20 visual output params, in output order (p0..p19). Names are verbatim
|
||||
* from the a-immersive original (`VISUAL_PARAM_NAMES`, a-app.js:46).
|
||||
|
|
@ -157,7 +166,11 @@ export class FlowFieldVisualizer {
|
|||
private canvas: HTMLCanvasElement;
|
||||
private ctx: CanvasRenderingContext2D;
|
||||
private particles: Particle[] = [];
|
||||
private numParticles = 400;
|
||||
private colorBuckets: Particle[][] = Array.from(
|
||||
{ length: FLOW_COLOR_BUCKETS },
|
||||
() => [],
|
||||
);
|
||||
private numParticles = N_FLOW_PARTICLES;
|
||||
private time = 0;
|
||||
private width = 0;
|
||||
private height = 0;
|
||||
|
|
@ -210,8 +223,15 @@ export class FlowFieldVisualizer {
|
|||
|
||||
private initParticles(): void {
|
||||
this.particles = [];
|
||||
for (const bucket of this.colorBuckets) bucket.length = 0;
|
||||
for (let i = 0; i < this.numParticles; i++) {
|
||||
this.particles.push(this.makeParticle(i));
|
||||
const particle = this.makeParticle(i);
|
||||
this.particles.push(particle);
|
||||
const bucket = Math.min(
|
||||
FLOW_COLOR_BUCKETS - 1,
|
||||
Math.floor((i * FLOW_COLOR_BUCKETS) / this.numParticles),
|
||||
);
|
||||
this.colorBuckets[bucket].push(particle);
|
||||
}
|
||||
this.ctx.fillStyle = '#0d0d0d';
|
||||
this.ctx.fillRect(0, 0, this.width, this.height);
|
||||
|
|
@ -317,10 +337,14 @@ export class FlowFieldVisualizer {
|
|||
ctx.fillStyle = `rgba(13, 13, 13, ${params.fadeRate})`;
|
||||
ctx.fillRect(0, 0, width, height);
|
||||
|
||||
for (const p of this.particles) {
|
||||
const cx = width * 0.5;
|
||||
const cy = height * 0.5;
|
||||
const cx = width * 0.5;
|
||||
const cy = height * 0.5;
|
||||
const repulsorRadius = Math.min(width, height) * 0.28;
|
||||
|
||||
// Advance the simulation without touching Canvas state. Rendering is
|
||||
// batched by colour below, avoiding one dynamic colour parse + fill call
|
||||
// per particle.
|
||||
for (const p of this.particles) {
|
||||
// Sample flow field
|
||||
const nx = p.x * params.scale;
|
||||
const ny = p.y * params.scale;
|
||||
|
|
@ -378,7 +402,6 @@ export class FlowFieldVisualizer {
|
|||
nextY -= nyCenter * dispersionForce;
|
||||
|
||||
// Orbiting repulsor points carve dynamic voids and bursts.
|
||||
const repulsorRadius = Math.min(width, height) * 0.28;
|
||||
for (let r = 0; r < params.repulsorCount; r++) {
|
||||
const phase = this.time * params.repulsorOrbitRate + (r / 4) * TWO_PI;
|
||||
const wobble = 0.6 + 0.15 * r;
|
||||
|
|
@ -404,14 +427,28 @@ export class FlowFieldVisualizer {
|
|||
|
||||
p.age += 1;
|
||||
if (p.age >= p.life) this.respawnParticle(p);
|
||||
}
|
||||
|
||||
// Colour based on particle id + hue params
|
||||
const hue = (params.hueBase + (p.id / this.numParticles) * params.hueSpread) % 360;
|
||||
const lightness = 50 + Math.sin(p.id * 0.1 + this.time) * 15;
|
||||
|
||||
// Preserve the original id-ordered hue/lightness sweep with one
|
||||
// representative colour per contiguous band. Each band becomes one path
|
||||
// and one fill instead of every particle parsing its own HSL string.
|
||||
for (const bucket of this.colorBuckets) {
|
||||
if (bucket.length === 0) continue;
|
||||
const representative = bucket[Math.floor(bucket.length * 0.5)];
|
||||
const hue =
|
||||
(params.hueBase +
|
||||
(representative.id / this.numParticles) * params.hueSpread) %
|
||||
360;
|
||||
const lightness =
|
||||
50 + Math.sin(representative.id * 0.1 + this.time) * 15;
|
||||
ctx.fillStyle = `hsl(${hue}, 75%, ${lightness}%)`;
|
||||
ctx.beginPath();
|
||||
ctx.arc(p.x, p.y, params.particleSize, 0, TWO_PI);
|
||||
for (const p of bucket) {
|
||||
// Start an independent sub-path so adjacent circles never acquire a
|
||||
// connecting edge when the whole colour band is filled at once.
|
||||
ctx.moveTo(p.x + params.particleSize, p.y);
|
||||
ctx.arc(p.x, p.y, params.particleSize, 0, TWO_PI);
|
||||
}
|
||||
ctx.fill();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -379,17 +379,32 @@ export { applyCurve };
|
|||
* onto its own N outputs; the `i < engineOut.length` guard keeps it safe during
|
||||
* the async reshape window.
|
||||
*/
|
||||
export function shapeValues(params: MFParam[], engineOut: Float32Array | null): number[] {
|
||||
return params.map((p, i) => {
|
||||
if (p.status === 'off') return 0;
|
||||
export function shapeValuesInto<T extends { length: number; [index: number]: number }>(
|
||||
params: MFParam[],
|
||||
engineOut: Float32Array | null,
|
||||
out: T,
|
||||
): T {
|
||||
const count = Math.min(params.length, out.length);
|
||||
for (let i = 0; i < count; i++) {
|
||||
const p = params[i];
|
||||
if (p.status === 'off') {
|
||||
out[i] = 0;
|
||||
continue;
|
||||
}
|
||||
if (p.status === 'fixed') {
|
||||
return p.min + Math.max(0, Math.min(1, p.val ?? 0.5)) * (p.max - p.min);
|
||||
out[i] = p.min + Math.max(0, Math.min(1, p.val ?? 0.5)) * (p.max - p.min);
|
||||
continue;
|
||||
}
|
||||
const engineIndex = p.engineIndex ?? i;
|
||||
const raw = engineOut && engineIndex < engineOut.length ? engineOut[engineIndex] : 0.5;
|
||||
const v = p.min + applyCurve(raw, p.curve) * (p.max - p.min);
|
||||
return Math.max(0, Math.min(1, v));
|
||||
});
|
||||
out[i] = Math.max(0, Math.min(1, v));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function shapeValues(params: MFParam[], engineOut: Float32Array | null): number[] {
|
||||
return shapeValuesInto(params, engineOut, new Array<number>(params.length));
|
||||
}
|
||||
|
||||
/** Create a backend-agnostic output card after every schema output is active. */
|
||||
|
|
|
|||
|
|
@ -10,8 +10,9 @@
|
|||
* The engine imports NO React. The only React in `engine/` is the
|
||||
* EngineProvider/useEngine binding layer (separate files).
|
||||
*
|
||||
* `subscribe(cb)` + `version()` are the `useSyncExternalStore` contract: React
|
||||
* re-reads on a version bump but consumers read the live Float32Array
|
||||
* `subscribe(cb)` + `version()` expose structural/training state changes.
|
||||
* `subscribeOutputs(cb)` + `outputVersion()` expose a throttled live-output
|
||||
* channel for DOM consumers. Canvas consumers read the live Float32Array
|
||||
* imperatively via `getOutputs()` / `routedOutput()`.
|
||||
*/
|
||||
|
||||
|
|
@ -410,6 +411,16 @@ export class EngineApi {
|
|||
return this.spine.version();
|
||||
}
|
||||
|
||||
/** Subscribe to throttled live-output changes for non-canvas UI consumers. */
|
||||
subscribeOutputs(cb: () => void): () => void {
|
||||
return this.spine.subscribeOutputs(cb);
|
||||
}
|
||||
|
||||
/** Latest live-output revision; increments on every inference. */
|
||||
outputVersion(): number {
|
||||
return this.spine.outputVersion();
|
||||
}
|
||||
|
||||
/** Subscribe to a named engine event (`ml.*`, `feedback.*`, …). */
|
||||
on(event: string, fn: (payload?: unknown) => void): () => void {
|
||||
return this.spine.on(event, fn);
|
||||
|
|
|
|||
|
|
@ -55,7 +55,12 @@ export type {
|
|||
|
||||
export { EngineProvider, EngineContext } from './EngineProvider';
|
||||
export type { EngineProviderProps } from './EngineProvider';
|
||||
export { useEngine, useEngineOrThrow, useEngineVersion } from './useEngine';
|
||||
export {
|
||||
useEngine,
|
||||
useEngineOrThrow,
|
||||
useEngineVersion,
|
||||
useEngineOutputVersion,
|
||||
} from './useEngine';
|
||||
|
||||
// Pipeline config types + defaults. The PROCESSING lives in the C++/WASM core
|
||||
// (one-core-engine P4); configure via EngineApi.setInputConfig / setOutputConfig.
|
||||
|
|
|
|||
|
|
@ -10,9 +10,10 @@
|
|||
* all C++/WASM chains since one-core-engine P4) and fires the single
|
||||
* `backend.send` at the action TAIL, off React's render cycle.
|
||||
*
|
||||
* React subscribes via `useSyncExternalStore(subscribe, version)` — the version
|
||||
* counter, NOT the array — and reads the live `Float32Array` imperatively (so
|
||||
* canvases never re-render per frame).
|
||||
* Structural/training state uses `subscribe/version`; opt-in DOM output
|
||||
* consumers use the throttled `subscribeOutputs/outputVersion` channel.
|
||||
* Canvases read the live `Float32Array` imperatively and never re-render per
|
||||
* frame.
|
||||
*
|
||||
* Buffers are reused (no per-frame allocation): `routedBuf` is a single
|
||||
* Float32Array threaded through the output pipeline and handed to the backend.
|
||||
|
|
@ -69,7 +70,11 @@ export class Spine implements EngineSink {
|
|||
};
|
||||
|
||||
private listeners = new Set<() => void>();
|
||||
private outputListeners = new Set<() => void>();
|
||||
private eventListeners = new Map<string, Set<(payload?: unknown) => void>>();
|
||||
private outputVersion_ = 0;
|
||||
private lastOutputNotifyMs = -Infinity;
|
||||
private readonly outputNotifyIntervalMs = 1000 / 30;
|
||||
|
||||
// Engine handles wired in via `attach`.
|
||||
private iml: WasmIML | null = null;
|
||||
|
|
@ -235,7 +240,9 @@ export class Spine implements EngineSink {
|
|||
* the 2-D manifold / XY-pad path — delegates to {@link setInputs}.
|
||||
*/
|
||||
setInput(x: number, y: number): Float32Array | null {
|
||||
return this.setInputs([x, y]);
|
||||
this.rawInput[0] = x;
|
||||
this.rawInput[1] = y;
|
||||
return this.setInputs(this.rawInput);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -282,7 +289,7 @@ export class Spine implements EngineSink {
|
|||
|
||||
// 3. ml (inference into the reused buffer; no alloc).
|
||||
iml.processInto(this.mlBuf);
|
||||
this.liveOutputs.set(this.mlBuf.subarray(0, this.liveOutputs.length));
|
||||
this.liveOutputs.set(this.mlBuf);
|
||||
|
||||
// 4. routed (output chain, in place on the reused routedBuf → C++-side state).
|
||||
if (!this.routedBuf || this.routedBuf.length !== this.mlBuf.length) {
|
||||
|
|
@ -298,7 +305,7 @@ export class Spine implements EngineSink {
|
|||
// 5. single backend.send at the tail (off React render).
|
||||
if (this.backendSend && this.routedBuf) this.backendSend(this.routedBuf);
|
||||
|
||||
this.bump_();
|
||||
this.publishOutputs_();
|
||||
return this.routedBuf;
|
||||
}
|
||||
|
||||
|
|
@ -342,6 +349,18 @@ export class Spine implements EngineSink {
|
|||
|
||||
version = (): number => this.state_.version;
|
||||
|
||||
/**
|
||||
* Live-output subscription kept separate from structural/training state.
|
||||
* Notifications are capped at 30 Hz for DOM consumers; canvas consumers read
|
||||
* the reused output buffers directly in their own rAF loops.
|
||||
*/
|
||||
subscribeOutputs = (cb: () => void): (() => void) => {
|
||||
this.outputListeners.add(cb);
|
||||
return () => { this.outputListeners.delete(cb); };
|
||||
};
|
||||
|
||||
outputVersion = (): number => this.outputVersion_;
|
||||
|
||||
getState(): Readonly<SpineState> {
|
||||
return this.state_;
|
||||
}
|
||||
|
|
@ -357,4 +376,14 @@ export class Spine implements EngineSink {
|
|||
this.state_ = { ...this.state_, version: this.state_.version + 1 };
|
||||
for (const fn of this.listeners) fn();
|
||||
}
|
||||
|
||||
private publishOutputs_(): void {
|
||||
this.outputVersion_++;
|
||||
if (this.outputListeners.size === 0) return;
|
||||
const now =
|
||||
typeof performance !== 'undefined' ? performance.now() : Date.now();
|
||||
if (now - this.lastOutputNotifyMs < this.outputNotifyIntervalMs) return;
|
||||
this.lastOutputNotifyMs = now;
|
||||
for (const fn of this.outputListeners) fn();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,12 +3,9 @@
|
|||
*
|
||||
* `useEngine()` returns the EngineApi from context (or null before it loads).
|
||||
*
|
||||
* `useEngineVersion()` subscribes to engine state changes via
|
||||
* `useSyncExternalStore(engine.subscribe, engine.version)`. It returns the
|
||||
* VERSION COUNTER (a number), NOT the output array — so a component re-renders
|
||||
* when engine state changes but reads the live `Float32Array` imperatively
|
||||
* (`engine.getOutputs()` / `engine.routedOutput()`) inside a rAF loop or on
|
||||
* render. This keeps per-frame audio inference off React's render cycle.
|
||||
* `useEngineVersion()` subscribes only to structural/training state.
|
||||
* `useEngineOutputVersion()` is the opt-in, throttled live-output channel for
|
||||
* DOM consumers. Canvas consumers read live buffers imperatively instead.
|
||||
*/
|
||||
|
||||
import { useContext, useSyncExternalStore } from 'react';
|
||||
|
|
@ -41,3 +38,18 @@ export function useEngineVersion(engine: EngineApi | null): number {
|
|||
() => 0,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Subscribe to the engine's throttled live-output channel. Disabled consumers
|
||||
* neither subscribe nor re-render and report version 0.
|
||||
*/
|
||||
export function useEngineOutputVersion(
|
||||
engine: EngineApi | null,
|
||||
enabled = true,
|
||||
): number {
|
||||
return useSyncExternalStore(
|
||||
(cb) => (engine && enabled ? engine.subscribeOutputs(cb) : () => {}),
|
||||
() => (engine && enabled ? engine.outputVersion() : 0),
|
||||
() => 0,
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
* Sits ABOVE the engine: it owns a single rAF loop that, each frame,
|
||||
* 1. polls poll-based sources (gamepad buttons → actions),
|
||||
* 2. pulls every active source's axes into the shared `vector` (pull-based),
|
||||
* 3. blends/maps the N-dim vector down to the engine's input arity, and
|
||||
* 3. maps the N-dim vector onto the engine's input arity, and
|
||||
* 4. fires engine.setInputs(...) exactly once.
|
||||
*
|
||||
* The XY pad remains push-driven via the existing onMove handler — when it's the
|
||||
|
|
@ -14,14 +14,12 @@
|
|||
* composable and the channel layout coherent.
|
||||
*
|
||||
* ── Dedicated dimensions (no blending) ──────────────────────────────────────
|
||||
* The WASM net is over-provisioned to a 32-input head (= MAX_AXES; see
|
||||
* nisps/wasm/bindings.cpp). Each active axis drives its OWN engine input slot
|
||||
* 1:1 — a double-stick gamepad is 4 genuine dims, a learned MIDI surface is N
|
||||
* genuine dims. `compose()` simply forwards the active axes; the engine
|
||||
* zero-pads the remaining slots and a zero input is inert (0 × weight = 0), so
|
||||
* unused dimensions never perturb the net. We do NOT mean-blend (the previous
|
||||
* behaviour) — that diluted every source and biased the net toward idle
|
||||
* sources' resting values.
|
||||
* Each active axis drives its OWN engine input slot 1:1 — a double-stick
|
||||
* gamepad is 4 genuine dims, and a learned MIDI surface is N genuine dims.
|
||||
* `compose()` forwards the active axes and truncates only when a transient or
|
||||
* deliberately smaller model cannot accept all of them. We do NOT mean-blend
|
||||
* (the previous behaviour) — that diluted every source and biased the net
|
||||
* toward idle sources' resting values.
|
||||
*
|
||||
* Changing the ACTIVE layout runs through ConsoleApp's persistent I/O policy:
|
||||
* keep-capacity remaps stable dimensions in place until capacity is exceeded;
|
||||
|
|
@ -42,6 +40,7 @@ export class InputLayer {
|
|||
private sources: InputSource[] = [];
|
||||
private engine: InputEngineSink | null = null;
|
||||
private vector = new Float32Array(MAX_AXES);
|
||||
private reduced: number[] = [];
|
||||
private running = false;
|
||||
private rafId: number | null = null;
|
||||
private actionListeners = new Set<(a: InputAction) => void>();
|
||||
|
|
@ -136,7 +135,7 @@ export class InputLayer {
|
|||
}
|
||||
if (n === 0) return;
|
||||
|
||||
// 3. reduce to the engine's input arity (see file header — blend, not fake).
|
||||
// 3. reduce to the engine's current input arity (dedicated, not blended).
|
||||
const reduced = this.compose(n, engine.architecture.inputSize);
|
||||
|
||||
// 4. one engine write.
|
||||
|
|
@ -157,17 +156,14 @@ export class InputLayer {
|
|||
* the engine zero-pads the slots beyond `count` and a zero input is inert
|
||||
* (0 × weight = 0), so unused dimensions never perturb the net.
|
||||
*
|
||||
* The net's input arity is over-provisioned (32, = MAX_AXES), so `inputSize`
|
||||
* is effectively always ≥ n; the `min` only guards a transient where more
|
||||
* axes are active than the net can take. We deliberately do NOT mean-blend
|
||||
* (the old behaviour) — that diluted every source and biased the net toward
|
||||
* idle sources' resting values.
|
||||
* The ConsoleApp owns the reshape policy and grows the model when needed.
|
||||
* The `min` guards the short transition without blending or reassigning axes.
|
||||
*/
|
||||
private compose(n: number, inputSize: number): number[] {
|
||||
const count = Math.min(n, inputSize);
|
||||
const out = new Array<number>(count);
|
||||
for (let i = 0; i < count; i++) out[i] = this.vector[i];
|
||||
return out;
|
||||
this.reduced.length = count;
|
||||
for (let i = 0; i < count; i++) this.reduced[i] = this.vector[i];
|
||||
return this.reduced;
|
||||
}
|
||||
|
||||
// ---- actions + layout fan-out -------------------------------------------
|
||||
|
|
|
|||
65
manifold/tests/engine-notification-hot-path.test.ts
Normal file
65
manifold/tests/engine-notification-hot-path.test.ts
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
import { expect, test } from 'bun:test';
|
||||
import { Spine } from '../src/engine/spine';
|
||||
import type { WasmIML } from '../src/engine/wasm-iml';
|
||||
import { InputLayer } from '../src/inputs/input-layer';
|
||||
import { XYPadSource } from '../src/inputs/xy-pad-source';
|
||||
|
||||
function makeFakeIml(outputSize = 20): WasmIML {
|
||||
return {
|
||||
architecture: {
|
||||
inputSize: 2,
|
||||
hidden: [0, 0, 0] as [number, number, number],
|
||||
outputSize,
|
||||
numLayers: 3,
|
||||
},
|
||||
setInputConfig: () => {},
|
||||
setOutputConfig: () => {},
|
||||
setOutputFreezeMask: () => {},
|
||||
resetInput: () => {},
|
||||
resetOutput: () => {},
|
||||
processInput: (x: number, y: number) => ({ x, y, frozen: false }),
|
||||
setInput: () => {},
|
||||
processInto: (buf: Float32Array) => buf.fill(0.25),
|
||||
processOutput: () => {},
|
||||
} as unknown as WasmIML;
|
||||
}
|
||||
|
||||
test('live inference publishes through the output channel without notifying state subscribers', () => {
|
||||
const spine = new Spine();
|
||||
spine.attach(makeFakeIml(), null);
|
||||
spine.setState({ outputSize: 20 });
|
||||
let stateNotifications = 0;
|
||||
let outputNotifications = 0;
|
||||
spine.subscribe(() => stateNotifications++);
|
||||
spine.subscribeOutputs(() => outputNotifications++);
|
||||
|
||||
spine.setInputs([0.2, 0.8]);
|
||||
spine.setInputs([0.3, 0.7]);
|
||||
|
||||
expect(stateNotifications).toBe(0);
|
||||
expect(outputNotifications).toBe(1);
|
||||
expect(spine.outputVersion()).toBe(2);
|
||||
expect(Array.from(spine.outputs())).toEqual(new Array(20).fill(0.25));
|
||||
});
|
||||
|
||||
test('InputLayer reuses its reduced input vector across animation frames', () => {
|
||||
const layer = new InputLayer();
|
||||
const source = new XYPadSource();
|
||||
const writes: ReadonlyArray<number>[] = [];
|
||||
layer.attach({
|
||||
architecture: { inputSize: 2 },
|
||||
setInputs: (values) => writes.push(values),
|
||||
});
|
||||
layer.setSources([source]);
|
||||
|
||||
source.pushAxes(0.2, 0.8);
|
||||
layer.frame();
|
||||
source.pushAxes(0.3, 0.7);
|
||||
layer.frame();
|
||||
|
||||
expect(writes).toHaveLength(2);
|
||||
expect(writes[0]).toBe(writes[1]);
|
||||
expect(Array.from(writes[1])).toEqual(
|
||||
Array.from(new Float32Array([0.3, 0.7])),
|
||||
);
|
||||
});
|
||||
87
manifold/tests/particle-hot-path.test.ts
Normal file
87
manifold/tests/particle-hot-path.test.ts
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
import { expect, test } from 'bun:test';
|
||||
import {
|
||||
FLOW_COLOR_BUCKETS,
|
||||
FlowFieldVisualizer,
|
||||
N_FLOW_PARTICLES,
|
||||
} from '../src/console/flow-field';
|
||||
|
||||
interface FakeContext {
|
||||
fillStyles: string[];
|
||||
fillCount: number;
|
||||
arcCount: number;
|
||||
fillStyle: string;
|
||||
setTransform(): void;
|
||||
scale(): void;
|
||||
fillRect(): void;
|
||||
beginPath(): void;
|
||||
moveTo(): void;
|
||||
arc(): void;
|
||||
fill(): void;
|
||||
}
|
||||
|
||||
function makeContext(): FakeContext {
|
||||
const ctx = {
|
||||
fillStyles: [] as string[],
|
||||
fillCount: 0,
|
||||
arcCount: 0,
|
||||
setTransform() {},
|
||||
scale() {},
|
||||
fillRect() {},
|
||||
beginPath() {},
|
||||
moveTo() {},
|
||||
arc() {
|
||||
ctx.arcCount++;
|
||||
},
|
||||
fill() {
|
||||
ctx.fillCount++;
|
||||
},
|
||||
} as FakeContext;
|
||||
Object.defineProperty(ctx, 'fillStyle', {
|
||||
set(value: string) {
|
||||
ctx.fillStyles.push(value);
|
||||
},
|
||||
get() {
|
||||
return ctx.fillStyles.at(-1) ?? '';
|
||||
},
|
||||
});
|
||||
return ctx;
|
||||
}
|
||||
|
||||
test('particle draw batches dynamic colours instead of parsing one HSL string per particle', () => {
|
||||
const previousWindow = globalThis.window;
|
||||
Object.defineProperty(globalThis, 'window', {
|
||||
value: { devicePixelRatio: 1 },
|
||||
configurable: true,
|
||||
});
|
||||
|
||||
try {
|
||||
const ctx = makeContext();
|
||||
const canvas = {
|
||||
getContext: () => ctx,
|
||||
getBoundingClientRect: () => ({ width: 640, height: 480 }),
|
||||
width: 0,
|
||||
height: 0,
|
||||
} as unknown as HTMLCanvasElement;
|
||||
const visualizer = new FlowFieldVisualizer(canvas);
|
||||
|
||||
ctx.fillStyles.length = 0;
|
||||
ctx.fillCount = 0;
|
||||
ctx.arcCount = 0;
|
||||
visualizer.draw();
|
||||
|
||||
expect(ctx.arcCount).toBe(N_FLOW_PARTICLES);
|
||||
expect(ctx.fillCount).toBe(FLOW_COLOR_BUCKETS);
|
||||
expect(ctx.fillStyles.filter((value) => value.startsWith('hsl('))).toHaveLength(
|
||||
FLOW_COLOR_BUCKETS,
|
||||
);
|
||||
} finally {
|
||||
if (previousWindow === undefined) {
|
||||
delete (globalThis as { window?: Window }).window;
|
||||
} else {
|
||||
Object.defineProperty(globalThis, 'window', {
|
||||
value: previousWindow,
|
||||
configurable: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
Loading…
Reference in a new issue