w1n5t0n/memlnaut-nisps
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

merge(inputs): modular XY/MIDI/gamepad input layer + Inputs dock panel

Browse source
This commit is contained in:
monkey-w1n5t0n 2026-06-28 04:32:44 +02:00
commit 1ab269415e
11 changed files with 1257 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
manifold/src/console/ConsoleApp.tsx
View file

@ -54,6 +54,7 @@ import { FeedbackController, type ProtoFeedbackMode } from '../feedback';
import { DEFAULT_OUTPUT_MODE, OUTPUT_MODES, outputModeDescriptor } from './output-mode'; import { DEFAULT_OUTPUT_MODE, OUTPUT_MODES, outputModeDescriptor } from './output-mode';
import { useSettings, resolveInputMap } from '../settings/settings-store'; import { useSettings, resolveInputMap } from '../settings/settings-store';
import { useBackendManager } from '../backends'; import { useBackendManager } from '../backends';
import { useInputLayer } from '../inputs';
let SNAP_ID = 0; let SNAP_ID = 0;
@ -229,9 +230,17 @@ export function ConsoleApp({ focus: initialFocus = 'composite' }: ConsoleAppProp
// rAF loop; the Editor serial protocol remains its own panel. // rAF loop; the Editor serial protocol remains its own panel.
const setOutputMode = (m: OutputMode) => setOutputModeState(m); const setOutputMode = (m: OutputMode) => setOutputModeState(m);
// Drive a pad/joystick/manifold move through the real engine, then mirror the // Modular input layer (workstream F): composes the active input SOURCE(s)
// raw position into React state for readouts. // (XY pad / MIDI / gamepad) into the engine. The XY pad is push-driven via
// `pushPad`; MIDI + gamepad are pulled by the layer's own rAF loop.
const inputs = useInputLayer(engine);
// Drive a pad/joystick/manifold move through the input layer's XY-pad source,
// then mirror the raw position into React state for readouts. The layer's loop
// composes it with any other active sources and writes to the engine; we still
// do a direct setInput so the pad feels instant even when it's the sole source.
const onMove = (x: number, y: number) => { const onMove = (x: number, y: number) => {
inputs.pushPad(x, y);
engine?.setInput(x, y); engine?.setInput(x, y);
setPos([x, y]); setPos([x, y]);
}; };
@ -579,6 +588,7 @@ export function ConsoleApp({ focus: initialFocus = 'composite' }: ConsoleAppProp
setOscSendRaw, setOscSendRaw,
setParams: (next: MFParam[]) => setParams(next), setParams: (next: MFParam[]) => setParams(next),
markers, markers,
inputs,
health, health,
gradient: gradient.norms, gradient: gradient.norms,
gradientStatus: gradient.status, gradientStatus: gradient.status,

186
manifold/src/console/Drawers.tsx
View file

@ -225,37 +225,175 @@ function LearningDrawer(ctx: ConsoleCtx, depth: DrawerDepth) {
} }
// =========================================================================== // ===========================================================================
// 2. INPUTS (dock-spec §2 — workstream F territory, referenced) // 2. INPUTS (workstream F; inputs-spec — modular input layer)
// =========================================================================== // ===========================================================================
const STATUS_TONE: Record<string, string> = {
ready: 'var(--accent)',
connecting: 'var(--warn)',
unavailable: 'var(--fg-dim)',
error: 'var(--danger)',
idle: 'var(--fg-dim)',
};
function InputsDrawer(ctx: ConsoleCtx, depth: DrawerDepth) { function InputsDrawer(ctx: ConsoleCtx, depth: DrawerDepth) {
const src = const inp = ctx.inputs;
ctx.mode.input === 'joystick' ? 'Joystick' : ctx.mode.input === 'audio_in' ? 'Mic (1-input)' : 'XY pad'; const enabledCount = inp.sources.filter((s) => s.enabled).length;
const reshaping = inp.axisCount > inp.engineInputSize;
return ( return (
<> <>
<div style={{ display: 'flex', gap: 6, flexWrap: 'wrap' }}> <div style={{ display: 'flex', gap: 6, flexWrap: 'wrap', alignItems: 'center' }}>
<Badge tone="info">{src}</Badge> <Badge tone="info">
<Chip>2 inputs</Chip> {enabledCount === 0 ? 'no source' : `${enabledCount} source${enabledCount > 1 ? 's' : ''}`}
</Badge>
<Chip tone="var(--accent)">{inp.axisCount} axes</Chip>
<Chip>engine: {inp.engineInputSize}-in</Chip>
{reshaping && <Chip tone="var(--warn)">blended → {inp.engineInputSize}</Chip>}
</div> </div>
<SectionLabel>Source</SectionLabel>
<Segmented <SectionLabel>Sources</SectionLabel>
value={ctx.mode.input} <div style={{ display: 'flex', flexDirection: 'column', gap: 6 }}>
onChange={() => { {inp.sources.map((s) => (
/* TODO(workstream F, inputs-spec): MIDI / gamepad / hands sources land here; the <div
input source is currently fixed by the mode. engine.setInput(x,y) is the only door. */ key={s.kind}
}} style={{
options={[ display: 'flex',
{ value: 'xy', label: 'XY pad' }, alignItems: 'center',
{ value: 'joystick', label: 'Joystick' }, justifyContent: 'space-between',
{ value: 'audio_in', label: 'Mic' }, gap: 8,
]} padding: '4px 8px',
/> background: 'var(--bg-2)',
border: '1px solid var(--line)',
borderRadius: 'var(--r-1)',
}}
>
<div style={{ display: 'flex', flexDirection: 'column', gap: 2 }}>
<span style={{ fontSize: 'var(--fs-xs)', color: 'var(--fg-mute)' }}>
{s.label}
{s.enabled ? ` · ${s.axisCount} ax` : ''}
</span>
{depth !== 'peek' && (
<span style={{ fontSize: 9, color: STATUS_TONE[s.status.state] ?? 'var(--fg-dim)' }}>
{s.status.message}
</span>
)}
</div>
<Switch checked={s.enabled} onChange={(v) => inp.setEnabled(s.kind, v)} label="" />
</div>
))}
</div>
{depth !== 'peek' && ( {depth !== 'peek' && (
<p style={{ fontSize: 'var(--fs-xs)', color: 'var(--fg-dim)', margin: 0, lineHeight: 1.6 }}> <>
v1 browser is fixed-2-input ({`MLP<2,…>`}). The modular N×M input matrix, per-axis pipeline {/* ---- Gamepad config ---- */}
(deadzone → zoom → curve → smoothing → momentum) and MIDI / gamepad sources are owned by {inp.sources.find((s) => s.kind === 'gamepad')?.enabled && (
workstream F (inputs-spec); this drawer fixes the dock shape only. <>
</p> <SectionLabel>Gamepad · sticks</SectionLabel>
<Segmented
value={inp.gamepadStickMode}
onChange={inp.setGamepadStickMode}
options={[
{ value: 'single', label: 'Single (2 ax)' },
{ value: 'double', label: 'Double (4 ax)' },
]}
/>
</>
)}
{/* ---- MIDI learn-map ---- */}
{inp.sources.find((s) => s.kind === 'midi')?.enabled && (
<>
<SectionLabel>MIDI · learn-map</SectionLabel>
<div style={{ display: 'flex', gap: 6, alignItems: 'center', flexWrap: 'wrap' }}>
<Button
size="sm"
variant={inp.midiLearnArmed ? 'primary' : 'secondary'}
onClick={() => inp.armMidiLearn(!inp.midiLearnArmed)}
>
{inp.midiLearnArmed ? 'learning… (move a control)' : 'Learn axis'}
</Button>
{inp.midiBindings.length > 0 && (
<Button size="sm" variant="secondary" onClick={inp.clearMidiBindings}>
Clear all
</Button>
)}
</div>
{inp.midiBindings.length === 0 ? (
<p style={{ fontSize: 9, color: 'var(--fg-dim)', margin: 0 }}>
No axes learned yet — arm Learn, then wiggle a knob or hit a pad. CCs map to a
continuous axis; notes map to a gate (1 while held).
</p>
) : (
<div style={{ display: 'flex', flexDirection: 'column', gap: 3 }}>
{inp.midiBindings.map((b, i) => (
<div
key={`${b.kind}-${b.number}-${b.channel}`}
style={{
display: 'flex',
justifyContent: 'space-between',
alignItems: 'center',
fontSize: 'var(--fs-xs)',
color: 'var(--fg-mute)',
}}
>
<span>
{b.label} · {b.value.toFixed(2)}
</span>
<button
type="button"
onClick={() => inp.clearMidiBinding(i)}
style={{
background: 'transparent',
border: 0,
color: 'var(--danger)',
cursor: 'pointer',
fontSize: 'var(--fs-xs)',
}}
>
✕
</button>
</div>
))}
</div>
)}
{inp.midiInputs.length > 0 && (
<p style={{ fontSize: 9, color: 'var(--fg-dim)', margin: 0 }}>
Listening on: {inp.midiInputs.map((p) => p.name).join(', ')}
</p>
)}
</>
)}
{/* ---- Channel layout ---- */}
<SectionLabel>Channel layout</SectionLabel>
{inp.channelLayout.length === 0 ? (
<p style={{ fontSize: 9, color: 'var(--fg-dim)', margin: 0 }}>
No active axes. Enable a source above.
</p>
) : (
<div style={{ display: 'flex', flexWrap: 'wrap', gap: 4 }}>
{inp.channelLayout.map((c, i) => (
<Chip key={i}>
{i}: {c.source}·{c.label}
</Chip>
))}
</div>
)}
<p style={{ fontSize: 9, color: 'var(--fg-dim)', margin: 0, lineHeight: 1.6 }}>
The active sources concatenate into one input vector at the head of the spine.
{reshaping
? ` The browser WASM is a fixed ${inp.engineInputSize}-input head (MLP<2,…>), so the
${inp.axisCount} axes are blended down to ${inp.engineInputSize} (even→X / odd→Y mean).`
: ''}{' '}
{/* TODO(workstream F, docs/redesign/inputs-spec.md — "multiple WASM modules +
warm-start"): give every axis its own genuine input dimension by (re)loading a
WASM module whose MLP arity matches axisCount and warm-starting from the prior
net. Deferred — the reduction lives in InputLayer.compose(). */}
True per-axis dimensions land with the multi-WASM reshape (inputs-spec).
</p>
</>
)} )}
</> </>
); );

5
manifold/src/console/types.ts
View file

@ -5,6 +5,7 @@ import type { MFMode, MFParam } from './model';
import type { BackendId } from '../dock/output-state'; import type { BackendId } from '../dock/output-state';
import type { FeedbackMode } from '../engine/types'; import type { FeedbackMode } from '../engine/types';
import type { BackendStatus } from '../backends/backend'; import type { BackendStatus } from '../backends/backend';
import type { UseInputLayer } from '../inputs';
/** The two product feedback modes (dock-spec §1.1; rl-feedback-design §0). */ /** The two product feedback modes (dock-spec §1.1; rl-feedback-design §0). */
export type FeedbackModeUI = 'explore-and-place' | 'geometric-dislike'; export type FeedbackModeUI = 'explore-and-place' | 'geometric-dislike';
@ -110,6 +111,10 @@ export interface ConsoleCtx {
/** Markers plotted at the input location where each feedback was given. */ /** Markers plotted at the input location where each feedback was given. */
markers: FeedbackMarker[]; markers: FeedbackMarker[];
// ---- Modular input layer (workstream F; inputs-spec) ----
/** The composed input layer: source enable/config/status + channel layout. */
inputs: UseInputLayer;
health: number; health: number;
gradient: number[]; gradient: number[];
gradientStatus: string[]; gradientStatus: string[];

56
manifold/src/inputs/base-source.ts Normal file
View file

@ -0,0 +1,56 @@
/**
* BaseSource — shared status + action plumbing for InputSource adapters.
* Subclasses implement axisCount/axisLabels/sample/start/stop.
*/
import type {
InputAction,
InputSource,
InputSourceKind,
InputSourceStatus,
} from './types';
export abstract class BaseSource implements InputSource {
abstract readonly kind: InputSourceKind;
abstract readonly label: string;
protected statusState: InputSourceStatus;
private statusListeners = new Set<(s: InputSourceStatus) => void>();
private actionListeners = new Set<(a: InputAction) => void>();
constructor(initial: InputSourceStatus = { state: 'idle', message: 'idle' }) {
this.statusState = initial;
}
abstract axisCount(): number;
abstract axisLabels(): string[];
abstract sample(out: Float32Array, offset: number): number;
abstract start(): Promise<void> | void;
abstract stop(): Promise<void> | void;
status(): InputSourceStatus {
return this.statusState;
}
onStatusChange(cb: (s: InputSourceStatus) => void): () => void {
this.statusListeners.add(cb);
return () => {
this.statusListeners.delete(cb);
};
}
onAction(cb: (a: InputAction) => void): () => void {
this.actionListeners.add(cb);
return () => {
this.actionListeners.delete(cb);
};
}
protected setStatus(s: InputSourceStatus): void {
this.statusState = s;
for (const cb of this.statusListeners) cb(s);
}
protected emitAction(a: InputAction): void {
for (const cb of this.actionListeners) cb(a);
}
}

162
manifold/src/inputs/gamepad-source.ts Normal file
View file

@ -0,0 +1,162 @@
/**
* GamepadSource — physical gamepad via the Gamepad API.
*
* Stick mode:
* - 'single' → 2 axes (left stick X/Y)
* - 'double' → 4 axes (left stick X/Y + right stick X/Y)
*
* Standard-mapping gamepad axes are [-1,1]; we remap to [0,1] for the ML head
* and apply a small radial deadzone (sticks rarely rest at exact centre). A
* y-axis flip makes "stick up" = 1 (the API has up = -1).
*
* Pull-based: the Gamepad API is itself a poll-based snapshot
* (`navigator.getGamepads()`), so `sample()` reads the live snapshot directly —
* no event latching needed for axes. Buttons ARE edge-detected each frame (in
* `poll()`, driven by the InputLayer loop) and surfaced as discrete actions.
*
* Graceful degrade: if the Gamepad API is missing OR no pad is connected, the
* source reports `unavailable`/`connecting` and contributes its axes as 0.5
* (centre) so it never destabilises the composed vector.
*/
import { BaseSource } from './base-source';
import type { InputSourceKind } from './types';
export type StickMode = 'single' | 'double';
const DEADZONE = 0.08;
export class GamepadSource extends BaseSource {
readonly kind: InputSourceKind = 'gamepad';
readonly label = 'Gamepad';
private stickMode: StickMode = 'single';
private padIndex: number | null = null;
private running = false;
/** Per-button last pressed-state for edge detection. */
private buttonsDown: boolean[] = [];
private onConnect = (e: GamepadEvent) => {
if (this.padIndex === null) this.padIndex = e.gamepad.index;
this.setStatus({ state: 'ready', message: `Gamepad: ${e.gamepad.id}` });
};
private onDisconnect = (e: GamepadEvent) => {
if (e.gamepad.index === this.padIndex) {
this.padIndex = null;
this.setStatus({ state: 'connecting', message: 'Gamepad disconnected — press a button' });
}
};
isAvailable(): boolean {
return typeof navigator !== 'undefined' && typeof navigator.getGamepads === 'function';
}
setStickMode(mode: StickMode): void {
this.stickMode = mode;
}
getStickMode(): StickMode {
return this.stickMode;
}
axisCount(): number {
return this.stickMode === 'double' ? 4 : 2;
}
axisLabels(): string[] {
return this.stickMode === 'double'
? ['L-X', 'L-Y', 'R-X', 'R-Y']
: ['L-X', 'L-Y'];
}
private activePad(): Gamepad | null {
if (!this.isAvailable()) return null;
const pads = navigator.getGamepads();
if (this.padIndex !== null) {
const p = pads[this.padIndex];
if (p) return p;
}
// Fall back to the first connected pad.
for (const p of pads) {
if (p) {
this.padIndex = p.index;
return p;
}
}
return null;
}
sample(out: Float32Array, offset: number): number {
const n = this.axisCount();
const pad = this.activePad();
if (!pad) {
for (let i = 0; i < n; i++) out[offset + i] = 0.5; // centre when absent
return n;
}
// Left stick = axes 0,1; right stick = axes 2,3 (standard mapping).
out[offset] = remap(pad.axes[0] ?? 0);
out[offset + 1] = remap(-(pad.axes[1] ?? 0)); // flip: up = 1
if (n === 4) {
out[offset + 2] = remap(pad.axes[2] ?? 0);
out[offset + 3] = remap(-(pad.axes[3] ?? 0));
}
return n;
}
/** Edge-detect button presses → discrete actions. Driven by InputLayer loop. */
poll(): void {
if (!this.running) return;
const pad = this.activePad();
if (!pad) return;
if (this.buttonsDown.length !== pad.buttons.length) {
this.buttonsDown = pad.buttons.map((b) => b.pressed);
return; // first frame after (re)connect: prime, don't fire
}
for (let i = 0; i < pad.buttons.length; i++) {
const pressed = pad.buttons[i].pressed;
if (pressed && !this.buttonsDown[i]) {
this.emitAction({
source: this.kind,
id: `button:${i}`,
label: `Button ${i}`,
value: pad.buttons[i].value || 1,
});
}
this.buttonsDown[i] = pressed;
}
}
start(): void {
if (this.running) return;
this.running = true;
if (!this.isAvailable()) {
this.setStatus({ state: 'unavailable', message: 'Gamepad API not supported in this browser' });
return;
}
window.addEventListener('gamepadconnected', this.onConnect);
window.addEventListener('gamepaddisconnected', this.onDisconnect);
if (this.activePad()) {
this.setStatus({ state: 'ready', message: 'Gamepad ready' });
} else {
this.setStatus({ state: 'connecting', message: 'Press a button on your gamepad to connect' });
}
}
stop(): void {
this.running = false;
if (typeof window !== 'undefined') {
window.removeEventListener('gamepadconnected', this.onConnect);
window.removeEventListener('gamepaddisconnected', this.onDisconnect);
}
this.buttonsDown = [];
this.setStatus({ state: 'idle', message: 'Gamepad off' });
}
}
/** Map a [-1,1] stick axis (with radial deadzone) to [0,1]. */
function remap(v: number): number {
let x = v;
if (x > -DEADZONE && x < DEADZONE) x = 0;
else x = x > 0 ? (x - DEADZONE) / (1 - DEADZONE) : (x + DEADZONE) / (1 - DEADZONE);
const out = (x + 1) / 2;
return out < 0 ? 0 : out > 1 ? 1 : out;
}

24
manifold/src/inputs/index.ts Normal file
View file

@ -0,0 +1,24 @@
/**
* Modular INPUT layer (workstream F) — public surface.
*
* The user picks the input SOURCE(s) feeding the ML head (XY pad / MIDI /
* gamepad, or a combination); the InputLayer composes their axes into one
* N-dim vector at the head of the reactive spine. See input-layer.ts for the
* arity-reduction + the documented multi-WASM reshape TODO.
*/
export type {
InputSource,
InputSourceKind,
InputSourceState,
InputSourceStatus,
InputAction,
} from './types';
export { InputLayer } from './input-layer';
export type { InputEngineSink } from './input-layer';
export { XYPadSource } from './xy-pad-source';
export { WebMidiInputSource } from './midi-input-source';
export type { MidiBinding, MidiBindingKind } from './midi-input-source';
export { GamepadSource } from './gamepad-source';
export type { StickMode } from './gamepad-source';
export { useInputLayer } from './useInputLayer';
export type { UseInputLayer, SourceView } from './useInputLayer';

225
manifold/src/inputs/input-layer.ts Normal file
View file

@ -0,0 +1,225 @@
/**
* InputLayer — composes the active InputSource adapters into ONE N-dim input
* vector at the head of the reactive spine, and drives engine.setInputs.
*
* Sits ABOVE the engine: it owns a single rAF loop that, each frame,
* 1. polls poll-based sources (gamepad buttons → actions),
* 2. pulls every active source's axes into the shared `vector` (pull-based),
* 3. blends/maps the N-dim vector down to the engine's input arity, and
* 4. fires engine.setInputs(...) exactly once.
*
* The XY pad remains push-driven via the existing onMove handler — when it's the
* only active source the loop is effectively a no-op for it (it already latched
* its value), but routing everything through one compose path keeps sources
* composable and the channel layout coherent.
*
* ── Arity mismatch (the WASM reshape TODO) ──────────────────────────────────
* The browser WASM is fixed at MLP<2, …, 126> — a TWO-input head. When the
* composed vector has > 2 axes (double-stick gamepad = 4, MIDI learn-map = many)
* we must reduce to 2 to feed today's engine. We do NOT fake a wider net.
*
* chosen reduction (this pass): pairwise BLEND.
* inX = mean(axis[0], axis[2], axis[4], …) // even axes
* inY = mean(axis[1], axis[3], axis[5], …) // odd axes
* so a single stick passes straight through (axis0→X, axis1→Y), a double
* stick averages L/R into one XY, and MIDI axes fold into X/Y by parity.
*
* TODO(workstream F, docs/redesign/inputs-spec.md — "multiple WASM modules +
* warm-start"): the real fix is to (re)load a WASM module whose MLP input arity
* matches the composed axis count and warm-start its weights from the prior net,
* so every axis gets its own genuine input dimension instead of being blended.
* That is a larger build (multiple .wasm artefacts or a runtime-variadic head)
* and is deliberately deferred — this layer is wired so that swapping the
* reduction for a true reshape is a localised change in `compose()`.
*/
import type { InputAction, InputSource } from './types';
/** Minimal engine surface the layer needs (keeps this framework/engine-neutral). */
export interface InputEngineSink {
setInputs(arr: ReadonlyArray<number>): void;
readonly architecture: { inputSize: number };
}
const MAX_AXES = 32;
export class InputLayer {
private sources: InputSource[] = [];
private engine: InputEngineSink | null = null;
private vector = new Float32Array(MAX_AXES);
private running = false;
private rafId: number | null = null;
private actionListeners = new Set<(a: InputAction) => void>();
private layoutListeners = new Set<() => void>();
private unsubActions = new Map<InputSource, () => void>();
attach(engine: InputEngineSink): void {
this.engine = engine;
}
/** Replace the active source set. Sources already started stay started. */
setSources(sources: InputSource[]): void {
// Unsubscribe actions from sources being dropped.
for (const [src, unsub] of this.unsubActions) {
if (!sources.includes(src)) {
unsub();
this.unsubActions.delete(src);
}
}
// Subscribe new sources' actions through to our fan-out.
for (const src of sources) {
if (!this.unsubActions.has(src)) {
this.unsubActions.set(src, src.onAction((a) => this.fanAction(a)));
}
}
this.sources = sources;
this.notifyLayout();
}
getSources(): ReadonlyArray<InputSource> {
return this.sources;
}
/** Total composed axis count across active sources. */
axisCount(): number {
let n = 0;
for (const s of this.sources) n += s.axisCount();
return Math.min(n, MAX_AXES);
}
/** Per-axis labels in composed order ("XY:X", "Gamepad:L-X", …). */
channelLayout(): { source: string; label: string }[] {
const out: { source: string; label: string }[] = [];
for (const s of this.sources) {
const labels = s.axisLabels();
for (const l of labels) {
if (out.length >= MAX_AXES) break;
out.push({ source: s.label, label: l });
}
}
return out;
}
// ---- loop ----------------------------------------------------------------
start(): void {
if (this.running) return;
this.running = true;
const tick = () => {
if (!this.running) return;
this.frame();
this.rafId = requestAnimationFrame(tick);
};
this.rafId = requestAnimationFrame(tick);
}
stop(): void {
this.running = false;
if (this.rafId !== null) {
cancelAnimationFrame(this.rafId);
this.rafId = null;
}
}
/** One composition tick (also callable directly from a push handler/tests). */
frame(): void {
const engine = this.engine;
if (!engine || this.sources.length === 0) return;
// 1. poll poll-based sources (gamepad button edges → actions).
for (const s of this.sources) {
const maybePoll = s as { poll?: () => void };
if (typeof maybePoll.poll === 'function') maybePoll.poll();
}
// 2. pull each source's axes into the shared vector.
let n = 0;
for (const s of this.sources) {
if (n >= MAX_AXES) break;
n += s.sample(this.vector, n);
}
if (n === 0) return;
// 3. reduce to the engine's input arity (see file header — blend, not fake).
const reduced = this.compose(n, engine.architecture.inputSize);
// 4. one engine write.
engine.setInputs(reduced);
}
/**
* Reduce the composed N-axis vector to the engine's input arity.
*
* For the fixed 2-input WASM, fold by parity (even→X, odd→Y) via mean. If a
* future multi-module engine reports inputSize >= n, this passes axes through
* 1:1 (truncated/padded) — the seam where the real reshape lands.
*/
private compose(n: number, inputSize: number): number[] {
if (inputSize >= n) {
// True passthrough path (future multi-module head). Pad with 0.5.
const out = new Array<number>(inputSize);
for (let i = 0; i < inputSize; i++) out[i] = i < n ? this.vector[i] : 0.5;
return out;
}
if (inputSize === 2) {
let sx = 0;
let sy = 0;
let cx = 0;
let cy = 0;
for (let i = 0; i < n; i++) {
if ((i & 1) === 0) {
sx += this.vector[i];
cx++;
} else {
sy += this.vector[i];
cy++;
}
}
return [cx ? sx / cx : 0.5, cy ? sy / cy : 0.5];
}
// Generic fallback for any other fixed arity: chunked mean.
const out = new Array<number>(inputSize).fill(0.5);
const per = Math.ceil(n / inputSize);
for (let k = 0; k < inputSize; k++) {
let s = 0;
let c = 0;
for (let i = k * per; i < Math.min((k + 1) * per, n); i++) {
s += this.vector[i];
c++;
}
if (c) out[k] = s / c;
}
return out;
}
// ---- actions + layout fan-out -------------------------------------------
onAction(cb: (a: InputAction) => void): () => void {
this.actionListeners.add(cb);
return () => {
this.actionListeners.delete(cb);
};
}
onLayoutChange(cb: () => void): () => void {
this.layoutListeners.add(cb);
return () => {
this.layoutListeners.delete(cb);
};
}
private fanAction(a: InputAction): void {
for (const cb of this.actionListeners) cb(a);
}
private notifyLayout(): void {
for (const cb of this.layoutListeners) cb();
}
dispose(): void {
this.stop();
for (const unsub of this.unsubActions.values()) unsub();
this.unsubActions.clear();
this.actionListeners.clear();
this.layoutListeners.clear();
}
}

219
manifold/src/inputs/midi-input-source.ts Normal file
View file

@ -0,0 +1,219 @@
/**
* WebMidiInputSource — a MIDI controller as an N-axis input source (NEW).
*
* Uses `navigator.requestMIDIAccess()` (feature-detected; graceful degrade) and
* listens on every input port. Incoming Control-Change and Note messages drive
* "learned" axes:
*
* - **CC axis**: a learned CC# (per channel) → its 7-bit value mapped to [0,1].
* - **Note axis**: a learned note → 1 while held (note-on, velocity>0), 0 on
* note-off. Note-on ALSO surfaces a discrete action (so a pad can fire
* commit/perturb without the keyboard).
*
* **Learn-map.** When `armLearn()` is active, the NEXT distinct CC or note seen
* is bound to a new axis (appended). This is the standard "MIDI learn" gesture:
* arm → wiggle the knob/pad → it captures. Axes can be cleared individually.
* The bindings are exposed for the dock channel-layout view.
*
* Pull-based: messages latch the latest per-binding value into `values`;
* `sample()` copies them out. Hot path performs no IO/allocation.
*
* NOTE: the canonical learn-map spec was `docs/redesign/inputs-spec.md`, which
* is absent from this tree. The shape here follows the WebMidiBackend output
* pattern (src/backends/midi-backend.ts) for symmetry.
*/
import { BaseSource } from './base-source';
import type { InputSourceKind } from './types';
export type MidiBindingKind = 'cc' | 'note';
export interface MidiBinding {
kind: MidiBindingKind;
/** CC number or note number. */
number: number;
/** 1-based MIDI channel (1..16). */
channel: number;
/** Latest normalised value ∈ [0,1]. */
value: number;
/** Human label ("CC74 ch1", "Note 60 ch1"). */
label: string;
}
const STATUS_BYTE_CC = 0xb0;
const STATUS_BYTE_NOTE_ON = 0x90;
const STATUS_BYTE_NOTE_OFF = 0x80;
export class WebMidiInputSource extends BaseSource {
readonly kind: InputSourceKind = 'midi';
readonly label = 'MIDI';
private access: MIDIAccess | null = null;
private inputs: MIDIInput[] = [];
private bindings: MidiBinding[] = [];
private learnArmed = false;
private bindingsListeners = new Set<(b: MidiBinding[]) => void>();
isAvailable(): boolean {
return typeof navigator !== 'undefined' && typeof navigator.requestMIDIAccess === 'function';
}
axisCount(): number {
return this.bindings.length;
}
axisLabels(): string[] {
return this.bindings.map((b) => b.label);
}
sample(out: Float32Array, offset: number): number {
const n = this.bindings.length;
for (let i = 0; i < n; i++) out[offset + i] = this.bindings[i].value;
return n;
}
// ---- Learn-map API (consumed by the dock) -------------------------------
/** Arm/disarm MIDI-learn: the next distinct CC/note is captured as an axis. */
armLearn(armed: boolean): void {
this.learnArmed = armed;
this.setStatus(
armed
? { state: 'ready', message: 'Learn armed — move a knob or hit a pad' }
: this.readyStatus(),
);
}
isLearnArmed(): boolean {
return this.learnArmed;
}
getBindings(): ReadonlyArray<MidiBinding> {
return this.bindings;
}
/** Remove the binding (axis) at index. */
clearBinding(index: number): void {
if (index < 0 || index >= this.bindings.length) return;
this.bindings.splice(index, 1);
this.notifyBindings();
}
clearAllBindings(): void {
this.bindings = [];
this.notifyBindings();
}
onBindingsChange(cb: (b: MidiBinding[]) => void): () => void {
this.bindingsListeners.add(cb);
return () => {
this.bindingsListeners.delete(cb);
};
}
// ---- Lifecycle ----------------------------------------------------------
async start(): Promise<void> {
if (!this.isAvailable()) {
this.setStatus({ state: 'unavailable', message: 'Web MIDI not supported in this browser' });
return;
}
this.setStatus({ state: 'connecting', message: 'Requesting MIDI access…' });
try {
this.access = await navigator.requestMIDIAccess!({ sysex: false });
this.access.onstatechange = () => this.rewire();
this.rewire();
this.setStatus(this.readyStatus());
} catch (err) {
this.setStatus({ state: 'error', message: `MIDI access denied: ${(err as Error).message}` });
}
}
stop(): void {
for (const inp of this.inputs) inp.onmidimessage = null;
this.inputs = [];
if (this.access) this.access.onstatechange = null;
this.access = null;
this.learnArmed = false;
this.setStatus({ state: 'idle', message: 'MIDI input off' });
}
/** Enumerate the connected input ports (for the dock readout). */
listInputs(): { id: string; name: string }[] {
if (!this.access) return [];
const out: { id: string; name: string }[] = [];
this.access.inputs.forEach((inp, id) => out.push({ id, name: inp.name ?? `MIDI Input ${id}` }));
return out;
}
private rewire(): void {
if (!this.access) return;
for (const inp of this.inputs) inp.onmidimessage = null;
this.inputs = [];
this.access.inputs.forEach((inp) => {
inp.onmidimessage = (e) => this.onMessage(e);
this.inputs.push(inp);
});
if (this.statusState.state !== 'connecting') this.setStatus(this.readyStatus());
}
private readyStatus(): { state: 'ready'; message: string } {
const n = this.inputs.length;
return {
state: 'ready',
message: n ? `MIDI in — ${n} port${n > 1 ? 's' : ''}, ${this.bindings.length} axes` : 'MIDI ready — no input ports',
};
}
private onMessage(e: MIDIMessageEvent): void {
const data = e.data;
if (!data || data.length < 2) return;
const status = data[0] & 0xf0;
const channel = (data[0] & 0x0f) + 1;
const d1 = data[1];
const d2 = data.length > 2 ? data[2] : 0;
if (status === STATUS_BYTE_CC) {
this.handleBindable('cc', d1, channel, d2 / 127);
} else if (status === STATUS_BYTE_NOTE_ON && d2 > 0) {
this.handleBindable('note', d1, channel, 1);
this.emitAction({ source: this.kind, id: `note:${d1}`, label: `Note ${d1}`, value: d2 / 127 });
} else if (status === STATUS_BYTE_NOTE_OFF || (status === STATUS_BYTE_NOTE_ON && d2 === 0)) {
this.handleBindable('note', d1, channel, 0, /*onlyUpdate*/ true);
}
}
/**
* Route an incoming bindable message: update a matching binding's value, or —
* if learn is armed — create a new axis binding for it.
*/
private handleBindable(
kind: MidiBindingKind,
number: number,
channel: number,
value: number,
onlyUpdate = false,
): void {
const existing = this.bindings.find(
(b) => b.kind === kind && b.number === number && b.channel === channel,
);
if (existing) {
existing.value = value;
this.notifyBindings();
return;
}
if (onlyUpdate) return; // note-off for an unbound note: ignore
if (this.learnArmed) {
const label =
kind === 'cc' ? `CC${number} ch${channel}` : `Note ${number} ch${channel}`;
this.bindings.push({ kind, number, channel, value, label });
this.learnArmed = false; // learn one binding per arm
this.setStatus(this.readyStatus());
this.notifyBindings();
}
}
private notifyBindings(): void {
const snapshot = this.bindings.map((b) => ({ ...b }));
for (const cb of this.bindingsListeners) cb(snapshot);
}
}

101
manifold/src/inputs/types.ts Normal file
View file

@ -0,0 +1,101 @@
/**
* Modular INPUT layer — adapter interface (workstream F).
*
* The redesign calls for the user to choose the input SOURCE(s) feeding the ML
* head: the XY pad, a MIDI controller (CCs / notes learned onto axes), or a
* gamepad (single-stick = 2 axes, double-stick = 4). Sources are composable —
* the active sources are concatenated into one N-dim input vector at the head of
* the reactive spine (engine.setInput / setInputs).
*
* Design intent (the canonical spec was `docs/redesign/inputs-spec.md`, which is
* NOT present in this tree — see TODO refs below):
*
* - **Pull-based.** The InputLayer owns a single rAF loop and pulls every
* active source's current state once per frame via `sample(out, offset)`.
* Event-driven sources (MIDI, gamepad) latch their latest state into internal
* fields between frames; the pad pushes directly (`pushAxes`) but still reads
* back through the same path. No source schedules its own engine writes.
* - **N axes + discrete actions.** Each source declares an axis COUNT (its
* dimensionality) and writes its axes into a slice of the shared vector. It
* may also surface momentary discrete ACTIONS (e.g. a MIDI note, a gamepad
* button) that the layer fans out to listeners (used for verdict
* commit/perturb without a keyboard).
* - **Graceful degrade.** A source that the platform can't provide
* (`navigator.requestMIDIAccess` missing, no gamepad connected) reports an
* `unavailable` status and contributes zero axes rather than throwing.
*
* Axis values are normalised to [0,1] (the ML head's input domain); the spine's
* input-pipeline then applies deadzone/zoom/curve/etc.
*/
/** Lifecycle / availability of a source, surfaced to the dock UI. */
export type InputSourceState =
| 'idle' // constructed, not yet started
| 'connecting' // async permission / device handshake in flight
| 'ready' // live, producing axes
| 'unavailable' // platform/feature not present (graceful degrade)
| 'error'; // start failed
export interface InputSourceStatus {
state: InputSourceState;
message: string;
}
/** Stable identity of a source kind. */
export type InputSourceKind = 'xy-pad' | 'midi' | 'gamepad';
/**
* A momentary discrete action surfaced by a source (e.g. a MIDI note-on or a
* gamepad face-button press). Fanned out to InputLayer action listeners so the
* console can bind them to commit / perturb / reroll without the keyboard.
*/
export interface InputAction {
source: InputSourceKind;
/** Stable per-source action id ("note:36", "button:0", …). */
id: string;
/** Human label for the learn-map UI. */
label: string;
/** 0..1 velocity / analogue value where meaningful (else 1 for a press). */
value: number;
}
/**
* Pull-based input source adapter.
*
* Sources are framework-neutral (NO React). The InputLayer composes them; the
* dock store wraps the layer for React.
*/
export interface InputSource {
readonly kind: InputSourceKind;
/** Short human label for the dock ("XY pad", "MIDI", "Gamepad"). */
readonly label: string;
/**
* Number of axes this source currently contributes to the input vector.
* MAY change at runtime (e.g. gamepad single↔double stick, MIDI learn-map
* growing). The layer re-reads this each composition.
*/
axisCount(): number;
/** Per-axis labels for the channel-layout view (length === axisCount()). */
axisLabels(): string[];
/**
* Write this source's current axes (each ∈ [0,1]) into `out` starting at
* `offset`. Returns the number of axes written (=== axisCount()). Pull-based:
* reads latched internal state, performs NO IO. Hot path — no allocation.
*/
sample(out: Float32Array, offset: number): number;
/** Begin producing (request permissions, attach listeners). Idempotent. */
start(): Promise<void> | void;
/** Stop producing + release resources. Idempotent. */
stop(): Promise<void> | void;
status(): InputSourceStatus;
onStatusChange(cb: (s: InputSourceStatus) => void): () => void;
/** Subscribe to discrete actions (notes / buttons). Returns unsubscribe. */
onAction(cb: (a: InputAction) => void): () => void;
}

235
manifold/src/inputs/useInputLayer.ts Normal file
View file

@ -0,0 +1,235 @@
/**
* useInputLayer — the thin React binding over the framework-neutral
* {@link InputLayer} + source adapters.
*
* Owns:
* - ONE InputLayer + one instance of each source (XY pad / MIDI / gamepad),
* created per engine and attached to it.
* - Which sources are ENABLED (the dock toggles these); enabling starts a
* source (async for MIDI) and adds it to the layer's composed set.
* - Per-source config (gamepad stick mode; MIDI learn arm + bindings).
* - The composed channel layout + per-source status, surfaced for the drawer.
*
* The XY pad source is the one consumers push into directly: `pushPad(x,y)` is
* called from ConsoleApp.onMove so the existing pad keeps working unchanged
* while still composing with the other sources.
*
* Discrete actions (MIDI notes / gamepad buttons) are fanned out via
* `onAction` so the console can later bind them to verdicts (commit/perturb).
*/
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import type { EngineApi } from '../engine';
import { InputLayer } from './input-layer';
import { XYPadSource } from './xy-pad-source';
import { WebMidiInputSource, type MidiBinding } from './midi-input-source';
import { GamepadSource, type StickMode } from './gamepad-source';
import type { InputAction, InputSource, InputSourceKind, InputSourceStatus } from './types';
export interface SourceView {
kind: InputSourceKind;
label: string;
enabled: boolean;
status: InputSourceStatus;
axisCount: number;
}
export interface UseInputLayer {
/** Push the on-screen XY pad position (∈ [0,1]) — call from onMove. */
pushPad: (x: number, y: number) => void;
/** Per-source enable + status + axis count for the dock. */
sources: SourceView[];
/** Toggle a source on/off. */
setEnabled: (kind: InputSourceKind, enabled: boolean) => void;
/** Composed channel layout (per-axis source+label). */
channelLayout: { source: string; label: string }[];
/** Total composed axis count. */
axisCount: number;
/** Engine input arity (the fixed WASM head = 2). */
engineInputSize: number;
// ---- gamepad config ----
gamepadStickMode: StickMode;
setGamepadStickMode: (m: StickMode) => void;
// ---- midi learn-map ----
midiLearnArmed: boolean;
armMidiLearn: (armed: boolean) => void;
midiBindings: MidiBinding[];
clearMidiBinding: (i: number) => void;
clearMidiBindings: () => void;
midiInputs: { id: string; name: string }[];
/** Subscribe to discrete actions (notes/buttons). */
onAction: (cb: (a: InputAction) => void) => () => void;
}
export function useInputLayer(engine: EngineApi | null): UseInputLayer {
// One layer + one of each source, created once per engine.
const layerRef = useRef<InputLayer | null>(null);
const padRef = useRef<XYPadSource | null>(null);
const midiRef = useRef<WebMidiInputSource | null>(null);
const gamepadRef = useRef<GamepadSource | null>(null);
if (!layerRef.current) {
layerRef.current = new InputLayer();
padRef.current = new XYPadSource();
midiRef.current = new WebMidiInputSource();
gamepadRef.current = new GamepadSource();
}
const layer = layerRef.current!;
const pad = padRef.current!;
const midi = midiRef.current!;
const gamepad = gamepadRef.current!;
// Enabled set — pad on by default (parity with today's behaviour).
const [enabled, setEnabledSet] = useState<Record<InputSourceKind, boolean>>({
'xy-pad': true,
midi: false,
gamepad: false,
});
const [statuses, setStatuses] = useState<Record<InputSourceKind, InputSourceStatus>>({
'xy-pad': pad.status(),
midi: midi.status(),
gamepad: gamepad.status(),
});
const [layoutTick, setLayoutTick] = useState(0);
const [gamepadStickMode, setGamepadStickModeState] = useState<StickMode>('single');
const [midiLearnArmed, setMidiLearnArmed] = useState(false);
const [midiBindings, setMidiBindings] = useState<MidiBinding[]>([]);
const [midiInputs, setMidiInputs] = useState<{ id: string; name: string }[]>([]);
// Attach to engine; start the pad immediately. Wire status/binding listeners.
useEffect(() => {
if (!engine) return;
layer.attach(engine);
pad.start();
layer.setSources([pad]);
layer.start();
const unsubs: (() => void)[] = [];
const wireStatus = (s: InputSource) =>
unsubs.push(
s.onStatusChange((st) => setStatuses((m) => ({ ...m, [s.kind]: st }))),
);
wireStatus(pad);
wireStatus(midi);
wireStatus(gamepad);
unsubs.push(layer.onLayoutChange(() => setLayoutTick((t) => t + 1)));
unsubs.push(
midi.onBindingsChange((b) => {
setMidiBindings(b);
setLayoutTick((t) => t + 1);
setMidiLearnArmed(midi.isLearnArmed());
}),
);
return () => {
for (const u of unsubs) u();
layer.dispose();
void midi.stop();
gamepad.stop();
};
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [engine]);
// Recompose the active source set whenever the enabled set changes.
useEffect(() => {
const active: InputSource[] = [];
if (enabled['xy-pad']) active.push(pad);
if (enabled.midi) active.push(midi);
if (enabled.gamepad) active.push(gamepad);
layer.setSources(active);
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [enabled]);
const setEnabled = useCallback(
(kind: InputSourceKind, on: boolean) => {
setEnabledSet((m) => ({ ...m, [kind]: on }));
if (kind === 'midi') {
if (on) {
void midi.start().then(() => setMidiInputs(midi.listInputs()));
} else {
void midi.stop();
}
} else if (kind === 'gamepad') {
if (on) gamepad.start();
else gamepad.stop();
} else if (kind === 'xy-pad') {
if (on) pad.start();
else pad.stop();
}
},
[midi, gamepad, pad],
);
const setGamepadStickMode = useCallback(
(m: StickMode) => {
gamepad.setStickMode(m);
setGamepadStickModeState(m);
setLayoutTick((t) => t + 1);
},
[gamepad],
);
const armMidiLearn = useCallback(
(armed: boolean) => {
midi.armLearn(armed);
setMidiLearnArmed(armed);
},
[midi],
);
const clearMidiBinding = useCallback(
(i: number) => {
midi.clearBinding(i);
setMidiBindings([...midi.getBindings()]);
setLayoutTick((t) => t + 1);
},
[midi],
);
const clearMidiBindings = useCallback(() => {
midi.clearAllBindings();
setMidiBindings([]);
setLayoutTick((t) => t + 1);
}, [midi]);
const pushPad = useCallback((x: number, y: number) => pad.pushAxes(x, y), [pad]);
const onAction = useCallback((cb: (a: InputAction) => void) => layer.onAction(cb), [layer]);
const sources: SourceView[] = useMemo(
() =>
([pad, midi, gamepad] as InputSource[]).map((s) => ({
kind: s.kind,
label: s.label,
enabled: enabled[s.kind],
status: statuses[s.kind],
axisCount: enabled[s.kind] ? s.axisCount() : 0,
})),
// layoutTick forces recompute when axis counts shift (learn-map / stick mode).
// eslint-disable-next-line react-hooks/exhaustive-deps
[enabled, statuses, layoutTick, pad, midi, gamepad],
);
const channelLayout = useMemo(
() => layer.channelLayout(),
// eslint-disable-next-line react-hooks/exhaustive-deps
[layoutTick, enabled],
);
return {
pushPad,
sources,
setEnabled,
channelLayout,
axisCount: channelLayout.length,
engineInputSize: engine?.architecture.inputSize ?? 2,
gamepadStickMode,
setGamepadStickMode,
midiLearnArmed,
armMidiLearn,
midiBindings,
clearMidiBinding,
clearMidiBindings,
midiInputs,
onAction,
};
}

56
manifold/src/inputs/xy-pad-source.ts Normal file
View file

@ -0,0 +1,56 @@
/**
* XYPadSource — the existing on-screen XY pad / joystick / manifold drag,
* wrapped as a 2-axis InputSource.
*
* Push-based: the pad's `onMove(x, y)` handler calls `pushAxes(x, y)`, which
* latches the values. `sample()` reads them back through the standard pull path
* so the XY pad composes identically with poll-based sources (MIDI/gamepad).
*
* This is always available (it's an on-screen widget), so it starts `ready`.
*/
import { BaseSource } from './base-source';
import type { InputSourceKind } from './types';
export class XYPadSource extends BaseSource {
readonly kind: InputSourceKind = 'xy-pad';
readonly label = 'XY pad';
private x = 0.5;
private y = 0.5;
constructor() {
super({ state: 'ready', message: 'XY pad ready' });
}
axisCount(): number {
return 2;
}
axisLabels(): string[] {
return ['X', 'Y'];
}
/** Latch the latest pad position (∈ [0,1]). Called from the pad's onMove. */
pushAxes(x: number, y: number): void {
this.x = clamp01(x);
this.y = clamp01(y);
}
sample(out: Float32Array, offset: number): number {
out[offset] = this.x;
out[offset + 1] = this.y;
return 2;
}
start(): void {
this.setStatus({ state: 'ready', message: 'XY pad ready' });
}
stop(): void {
this.setStatus({ state: 'idle', message: 'XY pad off' });
}
}
function clamp01(v: number): number {
return v < 0 ? 0 : v > 1 ? 1 : v;
}