merge(vcv-bridge): browser drives+trains VCV module over WS-OSC (/nisps/input + /nisps/feedback)

# Conflicts:
#	manifold/src/backends/manager.ts
This commit is contained in:
monkey-w1n5t0n 2026-06-28 04:33:58 +02:00
commit 45e59e1a23
13 changed files with 552 additions and 14 deletions

View file

@ -63,6 +63,16 @@ function oscMessage(address, value) {
]); ]);
} }
/** A single OSC message carrying N floats (one ",fff…" message, not N messages). */
function oscMessageFloats(address, values) {
const tags = ',' + 'f'.repeat(values.length);
return Buffer.concat([
oscString(address),
oscString(tags),
...values.map((v) => oscFloat(v)),
]);
}
function oscMessageString(address, value) { function oscMessageString(address, value) {
return Buffer.concat([ return Buffer.concat([
oscString(address), oscString(address),
@ -145,6 +155,11 @@ function sendOSCString(address, value) {
udpSend.send(msg, OSC_PORT, OSC_HOST); udpSend.send(msg, OSC_PORT, OSC_HOST);
} }
function sendOSCFloats(address, values) {
const msg = oscMessageFloats(address, values);
udpSend.send(msg, OSC_PORT, OSC_HOST);
}
function sendOSCBundle(params) { function sendOSCBundle(params) {
const messages = params.map(([name, value]) => const messages = params.map(([name, value]) =>
oscMessage(`${OSC_PREFIX}/${name}`, value) oscMessage(`${OSC_PREFIX}/${name}`, value)
@ -218,6 +233,16 @@ wss.on('connection', (ws) => {
case 'weights': case 'weights':
sendOSCString(`${OSC_PREFIX}/weights`, JSON.stringify(data.payload)); sendOSCString(`${OSC_PREFIX}/weights`, JSON.stringify(data.payload));
return; return;
case 'input':
// Current input VECTOR as ONE multi-float message to /nisps/input.
if (Array.isArray(data.payload)) {
sendOSCFloats(`${OSC_PREFIX}/input`, data.payload);
}
return;
case 'feedback':
// Verdict op as OSC string to /nisps/feedback (trains the module).
sendOSCString(`${OSC_PREFIX}/feedback`, JSON.stringify(data.payload));
return;
case 'params': case 'params':
if (Array.isArray(data.payload)) { if (Array.isArray(data.payload)) {
if (USE_BUNDLES) { if (USE_BUNDLES) {

View file

@ -22,8 +22,11 @@
// /nisps/<param_name> <float> (webapp -> target) // /nisps/<param_name> <float> (webapp -> target)
// /nisps/state <string> (webapp -> target: full JSON state) // /nisps/state <string> (webapp -> target: full JSON state)
// /nisps/weights <string> (webapp -> target: weights JSON) // /nisps/weights <string> (webapp -> target: weights JSON)
// /nisps/input <f...f> (webapp -> target: input vector, ONE message;
// also target -> webapp for visualisation)
// /nisps/feedback <string> (webapp -> target: verdict op JSON —
// { op, spread, input[], output[] })
// /nisps/output <f...f> (target -> webapp: output float array) // /nisps/output <f...f> (target -> webapp: output float array)
// /nisps/input <f...f> (target -> webapp: input float array)
import { parseArgs } from "jsr:@std/cli@1/parse-args"; import { parseArgs } from "jsr:@std/cli@1/parse-args";
@ -104,6 +107,13 @@ function oscMessage(address: string, value: number): Uint8Array {
return concat(oscString(address), oscString(",f"), oscFloat(value)); return concat(oscString(address), oscString(",f"), oscFloat(value));
} }
/** A single OSC message carrying N floats (one ",fff…" message, not N messages). */
function oscMessageFloats(address: string, values: number[]): Uint8Array {
const tags = "," + "f".repeat(values.length);
const floatBufs = values.map((v) => oscFloat(v));
return concat(oscString(address), oscString(tags), ...floatBufs);
}
function oscMessageString(address: string, value: string): Uint8Array { function oscMessageString(address: string, value: string): Uint8Array {
return concat(oscString(address), oscString(",s"), oscString(value)); return concat(oscString(address), oscString(",s"), oscString(value));
} }
@ -197,6 +207,11 @@ function sendOSCString(address: string, value: string): void {
udpSend.send(msg, oscAddr); udpSend.send(msg, oscAddr);
} }
function sendOSCFloats(address: string, values: number[]): void {
const msg = oscMessageFloats(address, values);
udpSend.send(msg, oscAddr);
}
function sendOSCBundle(params: [string, number][]): void { function sendOSCBundle(params: [string, number][]): void {
const messages = params.map(([name, value]) => const messages = params.map(([name, value]) =>
oscMessage(`${OSC_PREFIX}/${name}`, value) oscMessage(`${OSC_PREFIX}/${name}`, value)
@ -249,6 +264,18 @@ function handleWs(ws: WebSocket): void {
// Send weights JSON as OSC string to /nisps/weights // Send weights JSON as OSC string to /nisps/weights
sendOSCString(`${OSC_PREFIX}/weights`, JSON.stringify(data.payload)); sendOSCString(`${OSC_PREFIX}/weights`, JSON.stringify(data.payload));
return; return;
case "input":
// Current input VECTOR as ONE multi-float message to /nisps/input
// (browser drives the VCV module's inputs in bridged mode).
if (Array.isArray(data.payload)) {
sendOSCFloats(`${OSC_PREFIX}/input`, data.payload as number[]);
}
return;
case "feedback":
// Verdict op as OSC string to /nisps/feedback (trains the module's
// embedded learner: { op, spread, input[], output[] }).
sendOSCString(`${OSC_PREFIX}/feedback`, JSON.stringify(data.payload));
return;
case "params": case "params":
// Legacy batch format embedded in structured message // Legacy batch format embedded in structured message
if (Array.isArray(data.payload)) { if (Array.isArray(data.payload)) {

View file

@ -1,7 +1,7 @@
# Output Backends (`manifold/src/backends/`) # Output Backends (`manifold/src/backends/`)
Real output transports for the Manifold app. Exactly one backend is *active* at a Real output transports for the Manifold app. Exactly one backend is *active* at a
time, chosen by the dock **Mode** (Particle / MIDI / OSC / Built-in Synth / time, chosen by the dock **Mode** (Particle / MIDI / OSC / VCV / Built-in Synth /
Editor → `BackendId`). The `BackendManager` consumes the engine spine and Editor → `BackendId`). The `BackendManager` consumes the engine spine and
forwards each routed output vector to the active backend's `send()`. forwards each routed output vector to the active backend's `send()`.
@ -17,6 +17,7 @@ See `docs/redesign/backends-spec.md` for the authoritative design.
| `midi-backend.ts` | `WebMidiBackend` — real Web MIDI CC out (per-output CC#/channel/range/name, throttled + dead-zone). | | `midi-backend.ts` | `WebMidiBackend` — real Web MIDI CC out (per-output CC#/channel/range/name, throttled + dead-zone). |
| `osc-client.ts` | `NispsOscClient` — WS transport to the Deno OSC bridge (JSON protocol, auto-reconnect). | | `osc-client.ts` | `NispsOscClient` — WS transport to the Deno OSC bridge (JSON protocol, auto-reconnect). |
| `osc-backend.ts` | `OscBridgeBackend` — OSC out over WS; per-output address path + physical range. | | `osc-backend.ts` | `OscBridgeBackend` — OSC out over WS; per-output address path + physical range. |
| `vcv-backend.ts` | `VcvBackend` — drives + **trains** the VCV Rack NISPS module over the OSC↔WS bridge (streams `/nisps/input`, forwards the verdict loop to `/nisps/feedback`, receives `/nisps/output` + `/nisps/state`). |
| `passthrough-backend.ts` | No-op sink for synth (plays in-engine) / particles (rAF consumer) / editor. | | `passthrough-backend.ts` | No-op sink for synth (plays in-engine) / particles (rAF consumer) / editor. |
| `presets.ts` | Named per-backend output-config presets (localStorage, per-backend namespace). | | `presets.ts` | Named per-backend output-config presets (localStorage, per-backend namespace). |
| `useBackendManager.ts` | Thin React binding: builds `BackendContext` from the store, switches Mode, surfaces status. | | `useBackendManager.ts` | Thin React binding: builds `BackendContext` from the store, switches Mode, surfaces status. |
@ -45,3 +46,32 @@ node bridge.mjs
Default bridge URL: `ws://localhost:8765` (configurable in the OSC config panel). Default bridge URL: `ws://localhost:8765` (configurable in the OSC config panel).
WS protocol (browser → bridge): `{ type:'params', payload:[[path,value],…] }`. WS protocol (browser → bridge): `{ type:'params', payload:[[path,value],…] }`.
## VCV bridge — module **and** bridge must both be running
When Mode = **VCV**, the browser is authoritative in *bridged mode*: the
`VcvBackend` connects over the **same Deno bridge** and
- streams the current input vector continuously to **`/nisps/input`** (the
browser drives the module's inputs),
- streams the routed per-output CV (uni 010 V / bipolar ±5 V), and
- forwards the **verdict loop** — thumbs-up/down + explore-and-place — to
**`/nisps/feedback`** (`{ op:'up'|'down'|'rand'|'clear', spread, input[],
output[] }`), so the module's embedded net trains in lock-step with the
browser session.
It receives **`/nisps/output`** + **`/nisps/state`** back for status /
visualisation. The verdict forwarding is gated in `BackendManager.forwardFeedback`
— a no-op unless VCV is the active backend (otherwise the browser engine is the
learner). Both processes are required:
```bash
# 1) the VCV Rack NISPS module (vcv/) — default UDP listen port 7001
# 2) the Deno bridge, pointed at the module's UDP port:
cd manifold/osc-bridge
deno run --allow-net bridge.ts --osc-port 7001
```
Default VCV bridge URL: `ws://localhost:8765` (configurable in the VCV config
panel). Until the WS connects the backend shows "bridge not running"; until the
module replies it shows "waiting for module…".

View file

@ -16,6 +16,8 @@ export { WebMidiBackend } from './midi-backend';
export type { MidiBackendConfig } from './midi-backend'; export type { MidiBackendConfig } from './midi-backend';
export { OscBridgeBackend } from './osc-backend'; export { OscBridgeBackend } from './osc-backend';
export type { OscBackendConfig } from './osc-backend'; export type { OscBackendConfig } from './osc-backend';
export { VcvBackend } from './vcv-backend';
export type { VcvBackendConfig, VcvFeedbackOp } from './vcv-backend';
export { NispsOscClient } from './osc-client'; export { NispsOscClient } from './osc-client';
export { PassthroughBackend } from './passthrough-backend'; export { PassthroughBackend } from './passthrough-backend';
export { ParticleBackend } from './particle-backend'; export { ParticleBackend } from './particle-backend';

View file

@ -22,12 +22,19 @@ import { WebMidiBackend } from './midi-backend';
import { OscBridgeBackend } from './osc-backend'; import { OscBridgeBackend } from './osc-backend';
import { PassthroughBackend } from './passthrough-backend'; import { PassthroughBackend } from './passthrough-backend';
import { ParticleBackend } from './particle-backend'; import { ParticleBackend } from './particle-backend';
import { VcvBackend, type VcvFeedbackOp } from './vcv-backend';
/** The slice of EngineApi the manager depends on (keeps it decoupled/testable). */ /** The slice of EngineApi the manager depends on (keeps it decoupled/testable). */
export interface ManagerEngine { export interface ManagerEngine {
subscribe(cb: () => void): () => void; subscribe(cb: () => void): () => void;
routedOutput(): Float32Array | null; routedOutput(): Float32Array | null;
audio: { setMuted(muted: boolean): void }; audio: { setMuted(muted: boolean): void };
/**
* Current control input vector (2-D for the fixed 2N MLP). Optional when
* present the VCV backend streams it to the module so the browser drives the
* module's inputs in bridged mode.
*/
inputVector?(): ReadonlyArray<number>;
} }
export class BackendManager { export class BackendManager {
@ -50,12 +57,17 @@ export class BackendManager {
['synth', backends?.synth ?? new PassthroughBackend('synth', 'Built-in Synth — audio plays in the engine')], ['synth', backends?.synth ?? new PassthroughBackend('synth', 'Built-in Synth — audio plays in the engine')],
['particles', backends?.particles ?? new ParticleBackend()], ['particles', backends?.particles ?? new ParticleBackend()],
['cvgate', backends?.cvgate ?? new PassthroughBackend('cvgate', 'CV / gate (via VCV bridge)')], ['cvgate', backends?.cvgate ?? new PassthroughBackend('cvgate', 'CV / gate (via VCV bridge)')],
['vcv', backends?.vcv ?? new PassthroughBackend('vcv', 'VCV bridge')], ['vcv', backends?.vcv ?? new VcvBackend()],
]); ]);
// Single subscription to the spine: forward routed → active backend. // Single subscription to the spine: forward routed → active backend. For the
// VCV backend we also stream the current input vector each tick so the
// browser drives the module's inputs in bridged mode.
this.unsub = this.engine.subscribe(() => { this.unsub = this.engine.subscribe(() => {
if (!this.active) return; if (!this.active) return;
if (this.active instanceof VcvBackend && this.engine.inputVector) {
this.active.setInputVector(this.engine.inputVector());
}
const routed = this.engine.routedOutput(); const routed = this.engine.routedOutput();
if (routed) this.active.send(routed); if (routed) this.active.send(routed);
}); });
@ -72,6 +84,25 @@ export class BackendManager {
return b instanceof OscBridgeBackend ? b : null; return b instanceof OscBridgeBackend ? b : null;
} }
vcv(): VcvBackend | null {
const b = this.backends.get('vcv');
return b instanceof VcvBackend ? b : null;
}
/**
* Forward a verdict op to the VCV module's embedded learner over the bridge.
* No-op unless the VCV backend is the ACTIVE one the verdict loop only
* trains the module when Mode = VCV (otherwise the browser engine is the
* learner). Returns true if it was forwarded.
*/
forwardFeedback(op: VcvFeedbackOp): boolean {
if (this.activeId !== 'vcv') return false;
const vcv = this.vcv();
if (!vcv) return false;
vcv.sendFeedback(op);
return true;
}
get(id: BackendId): OutputBackend | undefined { get(id: BackendId): OutputBackend | undefined {
return this.backends.get(id); return this.backends.get(id);
} }

View file

@ -123,6 +123,24 @@ export class NispsOscClient {
this.send({ type: 'weights', payload }); this.send({ type: 'weights', payload });
} }
/**
* Send the current input VECTOR as ONE multi-float OSC message to
* `<prefix>/input` (the bridge `input` verb). Used in VCV bridged mode so the
* browser drives the module's inputs. Distinct from `sendParams` (which emits
* one single-float message per entry).
*/
sendInput(values: ReadonlyArray<number>): void {
this.send({ type: 'input', payload: Array.from(values) });
}
/**
* Send a verdict op as a JSON string to `<prefix>/feedback` (the bridge
* `feedback` verb). Trains the VCV module's embedded learner over the bridge.
*/
sendFeedback(op: object): void {
this.send({ type: 'feedback', payload: op });
}
// ── Receive ──────────────────────────────────────────────────────── // ── Receive ────────────────────────────────────────────────────────
onOutputsReceived(cb: (v: number[]) => void): () => void { onOutputsReceived(cb: (v: number[]) => void): () => void {
this.outputsCbs.push(cb); this.outputsCbs.push(cb);

View file

@ -18,8 +18,8 @@ import { useEffect, useMemo, useRef, useState } from 'react';
import type { EngineApi } from '../engine'; import type { EngineApi } from '../engine';
import type { MFParam } from '../console/model'; import type { MFParam } from '../console/model';
import type { BackendContext, BackendStatus, OutputMapping } from './backend'; import type { BackendContext, BackendStatus, OutputMapping } from './backend';
import type { BackendId, MidiCcSpec, OscSpec } from '../dock/output-state'; import type { BackendId, MidiCcSpec, OscSpec, VcvSpec } from '../dock/output-state';
import { defaultMidiSpec, defaultOscSpec } from '../dock/output-state'; import { defaultMidiSpec, defaultOscSpec, defaultVcvSpec } from '../dock/output-state';
import { BackendManager } from './manager'; import { BackendManager } from './manager';
export interface MidiSettings { export interface MidiSettings {
@ -32,6 +32,13 @@ export interface OscSettings {
sendRaw: boolean; sendRaw: boolean;
} }
export interface VcvSettings {
/** Bridge WebSocket URL (the Deno bridge relays to the module over UDP). */
url: string;
/** Send raw 0..1 instead of the per-output bipolar/unipolar voltage range. */
sendRaw: boolean;
}
function toMapping(p: MFParam): OutputMapping { function toMapping(p: MFParam): OutputMapping {
return { return {
state: p.status, state: p.status,
@ -58,6 +65,7 @@ export function useBackendManager(
params: MFParam[], params: MFParam[],
midiSettings: MidiSettings, midiSettings: MidiSettings,
oscSettings: OscSettings, oscSettings: OscSettings,
vcvSettings: VcvSettings,
): UseBackendManager { ): UseBackendManager {
const managerRef = useRef<BackendManager | null>(null); const managerRef = useRef<BackendManager | null>(null);
const [status, setStatus] = useState<BackendStatus>({ state: 'idle', message: 'idle' }); const [status, setStatus] = useState<BackendStatus>({ state: 'idle', message: 'idle' });
@ -132,5 +140,13 @@ export function useBackendManager(
osc.setOscConfig(specs, { url: oscSettings.url, sendRaw: oscSettings.sendRaw }); osc.setOscConfig(specs, { url: oscSettings.url, sendRaw: oscSettings.sendRaw });
}, [manager, params, oscSettings.url, oscSettings.sendRaw]); }, [manager, params, oscSettings.url, oscSettings.sendRaw]);
// Push per-output VCV config (polarity) + bridge URL/raw whenever they change.
useEffect(() => {
const vcv = manager?.vcv();
if (!vcv) return;
const specs: VcvSpec[] = params.map((p) => p.vcv ?? defaultVcvSpec());
vcv.setVcvConfig(specs, { url: vcvSettings.url, sendRaw: vcvSettings.sendRaw });
}, [manager, params, vcvSettings.url, vcvSettings.sendRaw]);
return { manager, status, midiPorts, refreshMidiPorts }; return { manager, status, midiPorts, refreshMidiPorts };
} }

View file

@ -0,0 +1,265 @@
/**
* VcvBackend drives + trains the VCV Rack NISPS module over the OSCWS bridge
* (backends-spec §2.6; vcv/SPEC.md OSC verbs). In "bridged" mode the BROWSER is
* authoritative: it streams the current input vector to the module and forwards
* the verdict loop (thumbs up/down, explore-and-place) so the module's embedded
* net trains in lock-step with the browser session.
*
* Transport (reuses {@link NispsOscClient} the Deno bridge in
* manifold/osc-bridge, default ws://localhost:8765, default module UDP 7001):
*
* browser module
* /nisps/input <ff> the current 2-D input vector (drives the module)
* /nisps/output <ff> per-output values (CV) sent as params batch so
* the bridge maps each to /nisps/<name>; the module
* also derives its own outputs, but the browser
* value is authoritative in bridged mode
* /nisps/feedback {op,} verdict op (up | down | rand | clear) as a JSON
* string { op, spread, input[], output[] }
*
* module browser
* /nisps/output <ff> module's live outputs (status / visualisation)
* /nisps/input <ff> module's live inputs (echo / status)
* /nisps/state <json> module status snapshot (surfaced as a message)
*
* The bridge process AND the VCV module must both be running until the WS
* connects we surface "bridge not running"; until the module replies we stay
* "connected, waiting for module".
*
* British spelling in product copy; the synth is the "Built-in Synth", never
* "C15".
*/
import type { BackendContext, BackendStatus, OutputBackend } from './backend';
import { isSilent, mapOutput } from './mapping';
import { NispsOscClient } from './osc-client';
import type { VcvSpec } from '../dock/output-state';
const SEND_INTERVAL_MS = 50;
const DEAD_ZONE = 0.002; // on the normalised value, pre physical-scale
/** A verdict op forwarded to the module's embedded learner. */
export interface VcvFeedbackOp {
op: 'up' | 'down' | 'rand' | 'clear';
/** Master spread (0..1) — mirrors the engine spread knob. */
spread: number;
/** The control input the verdict was given at (2-D). */
input: number[];
/** The heard output vector at that input (≤126 dims). */
output: number[];
}
export interface VcvBackendConfig {
/** Bridge WebSocket URL (the bridge then relays to the module over UDP). */
url: string;
/** Send raw normalised 0..1 instead of the per-output bipolar/unipolar range. */
sendRaw: boolean;
}
export class VcvBackend implements OutputBackend {
readonly id = 'vcv' as const;
private client = new NispsOscClient();
private ctx: BackendContext | null = null;
private specs: VcvSpec[] = [];
private sendRaw = false;
/** Latest input vector the browser is driving the module with (2-D). */
private inputVec: number[] = [0.5, 0.5];
private lastSent: Float32Array = new Float32Array(0); // last normalised output
private batch: Array<[string, number]> = []; // reused outer; entries reused
private lastSendMs = 0;
private lastInputSent: [number, number] = [-1, -1];
/** Latest module-reported outputs (for visualisation), null until first echo. */
private moduleOutputs: number[] | null = null;
private gotModuleReply = false;
private statusState: BackendStatus = { state: 'idle', message: 'VCV idle' };
private statusListeners = new Set<(s: BackendStatus) => void>();
private outputListeners = new Set<(v: number[]) => void>();
private offConn: (() => void) | null = null;
private offInfo: (() => void) | null = null;
private offOutputs: (() => void) | null = null;
private offInputs: (() => void) | null = null;
isAvailable(): boolean {
return typeof WebSocket !== 'undefined';
}
async start(ctx: BackendContext): Promise<void> {
this.ctx = ctx;
this.lastSent = new Float32Array(ctx.outputCount).fill(-1);
if (!this.isAvailable()) {
this.setStatus({ state: 'unavailable', message: 'WebSocket not available' });
return;
}
this.offConn = this.client.onConnectionChange((connected) => {
if (!connected) {
this.gotModuleReply = false;
this.setStatus({ state: 'error', message: `VCV bridge not running — start it (${this.client.url})` });
return;
}
this.setStatus({
state: this.gotModuleReply ? 'ready' : 'connecting',
message: this.gotModuleReply
? `VCV module connected (${this.client.url})`
: `Bridge connected (${this.client.url}) — waiting for module…`,
});
});
this.offInfo = this.client.onInfo((m) => {
if (this.client.connected && !this.gotModuleReply) {
this.setStatus({ state: 'connecting', message: m });
}
});
// Module → browser: a reply on either channel proves the module is alive.
this.offOutputs = this.client.onOutputsReceived((v) => this.onModuleReply(v, true));
this.offInputs = this.client.onInputsReceived((v) => this.onModuleReply(v, false));
this.setStatus({ state: 'connecting', message: `Connecting to VCV bridge (${this.client.url})…` });
this.client.connect({ reconnect: true }).catch(() => {
this.setStatus({ state: 'error', message: `VCV bridge not running — start it (${this.client.url})` });
});
}
setContext(ctx: BackendContext): void {
this.ctx = ctx;
if (this.lastSent.length !== ctx.outputCount) {
this.lastSent = new Float32Array(ctx.outputCount).fill(-1);
}
}
/** Update per-output VCV specs (polarity) + bridge URL/raw toggle. */
setVcvConfig(specs: VcvSpec[], cfg: VcvBackendConfig): void {
this.specs = specs;
this.sendRaw = cfg.sendRaw;
if (cfg.url !== this.client.url) {
this.client.setUrl(cfg.url);
this.gotModuleReply = false;
if (this.isAvailable()) {
this.setStatus({ state: 'connecting', message: `Connecting to VCV bridge (${cfg.url})…` });
this.client.connect({ reconnect: true }).catch(() => {
this.setStatus({ state: 'error', message: `VCV bridge not running — start it (${cfg.url})` });
});
}
}
this.lastSent.fill(-1);
}
/**
* Set the input vector the browser drives the module with (bridged mode). The
* next `send()` streams it to /nisps/input. Copied caller may mutate.
*/
setInputVector(vec: ReadonlyArray<number>): void {
if (this.inputVec.length !== vec.length) this.inputVec = new Array(vec.length);
for (let i = 0; i < vec.length; i++) this.inputVec[i] = vec[i];
}
/**
* Forward a verdict op to the module's embedded learner over /nisps/feedback.
* Hooked from the BackendManager when the verdict loop fires in VCV mode, so
* thumbs-up/down + explore-and-place train the module across the bridge.
*/
sendFeedback(op: VcvFeedbackOp): void {
if (!this.client.connected) return;
// → bridge `feedback` verb → OSC string to /nisps/feedback.
this.client.sendFeedback(op);
}
send(routed: Float32Array): void {
const ctx = this.ctx;
if (!ctx || !this.client.connected) return;
const now = typeof performance !== 'undefined' ? performance.now() : Date.now();
if (now - this.lastSendMs < SEND_INTERVAL_MS) return;
this.lastSendMs = now;
// 1) Stream the current input vector so the browser drives the module.
const ix = this.inputVec[0] ?? 0.5;
const iy = this.inputVec[1] ?? 0.5;
if (Math.abs(ix - this.lastInputSent[0]) >= DEAD_ZONE || Math.abs(iy - this.lastInputSent[1]) >= DEAD_ZONE) {
this.lastInputSent[0] = ix;
this.lastInputSent[1] = iy;
// → bridge `input` verb → ONE multi-float message to /nisps/input.
this.client.sendInput(this.inputVec);
}
// 2) Stream the routed per-output values (authoritative CV in bridged mode).
const n = Math.min(routed.length, ctx.mappings.length);
this.batch.length = 0;
for (let i = 0; i < n; i++) {
const m = ctx.mappings[i];
if (isSilent(m)) continue;
const mapped = mapOutput(routed[i], m); // 0..1 in [min,max]
const prev = this.lastSent[i];
if (prev >= 0 && Math.abs(mapped - prev) < DEAD_ZONE) continue;
this.lastSent[i] = mapped;
const spec = this.specs[i];
const value = this.sendRaw ? mapped : this.toVoltage(mapped, spec);
const name = ctx.names[i] ? sanitise(ctx.names[i]) : `out${i}`;
this.batch.push([name, value]);
}
if (this.batch.length) this.client.sendParams(this.batch);
}
/** Map a 0..1 value into the per-output VCV voltage range (uni/bipolar). */
private toVoltage(v: number, spec: VcvSpec | undefined): number {
// Unipolar 0..10 V; bipolar ±5 V (dock-spec §4.3 polarity).
return spec?.bipolar ? v * 10 - 5 : v * 10;
}
/** Latest module-reported output vector for visualisation (may be null). */
moduleStatusOutputs(): number[] | null {
return this.moduleOutputs;
}
/** Subscribe to module-reported outputs (visualisation feed). */
onModuleOutputs(cb: (v: number[]) => void): () => void {
this.outputListeners.add(cb);
return () => this.outputListeners.delete(cb);
}
private onModuleReply(v: number[], isOutput: boolean): void {
if (!this.gotModuleReply) {
this.gotModuleReply = true;
this.setStatus({ state: 'ready', message: `VCV module connected (${this.client.url})` });
}
if (isOutput) {
this.moduleOutputs = v;
for (const cb of this.outputListeners) cb(v);
}
}
async teardown(): Promise<void> {
this.offConn?.();
this.offInfo?.();
this.offOutputs?.();
this.offInputs?.();
this.offConn = null;
this.offInfo = null;
this.offOutputs = null;
this.offInputs = null;
this.client.disconnect();
this.gotModuleReply = false;
this.setStatus({ state: 'idle', message: 'VCV idle' });
}
status(): BackendStatus {
return this.statusState;
}
onStatusChange(cb: (s: BackendStatus) => void): () => void {
this.statusListeners.add(cb);
return () => this.statusListeners.delete(cb);
}
private setStatus(s: BackendStatus): void {
this.statusState = s;
for (const cb of this.statusListeners) cb(s);
}
}
/** Sanitise an output name into an OSC-path-safe token (the bridge prefixes it). */
function sanitise(name: string): string {
return name.toLowerCase().replace(/[^a-z0-9_]+/g, '_').replace(/^_+|_+$/g, '') || 'out';
}

View file

@ -131,6 +131,10 @@ export function ConsoleApp({ focus: initialFocus = 'composite' }: ConsoleAppProp
const [midiCcCount, setMidiCcCount] = useState(8); const [midiCcCount, setMidiCcCount] = useState(8);
const [oscUrl, setOscUrl] = useState('ws://localhost:8765'); const [oscUrl, setOscUrl] = useState('ws://localhost:8765');
const [oscSendRaw, setOscSendRaw] = useState(false); const [oscSendRaw, setOscSendRaw] = useState(false);
// VCV bridge: WS URL of the Deno bridge that relays to the VCV module over UDP
// (default module UDP 7001). Independent of the OSC backend's bridge.
const [vcvUrl, setVcvUrl] = useState('ws://localhost:8765');
const [vcvSendRaw, setVcvSendRaw] = useState(false);
// Feedback markers plotted on the 2D map (both polarities; session-scoped). // Feedback markers plotted on the 2D map (both polarities; session-scoped).
const [markers, setMarkers] = useState<FeedbackMarker[]>([]); const [markers, setMarkers] = useState<FeedbackMarker[]>([]);
const [volume, setVolume] = useState(0.8); const [volume, setVolume] = useState(0.8);
@ -264,15 +268,28 @@ export function ConsoleApp({ focus: initialFocus = 'composite' }: ConsoleAppProp
// spine and forwards routed outputs to the active backend; switching Mode // spine and forwards routed outputs to the active backend; switching Mode
// tears down the old backend, starts the new one, and gates synth audio // tears down the old backend, starts the new one, and gates synth audio
// (mute on non-synth modes). MIDI/OSC config + names ride the shared params. // (mute on non-synth modes). MIDI/OSC config + names ride the shared params.
const { status: backendStatus, midiPorts, refreshMidiPorts } = useBackendManager( const { manager: backendManager, status: backendStatus, midiPorts, refreshMidiPorts } = useBackendManager(
engine, engine,
outputBackend, outputBackend,
modeId, modeId,
params, params,
{ outputId: midiOutputId, ccCount: midiCcCount }, { outputId: midiOutputId, ccCount: midiCcCount },
{ url: oscUrl, sendRaw: oscSendRaw }, { url: oscUrl, sendRaw: oscSendRaw },
{ url: vcvUrl, sendRaw: vcvSendRaw },
); );
// VCV bridge: forward a verdict op to the module's embedded learner. No-op
// unless Mode = VCV (the manager gates on the active backend). This is how
// thumbs-up/down + explore-and-place TRAIN the module across the bridge.
const forwardVcvFeedback = (op: 'up' | 'down' | 'rand' | 'clear') => {
backendManager?.forwardFeedback({
op,
spread: spread ? 1 : 0.6,
input: [pos[0], pos[1]],
output: Array.from(engine?.getOutputs() ?? new Float32Array(0)),
});
};
// values come from the REAL engine output, shaped per-param. Recomputed when // values come from the REAL engine output, shaped per-param. Recomputed when
// the engine version bumps (new inference / weights) or params change. // the engine version bumps (new inference / weights) or params change.
const values = useMemo( const values = useMemo(
@ -326,6 +343,8 @@ export function ConsoleApp({ focus: initialFocus = 'composite' }: ConsoleAppProp
c?.like(pos, engine?.getOutputs() ?? new Float32Array(0)); c?.like(pos, engine?.getOutputs() ?? new Float32Array(0));
pushMarker(pos, 'positive'); pushMarker(pos, 'positive');
} }
// VCV bridged mode: also train the module — thumbs-up = positive verdict.
forwardVcvFeedback('up');
syncController(); syncController();
setNoiseCap((n) => Math.max(0.02, n * 0.7)); setNoiseCap((n) => Math.max(0.02, n * 0.7));
setHealth((h) => Math.min(1, h + 0.08)); setHealth((h) => Math.min(1, h + 0.08));
@ -350,12 +369,16 @@ export function ConsoleApp({ focus: initialFocus = 'composite' }: ConsoleAppProp
} else { } else {
pushSnap('explore'); pushSnap('explore');
c?.enterExplore(); c?.enterExplore();
// VCV bridged mode: entering explore re-rolls the module's net too.
forwardVcvFeedback('rand');
} }
} else { } else {
// Geometric dislike: push the current mapping away from this sound. // Geometric dislike: push the current mapping away from this sound.
pushSnap('dislike '); pushSnap('dislike ');
c?.dislike(pos, engine?.getOutputs() ?? new Float32Array(0), noiseCap, spread ? 1 : 0.6); c?.dislike(pos, engine?.getOutputs() ?? new Float32Array(0), noiseCap, spread ? 1 : 0.6);
pushMarker(pos, 'negative'); pushMarker(pos, 'negative');
// VCV bridged mode: thumbs-down = negative verdict.
forwardVcvFeedback('down');
setSeed((s) => s + (Math.random() - 0.5) * (noiseCap * 4 + 0.3)); setSeed((s) => s + (Math.random() - 0.5) * (noiseCap * 4 + 0.3));
setNoiseCap((n) => Math.min(0.5, n + 0.06)); setNoiseCap((n) => Math.min(0.5, n + 0.06));
setHealth((h) => Math.max(0.1, h - 0.06)); setHealth((h) => Math.max(0.1, h - 0.06));
@ -376,6 +399,8 @@ export function ConsoleApp({ focus: initialFocus = 'composite' }: ConsoleAppProp
// Outside a scratchpad session a re-roll randomises the real net directly. // Outside a scratchpad session a re-roll randomises the real net directly.
engine?.randomise(spread ? 1 : 0.6); engine?.randomise(spread ? 1 : 0.6);
} }
// VCV bridged mode: re-roll the module's net too.
forwardVcvFeedback('rand');
syncController(); syncController();
setSeed(Math.random() * 6); setSeed(Math.random() * 6);
setNoiseCap(0.4); setNoiseCap(0.4);
@ -586,6 +611,10 @@ export function ConsoleApp({ focus: initialFocus = 'composite' }: ConsoleAppProp
setOscUrl, setOscUrl,
oscSendRaw, oscSendRaw,
setOscSendRaw, setOscSendRaw,
vcvUrl,
setVcvUrl,
vcvSendRaw,
setVcvSendRaw,
setParams: (next: MFParam[]) => setParams(next), setParams: (next: MFParam[]) => setParams(next),
markers, markers,
inputs, inputs,

View file

@ -100,6 +100,12 @@ export interface ConsoleCtx {
setOscUrl: (u: string) => void; setOscUrl: (u: string) => void;
oscSendRaw: boolean; oscSendRaw: boolean;
setOscSendRaw: (v: boolean) => void; setOscSendRaw: (v: boolean) => void;
/** VCV backend settings (bridge URL + send-raw toggle). The Deno bridge
* relays to the VCV module over UDP (default module port 7001). */
vcvUrl: string;
setVcvUrl: (u: string) => void;
vcvSendRaw: boolean;
setVcvSendRaw: (v: boolean) => void;
/** Replace the whole params array (used when restoring a named preset). */ /** Replace the whole params array (used when restoring a named preset). */
setParams: (next: MFParam[]) => void; setParams: (next: MFParam[]) => void;

View file

@ -6,7 +6,9 @@
* namespace) on top, then a per-backend config section: * namespace) on top, then a per-backend config section:
* - MIDI output-port picker, number-of-CCs, per-output CC#/channel/name. * - MIDI output-port picker, number-of-CCs, per-output CC#/channel/name.
* - OSC bridge URL + connect status + send-raw toggle, per-output path/range. * - OSC bridge URL + connect status + send-raw toggle, per-output path/range.
* - VCV/CV per-output polarity (delegates to the existing BackendAdvanced body). * - VCV/CV bridge URL + connect status + send-raw toggle, per-output polarity
* (uni 010 V / bipolar ±5 V). The browser drives + trains the VCV
* module over the same Deno bridge.
* - Synth/Particle/Editor handled by ModeConfig in Drawers (no extra config here). * - Synth/Particle/Editor handled by ModeConfig in Drawers (no extra config here).
* *
* Everything is editable inline; writes go through the shared MFParam store * Everything is editable inline; writes go through the shared MFParam store
@ -16,7 +18,7 @@
import { useEffect, useState } from 'react'; import { useEffect, useState } from 'react';
import type { ConsoleCtx } from '../console/types'; import type { ConsoleCtx } from '../console/types';
import type { BackendId } from './output-state'; import type { BackendId } from './output-state';
import { defaultMidiSpec, defaultOscSpec } from './output-state'; import { defaultMidiSpec, defaultOscSpec, defaultVcvSpec } from './output-state';
import { import {
applyPreset, applyPreset,
deletePreset, deletePreset,
@ -107,6 +109,7 @@ function PresetBar({ ctx, backend }: { ctx: ConsoleCtx; backend: BackendId }) {
const backendSettings = (): Record<string, unknown> => { const backendSettings = (): Record<string, unknown> => {
if (backend === 'midi') return { outputId: ctx.midiOutputId, ccCount: ctx.midiCcCount }; if (backend === 'midi') return { outputId: ctx.midiOutputId, ccCount: ctx.midiCcCount };
if (backend === 'osc') return { url: ctx.oscUrl, sendRaw: ctx.oscSendRaw }; if (backend === 'osc') return { url: ctx.oscUrl, sendRaw: ctx.oscSendRaw };
if (backend === 'vcv') return { url: ctx.vcvUrl, sendRaw: ctx.vcvSendRaw };
return {}; return {};
}; };
@ -118,6 +121,9 @@ function PresetBar({ ctx, backend }: { ctx: ConsoleCtx; backend: BackendId }) {
} else if (backend === 'osc') { } else if (backend === 'osc') {
if ('url' in s) ctx.setOscUrl(String(s.url)); if ('url' in s) ctx.setOscUrl(String(s.url));
if ('sendRaw' in s) ctx.setOscSendRaw(Boolean(s.sendRaw)); if ('sendRaw' in s) ctx.setOscSendRaw(Boolean(s.sendRaw));
} else if (backend === 'vcv') {
if ('url' in s) ctx.setVcvUrl(String(s.url));
if ('sendRaw' in s) ctx.setVcvSendRaw(Boolean(s.sendRaw));
} }
}; };
@ -381,6 +387,76 @@ function OscConfig({ ctx }: { ctx: ConsoleCtx }) {
); );
} }
// ---- VCV / CV config (backends-spec §2.6 / §4.3) ---------------------------
function VcvConfig({ ctx }: { ctx: ConsoleCtx }) {
const s = ctx.backendStatus;
const statusColor = s.state === 'ready' ? 'var(--good)' : s.state === 'connecting' ? 'var(--warn)' : 'var(--danger)';
const [draftUrl, setDraftUrl] = useState(ctx.vcvUrl);
useEffect(() => setDraftUrl(ctx.vcvUrl), [ctx.vcvUrl]);
return (
<>
<SectionLabel>VCV bridge</SectionLabel>
<div style={{ display: 'flex', gap: 8, alignItems: 'center', flexWrap: 'wrap' }}>
<input
style={{ ...cellInput, width: 200 }}
value={draftUrl}
onChange={(e) => setDraftUrl(e.target.value)}
onBlur={() => ctx.setVcvUrl(draftUrl)}
placeholder="ws://localhost:8765"
/>
<button type="button" style={btn('var(--accent)')} onClick={() => ctx.setVcvUrl(draftUrl)}>
Connect
</button>
<label style={{ display: 'flex', gap: 6, alignItems: 'center', fontSize: 'var(--fs-xs)', color: 'var(--fg-dim)' }}>
<input type="checkbox" checked={ctx.vcvSendRaw} onChange={(e) => ctx.setVcvSendRaw(e.target.checked)} />
send raw 0..1
</label>
<span style={{ fontSize: 9, color: statusColor }}>{s.message}</span>
</div>
<p style={{ fontSize: 9, color: 'var(--fg-dim)', margin: 0, lineHeight: 1.6 }}>
The VCV Rack NISPS module AND the Deno OSC bridge (manifold/osc-bridge) must both be running. The browser drives
the module's inputs and forwards the verdict loop (thumbs up/down, explore-and-place) over the bridge, so the
module's embedded net trains in lock-step. Default module UDP port 7001.
</p>
<SectionLabel>Per-output polarity</SectionLabel>
<div style={{ maxHeight: 320, overflow: 'auto' }}>
<table style={{ width: '100%', borderCollapse: 'collapse' }}>
<thead>
<tr>
<Th>output</Th>
<Th>range</Th>
</tr>
</thead>
<tbody>
{ctx.params.map((p, i) => {
const v = p.vcv ?? defaultVcvSpec();
return (
<tr key={i}>
<td style={{ padding: '3px 6px', fontSize: 'var(--fs-xs)', color: 'var(--fg-dim)' }}>{p.name}</td>
<td style={{ padding: '3px 6px', width: 110 }}>
<button
type="button"
onClick={() => ctx.setParam(i, { vcv: { bipolar: !v.bipolar } })}
style={{
...btn(v.bipolar ? 'var(--danger)' : 'var(--fg-mute)'),
width: '100%',
}}
>
{v.bipolar ? '±5 V' : '010 V'}
</button>
</td>
</tr>
);
})}
</tbody>
</table>
</div>
</>
);
}
// ---- Public entry ---------------------------------------------------------- // ---- Public entry ----------------------------------------------------------
export interface OutputsBackendConfigProps { export interface OutputsBackendConfigProps {
@ -390,14 +466,14 @@ export interface OutputsBackendConfigProps {
/** The specialised, editable per-backend config + preset bar for the Outputs panel. */ /** The specialised, editable per-backend config + preset bar for the Outputs panel. */
export function OutputsBackendConfig({ ctx, backend }: OutputsBackendConfigProps) { export function OutputsBackendConfig({ ctx, backend }: OutputsBackendConfigProps) {
// Only MIDI / OSC carry a config + preset surface here; synth/particle/editor // MIDI / OSC / VCV carry a config + preset surface here; synth/particle/editor
// config is rendered by ModeConfig in Drawers. VCV/CV polarity stays in the // config is rendered by ModeConfig in Drawers. The full-depth BackendAdvanced
// full-depth BackendAdvanced modal. // modal reuses the same per-channel sections.
if (backend !== 'midi' && backend !== 'osc') return null; if (backend !== 'midi' && backend !== 'osc' && backend !== 'vcv') return null;
return ( return (
<div style={{ display: 'flex', flexDirection: 'column', gap: 6 }}> <div style={{ display: 'flex', flexDirection: 'column', gap: 6 }}>
<PresetBar ctx={ctx} backend={backend} /> <PresetBar ctx={ctx} backend={backend} />
{backend === 'midi' ? <MidiConfig ctx={ctx} /> : <OscConfig ctx={ctx} />} {backend === 'midi' ? <MidiConfig ctx={ctx} /> : backend === 'osc' ? <OscConfig ctx={ctx} /> : <VcvConfig ctx={ctx} />}
</div> </div>
); );
} }

View file

@ -130,3 +130,8 @@ export function defaultMidiSpec(index: number): MidiCcSpec {
export function defaultOscSpec(name: string): OscSpec { export function defaultOscSpec(name: string): OscSpec {
return { path: `/nisps/${name.toLowerCase()}`, rangeMin: 0, rangeMax: 1 }; return { path: `/nisps/${name.toLowerCase()}`, rangeMin: 0, rangeMax: 1 };
} }
/** Default VCV spec for an output — unipolar 010 V by default. */
export function defaultVcvSpec(): VcvSpec {
return { bipolar: false };
}

View file

@ -179,6 +179,14 @@ export class EngineApi {
return this.spine.routedOutput(); return this.spine.routedOutput();
} }
/**
* Current control input vector (2-D for the fixed 2N MLP). Used by the VCV
* backend (via BackendManager) to drive the module's inputs over the bridge.
*/
inputVector(): ReadonlyArray<number> {
return [this.spine.lastRawX, this.spine.lastRawY];
}
/** /**
* Re-run the LAST raw input through the spine used after a weight change * Re-run the LAST raw input through the spine used after a weight change
* (train / randomise / feedback) so outputs + audio reflect the new MLP * (train / randomise / feedback) so outputs + audio reflect the new MLP