From 7d36d3d18de64ec371601d5479ede95ee8d1e6ed Mon Sep 17 00:00:00 2001 From: monkey-w1n5t0n Date: Sun, 28 Jun 2026 04:28:45 +0200 Subject: [PATCH] feat(osc-bridge): add /nisps/input (multi-float) + /nisps/feedback (JSON) verbs; client sendInput/sendFeedback; VcvBackend uses them; document VCV runtime --- manifold/osc-bridge/bridge.mjs | 25 ++++++++++++++++++++++ manifold/osc-bridge/bridge.ts | 29 ++++++++++++++++++++++++- manifold/src/backends/README.md | 32 +++++++++++++++++++++++++++- manifold/src/backends/osc-client.ts | 18 ++++++++++++++++ manifold/src/backends/vcv-backend.ts | 17 ++++++--------- 5 files changed, 108 insertions(+), 13 deletions(-) diff --git a/manifold/osc-bridge/bridge.mjs b/manifold/osc-bridge/bridge.mjs index 2439f8e..a87ee0f 100644 --- a/manifold/osc-bridge/bridge.mjs +++ b/manifold/osc-bridge/bridge.mjs @@ -63,6 +63,16 @@ function oscMessage(address, value) { ]); } +/** A single OSC message carrying N floats (one ",fff…" message, not N messages). */ +function oscMessageFloats(address, values) { + const tags = ',' + 'f'.repeat(values.length); + return Buffer.concat([ + oscString(address), + oscString(tags), + ...values.map((v) => oscFloat(v)), + ]); +} + function oscMessageString(address, value) { return Buffer.concat([ oscString(address), @@ -145,6 +155,11 @@ function sendOSCString(address, value) { udpSend.send(msg, OSC_PORT, OSC_HOST); } +function sendOSCFloats(address, values) { + const msg = oscMessageFloats(address, values); + udpSend.send(msg, OSC_PORT, OSC_HOST); +} + function sendOSCBundle(params) { const messages = params.map(([name, value]) => oscMessage(`${OSC_PREFIX}/${name}`, value) @@ -218,6 +233,16 @@ wss.on('connection', (ws) => { case 'weights': sendOSCString(`${OSC_PREFIX}/weights`, JSON.stringify(data.payload)); return; + case 'input': + // Current input VECTOR as ONE multi-float message to /nisps/input. + if (Array.isArray(data.payload)) { + sendOSCFloats(`${OSC_PREFIX}/input`, data.payload); + } + return; + case 'feedback': + // Verdict op as OSC string to /nisps/feedback (trains the module). + sendOSCString(`${OSC_PREFIX}/feedback`, JSON.stringify(data.payload)); + return; case 'params': if (Array.isArray(data.payload)) { if (USE_BUNDLES) { diff --git a/manifold/osc-bridge/bridge.ts b/manifold/osc-bridge/bridge.ts index a7cf565..0d9e926 100644 --- a/manifold/osc-bridge/bridge.ts +++ b/manifold/osc-bridge/bridge.ts @@ -22,8 +22,11 @@ // /nisps/ (webapp -> target) // /nisps/state (webapp -> target: full JSON state) // /nisps/weights (webapp -> target: weights JSON) +// /nisps/input (webapp -> target: input vector, ONE message; +// also target -> webapp for visualisation) +// /nisps/feedback (webapp -> target: verdict op JSON — +// { op, spread, input[], output[] }) // /nisps/output (target -> webapp: output float array) -// /nisps/input (target -> webapp: input float array) import { parseArgs } from "jsr:@std/cli@1/parse-args"; @@ -104,6 +107,13 @@ function oscMessage(address: string, value: number): Uint8Array { return concat(oscString(address), oscString(",f"), oscFloat(value)); } +/** A single OSC message carrying N floats (one ",fff…" message, not N messages). */ +function oscMessageFloats(address: string, values: number[]): Uint8Array { + const tags = "," + "f".repeat(values.length); + const floatBufs = values.map((v) => oscFloat(v)); + return concat(oscString(address), oscString(tags), ...floatBufs); +} + function oscMessageString(address: string, value: string): Uint8Array { return concat(oscString(address), oscString(",s"), oscString(value)); } @@ -197,6 +207,11 @@ function sendOSCString(address: string, value: string): void { udpSend.send(msg, oscAddr); } +function sendOSCFloats(address: string, values: number[]): void { + const msg = oscMessageFloats(address, values); + udpSend.send(msg, oscAddr); +} + function sendOSCBundle(params: [string, number][]): void { const messages = params.map(([name, value]) => oscMessage(`${OSC_PREFIX}/${name}`, value) @@ -249,6 +264,18 @@ function handleWs(ws: WebSocket): void { // Send weights JSON as OSC string to /nisps/weights sendOSCString(`${OSC_PREFIX}/weights`, JSON.stringify(data.payload)); return; + case "input": + // Current input VECTOR as ONE multi-float message to /nisps/input + // (browser drives the VCV module's inputs in bridged mode). + if (Array.isArray(data.payload)) { + sendOSCFloats(`${OSC_PREFIX}/input`, data.payload as number[]); + } + return; + case "feedback": + // Verdict op as OSC string to /nisps/feedback (trains the module's + // embedded learner: { op, spread, input[], output[] }). + sendOSCString(`${OSC_PREFIX}/feedback`, JSON.stringify(data.payload)); + return; case "params": // Legacy batch format embedded in structured message if (Array.isArray(data.payload)) { diff --git a/manifold/src/backends/README.md b/manifold/src/backends/README.md index 48af9c5..961c9b2 100644 --- a/manifold/src/backends/README.md +++ b/manifold/src/backends/README.md @@ -1,7 +1,7 @@ # Output Backends (`manifold/src/backends/`) Real output transports for the Manifold app. Exactly one backend is *active* at a -time, chosen by the dock **Mode** (Particle / MIDI / OSC / Built-in Synth / +time, chosen by the dock **Mode** (Particle / MIDI / OSC / VCV / Built-in Synth / Editor → `BackendId`). The `BackendManager` consumes the engine spine and forwards each routed output vector to the active backend's `send()`. @@ -17,6 +17,7 @@ See `docs/redesign/backends-spec.md` for the authoritative design. | `midi-backend.ts` | `WebMidiBackend` — real Web MIDI CC out (per-output CC#/channel/range/name, throttled + dead-zone). | | `osc-client.ts` | `NispsOscClient` — WS transport to the Deno OSC bridge (JSON protocol, auto-reconnect). | | `osc-backend.ts` | `OscBridgeBackend` — OSC out over WS; per-output address path + physical range. | +| `vcv-backend.ts` | `VcvBackend` — drives + **trains** the VCV Rack NISPS module over the OSC↔WS bridge (streams `/nisps/input`, forwards the verdict loop to `/nisps/feedback`, receives `/nisps/output` + `/nisps/state`). | | `passthrough-backend.ts` | No-op sink for synth (plays in-engine) / particles (rAF consumer) / editor. | | `presets.ts` | Named per-backend output-config presets (localStorage, per-backend namespace). | | `useBackendManager.ts` | Thin React binding: builds `BackendContext` from the store, switches Mode, surfaces status. | @@ -45,3 +46,32 @@ node bridge.mjs Default bridge URL: `ws://localhost:8765` (configurable in the OSC config panel). WS protocol (browser → bridge): `{ type:'params', payload:[[path,value],…] }`. + +## VCV bridge — module **and** bridge must both be running + +When Mode = **VCV**, the browser is authoritative in *bridged mode*: the +`VcvBackend` connects over the **same Deno bridge** and + +- streams the current input vector continuously to **`/nisps/input`** (the + browser drives the module's inputs), +- streams the routed per-output CV (uni 0–10 V / bipolar ±5 V), and +- forwards the **verdict loop** — thumbs-up/down + explore-and-place — to + **`/nisps/feedback`** (`{ op:'up'|'down'|'rand'|'clear', spread, input[], + output[] }`), so the module's embedded net trains in lock-step with the + browser session. + +It receives **`/nisps/output`** + **`/nisps/state`** back for status / +visualisation. The verdict forwarding is gated in `BackendManager.forwardFeedback` +— a no-op unless VCV is the active backend (otherwise the browser engine is the +learner). Both processes are required: + +```bash +# 1) the VCV Rack NISPS module (vcv/) — default UDP listen port 7001 +# 2) the Deno bridge, pointed at the module's UDP port: +cd manifold/osc-bridge +deno run --allow-net bridge.ts --osc-port 7001 +``` + +Default VCV bridge URL: `ws://localhost:8765` (configurable in the VCV config +panel). Until the WS connects the backend shows "bridge not running"; until the +module replies it shows "waiting for module…". diff --git a/manifold/src/backends/osc-client.ts b/manifold/src/backends/osc-client.ts index ac43f98..1a4b66d 100644 --- a/manifold/src/backends/osc-client.ts +++ b/manifold/src/backends/osc-client.ts @@ -123,6 +123,24 @@ export class NispsOscClient { this.send({ type: 'weights', payload }); } + /** + * Send the current input VECTOR as ONE multi-float OSC message to + * `/input` (the bridge `input` verb). Used in VCV bridged mode so the + * browser drives the module's inputs. Distinct from `sendParams` (which emits + * one single-float message per entry). + */ + sendInput(values: ReadonlyArray): void { + this.send({ type: 'input', payload: Array.from(values) }); + } + + /** + * Send a verdict op as a JSON string to `/feedback` (the bridge + * `feedback` verb). Trains the VCV module's embedded learner over the bridge. + */ + sendFeedback(op: object): void { + this.send({ type: 'feedback', payload: op }); + } + // ── Receive ──────────────────────────────────────────────────────── onOutputsReceived(cb: (v: number[]) => void): () => void { this.outputsCbs.push(cb); diff --git a/manifold/src/backends/vcv-backend.ts b/manifold/src/backends/vcv-backend.ts index 4a9fe5d..32030c8 100644 --- a/manifold/src/backends/vcv-backend.ts +++ b/manifold/src/backends/vcv-backend.ts @@ -14,8 +14,8 @@ * the bridge maps each to /nisps/; the module * also derives its own outputs, but the browser * value is authoritative in bridged mode - * /nisps/feedback {op,…} verdict op (up | down | rand | clear) as JSON - * state — { op, spread, input[], output[] } + * /nisps/feedback {op,…} verdict op (up | down | rand | clear) as a JSON + * string — { op, spread, input[], output[] } * * module → browser * /nisps/output module's live outputs (status / visualisation) @@ -162,10 +162,8 @@ export class VcvBackend implements OutputBackend { */ sendFeedback(op: VcvFeedbackOp): void { if (!this.client.connected) return; - // The bridge ships `{ type:'state', payload }` as an OSC string to - // /nisps/state. We reuse that string channel for /nisps/feedback by tagging - // the payload with a `feedback` envelope the module routes accordingly. - this.client.sendState({ feedback: op }); + // → bridge `feedback` verb → OSC string to /nisps/feedback. + this.client.sendFeedback(op); } send(routed: Float32Array): void { @@ -182,11 +180,8 @@ export class VcvBackend implements OutputBackend { if (Math.abs(ix - this.lastInputSent[0]) >= DEAD_ZONE || Math.abs(iy - this.lastInputSent[1]) >= DEAD_ZONE) { this.lastInputSent[0] = ix; this.lastInputSent[1] = iy; - // The bridge maps `input` → /nisps/input (its `inputs` relay path). - this.client.sendParams([ - ['input', ix], - ['input', iy], - ]); + // → bridge `input` verb → ONE multi-float message to /nisps/input. + this.client.sendInput(this.inputVec); } // 2) Stream the routed per-output values (authoritative CV in bridged mode).