From f05669969faa9b7dc3a112e39669ee42160f1f63 Mon Sep 17 00:00:00 2001 From: monkey-w1n5t0n Date: Sun, 28 Jun 2026 04:21:16 +0200 Subject: [PATCH 1/4] feat(backends): add VcvBackend driving + training the VCV module over the OSC-WS bridge --- manifold/src/backends/vcv-backend.ts | 270 +++++++++++++++++++++++++++ 1 file changed, 270 insertions(+) create mode 100644 manifold/src/backends/vcv-backend.ts diff --git a/manifold/src/backends/vcv-backend.ts b/manifold/src/backends/vcv-backend.ts new file mode 100644 index 0000000..4a9fe5d --- /dev/null +++ b/manifold/src/backends/vcv-backend.ts @@ -0,0 +1,270 @@ +/** + * VcvBackend — drives + trains the VCV Rack NISPS module over the OSC↔WS bridge + * (backends-spec §2.6; vcv/SPEC.md OSC verbs). In "bridged" mode the BROWSER is + * authoritative: it streams the current input vector to the module and forwards + * the verdict loop (thumbs up/down, explore-and-place) so the module's embedded + * net trains in lock-step with the browser session. + * + * Transport (reuses {@link NispsOscClient} → the Deno bridge in + * manifold/osc-bridge, default ws://localhost:8765, default module UDP 7001): + * + * browser → module + * /nisps/input the current 2-D input vector (drives the module) + * /nisps/output per-output values (CV) — sent as params batch so + * the bridge maps each to /nisps/; the module + * also derives its own outputs, but the browser + * value is authoritative in bridged mode + * /nisps/feedback {op,…} verdict op (up | down | rand | clear) as JSON + * state — { op, spread, input[], output[] } + * + * module → browser + * /nisps/output module's live outputs (status / visualisation) + * /nisps/input module's live inputs (echo / status) + * /nisps/state module status snapshot (surfaced as a message) + * + * The bridge process AND the VCV module must both be running — until the WS + * connects we surface "bridge not running"; until the module replies we stay + * "connected, waiting for module". + * + * British spelling in product copy; the synth is the "Built-in Synth", never + * "C15". + */ +import type { BackendContext, BackendStatus, OutputBackend } from './backend'; +import { isSilent, mapOutput } from './mapping'; +import { NispsOscClient } from './osc-client'; +import type { VcvSpec } from '../dock/output-state'; + +const SEND_INTERVAL_MS = 50; +const DEAD_ZONE = 0.002; // on the normalised value, pre physical-scale + +/** A verdict op forwarded to the module's embedded learner. */ +export interface VcvFeedbackOp { + op: 'up' | 'down' | 'rand' | 'clear'; + /** Master spread (0..1) — mirrors the engine spread knob. */ + spread: number; + /** The control input the verdict was given at (2-D). */ + input: number[]; + /** The heard output vector at that input (≤126 dims). */ + output: number[]; +} + +export interface VcvBackendConfig { + /** Bridge WebSocket URL (the bridge then relays to the module over UDP). */ + url: string; + /** Send raw normalised 0..1 instead of the per-output bipolar/unipolar range. */ + sendRaw: boolean; +} + +export class VcvBackend implements OutputBackend { + readonly id = 'vcv' as const; + + private client = new NispsOscClient(); + private ctx: BackendContext | null = null; + private specs: VcvSpec[] = []; + private sendRaw = false; + + /** Latest input vector the browser is driving the module with (2-D). */ + private inputVec: number[] = [0.5, 0.5]; + + private lastSent: Float32Array = new Float32Array(0); // last normalised output + private batch: Array<[string, number]> = []; // reused outer; entries reused + private lastSendMs = 0; + private lastInputSent: [number, number] = [-1, -1]; + + /** Latest module-reported outputs (for visualisation), null until first echo. */ + private moduleOutputs: number[] | null = null; + private gotModuleReply = false; + + private statusState: BackendStatus = { state: 'idle', message: 'VCV idle' }; + private statusListeners = new Set<(s: BackendStatus) => void>(); + private outputListeners = new Set<(v: number[]) => void>(); + private offConn: (() => void) | null = null; + private offInfo: (() => void) | null = null; + private offOutputs: (() => void) | null = null; + private offInputs: (() => void) | null = null; + + isAvailable(): boolean { + return typeof WebSocket !== 'undefined'; + } + + async start(ctx: BackendContext): Promise { + this.ctx = ctx; + this.lastSent = new Float32Array(ctx.outputCount).fill(-1); + if (!this.isAvailable()) { + this.setStatus({ state: 'unavailable', message: 'WebSocket not available' }); + return; + } + this.offConn = this.client.onConnectionChange((connected) => { + if (!connected) { + this.gotModuleReply = false; + this.setStatus({ state: 'error', message: `VCV bridge not running — start it (${this.client.url})` }); + return; + } + this.setStatus({ + state: this.gotModuleReply ? 'ready' : 'connecting', + message: this.gotModuleReply + ? `VCV module connected (${this.client.url})` + : `Bridge connected (${this.client.url}) — waiting for module…`, + }); + }); + this.offInfo = this.client.onInfo((m) => { + if (this.client.connected && !this.gotModuleReply) { + this.setStatus({ state: 'connecting', message: m }); + } + }); + // Module → browser: a reply on either channel proves the module is alive. + this.offOutputs = this.client.onOutputsReceived((v) => this.onModuleReply(v, true)); + this.offInputs = this.client.onInputsReceived((v) => this.onModuleReply(v, false)); + + this.setStatus({ state: 'connecting', message: `Connecting to VCV bridge (${this.client.url})…` }); + this.client.connect({ reconnect: true }).catch(() => { + this.setStatus({ state: 'error', message: `VCV bridge not running — start it (${this.client.url})` }); + }); + } + + setContext(ctx: BackendContext): void { + this.ctx = ctx; + if (this.lastSent.length !== ctx.outputCount) { + this.lastSent = new Float32Array(ctx.outputCount).fill(-1); + } + } + + /** Update per-output VCV specs (polarity) + bridge URL/raw toggle. */ + setVcvConfig(specs: VcvSpec[], cfg: VcvBackendConfig): void { + this.specs = specs; + this.sendRaw = cfg.sendRaw; + if (cfg.url !== this.client.url) { + this.client.setUrl(cfg.url); + this.gotModuleReply = false; + if (this.isAvailable()) { + this.setStatus({ state: 'connecting', message: `Connecting to VCV bridge (${cfg.url})…` }); + this.client.connect({ reconnect: true }).catch(() => { + this.setStatus({ state: 'error', message: `VCV bridge not running — start it (${cfg.url})` }); + }); + } + } + this.lastSent.fill(-1); + } + + /** + * Set the input vector the browser drives the module with (bridged mode). The + * next `send()` streams it to /nisps/input. Copied — caller may mutate. + */ + setInputVector(vec: ReadonlyArray): void { + if (this.inputVec.length !== vec.length) this.inputVec = new Array(vec.length); + for (let i = 0; i < vec.length; i++) this.inputVec[i] = vec[i]; + } + + /** + * Forward a verdict op to the module's embedded learner over /nisps/feedback. + * Hooked from the BackendManager when the verdict loop fires in VCV mode, so + * thumbs-up/down + explore-and-place train the module across the bridge. + */ + sendFeedback(op: VcvFeedbackOp): void { + if (!this.client.connected) return; + // The bridge ships `{ type:'state', payload }` as an OSC string to + // /nisps/state. We reuse that string channel for /nisps/feedback by tagging + // the payload with a `feedback` envelope the module routes accordingly. + this.client.sendState({ feedback: op }); + } + + send(routed: Float32Array): void { + const ctx = this.ctx; + if (!ctx || !this.client.connected) return; + + const now = typeof performance !== 'undefined' ? performance.now() : Date.now(); + if (now - this.lastSendMs < SEND_INTERVAL_MS) return; + this.lastSendMs = now; + + // 1) Stream the current input vector so the browser drives the module. + const ix = this.inputVec[0] ?? 0.5; + const iy = this.inputVec[1] ?? 0.5; + if (Math.abs(ix - this.lastInputSent[0]) >= DEAD_ZONE || Math.abs(iy - this.lastInputSent[1]) >= DEAD_ZONE) { + this.lastInputSent[0] = ix; + this.lastInputSent[1] = iy; + // The bridge maps `input` → /nisps/input (its `inputs` relay path). + this.client.sendParams([ + ['input', ix], + ['input', iy], + ]); + } + + // 2) Stream the routed per-output values (authoritative CV in bridged mode). + const n = Math.min(routed.length, ctx.mappings.length); + this.batch.length = 0; + for (let i = 0; i < n; i++) { + const m = ctx.mappings[i]; + if (isSilent(m)) continue; + const mapped = mapOutput(routed[i], m); // 0..1 in [min,max] + const prev = this.lastSent[i]; + if (prev >= 0 && Math.abs(mapped - prev) < DEAD_ZONE) continue; + this.lastSent[i] = mapped; + const spec = this.specs[i]; + const value = this.sendRaw ? mapped : this.toVoltage(mapped, spec); + const name = ctx.names[i] ? sanitise(ctx.names[i]) : `out${i}`; + this.batch.push([name, value]); + } + if (this.batch.length) this.client.sendParams(this.batch); + } + + /** Map a 0..1 value into the per-output VCV voltage range (uni/bipolar). */ + private toVoltage(v: number, spec: VcvSpec | undefined): number { + // Unipolar 0..10 V; bipolar ±5 V (dock-spec §4.3 polarity). + return spec?.bipolar ? v * 10 - 5 : v * 10; + } + + /** Latest module-reported output vector for visualisation (may be null). */ + moduleStatusOutputs(): number[] | null { + return this.moduleOutputs; + } + + /** Subscribe to module-reported outputs (visualisation feed). */ + onModuleOutputs(cb: (v: number[]) => void): () => void { + this.outputListeners.add(cb); + return () => this.outputListeners.delete(cb); + } + + private onModuleReply(v: number[], isOutput: boolean): void { + if (!this.gotModuleReply) { + this.gotModuleReply = true; + this.setStatus({ state: 'ready', message: `VCV module connected (${this.client.url})` }); + } + if (isOutput) { + this.moduleOutputs = v; + for (const cb of this.outputListeners) cb(v); + } + } + + async teardown(): Promise { + this.offConn?.(); + this.offInfo?.(); + this.offOutputs?.(); + this.offInputs?.(); + this.offConn = null; + this.offInfo = null; + this.offOutputs = null; + this.offInputs = null; + this.client.disconnect(); + this.gotModuleReply = false; + this.setStatus({ state: 'idle', message: 'VCV idle' }); + } + + status(): BackendStatus { + return this.statusState; + } + + onStatusChange(cb: (s: BackendStatus) => void): () => void { + this.statusListeners.add(cb); + return () => this.statusListeners.delete(cb); + } + + private setStatus(s: BackendStatus): void { + this.statusState = s; + for (const cb of this.statusListeners) cb(s); + } +} + +/** Sanitise an output name into an OSC-path-safe token (the bridge prefixes it). */ +function sanitise(name: string): string { + return name.toLowerCase().replace(/[^a-z0-9_]+/g, '_').replace(/^_+|_+$/g, '') || 'out'; +} From ecb5bf2bb6058b785ae7732fe1a05598bd951ca1 Mon Sep 17 00:00:00 2001 From: monkey-w1n5t0n Date: Sun, 28 Jun 2026 04:22:05 +0200 Subject: [PATCH 2/4] feat(backends): register VcvBackend in manager + roster; add defaultVcvSpec, feedback forwarding, input-vector streaming --- manifold/src/backends/index.ts | 2 ++ manifold/src/backends/manager.ts | 35 +++++++++++++++++++++++++++++-- manifold/src/dock/output-state.ts | 5 +++++ 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/manifold/src/backends/index.ts b/manifold/src/backends/index.ts index a0bea84..ca2431c 100644 --- a/manifold/src/backends/index.ts +++ b/manifold/src/backends/index.ts @@ -16,6 +16,8 @@ export { WebMidiBackend } from './midi-backend'; export type { MidiBackendConfig } from './midi-backend'; export { OscBridgeBackend } from './osc-backend'; export type { OscBackendConfig } from './osc-backend'; +export { VcvBackend } from './vcv-backend'; +export type { VcvBackendConfig, VcvFeedbackOp } from './vcv-backend'; export { NispsOscClient } from './osc-client'; export { PassthroughBackend } from './passthrough-backend'; export { diff --git a/manifold/src/backends/manager.ts b/manifold/src/backends/manager.ts index 4afe0c2..4112258 100644 --- a/manifold/src/backends/manager.ts +++ b/manifold/src/backends/manager.ts @@ -21,12 +21,19 @@ import type { BackendId } from '../dock/output-state'; import { WebMidiBackend } from './midi-backend'; import { OscBridgeBackend } from './osc-backend'; import { PassthroughBackend } from './passthrough-backend'; +import { VcvBackend, type VcvFeedbackOp } from './vcv-backend'; /** The slice of EngineApi the manager depends on (keeps it decoupled/testable). */ export interface ManagerEngine { subscribe(cb: () => void): () => void; routedOutput(): Float32Array | null; audio: { setMuted(muted: boolean): void }; + /** + * Current control input vector (2-D for the fixed 2→N MLP). Optional — when + * present the VCV backend streams it to the module so the browser drives the + * module's inputs in bridged mode. + */ + inputVector?(): ReadonlyArray; } export class BackendManager { @@ -49,12 +56,17 @@ export class BackendManager { ['synth', backends?.synth ?? new PassthroughBackend('synth', 'Built-in Synth — audio plays in the engine')], ['particles', backends?.particles ?? new PassthroughBackend('particles', 'Particle visualiser')], ['cvgate', backends?.cvgate ?? new PassthroughBackend('cvgate', 'CV / gate (via VCV bridge)')], - ['vcv', backends?.vcv ?? new PassthroughBackend('vcv', 'VCV bridge')], + ['vcv', backends?.vcv ?? new VcvBackend()], ]); - // Single subscription to the spine: forward routed → active backend. + // Single subscription to the spine: forward routed → active backend. For the + // VCV backend we also stream the current input vector each tick so the + // browser drives the module's inputs in bridged mode. this.unsub = this.engine.subscribe(() => { if (!this.active) return; + if (this.active instanceof VcvBackend && this.engine.inputVector) { + this.active.setInputVector(this.engine.inputVector()); + } const routed = this.engine.routedOutput(); if (routed) this.active.send(routed); }); @@ -71,6 +83,25 @@ export class BackendManager { return b instanceof OscBridgeBackend ? b : null; } + vcv(): VcvBackend | null { + const b = this.backends.get('vcv'); + return b instanceof VcvBackend ? b : null; + } + + /** + * Forward a verdict op to the VCV module's embedded learner over the bridge. + * No-op unless the VCV backend is the ACTIVE one — the verdict loop only + * trains the module when Mode = VCV (otherwise the browser engine is the + * learner). Returns true if it was forwarded. + */ + forwardFeedback(op: VcvFeedbackOp): boolean { + if (this.activeId !== 'vcv') return false; + const vcv = this.vcv(); + if (!vcv) return false; + vcv.sendFeedback(op); + return true; + } + get(id: BackendId): OutputBackend | undefined { return this.backends.get(id); } diff --git a/manifold/src/dock/output-state.ts b/manifold/src/dock/output-state.ts index 52c912d..b240a97 100644 --- a/manifold/src/dock/output-state.ts +++ b/manifold/src/dock/output-state.ts @@ -130,3 +130,8 @@ export function defaultMidiSpec(index: number): MidiCcSpec { export function defaultOscSpec(name: string): OscSpec { return { path: `/nisps/${name.toLowerCase()}`, rangeMin: 0, rangeMax: 1 }; } + +/** Default VCV spec for an output — unipolar 0–10 V by default. */ +export function defaultVcvSpec(): VcvSpec { + return { bipolar: false }; +} From 964e37551f19c35e5f93ef0554772af736847067 Mon Sep 17 00:00:00 2001 From: monkey-w1n5t0n Date: Sun, 28 Jun 2026 04:25:39 +0200 Subject: [PATCH 3/4] feat(dock): wire VCV bridge URL/connect + per-output polarity; forward verdict loop to /nisps/feedback in VCV mode --- manifold/src/backends/useBackendManager.ts | 20 ++++- manifold/src/console/ConsoleApp.tsx | 31 +++++++- manifold/src/console/types.ts | 6 ++ manifold/src/dock/OutputsBackendConfig.tsx | 90 ++++++++++++++++++++-- manifold/src/engine/engine-api.ts | 8 ++ 5 files changed, 145 insertions(+), 10 deletions(-) diff --git a/manifold/src/backends/useBackendManager.ts b/manifold/src/backends/useBackendManager.ts index 875b37a..096e8e8 100644 --- a/manifold/src/backends/useBackendManager.ts +++ b/manifold/src/backends/useBackendManager.ts @@ -18,8 +18,8 @@ import { useEffect, useMemo, useRef, useState } from 'react'; import type { EngineApi } from '../engine'; import type { MFParam } from '../console/model'; import type { BackendContext, BackendStatus, OutputMapping } from './backend'; -import type { BackendId, MidiCcSpec, OscSpec } from '../dock/output-state'; -import { defaultMidiSpec, defaultOscSpec } from '../dock/output-state'; +import type { BackendId, MidiCcSpec, OscSpec, VcvSpec } from '../dock/output-state'; +import { defaultMidiSpec, defaultOscSpec, defaultVcvSpec } from '../dock/output-state'; import { BackendManager } from './manager'; export interface MidiSettings { @@ -32,6 +32,13 @@ export interface OscSettings { sendRaw: boolean; } +export interface VcvSettings { + /** Bridge WebSocket URL (the Deno bridge relays to the module over UDP). */ + url: string; + /** Send raw 0..1 instead of the per-output bipolar/unipolar voltage range. */ + sendRaw: boolean; +} + function toMapping(p: MFParam): OutputMapping { return { state: p.status, @@ -58,6 +65,7 @@ export function useBackendManager( params: MFParam[], midiSettings: MidiSettings, oscSettings: OscSettings, + vcvSettings: VcvSettings, ): UseBackendManager { const managerRef = useRef(null); const [status, setStatus] = useState({ state: 'idle', message: 'idle' }); @@ -132,5 +140,13 @@ export function useBackendManager( osc.setOscConfig(specs, { url: oscSettings.url, sendRaw: oscSettings.sendRaw }); }, [manager, params, oscSettings.url, oscSettings.sendRaw]); + // Push per-output VCV config (polarity) + bridge URL/raw whenever they change. + useEffect(() => { + const vcv = manager?.vcv(); + if (!vcv) return; + const specs: VcvSpec[] = params.map((p) => p.vcv ?? defaultVcvSpec()); + vcv.setVcvConfig(specs, { url: vcvSettings.url, sendRaw: vcvSettings.sendRaw }); + }, [manager, params, vcvSettings.url, vcvSettings.sendRaw]); + return { manager, status, midiPorts, refreshMidiPorts }; } diff --git a/manifold/src/console/ConsoleApp.tsx b/manifold/src/console/ConsoleApp.tsx index ef3b4cb..151f5de 100644 --- a/manifold/src/console/ConsoleApp.tsx +++ b/manifold/src/console/ConsoleApp.tsx @@ -127,6 +127,10 @@ export function ConsoleApp({ focus: initialFocus = 'composite' }: ConsoleAppProp const [midiCcCount, setMidiCcCount] = useState(8); const [oscUrl, setOscUrl] = useState('ws://localhost:8765'); const [oscSendRaw, setOscSendRaw] = useState(false); + // VCV bridge: WS URL of the Deno bridge that relays to the VCV module over UDP + // (default module UDP 7001). Independent of the OSC backend's bridge. + const [vcvUrl, setVcvUrl] = useState('ws://localhost:8765'); + const [vcvSendRaw, setVcvSendRaw] = useState(false); // Feedback markers plotted on the 2D map (both polarities; session-scoped). const [markers, setMarkers] = useState([]); const [volume, setVolume] = useState(0.8); @@ -252,15 +256,28 @@ export function ConsoleApp({ focus: initialFocus = 'composite' }: ConsoleAppProp // spine and forwards routed outputs to the active backend; switching Mode // tears down the old backend, starts the new one, and gates synth audio // (mute on non-synth modes). MIDI/OSC config + names ride the shared params. - const { status: backendStatus, midiPorts, refreshMidiPorts } = useBackendManager( + const { manager: backendManager, status: backendStatus, midiPorts, refreshMidiPorts } = useBackendManager( engine, outputBackend, modeId, params, { outputId: midiOutputId, ccCount: midiCcCount }, { url: oscUrl, sendRaw: oscSendRaw }, + { url: vcvUrl, sendRaw: vcvSendRaw }, ); + // VCV bridge: forward a verdict op to the module's embedded learner. No-op + // unless Mode = VCV (the manager gates on the active backend). This is how + // thumbs-up/down + explore-and-place TRAIN the module across the bridge. + const forwardVcvFeedback = (op: 'up' | 'down' | 'rand' | 'clear') => { + backendManager?.forwardFeedback({ + op, + spread: spread ? 1 : 0.6, + input: [pos[0], pos[1]], + output: Array.from(engine?.getOutputs() ?? new Float32Array(0)), + }); + }; + // values come from the REAL engine output, shaped per-param. Recomputed when // the engine version bumps (new inference / weights) or params change. const values = useMemo( @@ -314,6 +331,8 @@ export function ConsoleApp({ focus: initialFocus = 'composite' }: ConsoleAppProp c?.like(pos, engine?.getOutputs() ?? new Float32Array(0)); pushMarker(pos, 'positive'); } + // VCV bridged mode: also train the module — thumbs-up = positive verdict. + forwardVcvFeedback('up'); syncController(); setNoiseCap((n) => Math.max(0.02, n * 0.7)); setHealth((h) => Math.min(1, h + 0.08)); @@ -338,12 +357,16 @@ export function ConsoleApp({ focus: initialFocus = 'composite' }: ConsoleAppProp } else { pushSnap('explore'); c?.enterExplore(); + // VCV bridged mode: entering explore re-rolls the module's net too. + forwardVcvFeedback('rand'); } } else { // Geometric dislike: push the current mapping away from this sound. pushSnap('dislike −'); c?.dislike(pos, engine?.getOutputs() ?? new Float32Array(0), noiseCap, spread ? 1 : 0.6); pushMarker(pos, 'negative'); + // VCV bridged mode: thumbs-down = negative verdict. + forwardVcvFeedback('down'); setSeed((s) => s + (Math.random() - 0.5) * (noiseCap * 4 + 0.3)); setNoiseCap((n) => Math.min(0.5, n + 0.06)); setHealth((h) => Math.max(0.1, h - 0.06)); @@ -364,6 +387,8 @@ export function ConsoleApp({ focus: initialFocus = 'composite' }: ConsoleAppProp // Outside a scratchpad session a re-roll randomises the real net directly. engine?.randomise(spread ? 1 : 0.6); } + // VCV bridged mode: re-roll the module's net too. + forwardVcvFeedback('rand'); syncController(); setSeed(Math.random() * 6); setNoiseCap(0.4); @@ -574,6 +599,10 @@ export function ConsoleApp({ focus: initialFocus = 'composite' }: ConsoleAppProp setOscUrl, oscSendRaw, setOscSendRaw, + vcvUrl, + setVcvUrl, + vcvSendRaw, + setVcvSendRaw, setParams: (next: MFParam[]) => setParams(next), markers, health, diff --git a/manifold/src/console/types.ts b/manifold/src/console/types.ts index 4d45dbd..554741f 100644 --- a/manifold/src/console/types.ts +++ b/manifold/src/console/types.ts @@ -99,6 +99,12 @@ export interface ConsoleCtx { setOscUrl: (u: string) => void; oscSendRaw: boolean; setOscSendRaw: (v: boolean) => void; + /** VCV backend settings (bridge URL + send-raw toggle). The Deno bridge + * relays to the VCV module over UDP (default module port 7001). */ + vcvUrl: string; + setVcvUrl: (u: string) => void; + vcvSendRaw: boolean; + setVcvSendRaw: (v: boolean) => void; /** Replace the whole params array (used when restoring a named preset). */ setParams: (next: MFParam[]) => void; diff --git a/manifold/src/dock/OutputsBackendConfig.tsx b/manifold/src/dock/OutputsBackendConfig.tsx index 116d9da..0644127 100644 --- a/manifold/src/dock/OutputsBackendConfig.tsx +++ b/manifold/src/dock/OutputsBackendConfig.tsx @@ -6,7 +6,9 @@ * namespace) on top, then a per-backend config section: * - MIDI → output-port picker, number-of-CCs, per-output CC#/channel/name. * - OSC → bridge URL + connect status + send-raw toggle, per-output path/range. - * - VCV/CV→ per-output polarity (delegates to the existing BackendAdvanced body). + * - VCV/CV→ bridge URL + connect status + send-raw toggle, per-output polarity + * (uni 0–10 V / bipolar ±5 V). The browser drives + trains the VCV + * module over the same Deno bridge. * - Synth/Particle/Editor → handled by ModeConfig in Drawers (no extra config here). * * Everything is editable inline; writes go through the shared MFParam store @@ -16,7 +18,7 @@ import { useEffect, useState } from 'react'; import type { ConsoleCtx } from '../console/types'; import type { BackendId } from './output-state'; -import { defaultMidiSpec, defaultOscSpec } from './output-state'; +import { defaultMidiSpec, defaultOscSpec, defaultVcvSpec } from './output-state'; import { applyPreset, deletePreset, @@ -107,6 +109,7 @@ function PresetBar({ ctx, backend }: { ctx: ConsoleCtx; backend: BackendId }) { const backendSettings = (): Record => { if (backend === 'midi') return { outputId: ctx.midiOutputId, ccCount: ctx.midiCcCount }; if (backend === 'osc') return { url: ctx.oscUrl, sendRaw: ctx.oscSendRaw }; + if (backend === 'vcv') return { url: ctx.vcvUrl, sendRaw: ctx.vcvSendRaw }; return {}; }; @@ -118,6 +121,9 @@ function PresetBar({ ctx, backend }: { ctx: ConsoleCtx; backend: BackendId }) { } else if (backend === 'osc') { if ('url' in s) ctx.setOscUrl(String(s.url)); if ('sendRaw' in s) ctx.setOscSendRaw(Boolean(s.sendRaw)); + } else if (backend === 'vcv') { + if ('url' in s) ctx.setVcvUrl(String(s.url)); + if ('sendRaw' in s) ctx.setVcvSendRaw(Boolean(s.sendRaw)); } }; @@ -381,6 +387,76 @@ function OscConfig({ ctx }: { ctx: ConsoleCtx }) { ); } +// ---- VCV / CV config (backends-spec §2.6 / §4.3) --------------------------- + +function VcvConfig({ ctx }: { ctx: ConsoleCtx }) { + const s = ctx.backendStatus; + const statusColor = s.state === 'ready' ? 'var(--good)' : s.state === 'connecting' ? 'var(--warn)' : 'var(--danger)'; + const [draftUrl, setDraftUrl] = useState(ctx.vcvUrl); + useEffect(() => setDraftUrl(ctx.vcvUrl), [ctx.vcvUrl]); + return ( + <> + VCV bridge +
+ setDraftUrl(e.target.value)} + onBlur={() => ctx.setVcvUrl(draftUrl)} + placeholder="ws://localhost:8765" + /> + + + {s.message} +
+

+ The VCV Rack NISPS module AND the Deno OSC bridge (manifold/osc-bridge) must both be running. The browser drives + the module's inputs and forwards the verdict loop (thumbs up/down, explore-and-place) over the bridge, so the + module's embedded net trains in lock-step. Default module UDP port 7001. +

+ + Per-output polarity +
+ + + + + + + + + {ctx.params.map((p, i) => { + const v = p.vcv ?? defaultVcvSpec(); + return ( + + + + + ); + })} + +
outputrange
{p.name} + +
+
+ + ); +} + // ---- Public entry ---------------------------------------------------------- export interface OutputsBackendConfigProps { @@ -390,14 +466,14 @@ export interface OutputsBackendConfigProps { /** The specialised, editable per-backend config + preset bar for the Outputs panel. */ export function OutputsBackendConfig({ ctx, backend }: OutputsBackendConfigProps) { - // Only MIDI / OSC carry a config + preset surface here; synth/particle/editor - // config is rendered by ModeConfig in Drawers. VCV/CV polarity stays in the - // full-depth BackendAdvanced modal. - if (backend !== 'midi' && backend !== 'osc') return null; + // MIDI / OSC / VCV carry a config + preset surface here; synth/particle/editor + // config is rendered by ModeConfig in Drawers. The full-depth BackendAdvanced + // modal reuses the same per-channel sections. + if (backend !== 'midi' && backend !== 'osc' && backend !== 'vcv') return null; return (
- {backend === 'midi' ? : } + {backend === 'midi' ? : backend === 'osc' ? : }
); } diff --git a/manifold/src/engine/engine-api.ts b/manifold/src/engine/engine-api.ts index 9e5fada..34bd987 100644 --- a/manifold/src/engine/engine-api.ts +++ b/manifold/src/engine/engine-api.ts @@ -141,6 +141,14 @@ export class EngineApi { return this.spine.routedOutput(); } + /** + * Current control input vector (2-D for the fixed 2→N MLP). Used by the VCV + * backend (via BackendManager) to drive the module's inputs over the bridge. + */ + inputVector(): ReadonlyArray { + return [this.spine.lastRawX, this.spine.lastRawY]; + } + /** * Re-run the LAST raw input through the spine — used after a weight change * (train / randomise / feedback) so outputs + audio reflect the new MLP From 7d36d3d18de64ec371601d5479ede95ee8d1e6ed Mon Sep 17 00:00:00 2001 From: monkey-w1n5t0n Date: Sun, 28 Jun 2026 04:28:45 +0200 Subject: [PATCH 4/4] feat(osc-bridge): add /nisps/input (multi-float) + /nisps/feedback (JSON) verbs; client sendInput/sendFeedback; VcvBackend uses them; document VCV runtime --- manifold/osc-bridge/bridge.mjs | 25 ++++++++++++++++++++++ manifold/osc-bridge/bridge.ts | 29 ++++++++++++++++++++++++- manifold/src/backends/README.md | 32 +++++++++++++++++++++++++++- manifold/src/backends/osc-client.ts | 18 ++++++++++++++++ manifold/src/backends/vcv-backend.ts | 17 ++++++--------- 5 files changed, 108 insertions(+), 13 deletions(-) diff --git a/manifold/osc-bridge/bridge.mjs b/manifold/osc-bridge/bridge.mjs index 2439f8e..a87ee0f 100644 --- a/manifold/osc-bridge/bridge.mjs +++ b/manifold/osc-bridge/bridge.mjs @@ -63,6 +63,16 @@ function oscMessage(address, value) { ]); } +/** A single OSC message carrying N floats (one ",fff…" message, not N messages). */ +function oscMessageFloats(address, values) { + const tags = ',' + 'f'.repeat(values.length); + return Buffer.concat([ + oscString(address), + oscString(tags), + ...values.map((v) => oscFloat(v)), + ]); +} + function oscMessageString(address, value) { return Buffer.concat([ oscString(address), @@ -145,6 +155,11 @@ function sendOSCString(address, value) { udpSend.send(msg, OSC_PORT, OSC_HOST); } +function sendOSCFloats(address, values) { + const msg = oscMessageFloats(address, values); + udpSend.send(msg, OSC_PORT, OSC_HOST); +} + function sendOSCBundle(params) { const messages = params.map(([name, value]) => oscMessage(`${OSC_PREFIX}/${name}`, value) @@ -218,6 +233,16 @@ wss.on('connection', (ws) => { case 'weights': sendOSCString(`${OSC_PREFIX}/weights`, JSON.stringify(data.payload)); return; + case 'input': + // Current input VECTOR as ONE multi-float message to /nisps/input. + if (Array.isArray(data.payload)) { + sendOSCFloats(`${OSC_PREFIX}/input`, data.payload); + } + return; + case 'feedback': + // Verdict op as OSC string to /nisps/feedback (trains the module). + sendOSCString(`${OSC_PREFIX}/feedback`, JSON.stringify(data.payload)); + return; case 'params': if (Array.isArray(data.payload)) { if (USE_BUNDLES) { diff --git a/manifold/osc-bridge/bridge.ts b/manifold/osc-bridge/bridge.ts index a7cf565..0d9e926 100644 --- a/manifold/osc-bridge/bridge.ts +++ b/manifold/osc-bridge/bridge.ts @@ -22,8 +22,11 @@ // /nisps/ (webapp -> target) // /nisps/state (webapp -> target: full JSON state) // /nisps/weights (webapp -> target: weights JSON) +// /nisps/input (webapp -> target: input vector, ONE message; +// also target -> webapp for visualisation) +// /nisps/feedback (webapp -> target: verdict op JSON — +// { op, spread, input[], output[] }) // /nisps/output (target -> webapp: output float array) -// /nisps/input (target -> webapp: input float array) import { parseArgs } from "jsr:@std/cli@1/parse-args"; @@ -104,6 +107,13 @@ function oscMessage(address: string, value: number): Uint8Array { return concat(oscString(address), oscString(",f"), oscFloat(value)); } +/** A single OSC message carrying N floats (one ",fff…" message, not N messages). */ +function oscMessageFloats(address: string, values: number[]): Uint8Array { + const tags = "," + "f".repeat(values.length); + const floatBufs = values.map((v) => oscFloat(v)); + return concat(oscString(address), oscString(tags), ...floatBufs); +} + function oscMessageString(address: string, value: string): Uint8Array { return concat(oscString(address), oscString(",s"), oscString(value)); } @@ -197,6 +207,11 @@ function sendOSCString(address: string, value: string): void { udpSend.send(msg, oscAddr); } +function sendOSCFloats(address: string, values: number[]): void { + const msg = oscMessageFloats(address, values); + udpSend.send(msg, oscAddr); +} + function sendOSCBundle(params: [string, number][]): void { const messages = params.map(([name, value]) => oscMessage(`${OSC_PREFIX}/${name}`, value) @@ -249,6 +264,18 @@ function handleWs(ws: WebSocket): void { // Send weights JSON as OSC string to /nisps/weights sendOSCString(`${OSC_PREFIX}/weights`, JSON.stringify(data.payload)); return; + case "input": + // Current input VECTOR as ONE multi-float message to /nisps/input + // (browser drives the VCV module's inputs in bridged mode). + if (Array.isArray(data.payload)) { + sendOSCFloats(`${OSC_PREFIX}/input`, data.payload as number[]); + } + return; + case "feedback": + // Verdict op as OSC string to /nisps/feedback (trains the module's + // embedded learner: { op, spread, input[], output[] }). + sendOSCString(`${OSC_PREFIX}/feedback`, JSON.stringify(data.payload)); + return; case "params": // Legacy batch format embedded in structured message if (Array.isArray(data.payload)) { diff --git a/manifold/src/backends/README.md b/manifold/src/backends/README.md index 48af9c5..961c9b2 100644 --- a/manifold/src/backends/README.md +++ b/manifold/src/backends/README.md @@ -1,7 +1,7 @@ # Output Backends (`manifold/src/backends/`) Real output transports for the Manifold app. Exactly one backend is *active* at a -time, chosen by the dock **Mode** (Particle / MIDI / OSC / Built-in Synth / +time, chosen by the dock **Mode** (Particle / MIDI / OSC / VCV / Built-in Synth / Editor → `BackendId`). The `BackendManager` consumes the engine spine and forwards each routed output vector to the active backend's `send()`. @@ -17,6 +17,7 @@ See `docs/redesign/backends-spec.md` for the authoritative design. | `midi-backend.ts` | `WebMidiBackend` — real Web MIDI CC out (per-output CC#/channel/range/name, throttled + dead-zone). | | `osc-client.ts` | `NispsOscClient` — WS transport to the Deno OSC bridge (JSON protocol, auto-reconnect). | | `osc-backend.ts` | `OscBridgeBackend` — OSC out over WS; per-output address path + physical range. | +| `vcv-backend.ts` | `VcvBackend` — drives + **trains** the VCV Rack NISPS module over the OSC↔WS bridge (streams `/nisps/input`, forwards the verdict loop to `/nisps/feedback`, receives `/nisps/output` + `/nisps/state`). | | `passthrough-backend.ts` | No-op sink for synth (plays in-engine) / particles (rAF consumer) / editor. | | `presets.ts` | Named per-backend output-config presets (localStorage, per-backend namespace). | | `useBackendManager.ts` | Thin React binding: builds `BackendContext` from the store, switches Mode, surfaces status. | @@ -45,3 +46,32 @@ node bridge.mjs Default bridge URL: `ws://localhost:8765` (configurable in the OSC config panel). WS protocol (browser → bridge): `{ type:'params', payload:[[path,value],…] }`. + +## VCV bridge — module **and** bridge must both be running + +When Mode = **VCV**, the browser is authoritative in *bridged mode*: the +`VcvBackend` connects over the **same Deno bridge** and + +- streams the current input vector continuously to **`/nisps/input`** (the + browser drives the module's inputs), +- streams the routed per-output CV (uni 0–10 V / bipolar ±5 V), and +- forwards the **verdict loop** — thumbs-up/down + explore-and-place — to + **`/nisps/feedback`** (`{ op:'up'|'down'|'rand'|'clear', spread, input[], + output[] }`), so the module's embedded net trains in lock-step with the + browser session. + +It receives **`/nisps/output`** + **`/nisps/state`** back for status / +visualisation. The verdict forwarding is gated in `BackendManager.forwardFeedback` +— a no-op unless VCV is the active backend (otherwise the browser engine is the +learner). Both processes are required: + +```bash +# 1) the VCV Rack NISPS module (vcv/) — default UDP listen port 7001 +# 2) the Deno bridge, pointed at the module's UDP port: +cd manifold/osc-bridge +deno run --allow-net bridge.ts --osc-port 7001 +``` + +Default VCV bridge URL: `ws://localhost:8765` (configurable in the VCV config +panel). Until the WS connects the backend shows "bridge not running"; until the +module replies it shows "waiting for module…". diff --git a/manifold/src/backends/osc-client.ts b/manifold/src/backends/osc-client.ts index ac43f98..1a4b66d 100644 --- a/manifold/src/backends/osc-client.ts +++ b/manifold/src/backends/osc-client.ts @@ -123,6 +123,24 @@ export class NispsOscClient { this.send({ type: 'weights', payload }); } + /** + * Send the current input VECTOR as ONE multi-float OSC message to + * `/input` (the bridge `input` verb). Used in VCV bridged mode so the + * browser drives the module's inputs. Distinct from `sendParams` (which emits + * one single-float message per entry). + */ + sendInput(values: ReadonlyArray): void { + this.send({ type: 'input', payload: Array.from(values) }); + } + + /** + * Send a verdict op as a JSON string to `/feedback` (the bridge + * `feedback` verb). Trains the VCV module's embedded learner over the bridge. + */ + sendFeedback(op: object): void { + this.send({ type: 'feedback', payload: op }); + } + // ── Receive ──────────────────────────────────────────────────────── onOutputsReceived(cb: (v: number[]) => void): () => void { this.outputsCbs.push(cb); diff --git a/manifold/src/backends/vcv-backend.ts b/manifold/src/backends/vcv-backend.ts index 4a9fe5d..32030c8 100644 --- a/manifold/src/backends/vcv-backend.ts +++ b/manifold/src/backends/vcv-backend.ts @@ -14,8 +14,8 @@ * the bridge maps each to /nisps/; the module * also derives its own outputs, but the browser * value is authoritative in bridged mode - * /nisps/feedback {op,…} verdict op (up | down | rand | clear) as JSON - * state — { op, spread, input[], output[] } + * /nisps/feedback {op,…} verdict op (up | down | rand | clear) as a JSON + * string — { op, spread, input[], output[] } * * module → browser * /nisps/output module's live outputs (status / visualisation) @@ -162,10 +162,8 @@ export class VcvBackend implements OutputBackend { */ sendFeedback(op: VcvFeedbackOp): void { if (!this.client.connected) return; - // The bridge ships `{ type:'state', payload }` as an OSC string to - // /nisps/state. We reuse that string channel for /nisps/feedback by tagging - // the payload with a `feedback` envelope the module routes accordingly. - this.client.sendState({ feedback: op }); + // → bridge `feedback` verb → OSC string to /nisps/feedback. + this.client.sendFeedback(op); } send(routed: Float32Array): void { @@ -182,11 +180,8 @@ export class VcvBackend implements OutputBackend { if (Math.abs(ix - this.lastInputSent[0]) >= DEAD_ZONE || Math.abs(iy - this.lastInputSent[1]) >= DEAD_ZONE) { this.lastInputSent[0] = ix; this.lastInputSent[1] = iy; - // The bridge maps `input` → /nisps/input (its `inputs` relay path). - this.client.sendParams([ - ['input', ix], - ['input', iy], - ]); + // → bridge `input` verb → ONE multi-float message to /nisps/input. + this.client.sendInput(this.inputVec); } // 2) Stream the routed per-output values (authoritative CV in bridged mode).