diff --git a/playground/js/nisps/iml.js b/playground/js/nisps/iml.js index eea2005..4e095b0 100644 --- a/playground/js/nisps/iml.js +++ b/playground/js/nisps/iml.js @@ -215,4 +215,101 @@ export class IML { get exampleCount() { return this.dataset.size; } + + /** + * Export state as a .nisps-compatible JSON object. + * The weight format matches the VCV Rack serialization: a 3D array of + * connection weights [layer][node][weight], without bias values. + * Bias is stored in a separate parallel structure for lossless round-trip + * within the webapp, but VCV Rack will ignore it. + * + * @returns {object} .nisps JSON object + */ + exportState() { + const internalWeights = this.mlp.getWeights(); + + // Convert from JS format [{weights, bias}, ...] per layer + // to .nisps format: float[][][] (connection weights only) + const weights = internalWeights.map(layer => + layer.map(node => node.weights) + ); + + // Also capture bias values for lossless webapp round-trip + const biases = internalWeights.map(layer => + layer.map(node => node.bias) + ); + + // Features stored without bias term + const features = this.dataset.features.map(f => [...f]); + const labels = this.dataset.labels.map(l => [...l]); + + const activations = this.mlp.layers.map(layer => layer.activationName); + + return { + version: 1, + weights: weights, + biases: biases, + examples: { features, labels }, + mlpConfig: { + layers: [...this.mlp.layersNodes], + activations: activations, + }, + }; + } + + /** + * Import state from a .nisps JSON object. + * Accepts both the VCV Rack format (3D weight array without bias) and + * the webapp extended format (with separate biases array). + * + * @param {object} state - Parsed .nisps JSON + * @throws {Error} If version is invalid or architecture mismatches + */ + importState(state) { + if (!state || !state.version || state.version < 1) { + throw new Error('Invalid .nisps format: missing or unsupported version'); + } + + // Validate architecture compatibility if mlpConfig is present + if (state.mlpConfig && state.mlpConfig.layers) { + const expected = this.mlp.layersNodes; + const actual = state.mlpConfig.layers; + if (expected.length !== actual.length || + !expected.every((v, i) => v === actual[i])) { + throw new Error( + `Architecture mismatch: expected [${expected}], got [${actual}]` + ); + } + } + + // Load weights + if (state.weights) { + // Convert from .nisps 3D format to JS internal format + const internalWeights = state.weights.map((layer, li) => + layer.map((nodeWeights, ni) => ({ + weights: Array.isArray(nodeWeights) ? [...nodeWeights] : nodeWeights, + bias: (state.biases && state.biases[li] && state.biases[li][ni] !== undefined) + ? state.biases[li][ni] + : 0, + })) + ); + this.mlp.setWeights(internalWeights); + } + + // Load examples + if (state.examples) { + this.dataset.clear(); + const { features, labels } = state.examples; + if (features && labels) { + const count = Math.min(features.length, labels.length); + for (let i = 0; i < count; i++) { + this.dataset.add(features[i], labels[i]); + } + } + } + + // Re-run inference with current inputs + this.inputUpdated = true; + this.process(); + } } diff --git a/playground/js/nisps/layer.js b/playground/js/nisps/layer.js index 4f9e165..b39eaa9 100644 --- a/playground/js/nisps/layer.js +++ b/playground/js/nisps/layer.js @@ -34,6 +34,7 @@ export class Layer { this.numNodes = numNodes; this.nodes = []; + this.activationName = activationName; const pair = activationPairs[activationName]; this.activationFn = pair[0]; this.derivActivationFn = pair[1]; diff --git a/playground/js/nisps/osc-client.js b/playground/js/nisps/osc-client.js new file mode 100644 index 0000000..9351ab0 --- /dev/null +++ b/playground/js/nisps/osc-client.js @@ -0,0 +1,241 @@ +// osc-client.js — WebSocket client for NISPS <-> OSC bridge +// Connects the webapp to VCV Rack MEMLNaut module (or any OSC target) +// via the bridge server (bridge.ts / bridge.mjs). +// +// Usage: +// import { NispsOscClient } from './osc-client.js'; +// const osc = new NispsOscClient('ws://localhost:8765'); +// osc.onOutputsReceived(values => console.log('outputs:', values)); +// osc.onInputsReceived(values => console.log('inputs:', values)); +// await osc.connect(); +// osc.sendState(stateJson); + +export class NispsOscClient extends EventTarget { + /** + * @param {string} wsUrl WebSocket URL of the bridge server + */ + constructor(wsUrl = 'ws://localhost:8765') { + super(); + this._wsUrl = wsUrl; + this._ws = null; + this._connected = false; + this._reconnect = false; + this._reconnectDelay = 1000; + this._reconnectTimer = null; + + // Registered callbacks + this._outputsCallbacks = []; + this._inputsCallbacks = []; + this._infoCallbacks = []; + } + + /** Current connection state */ + get connected() { return this._connected; } + + /** The WebSocket URL */ + get url() { return this._wsUrl; } + set url(val) { + if (this._connected) { + this.disconnect(); + } + this._wsUrl = val; + } + + /** + * Connect to the bridge server. + * @param {object} [opts] + * @param {boolean} [opts.reconnect=true] Auto-reconnect on disconnect + * @returns {Promise} Resolves when connected + */ + connect({ reconnect = true } = {}) { + this._reconnect = reconnect; + + return new Promise((resolve, reject) => { + if (this._connected && this._ws) { + resolve(); + return; + } + + try { + this._ws = new WebSocket(this._wsUrl); + } catch (err) { + reject(err); + return; + } + + this._ws.onopen = () => { + this._connected = true; + this._reconnectDelay = 1000; // reset backoff + this.dispatchEvent(new CustomEvent('connected')); + resolve(); + }; + + this._ws.onclose = () => { + const wasConnected = this._connected; + this._connected = false; + this._ws = null; + this.dispatchEvent(new CustomEvent('disconnected')); + + if (this._reconnect) { + this._scheduleReconnect(); + } + + if (!wasConnected) { + reject(new Error('WebSocket closed before connecting')); + } + }; + + this._ws.onerror = (e) => { + this.dispatchEvent(new CustomEvent('error', { detail: e })); + }; + + this._ws.onmessage = (e) => { + this._handleMessage(e.data); + }; + }); + } + + /** Disconnect from the bridge. */ + disconnect() { + this._reconnect = false; + if (this._reconnectTimer) { + clearTimeout(this._reconnectTimer); + this._reconnectTimer = null; + } + if (this._ws) { + this._ws.onclose = null; // prevent reconnect trigger + this._ws.close(); + this._ws = null; + } + this._connected = false; + this.dispatchEvent(new CustomEvent('disconnected')); + } + + // ── Send methods ───────────────────────────────────────────────── + + /** + * Send full .nisps state JSON to the VCV module. + * The module will call dataFromJson to apply it. + * @param {object|string} stateJson + */ + sendState(stateJson) { + const payload = typeof stateJson === 'string' ? JSON.parse(stateJson) : stateJson; + this._send({ type: 'state', payload }); + } + + /** + * Send just weights to the VCV module. + * @param {object} weightsObj { weights: [[[...]]] } + */ + sendWeights(weightsObj) { + const payload = typeof weightsObj === 'string' ? JSON.parse(weightsObj) : weightsObj; + this._send({ type: 'weights', payload }); + } + + /** + * Send individual parameter updates (legacy format, for synth params). + * @param {Array<[string, number]>} params e.g. [["Env_A_Att", 0.35], ...] + */ + sendParams(params) { + this._send({ type: 'params', payload: params }); + } + + /** + * Send a raw param batch in legacy format (backwards compatible). + * @param {Array<[string, number]>} batch + */ + sendParamBatch(batch) { + this._send(batch); + } + + // ── Receive handlers ───────────────────────────────────────────── + + /** + * Register a callback for output values from VCV. + * @param {function(number[]): void} callback + * @returns {function} unsubscribe function + */ + onOutputsReceived(callback) { + this._outputsCallbacks.push(callback); + return () => { + const idx = this._outputsCallbacks.indexOf(callback); + if (idx >= 0) this._outputsCallbacks.splice(idx, 1); + }; + } + + /** + * Register a callback for input values from VCV. + * @param {function(number[]): void} callback + * @returns {function} unsubscribe function + */ + onInputsReceived(callback) { + this._inputsCallbacks.push(callback); + return () => { + const idx = this._inputsCallbacks.indexOf(callback); + if (idx >= 0) this._inputsCallbacks.splice(idx, 1); + }; + } + + /** + * Register a callback for bridge info messages. + * @param {function(string): void} callback + * @returns {function} unsubscribe function + */ + onInfo(callback) { + this._infoCallbacks.push(callback); + return () => { + const idx = this._infoCallbacks.indexOf(callback); + if (idx >= 0) this._infoCallbacks.splice(idx, 1); + }; + } + + // ── Internal ───────────────────────────────────────────────────── + + _send(data) { + if (!this._ws || this._ws.readyState !== WebSocket.OPEN) return; + this._ws.send(JSON.stringify(data)); + } + + _handleMessage(raw) { + try { + const msg = JSON.parse(raw); + + switch (msg.type) { + case 'outputs': + for (const cb of this._outputsCallbacks) cb(msg.values); + this.dispatchEvent(new CustomEvent('outputs', { detail: msg.values })); + break; + + case 'inputs': + for (const cb of this._inputsCallbacks) cb(msg.values); + this.dispatchEvent(new CustomEvent('inputs', { detail: msg.values })); + break; + + case 'info': + for (const cb of this._infoCallbacks) cb(msg.message); + this.dispatchEvent(new CustomEvent('info', { detail: msg.message })); + break; + + case 'osc': + // Generic OSC message passthrough + this.dispatchEvent(new CustomEvent('osc', { detail: msg })); + break; + } + } catch { + // ignore parse errors + } + } + + _scheduleReconnect() { + if (this._reconnectTimer) return; + this._reconnectTimer = setTimeout(() => { + this._reconnectTimer = null; + if (!this._connected && this._reconnect) { + this.connect({ reconnect: true }).catch(() => { + // increase backoff, max 30s + this._reconnectDelay = Math.min(this._reconnectDelay * 1.5, 30000); + }); + } + }, this._reconnectDelay); + } +} diff --git a/playground/osc-bridge/bridge.mjs b/playground/osc-bridge/bridge.mjs index 0e9904a..2439f8e 100644 --- a/playground/osc-bridge/bridge.mjs +++ b/playground/osc-bridge/bridge.mjs @@ -1,21 +1,20 @@ #!/usr/bin/env node -// NISPS → OSC Bridge -// WebSocket server that receives parameter updates from the browser -// and forwards them as OSC messages to any OSC-capable software. +// NISPS <-> OSC Bridge (bidirectional) +// WebSocket server that bridges the browser webapp and OSC-capable software +// (VCV Rack MEMLNaut module, SuperCollider, etc.). // // Usage: -// node bridge.mjs # defaults: ws:8765, osc:127.0.0.1:57120 +// node bridge.mjs # defaults // node bridge.mjs --osc-host 192.168.1.5 # send to another machine -// node bridge.mjs --osc-port 9000 # SuperCollider on custom port +// node bridge.mjs --osc-port 9000 # target port // node bridge.mjs --ws-port 8765 # WebSocket listen port -// node bridge.mjs --osc-prefix /nisps # OSC address prefix (default: /nisps) -// node bridge.mjs --bundle # send OSC bundles instead of individual messages +// node bridge.mjs --listen-port 9001 # UDP listen port for incoming OSC +// node bridge.mjs --osc-prefix /nisps # OSC address prefix +// node bridge.mjs --bundle # send OSC bundles // -// OSC address format: -// /nisps/ -// e.g. /nisps/Env_A_Att 0.35 -// /nisps/SVF_Cutoff 0.72 +// Webapp -> Bridge -> OSC target (param updates, state, weights) +// OSC target -> Bridge -> Webapp (output values, input values) import { createSocket } from 'node:dgram'; import { WebSocketServer } from 'ws'; @@ -31,15 +30,20 @@ const hasFlag = (name) => args.includes(`--${name}`); const WS_PORT = parseInt(flag('ws-port', '8765'), 10); const OSC_HOST = flag('osc-host', '127.0.0.1'); -const OSC_PORT = parseInt(flag('osc-port', '57120'), 10); +const OSC_PORT = parseInt(flag('osc-port', '9000'), 10); const OSC_PREFIX = flag('osc-prefix', '/nisps'); +const LISTEN_PORT = parseInt(flag('listen-port', '9001'), 10); const USE_BUNDLES = hasFlag('bundle'); // ---- OSC encoding (minimal, no dependencies) ---- +function oscPadded(len) { + return len + (4 - (len % 4)) % 4; +} + function oscString(str) { const len = str.length + 1; // null terminator - const padded = len + (4 - (len % 4)) % 4; + const padded = oscPadded(len); const buf = Buffer.alloc(padded); buf.write(str, 'ascii'); return buf; @@ -59,9 +63,16 @@ function oscMessage(address, value) { ]); } +function oscMessageString(address, value) { + return Buffer.concat([ + oscString(address), + oscString(',s'), + oscString(value), + ]); +} + function oscBundle(messages) { const header = oscString('#bundle'); - // NTP timestamp: immediately (1 in upper 32 bits) const timetag = Buffer.alloc(8); timetag.writeUInt32BE(1, 0); @@ -74,12 +85,64 @@ function oscBundle(messages) { return Buffer.concat(parts); } -// ---- UDP socket ---- -const udp = createSocket('udp4'); +// ---- OSC decoding ---- + +function readOscString(buf, offset) { + let end = offset; + while (end < buf.length && buf[end] !== 0) end++; + const str = buf.toString('ascii', offset, end); + const nextOffset = offset + oscPadded(end - offset + 1); + return [str, nextOffset]; +} + +function readOscFloat(buf, offset) { + if (offset + 4 > buf.length) return [0, offset + 4]; + return [buf.readFloatBE(offset), offset + 4]; +} + +function parseOscMessage(buf) { + if (buf.length < 4) return null; + let offset = 0; + + const [address, off1] = readOscString(buf, offset); + if (!address.startsWith('/')) return null; + offset = off1; + + const [tags, off2] = readOscString(buf, offset); + if (!tags.startsWith(',')) return null; + offset = off2; + + const types = tags.slice(1); + const args = []; + + for (const t of types) { + if (t === 'f') { + const [val, off] = readOscFloat(buf, offset); + args.push(val); + offset = off; + } else if (t === 's') { + const [val, off] = readOscString(buf, offset); + args.push(val); + offset = off; + } + } + + return { address, types, args }; +} + +// ---- UDP sockets ---- + +// Outgoing: sends OSC to target +const udpSend = createSocket('udp4'); function sendOSC(address, value) { const msg = oscMessage(address, value); - udp.send(msg, OSC_PORT, OSC_HOST); + udpSend.send(msg, OSC_PORT, OSC_HOST); +} + +function sendOSCString(address, value) { + const msg = oscMessageString(address, value); + udpSend.send(msg, OSC_PORT, OSC_HOST); } function sendOSCBundle(params) { @@ -87,35 +150,96 @@ function sendOSCBundle(params) { oscMessage(`${OSC_PREFIX}/${name}`, value) ); const bundle = oscBundle(messages); - udp.send(bundle, OSC_PORT, OSC_HOST); + udpSend.send(bundle, OSC_PORT, OSC_HOST); } +// Incoming: listens for OSC from target +const udpRecv = createSocket('udp4'); +udpRecv.bind(LISTEN_PORT, '0.0.0.0'); + +udpRecv.on('message', (buf, _rinfo) => { + const msg = parseOscMessage(buf); + if (!msg) return; + + const wsMsg = { type: 'osc', address: msg.address }; + + if (msg.address === `${OSC_PREFIX}/output` || msg.address === '/nisps/output') { + wsMsg.type = 'outputs'; + wsMsg.values = msg.args.filter(a => typeof a === 'number'); + } else if (msg.address === `${OSC_PREFIX}/input` || msg.address === '/nisps/input') { + wsMsg.type = 'inputs'; + wsMsg.values = msg.args.filter(a => typeof a === 'number'); + } else { + wsMsg.args = msg.args; + } + + broadcastToWs(JSON.stringify(wsMsg)); +}); + +udpRecv.on('error', (err) => { + console.error('[udp] Listen error:', err.message); +}); + // ---- WebSocket server ---- const wss = new WebSocketServer({ port: WS_PORT }); +const wsClients = new Set(); -let clientCount = 0; +function broadcastToWs(data) { + for (const ws of wsClients) { + try { + if (ws.readyState === 1) { // OPEN + ws.send(data); + } + } catch { + // ignore + } + } +} wss.on('connection', (ws) => { - clientCount++; - console.log(`[ws] Client connected (${clientCount} total)`); + wsClients.add(ws); + console.log(`[ws] Client connected (${wsClients.size} total)`); - // Tell the browser what we're targeting ws.send(JSON.stringify({ type: 'info', - message: `OSC → ${OSC_HOST}:${OSC_PORT} (prefix: ${OSC_PREFIX})`, + message: `OSC <-> ${OSC_HOST}:${OSC_PORT} (prefix: ${OSC_PREFIX}, listen: ${LISTEN_PORT})`, })); - ws.on('message', (data) => { + ws.on('message', (raw) => { try { - // Expects: [[paramName, value], ...] - const batch = JSON.parse(data); - if (!Array.isArray(batch)) return; + const data = JSON.parse(raw); - if (USE_BUNDLES) { - sendOSCBundle(batch); - } else { - for (const [name, value] of batch) { - sendOSC(`${OSC_PREFIX}/${name}`, value); + // Structured message format: { type, payload } + if (data && typeof data === 'object' && data.type) { + switch (data.type) { + case 'state': + sendOSCString(`${OSC_PREFIX}/state`, JSON.stringify(data.payload)); + return; + case 'weights': + sendOSCString(`${OSC_PREFIX}/weights`, JSON.stringify(data.payload)); + return; + case 'params': + if (Array.isArray(data.payload)) { + if (USE_BUNDLES) { + sendOSCBundle(data.payload); + } else { + for (const [name, value] of data.payload) { + sendOSC(`${OSC_PREFIX}/${name}`, value); + } + } + } + return; + } + } + + // Legacy format: [[paramName, value], ...] + if (Array.isArray(data)) { + if (USE_BUNDLES) { + sendOSCBundle(data); + } else { + for (const [name, value] of data) { + sendOSC(`${OSC_PREFIX}/${name}`, value); + } } } } catch (e) { @@ -124,22 +248,28 @@ wss.on('connection', (ws) => { }); ws.on('close', () => { - clientCount--; - console.log(`[ws] Client disconnected (${clientCount} remaining)`); + wsClients.delete(ws); + console.log(`[ws] Client disconnected (${wsClients.size} remaining)`); }); }); console.log(` -NISPS → OSC Bridge -────────────────── - WebSocket: ws://localhost:${WS_PORT} - OSC target: ${OSC_HOST}:${OSC_PORT} - Prefix: ${OSC_PREFIX} - Mode: ${USE_BUNDLES ? 'bundles' : 'individual messages'} +NISPS <-> OSC Bridge (bidirectional) +──────────────────────────────────── + WebSocket: ws://localhost:${WS_PORT} + OSC target: ${OSC_HOST}:${OSC_PORT} + OSC listen: 0.0.0.0:${LISTEN_PORT} + Prefix: ${OSC_PREFIX} + Mode: ${USE_BUNDLES ? 'bundles' : 'individual messages'} - OSC addresses: ${OSC_PREFIX}/ - e.g. ${OSC_PREFIX}/Env_A_Att 0.35 - ${OSC_PREFIX}/SVF_Cut 0.72 + Webapp -> VCV: + params: [[name, value], ...] or { type: "params", payload: [...] } + state: { type: "state", payload: } + weights: { type: "weights", payload: } - Waiting for browser connection... + VCV -> Webapp: + /nisps/output -> { type: "outputs", values: [...] } + /nisps/input -> { type: "inputs", values: [...] } + + Waiting for connections... `); diff --git a/playground/osc-bridge/bridge.ts b/playground/osc-bridge/bridge.ts index 3ef803e..a7cf565 100644 --- a/playground/osc-bridge/bridge.ts +++ b/playground/osc-bridge/bridge.ts @@ -1,38 +1,42 @@ #!/usr/bin/env -S deno run --allow-net --unstable-net -// NISPS → OSC Bridge -// Zero-dependency bridge: WebSocket server receives parameter updates from the -// browser and forwards them as OSC messages to any OSC-capable software. +// NISPS <-> OSC Bridge (bidirectional) +// WebSocket server that bridges the browser webapp and OSC-capable software +// (VCV Rack MEMLNaut module, SuperCollider, etc.). +// +// Webapp -> Bridge -> OSC target (param updates, state, weights) +// OSC target -> Bridge -> Webapp (output values, input values) // // Run with Deno: // deno run --allow-net bridge.ts // -// Or use the compiled binary: -// ./nisps-osc-bridge -// // Options: // --osc-host 192.168.1.5 Target IP (default: 127.0.0.1) -// --osc-port 9000 Target port (default: 57120 / SuperCollider) +// --osc-port 9000 Target port (default: 9000 / VCV MEMLNaut) // --osc-prefix /my Address prefix (default: /nisps) -// --ws-port 8000 WebSocket listen port (default: 8765) +// --ws-port 8765 WebSocket listen port (default: 8765) +// --listen-port 9001 UDP port for incoming OSC (default: 9001) // --bundle Send OSC bundles instead of individual messages // // OSC address format: -// /nisps/ -// e.g. /nisps/Env_A_Att 0.35 -// /nisps/SVF_Flt_Cut 0.72 +// /nisps/ (webapp -> target) +// /nisps/state (webapp -> target: full JSON state) +// /nisps/weights (webapp -> target: weights JSON) +// /nisps/output (target -> webapp: output float array) +// /nisps/input (target -> webapp: input float array) import { parseArgs } from "jsr:@std/cli@1/parse-args"; // ---- CLI args ---- const args = parseArgs(Deno.args, { - string: ["osc-host", "osc-port", "osc-prefix", "ws-port"], + string: ["osc-host", "osc-port", "osc-prefix", "ws-port", "listen-port"], boolean: ["bundle", "help"], default: { "osc-host": "127.0.0.1", - "osc-port": "57120", + "osc-port": "9000", "osc-prefix": "/nisps", "ws-port": "8765", + "listen-port": "9001", "bundle": false, "help": false, }, @@ -40,17 +44,18 @@ const args = parseArgs(Deno.args, { if (args.help) { console.log(` -NISPS → OSC Bridge +NISPS <-> OSC Bridge (bidirectional) Usage: nisps-osc-bridge [options] Options: - --osc-host Target IP address (default: 127.0.0.1) - --osc-port Target UDP port (default: 57120) - --osc-prefix OSC address prefix (default: /nisps) - --ws-port WebSocket listen port (default: 8765) - --bundle Send OSC bundles instead of individual messages - --help Show this help + --osc-host Target IP address (default: 127.0.0.1) + --osc-port Target UDP port (default: 9000) + --osc-prefix OSC address prefix (default: /nisps) + --ws-port WebSocket listen port (default: 8765) + --listen-port UDP listen port for incoming OSC (default: 9001) + --bundle Send OSC bundles instead of individual messages + --help Show this help `); Deno.exit(0); } @@ -59,15 +64,20 @@ const WS_PORT = parseInt(args["ws-port"]); const OSC_HOST = args["osc-host"]; const OSC_PORT = parseInt(args["osc-port"]); const OSC_PREFIX = args["osc-prefix"]; +const LISTEN_PORT = parseInt(args["listen-port"]); const USE_BUNDLES = args.bundle; // ---- OSC encoding (zero dependencies) ---- +function oscPadded(len: number): number { + return len + (4 - (len % 4)) % 4; +} + function oscString(str: string): Uint8Array { const encoder = new TextEncoder(); const strBytes = encoder.encode(str); const len = strBytes.length + 1; // null terminator - const padded = len + (4 - (len % 4)) % 4; + const padded = oscPadded(len); const buf = new Uint8Array(padded); buf.set(strBytes); return buf; @@ -94,6 +104,10 @@ function oscMessage(address: string, value: number): Uint8Array { return concat(oscString(address), oscString(",f"), oscFloat(value)); } +function oscMessageString(address: string, value: string): Uint8Array { + return concat(oscString(address), oscString(",s"), oscString(value)); +} + function u32be(val: number): Uint8Array { const buf = new ArrayBuffer(4); new DataView(buf).setUint32(0, val, false); @@ -113,13 +127,74 @@ function oscBundle(messages: Uint8Array[]): Uint8Array { return concat(...parts); } -// ---- UDP socket ---- -const udp = Deno.listenDatagram({ port: 0, transport: "udp", hostname: "0.0.0.0" }); +// ---- OSC decoding ---- + +function readOscString(buf: Uint8Array, offset: number): [string, number] { + let end = offset; + while (end < buf.length && buf[end] !== 0) end++; + const decoder = new TextDecoder(); + const str = decoder.decode(buf.slice(offset, end)); + const nextOffset = offset + oscPadded(end - offset + 1); + return [str, nextOffset]; +} + +function readOscFloat(buf: Uint8Array, offset: number): [number, number] { + if (offset + 4 > buf.length) return [0, offset + 4]; + const view = new DataView(buf.buffer, buf.byteOffset + offset, 4); + return [view.getFloat32(0, false), offset + 4]; +} + +interface ParsedOscMessage { + address: string; + types: string; + args: (number | string)[]; +} + +function parseOscMessage(buf: Uint8Array): ParsedOscMessage | null { + if (buf.length < 4) return null; + let offset = 0; + + const [address, off1] = readOscString(buf, offset); + if (!address.startsWith("/")) return null; + offset = off1; + + const [tags, off2] = readOscString(buf, offset); + if (!tags.startsWith(",")) return null; + offset = off2; + + const types = tags.slice(1); + const args: (number | string)[] = []; + + for (const t of types) { + if (t === "f") { + const [val, off] = readOscFloat(buf, offset); + args.push(val); + offset = off; + } else if (t === "s") { + const [val, off] = readOscString(buf, offset); + args.push(val); + offset = off; + } + // skip unknown types + } + + return { address, types, args }; +} + +// ---- UDP sockets ---- + +// Outgoing: sends OSC to the target (VCV module) +const udpSend = Deno.listenDatagram({ port: 0, transport: "udp", hostname: "0.0.0.0" }); const oscAddr: Deno.NetAddr = { transport: "udp", hostname: OSC_HOST, port: OSC_PORT }; function sendOSC(address: string, value: number): void { const msg = oscMessage(address, value); - udp.send(msg, oscAddr); + udpSend.send(msg, oscAddr); +} + +function sendOSCString(address: string, value: string): void { + const msg = oscMessageString(address, value); + udpSend.send(msg, oscAddr); } function sendOSCBundle(params: [string, number][]): void { @@ -127,33 +202,76 @@ function sendOSCBundle(params: [string, number][]): void { oscMessage(`${OSC_PREFIX}/${name}`, value) ); const bundle = oscBundle(messages); - udp.send(bundle, oscAddr); + udpSend.send(bundle, oscAddr); } -// ---- WebSocket server (Deno built-in) ---- -let clientCount = 0; +// Incoming: listens for OSC from the target (VCV module) +const udpRecv = Deno.listenDatagram({ port: LISTEN_PORT, transport: "udp", hostname: "0.0.0.0" }); + +// ---- WebSocket server ---- +const wsClients: Set = new Set(); + +function broadcastToWs(data: string): void { + for (const ws of wsClients) { + try { + if (ws.readyState === WebSocket.OPEN) { + ws.send(data); + } + } catch { + // ignore send errors + } + } +} function handleWs(ws: WebSocket): void { - clientCount++; - console.log(`[ws] Client connected (${clientCount} total)`); + wsClients.add(ws); + console.log(`[ws] Client connected (${wsClients.size} total)`); ws.onopen = () => { ws.send(JSON.stringify({ type: "info", - message: `OSC → ${OSC_HOST}:${OSC_PORT} (prefix: ${OSC_PREFIX})`, + message: `OSC <-> ${OSC_HOST}:${OSC_PORT} (prefix: ${OSC_PREFIX}, listen: ${LISTEN_PORT})`, })); }; ws.onmessage = (e) => { try { - const batch = JSON.parse(e.data as string); - if (!Array.isArray(batch)) return; + const data = JSON.parse(e.data as string); - if (USE_BUNDLES) { - sendOSCBundle(batch); - } else { - for (const [name, value] of batch) { - sendOSC(`${OSC_PREFIX}/${name}`, value); + // New structured message format: { type, payload } + if (data && typeof data === "object" && data.type) { + switch (data.type) { + case "state": + // Send full state JSON as OSC string to /nisps/state + sendOSCString(`${OSC_PREFIX}/state`, JSON.stringify(data.payload)); + return; + case "weights": + // Send weights JSON as OSC string to /nisps/weights + sendOSCString(`${OSC_PREFIX}/weights`, JSON.stringify(data.payload)); + return; + case "params": + // Legacy batch format embedded in structured message + if (Array.isArray(data.payload)) { + if (USE_BUNDLES) { + sendOSCBundle(data.payload); + } else { + for (const [name, value] of data.payload) { + sendOSC(`${OSC_PREFIX}/${name}`, value); + } + } + } + return; + } + } + + // Legacy format: [[paramName, value], ...] + if (Array.isArray(data)) { + if (USE_BUNDLES) { + sendOSCBundle(data); + } else { + for (const [name, value] of data) { + sendOSC(`${OSC_PREFIX}/${name}`, value); + } } } } catch (err) { @@ -162,13 +280,45 @@ function handleWs(ws: WebSocket): void { }; ws.onclose = () => { - clientCount--; - console.log(`[ws] Client disconnected (${clientCount} remaining)`); + wsClients.delete(ws); + console.log(`[ws] Client disconnected (${wsClients.size} remaining)`); }; } +// ---- UDP receive loop (OSC from VCV -> relay to WebSocket clients) ---- + +async function udpReceiveLoop(): Promise { + for await (const [data, _addr] of udpRecv) { + const msg = parseOscMessage(data); + if (!msg) continue; + + // Relay parsed OSC messages to all connected WebSocket clients + const wsMsg: Record = { type: "osc", address: msg.address }; + + if (msg.address === `${OSC_PREFIX}/output` || msg.address === "/nisps/output") { + // Float array of outputs + wsMsg.type = "outputs"; + wsMsg.values = msg.args.filter((a): a is number => typeof a === "number"); + } else if (msg.address === `${OSC_PREFIX}/input` || msg.address === "/nisps/input") { + // Float array of inputs + wsMsg.type = "inputs"; + wsMsg.values = msg.args.filter((a): a is number => typeof a === "number"); + } else { + // Generic OSC message + wsMsg.args = msg.args; + } + + broadcastToWs(JSON.stringify(wsMsg)); + } +} + +// Start UDP receive loop +udpReceiveLoop().catch((err) => { + console.error("[udp] Receive loop error:", err); +}); + +// Start WebSocket server Deno.serve({ port: WS_PORT }, (req) => { - // Only accept WebSocket upgrades const upgrade = req.headers.get("upgrade") || ""; if (upgrade.toLowerCase() !== "websocket") { return new Response("NISPS OSC Bridge — connect via WebSocket", { status: 200 }); @@ -179,16 +329,22 @@ Deno.serve({ port: WS_PORT }, (req) => { }); console.log(` -NISPS → OSC Bridge -────────────────── - WebSocket: ws://localhost:${WS_PORT} - OSC target: ${OSC_HOST}:${OSC_PORT} - Prefix: ${OSC_PREFIX} - Mode: ${USE_BUNDLES ? "bundles" : "individual messages"} +NISPS <-> OSC Bridge (bidirectional) +──────────────────────────────────── + WebSocket: ws://localhost:${WS_PORT} + OSC target: ${OSC_HOST}:${OSC_PORT} + OSC listen: 0.0.0.0:${LISTEN_PORT} + Prefix: ${OSC_PREFIX} + Mode: ${USE_BUNDLES ? "bundles" : "individual messages"} - OSC addresses: ${OSC_PREFIX}/ - e.g. ${OSC_PREFIX}/Env_A_Att 0.35 - ${OSC_PREFIX}/SVF_Flt_Cut 0.72 + Webapp -> VCV: + params: [[name, value], ...] or { type: "params", payload: [...] } + state: { type: "state", payload: } + weights: { type: "weights", payload: } - Waiting for browser connection... + VCV -> Webapp: + /nisps/output -> { type: "outputs", values: [...] } + /nisps/input -> { type: "inputs", values: [...] } + + Waiting for connections... `); diff --git a/playground/osc-bridge/package-lock.json b/playground/osc-bridge/package-lock.json new file mode 100644 index 0000000..34e917e --- /dev/null +++ b/playground/osc-bridge/package-lock.json @@ -0,0 +1,36 @@ +{ + "name": "nisps-osc-bridge", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "nisps-osc-bridge", + "version": "0.1.0", + "dependencies": { + "ws": "^8.18.0" + } + }, + "node_modules/ws": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.20.0.tgz", + "integrity": "sha512-sAt8BhgNbzCtgGbt2OxmpuryO63ZoDk/sqaB/znQm94T4fCEsy/yV+7CdC1kJhOU9lboAEU7R3kquuycDoibVA==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + } + } +} diff --git a/vcv/BUILDING.md b/vcv/BUILDING.md new file mode 100644 index 0000000..fb05a97 --- /dev/null +++ b/vcv/BUILDING.md @@ -0,0 +1,68 @@ +# Building MEMLNaut VCV Plugin + +## Prerequisites + +- **VCV Rack 2 SDK** — download from https://vcvrack.com/manual/PluginDevelopmentTutorial or build from source +- **C++20 compiler** — GCC 10+, Clang 11+, or MSVC 19.29+ (required by nisps-core for `std::span` and concepts) +- **GNU Make** +- **zip** (for distribution packaging only) + +## Build Steps + +```bash +# 1. Set the SDK path (adjust to your installation) +export RACK_DIR=/path/to/Rack-SDK + +# 2. Build the plugin +cd vcv +make + +# This produces plugin.so (Linux), plugin.dylib (macOS), or plugin.dll (Windows) +``` + +The Makefile adds `-std=c++20` and includes nisps-core headers from `../nisps-core/include`. The VCV SDK's default `-std=c++11` flag is filtered out to avoid conflicts. + +## Local Installation + +```bash +# Option A: use the SDK's install target +make install +# Copies the plugin to ~/.local/share/Rack2/plugins-lin/ (or platform equivalent) + +# Option B: manual symlink (useful during development) +ln -s $(pwd) ~/.local/share/Rack2/plugins-lin/MEMLNaut +``` + +After installing, restart VCV Rack (or use the module browser refresh if available). The MEMLNaut module appears in the module browser. + +## Distribution Packaging + +```bash +# Build and package in one step +make -f Makefile.dist dist + +# Or package an already-built plugin +make -f Makefile.dist package-only + +# Output: dist/MEMLNaut-0.1.0-Linux-x86_64.zip (platform name varies) +``` + +The zip contains a `MEMLNaut/` directory with `plugin.so` (or `.dylib`/`.dll`), `plugin.json`, and `res/`. Users extract this into their VCV Rack plugins directory. + +## Cross-Compilation + +Cross-compilation is **not currently supported**. Building for each platform requires the native VCV Rack SDK and a matching C++20 toolchain. + +Options for multi-platform releases: + +- **GitHub Actions CI** — build on Linux, macOS, and Windows runners. The VCV SDK provides Docker images for consistent builds. +- **Docker** — VCV provides `ghcr.io/vcvrack/rack-plugin-toolchain` images for cross-platform builds from a Linux host. +- **Manual** — build natively on each target platform. + +The VCV Library submission process handles multi-platform builds automatically via their CI pipeline, but we are not submitting to the Library initially. + +## Troubleshooting + +- **`-std=c++11` conflicts**: The Makefile filters this out, but if you see C++20 errors, verify your `RACK_DIR` points to a v2 SDK and that your compiler supports C++20. +- **nisps-core not found**: The include path assumes nisps-core is at `../nisps-core/include` relative to the `vcv/` directory. Verify the path or adjust `-I` in the Makefile. +- **Plugin not appearing**: Check that the built `.so`/`.dylib`/`.dll` is in the correct plugins directory and that `plugin.json` is alongside it. diff --git a/vcv/Makefile.dist b/vcv/Makefile.dist new file mode 100644 index 0000000..8e01378 --- /dev/null +++ b/vcv/Makefile.dist @@ -0,0 +1,35 @@ +# Distribution packaging for MEMLNaut VCV plugin +# +# Usage: +# make -f Makefile.dist dist # Build plugin then package +# make -f Makefile.dist package-only # Package without rebuilding +# make -f Makefile.dist clean-dist # Remove dist/ directory + +PLUGIN_NAME = MEMLNaut +VERSION = 0.1.0 +PLATFORM = $(shell uname -s)-$(shell uname -m) + +# Delegate build to the main Makefile (which includes the VCV SDK plugin.mk) +all: + $(MAKE) -f Makefile + +# Build then package +dist: all package-only + +# Package the already-built plugin (no rebuild) +package-only: + @echo "Packaging $(PLUGIN_NAME) v$(VERSION) for $(PLATFORM)..." + mkdir -p dist/$(PLUGIN_NAME) + cp plugin.so dist/$(PLUGIN_NAME)/ 2>/dev/null || true + cp plugin.dylib dist/$(PLUGIN_NAME)/ 2>/dev/null || true + cp plugin.dll dist/$(PLUGIN_NAME)/ 2>/dev/null || true + cp plugin.json dist/$(PLUGIN_NAME)/ + cp -r res dist/$(PLUGIN_NAME)/ + cd dist && zip -r $(PLUGIN_NAME)-$(VERSION)-$(PLATFORM).zip $(PLUGIN_NAME) + rm -rf dist/$(PLUGIN_NAME) + @echo "Created dist/$(PLUGIN_NAME)-$(VERSION)-$(PLATFORM).zip" + +clean-dist: + rm -rf dist + +.PHONY: all dist package-only clean-dist diff --git a/vcv/NISPS-FORMAT.md b/vcv/NISPS-FORMAT.md new file mode 100644 index 0000000..e3987c4 --- /dev/null +++ b/vcv/NISPS-FORMAT.md @@ -0,0 +1,206 @@ +# .nisps File Format Specification + +Version: 1 + +## Overview + +The `.nisps` format is a JSON file that captures the complete state of a NISPS interactive ML engine: network weights, training examples, I/O configuration, and MLP architecture metadata. It enables preset sharing between the VCV Rack module and the web playground. + +## Top-Level Structure + +```json +{ + "version": 1, + "noiseLevel": 0.1, + "slewMs": 10.0, + "outputRangeUnipolar": [true, false, ...], + "inputRangeUnipolar": [true, true, ...], + "weights": [[[...], ...], ...], + "examples": { + "features": [[...], ...], + "labels": [[...], ...] + }, + "mlpConfig": { + "layers": [3, 16, 24, 16, 12], + "activations": ["relu", "relu", "relu", "sigmoid"] + }, + "params": [0.5, 0.8, ...] +} +``` + +## Field Reference + +### `version` (integer, required) + +Format version number. Currently `1`. Loaders must reject files where `version < 1`. Future versions will increment this value; loaders should accept any version they understand and reject higher versions gracefully. + +### `noiseLevel` (float, optional) + +RL exploration noise amplitude. Range: 0.0-1.0. Default: `0.1`. Controls how much random perturbation is applied to weights during reinforcement learning exploration. + +### `slewMs` (float, optional) + +Output slew rate in milliseconds. Default: `10.0`. Smooths transitions between output values to prevent clicks/artifacts. In VCV Rack this is the time constant for exponential smoothing on output voltages. + +### `outputRangeUnipolar` (boolean[], optional) + +Per-output voltage range flag. Length must equal the number of MLP outputs. When `true`, the output maps to 0-10V (unipolar); when `false`, it maps to +/-5V (bipolar). Default: all `false`. + +This field is VCV Rack-specific. The web playground ignores it but should preserve it on round-trip. + +### `inputRangeUnipolar` (boolean[], optional) + +Per-input voltage range flag. Length must equal the maximum number of inputs (8 in VCV Rack). When `true`, the input expects 0-10V; when `false`, +/-5V. Default: all `false`. + +This field is VCV Rack-specific. The web playground ignores it but should preserve it on round-trip. + +### `weights` (float[][][], required for state restore) + +MLP weights as a 3D array: `weights[layer][node][weight]`. + +**Serialization order:** +- Outer dimension: layers, from first hidden layer to output layer. Length = number of layers - 1 (i.e., number of weight matrices). +- Middle dimension: nodes within that layer. Length = `mlpConfig.layers[i+1]` (the number of nodes in the destination layer). +- Inner dimension: connection weights from source nodes. Length = `mlpConfig.layers[i]` (the number of nodes in the source layer, including bias for the first layer). + +**Bias handling:** In the VCV Rack C++ implementation, bias is stored separately from connection weights and is **not** included in this array. The bias values are not serialized. On load, biases remain at their current values (typically 0). + +In the web playground, bias is also stored separately per node. The webapp's export/import methods handle the structural difference by serializing weights without bias (matching the VCV format) and preserving bias in a separate optional field. + +**Example for a `[3, 4, 2]` network** (3 inputs, 4 hidden, 2 outputs): +```json +"weights": [ + [ // layer 0: input -> hidden + [0.1, -0.3, 0.5], // hidden node 0: 3 weights from 3 inputs + [0.2, 0.4, -0.1], // hidden node 1 + [-0.6, 0.3, 0.2], // hidden node 2 + [0.1, -0.5, 0.7] // hidden node 3 + ], + [ // layer 1: hidden -> output + [0.3, -0.2, 0.4, 0.1], // output node 0: 4 weights from 4 hidden nodes + [-0.1, 0.5, -0.3, 0.2] // output node 1 + ] +] +``` + +### `examples` (object, optional) + +Training examples as parallel arrays. + +#### `examples.features` (float[][], required if examples present) + +Input feature vectors. Each inner array has length equal to `mlpConfig.layers[0]` minus the bias term. In the VCV module, features are stored **without** the bias term appended. + +#### `examples.labels` (float[][], required if examples present) + +Output label vectors. Each inner array has length equal to `mlpConfig.layers[last]`. Must have the same outer length as `features`. + +**Example:** +```json +"examples": { + "features": [ + [0.3, 0.7], + [0.8, 0.2] + ], + "labels": [ + [0.1, 0.9, 0.5, 0.3, 0.7, 0.2, 0.8, 0.4, 0.6, 0.1, 0.5, 0.3], + [0.9, 0.1, 0.4, 0.7, 0.3, 0.8, 0.2, 0.6, 0.4, 0.9, 0.5, 0.7] + ] +} +``` + +### `mlpConfig` (object, required) + +Architecture metadata for validation on load. + +#### `mlpConfig.layers` (integer[], required) + +Layer sizes including input (with bias) and output. For the default VCV configuration with 2 inputs and 12 outputs: `[3, 16, 24, 16, 12]`. The first element includes the +1 bias node. + +#### `mlpConfig.activations` (string[], optional) + +Activation function names per weight layer. Length = `layers.length - 1`. Valid values: `"relu"`, `"sigmoid"`, `"tanh"`, `"linear"`. Default: all hidden layers use `"relu"`, output layer uses `"sigmoid"`. + +The VCV Rack module currently does not serialize this field (the architecture is fixed). The web playground includes it for forward compatibility. + +### `params` (float[], optional) + +VCV Rack module parameter values (knob positions, attenuators, etc.). Array indexed by the module's parameter enum. Present only in files saved from the VCV Rack "Save .nisps preset" menu. + +The web playground may use this field to store synth parameter values for preset portability. Consumers that don't understand the parameter layout should ignore this field. + +## Validation Rules + +Loaders should check the following on import: + +1. **Version gate**: `version` must be present and >= 1. Reject unknown future versions. +2. **Weight dimensions**: If `weights` and `mlpConfig.layers` are both present, verify: + - `weights.length === mlpConfig.layers.length - 1` + - `weights[i].length === mlpConfig.layers[i + 1]` + - `weights[i][j].length === mlpConfig.layers[i]` +3. **Example dimensions**: If `examples` is present: + - `features.length === labels.length` + - Each feature vector length should equal `mlpConfig.layers[0]` (or `mlpConfig.layers[0] - 1` if bias is excluded) + - Each label vector length should equal `mlpConfig.layers[last]` +4. **Architecture compatibility**: If the loader's MLP has a different architecture than `mlpConfig.layers`, the file cannot be loaded directly. The loader should reject or warn. +5. **Numeric validity**: All weight and example values must be finite (not NaN or Infinity). + +## Compatibility Notes + +### VCV Rack -> Web Playground + +- The VCV module uses a `[3, 16, 24, 16, 12]` architecture (2 inputs + bias, 12 outputs). +- The web playground uses a `[3, 32, 48, 64, 126]` architecture (2 inputs + bias, 126 outputs). +- Direct weight transfer between these architectures is not possible. The `mlpConfig.layers` field enables loaders to detect this mismatch and report it. +- Training examples (features/labels) are also architecture-dependent due to different output counts. + +### Bias Values + +The VCV C++ serialization does not include node bias values in the `weights` array. Bias is stored separately in `Node::m_bias` but is not written to JSON. This means bias values are reset to their pre-load state when restoring from a `.nisps` file. In practice this has minimal impact because: +- The MLP uses leaky ReLU (hidden) and sigmoid (output) activations +- Training quickly adjusts bias values +- Initial bias is typically 0 + +### Future Extensions + +New fields may be added to the top level without incrementing the version number, as long as they are optional and backward-compatible. The version number increments only for breaking changes to existing field semantics. + +## Complete Example + +A minimal but complete `.nisps` file for a `[3, 4, 2]` network (2 inputs, 4 hidden nodes, 2 outputs): + +```json +{ + "version": 1, + "noiseLevel": 0.15, + "slewMs": 10.0, + "outputRangeUnipolar": [true, false], + "inputRangeUnipolar": [false, false], + "weights": [ + [ + [0.123, -0.456, 0.789], + [-0.321, 0.654, -0.987], + [0.111, -0.222, 0.333], + [-0.444, 0.555, -0.666] + ], + [ + [0.12, -0.34, 0.56, -0.78], + [0.91, -0.23, 0.45, -0.67] + ] + ], + "examples": { + "features": [ + [0.3, 0.7], + [0.8, 0.2] + ], + "labels": [ + [0.9, 0.1], + [0.2, 0.8] + ] + }, + "mlpConfig": { + "layers": [3, 4, 2], + "activations": ["relu", "sigmoid"] + } +} +``` diff --git a/vcv/README.md b/vcv/README.md new file mode 100644 index 0000000..aa397f7 --- /dev/null +++ b/vcv/README.md @@ -0,0 +1,267 @@ +# MEMLNaut for VCV Rack + +MEMLNaut is a CV-to-CV mapper powered by a neural network that you train in real time using reinforcement learning. Patch any CV sources into its 2 inputs, connect its 12 outputs to your synth parameters, and shape the mapping by giving thumbs-up/thumbs-down feedback. The module learns your preferences, producing complex, nonlinear modulation that evolves with your taste. It does not generate sound itself -- it generates control voltages. + +## Installation + +MEMLNaut is built from source against the VCV Rack 2 SDK. + +### Requirements + +- VCV Rack 2 (Community Edition or Pro) +- VCV Rack SDK (v2.x) +- C++20 compiler (GCC 10+, Clang 12+, or MSVC 2019+) +- GNU Make + +### Build + +```bash +git clone --recursive https://github.com/MusicallyEmbodiedML/MEMLNaut-NISPS.git +cd MEMLNaut-NISPS/vcv + +# Point to your Rack SDK (or set in environment) +export RACK_DIR=/path/to/Rack-SDK + +make +make install # copies plugin to your VCV Rack plugins directory +``` + +The default `RACK_DIR` is `~/.local/share/Rack2/Rack-SDK`. If your SDK lives there, you can skip the export. + +The `nisps-core` headers (the ML engine) are included in the parent repository and referenced automatically via the Makefile. + +## Quick Start + +1. Add **MEMLNaut** from the module browser (under Controller / Utility). +2. Patch two LFOs (or any CV source) into the **X** and **Y** inputs. +3. Connect several of the 12 outputs to parameters on your synth voice -- filter cutoff, oscillator pitch, waveshape, VCA level, etc. +4. Press **RAND** to randomize the network. You should hear your synth respond as the LFOs sweep. +5. Flip the **LEARN** switch on. +6. When you hear something you like, press **+** (thumbs up). When you hear something you dislike, press **-** (thumbs down). +7. Keep exploring. The mapping will converge toward sounds you prefer. + +## Panel Controls + +### Knobs + +| Control | Description | +|---------|-------------| +| **SPREAD** | Controls weight initialization scale, RL noise amplitude, and weight decay. Low values produce extreme, polarized mappings. High values produce balanced, subtle mappings. Default: 60%. | +| **RATE** | Inference rate. Full CCW = ~170 Hz (block rate, cheapest). Noon = ~2 kHz (good for CV). Full CW = 44.1 kHz (audio rate, most expensive). | + +### Buttons + +| Control | Description | +|---------|-------------| +| **+** | Thumbs up. Captures the current input/output pair as a training example and trains the network. Requires LEARN to be enabled. | +| **-** | Thumbs down. Increases exploration noise and perturbs the network weights to try something different. Requires LEARN to be enabled. | +| **LEARN** | Toggle switch. Enables/disables RL feedback. When off, the module still runs inference -- it just ignores +/- presses. | +| **RAND** | Randomize all network weights (using current SPREAD setting). | +| **CLEAR** | Long-press (~1 second) to clear all training examples and reset the network. | + +### Inputs + +| Port | Description | +|------|-------------| +| **X** | Primary CV input. Default range: 0-10V (unipolar). | +| **Y** | Secondary CV input. Default range: 0-10V (unipolar). | +| **SPREAD CV** | CV modulation of the SPREAD knob (added to knob value, 0-10V). | +| **LEARN** | Gate input. High = enable learning. Works alongside the LEARN toggle (either enables it). | +| **+ TRIG** | Trigger input for thumbs-up. Alternative to pressing the + button. | +| **- TRIG** | Trigger input for thumbs-down. Alternative to pressing the - button. | + +### Outputs + +| Port | Description | +|------|-------------| +| **OUT 1-12** | Raw MLP outputs, each with its own attenuverter trimpot and LED. Default: 0-10V unipolar. | +| **MEAN** | Mean of the 12 raw outputs (0-10V). | +| **STD** | Standard deviation of the 12 raw outputs (0-10V). | +| **DELTA** | Rate of change across all outputs (L2 norm of frame-to-frame difference). | +| **NOVELTY** | How far the current input is from any training example. 10V when untrained, drops as you add examples near the current position. | +| **CONFIDENCE** | Inverse of novelty. 0V when untrained, rises as examples accumulate near the current input. | + +Each of the 12 raw outputs has a **trimpot attenuverter** (-100% to +100%) for scaling and inverting individual outputs without external modules. + +### LEDs + +| LED | Meaning | +|-----|---------| +| **LEARN** (green) | Lit when learning is enabled. | +| **TRAIN** (yellow) | Flashes during background training. | +| **Output LEDs** (white) | Brightness tracks each output's current level. | + +### Display + +The built-in bar graph shows all 12 output levels in real time, color-coded by output index. The top-left corner shows the current noise level (N:). "TRAIN" appears in the top-right during active training. + +## RL Workflow + +The reinforcement learning loop works like this: + +1. **Start exploring.** Patch LFOs or sequencers into X and Y. Connect outputs to interesting synth parameters. Press RAND a few times to hear different random mappings. + +2. **Enable learning.** Flip the LEARN switch on (or send a gate to the LEARN input). + +3. **Thumbs up (+)** when you like what you hear. This: + - Saves the current input position and output values as a training example + - Trains the network to reproduce this mapping + - Slightly reduces exploration noise (the network becomes more "settled") + +4. **Thumbs down (-)** when you dislike what you hear. This: + - Increases exploration noise + - Perturbs the network weights to try a different mapping + - Does NOT save any training example + +5. **Repeat.** Over time, the network learns to produce outputs you tend to like across the input space. Regions near your thumbs-up examples will be stable; distant regions remain exploratory. + +6. **Disable learning** when you are happy with the mapping. The module continues running inference with the trained network. You now have a complex, personalized CV source. + +### Tips + +- Give thumbs-up at several different input positions to teach the network about different regions of the input space. +- The network holds up to 100 examples. Oldest examples are dropped when full (FIFO). +- Use the SPREAD knob to control how wild the exploration is. Low spread = dramatic changes. High spread = subtle refinements. +- The NOVELTY and CONFIDENCE outputs are useful for self-patching: route NOVELTY to control something that signals "unexplored territory." + +## Context Menu + +Right-click the module to access these settings: + +### Output Ranges + +Toggle each output between **unipolar (0-10V)** and **bipolar (+/-5V)**. Default is unipolar. Use bipolar for parameters that expect centered modulation (e.g., FM depth, panning). + +### Input Ranges + +Toggle each input between **unipolar (0-10V)** and **bipolar (+/-5V)**. Default is unipolar. Set to bipolar if your input source produces +/-5V signals (e.g., standard LFOs). + +### Output Slew + +Smoothing time applied when network weights change (after training or perturbation). Prevents audible clicks from sudden output jumps. Options: 0, 5, 10, 20, 50, 100 ms. Default: 10 ms. + +### Presets (.nisps) + +- **Save .nisps preset...** -- Export the full module state (weights, training examples, knob positions, ranges) to a `.nisps` JSON file. +- **Load .nisps preset...** -- Import a `.nisps` file, restoring the network and all settings. + +### OSC Bridge + +- **Enable OSC server** -- Start a UDP/WebSocket OSC server for live communication with the companion web app. +- **OSC listen port** -- Choose the port (default 9000). Change this if running multiple MEMLNaut instances. + +## Presets + +MEMLNaut uses `.nisps` files for saving and sharing trained networks. + +### What gets saved + +- All network weights (the learned mapping) +- All training examples (input/output pairs from thumbs-up) +- Knob positions (SPREAD, RATE, attenuverters) +- Input/output range settings +- Noise level and slew time + +### Saving and loading + +1. Right-click the module. +2. Under "Presets (.nisps)", choose **Save** or **Load**. +3. Pick a location and filename. + +### Sharing between VCV and the web playground + +The `.nisps` format is shared with the [NISPS web playground](https://musicallyembodiedml.github.io/memlnaut/). However, the VCV module and web app use different network architectures (VCV: 12 outputs, web: 126 outputs), so weights are not directly transferable between them. Training examples and configuration metadata are preserved for reference. The `mlpConfig.layers` field in the file lets each loader detect architecture mismatches. + +### Patch save/load + +Full module state is also saved automatically with your VCV Rack patch file. You do not need to manually export `.nisps` files to preserve your work between sessions. + +## OSC Integration + +MEMLNaut can communicate with the companion web app over OSC for live, bidirectional state sync. + +### Setup + +1. Right-click the module and enable **OSC server** (default port 9000). +2. In the web playground, open the OSC connection panel and connect to `localhost:9000`. +3. The web app connects via a WebSocket-to-OSC bridge. + +### What syncs + +| OSC Address | Direction | Content | +|-------------|-----------|---------| +| `/nisps/outputs` | VCV -> Web | Current output values (~10 times/sec) | +| `/nisps/inputs` | VCV -> Web | Current input values (~10 times/sec) | +| `/nisps/weights` | Both | Full weight transfer | +| `/nisps/state` | Both | Complete state sync (weights + examples + config) | + +### Multiple instances + +Each MEMLNaut instance needs its own OSC port. Use the port selector in the context menu (9000, 9001, 9002, 8000, 7000) to avoid conflicts. + +## Technical Details + +### Network architecture + +The module uses a multi-layer perceptron (MLP) with the following default architecture: + +``` +Inputs: 2 (+ 1 bias = 3 input nodes) +Hidden: 16 -> 24 -> 16 (3 hidden layers, ReLU activation) +Output: 12 (sigmoid activation, producing values in [0, 1]) +``` + +### Inference + +The MLP runs in the VCV `process()` callback. The RATE knob controls how often inference runs, from once per audio block (~170 Hz) to every sample (44.1 kHz). Between inference steps, outputs are linearly interpolated to avoid staircase artifacts. + +### Threading + +- **Audio thread**: Reads CV inputs, runs MLP inference, writes CV outputs. Never blocks. +- **Background thread**: Handles training (thumbs-up) and weight perturbation (thumbs-down). When complete, signals the audio thread to crossfade to the new outputs. +- Each module instance has its own independent background thread and ML engine. + +### Performance + +The default network is small (~20 KB of weights). At block-rate inference, CPU usage is negligible. At audio-rate inference with all 12 outputs patched, expect moderate CPU usage comparable to a complex oscillator module. Multiple instances scale linearly. + +## Building + +### Full build commands + +```bash +cd vcv +export RACK_DIR=/path/to/Rack-SDK +make # build the plugin +make install # install to VCV plugins directory +make clean # remove build artifacts +``` + +### SDK setup + +1. Download the VCV Rack SDK from https://vcvrack.com/manual/PluginDevelopmentTutorial +2. Extract it somewhere (e.g., `~/Rack-SDK`) +3. Set `RACK_DIR` to that path, or place it at `~/.local/share/Rack2/Rack-SDK` + +### Project structure + +``` +vcv/ + Makefile # Build configuration + plugin.json # Plugin manifest (name, version, tags) + src/ + plugin.cpp # Plugin initialization + plugin.hpp # Plugin globals + MEMLNaut.cpp # Module logic, UI, serialization + osc_server.hpp # OSC bridge server + res/ + MEMLNaut.svg # Panel artwork + dep/ # Build dependencies +``` + +### Dependencies + +- **VCV Rack SDK** (v2.x) -- provides the module framework +- **nisps-core** -- header-only C++20 ML library (included in the parent repo at `../nisps-core/`) + +No external package manager dependencies are required. The nisps-core headers are referenced directly from the Makefile. diff --git a/vcv/SPEC.md b/vcv/SPEC.md index b2b3864..a33b1bd 100644 --- a/vcv/SPEC.md +++ b/vcv/SPEC.md @@ -442,6 +442,106 @@ make install # Copies to VCV plugin directory --- +## Performance Characteristics + +### Inference Cost + +The MLP has architecture `[3, 16, 24, 16, 12]` (3 = 2 inputs + 1 bias node). Each layer's nodes compute a weighted sum of all inputs from the previous layer (including bias weight), then apply an activation function. + +| Layer transition | Nodes | Weights per node | Multiply-adds | Activations | +|-----------------|-------|-----------------|---------------|-------------| +| Input (3) -> Hidden 1 (16) | 16 | 3 | 48 | 16 (ReLU) | +| Hidden 1 (16) -> Hidden 2 (24) | 24 | 17 (16 + bias) | 408 | 24 (ReLU) | +| Hidden 2 (24) -> Hidden 3 (16) | 16 | 25 (24 + bias) | 400 | 16 (ReLU) | +| Hidden 3 (16) -> Output (12) | 12 | 17 (16 + bias) | 204 | 12 (sigmoid) | +| **Total** | | | **1,060** | **68** | + +One forward pass: ~1,060 multiply-adds + 68 activation evaluations (56 ReLU, 12 sigmoid). This is trivially cheap for any modern CPU. + +### Memory + +- ~20KB per MLP instance (weights + node state for [3, 16, 24, 16, 12]) +- 2 MLP instances per module (double-buffering for thread safety): ~40KB +- Dataset: up to 100 examples, each with 2 floats (inputs) + 12 floats (labels) = 5.6KB max +- Total per module instance: ~46KB — negligible + +### Threading + +- 1 background thread per module instance for training/perturbation +- No shared thread pool (simplicity over efficiency) +- 4 module instances = 4 threads + ~184KB total memory + +### Rate Knob Range + +| Knob position | Inference rate | Period (samples at 44.1kHz) | Behavior | +|--------------|---------------|----------------------------|----------| +| Full CCW (0.0) | ~170 Hz | 256 | Once per process block. Cheapest. | +| 12 o'clock (0.5) | ~2,756 Hz | 16 | Good for CV-rate modulation. | +| Full CW (1.0) | 44,100 Hz | 1 | Every sample. Audio-rate CV. | + +Mapping is exponential: `period = 256 * (1/256)^rate`, giving perceptually linear response. + +### CPU Estimate + +At default rate (~2kHz with knob at 0.5): ~2,000 forward passes/sec. Each pass is ~1,060 multiply-adds. Total: ~2.1M multiply-adds/sec. For context, a single modern CPU core can sustain billions of multiply-adds per second. Even at audio rate (44.1kHz = ~46M multiply-adds/sec), the MLP inference is a small fraction of available compute. + +The dominant CPU cost at audio rate is not the MLP math but the per-sample overhead in `process()` (derived output computation, slew interpolation, voltage scaling). At default rate this overhead is amortized across ~16 samples. + +--- + +## VCV Rack v1 Compatibility + +### v2-Specific APIs Used + +The following VCV Rack v2 APIs are used throughout `src/MEMLNaut.cpp`: + +| API | Usage | v1 Equivalent | +|-----|-------|--------------| +| `createPanel()` | `setPanel(createPanel(asset::plugin(...)))` — loads SVG panel | `SVGPanel` + `setPanel()` manual setup | +| `configButton()` | Configures momentary button params (RAND, +, -, CLEAR) | `configParam()` with min=0, max=1, default=0 | +| `configSwitch()` | Configures LEARN toggle with string labels | `configParam()` (no label strings) | +| `configParam()` | All knob/attenuverter configuration | Same name, but v2 added display formatting args | +| `configInput()` / `configOutput()` | Port labels for tooltips | Not available in v1 (no port tooltips) | +| `createCheckMenuItem()` | Context menu toggle items (output ranges, input ranges, OSC, slew) | Manual `MenuItem` subclass with `rightText` checkmark | +| `createSubmenuItem()` | Nested submenus (slew time, OSC port) | Manual `MenuItem` subclass overriding `createChildMenu()` | +| `createMenuItem()` | Simple menu actions (save/load preset) | Manual `MenuItem` subclass overriding `onAction()` | +| `createMenuLabel()` | Section headers in context menu | `MenuLabel` direct construction | +| `LedDisplay` | Base class for NanoVG bar graph display widget | `LedDisplayWidget` (similar but slightly different API) | +| `drawLayer()` | Layer-based drawing (layer 1 = foreground) | `draw()` only (no layer separation) | +| `createModel()` | Template model registration | Same syntax (available since late v1) | +| `string::f()` | Printf-style string formatting | `string::f()` (available in v1) | +| `dsp::BooleanTrigger` | Edge detection on boolean params | Available in v1 | +| `dsp::SchmittTrigger` | Edge detection on CV triggers | Available in v1 | +| `json_*` (jansson) | State serialization | Same (jansson is used in both v1 and v2) | +| `osdialog_*` | Native file dialogs for preset save/load | Same (osdialog bundled in both) | +| Standard widgets: `RoundBlackKnob`, `VCVButton`, `CKSS`, `Trimpot`, `PJ301MPort`, `SmallLight` | Panel components | All available in v1 (VCVButton may need renaming to `BefacoButton` or similar) | + +### APIs Without Direct v1 Equivalents + +- **`configInput()` / `configOutput()`**: v1 has no port tooltip system. These calls would simply be removed — ports would work but lack hover labels. +- **`configButton()` / `configSwitch()`**: Would revert to `configParam()` calls. Lose the semantic distinction and string labels. +- **`createCheckMenuItem()` / `createSubmenuItem()`**: The most labor-intensive change. Each menu item in v1 requires a dedicated `struct` subclass of `MenuItem` with `onAction()` and `rightText` overrides. Our context menu has ~20+ items. A v1 port would need ~20 small structs or a templated helper. + +### Estimated Effort + +- **Mechanical changes** (configButton -> configParam, remove configInput/configOutput labels): ~1 hour +- **Context menu rewrite** (createCheckMenuItem/createSubmenuItem -> manual MenuItem subclasses): ~3-4 hours. This is the bulk of the work — approximately 20 menu items each needing a small struct. +- **Display widget** (LedDisplay/drawLayer -> LedDisplayWidget/draw): ~30 minutes +- **Widget naming** (VCVButton and similar may have different names): ~30 minutes of research + find/replace +- **Testing**: ~2 hours (v1 has different SDK build, need separate build environment) +- **Total estimate**: ~8 hours of focused work + +### Recommendation + +**Ship v2-only.** Rationale: + +1. VCV Rack v1 userbase is declining — v2 Community Edition is free, removing the cost barrier that kept some users on v1. +2. The 8-hour port effort is not large, but maintaining two codepaths adds ongoing cost for every future feature. +3. nisps-core requires C++20 (`std::span`, concepts). The v1 SDK toolchain may not support C++20 on all platforms, which could require additional workarounds or feature-gating. +4. If v1 demand materializes, the port is straightforward and can be done as a one-time effort. + +--- + ## Open Questions (to resolve during implementation) 1. **MLP hidden layer sizing**: [16, 24, 16] is a guess. May need tuning based on real-world training performance with 12 outputs. diff --git a/vcv/res/MEMLNaut-expander.svg b/vcv/res/MEMLNaut-expander.svg new file mode 100644 index 0000000..0bf6259 --- /dev/null +++ b/vcv/res/MEMLNaut-expander.svg @@ -0,0 +1,74 @@ + + + + + + + + + + + + + MEMLNaut + EXP + + + + + + LINK + + + + + + + + + + IN + + + IN 3 + + + IN 4 + + + + IN 5 + + + IN 6 + + + + IN 7 + + + IN 8 + + + + + + reserved + + + + NISPS v0.1 + + + + + + + diff --git a/vcv/res/MEMLNaut-wide.svg b/vcv/res/MEMLNaut-wide.svg new file mode 100644 index 0000000..b095c9c --- /dev/null +++ b/vcv/res/MEMLNaut-wide.svg @@ -0,0 +1,217 @@ + + + + + + + + + + + + + MEMLNaut + Neural Interactive Shaping of Parameter Spaces + + + + + + + DISPLAY + input space / output map + + + + + + + SPREAD + + + + CV + + + RATE + + + + + + + + + + + + + − + + + LEARN + + + + RAND + + + CLR + + + + + + + + + IN + + + X + + + Y + + + IN 3 + + + IN 4 + + + IN 5 + + + IN 6 + + + + IN 7 + + + IN 8 + + + LEARN + + + +TRIG + + + −TRIG + + + + + + + + + + + OUT + + + + + + 1 + + + + + 2 + + + + + 3 + + + + + 4 + + + + + + 5 + + + + + 6 + + + + + 7 + + + + + 8 + + + + + + 9 + + + + + 10 + + + + + 11 + + + + + 12 + + + + + + + + + + MN + + + SD + + + ΔT + + + NV + + + CF + + + + + NISPS v0.1 + + + + + + + diff --git a/vcv/res/MEMLNaut.svg b/vcv/res/MEMLNaut.svg index 2eaf451..86cf8eb 100644 --- a/vcv/res/MEMLNaut.svg +++ b/vcv/res/MEMLNaut.svg @@ -1,5 +1,195 @@ - - - MEMLNaut - NISPS v0.1 + + + + + + + + + + + + + MEMLNaut + + + + + + + DISPLAY + + + + + + + SPREAD + + + + CV + + + RATE + + + + + + + + + + + + + − + + + LEARN + + + + + RAND + + + CLR + + + + + + + + + + + IN + + X + + + Y + + + LEARN + + + +TRIG + + + −TRIG + + + + + + + + + OUT + + + 1 + + + + + 2 + + + + + 3 + + + + + + 4 + + + + + 5 + + + + + 6 + + + + + + 7 + + + + + 8 + + + + + 9 + + + + + + 10 + + + + + 11 + + + + + 12 + + + + + + + + + + + MN + + + SD + + + ΔT + + + NV + + + CF + + + + + NISPS v0.1 + + + + + + diff --git a/vcv/src/MEMLNaut.cpp b/vcv/src/MEMLNaut.cpp index 0fe0139..4ab6e12 100644 --- a/vcv/src/MEMLNaut.cpp +++ b/vcv/src/MEMLNaut.cpp @@ -1,4 +1,5 @@ #include "plugin.hpp" +#include "osc_server.hpp" #include #include #include @@ -81,6 +82,68 @@ struct MEMLNaut : Module { float cachedConfidence = 0.f; // default: no confidence (0V) float lastInputs[MAX_ML_INPUTS] = {}; + // ── OSC bridge ──────────────────────────────────────────────────── + std::unique_ptr oscServer; + bool oscEnabled = false; + int oscPort = 9000; + int oscSendCounter = 0; + static constexpr int OSC_SEND_INTERVAL_SAMPLES = 4410; // ~100ms at 44.1kHz + + void startOsc() { + if (oscServer && oscServer->isRunning()) return; + oscServer = std::make_unique(); + + // When we receive full state JSON, apply it + oscServer->onState([this](const std::string& json) { + json_error_t error; + json_t* root = json_loads(json.c_str(), 0, &error); + if (!root) return; + dataFromJson(root); + json_decref(root); + }); + + // When we receive weights JSON, apply just the weights + oscServer->onWeights([this](const std::string& json) { + json_error_t error; + json_t* root = json_loads(json.c_str(), 0, &error); + if (!root) return; + json_t* jWeights = json_object_get(root, "weights"); + if (jWeights && json_is_array(jWeights)) { + nisps::MLP::mlp_weights weights; + for (size_t li = 0; li < json_array_size(jWeights); li++) { + json_t* jLayer = json_array_get(jWeights, li); + std::vector> layer; + for (size_t ni = 0; ni < json_array_size(jLayer); ni++) { + json_t* jNode = json_array_get(jLayer, ni); + std::vector node; + for (size_t wi = 0; wi < json_array_size(jNode); wi++) { + node.push_back(json_real_value(json_array_get(jNode, wi))); + } + layer.push_back(node); + } + weights.push_back(layer); + } + iml.set_weights(weights); + } + json_decref(root); + }); + + if (!oscServer->start(oscPort)) { + oscServer.reset(); + oscEnabled = false; + } else { + oscEnabled = true; + } + } + + void stopOsc() { + if (oscServer) { + oscServer->stop(); + oscServer.reset(); + } + oscEnabled = false; + } + // ── Triggers ────────────────────────────────────────────────────── dsp::BooleanTrigger randTrigger; dsp::BooleanTrigger thumbsUpTrigger; @@ -154,6 +217,7 @@ struct MEMLNaut : Module { } ~MEMLNaut() { + stopOsc(); shouldStop.store(true); jobCv.notify_one(); if (workerThread.joinable()) { @@ -432,6 +496,16 @@ struct MEMLNaut : Module { // Novelty + Confidence (computed on background thread, cached) outputs[OUTPUT_NOVELTY].setVoltage(cachedNovelty); outputs[OUTPUT_CONFIDENCE].setVoltage(cachedConfidence); + + // ── OSC send (throttled to ~100ms) ─────────────────────────── + if (oscServer && oscServer->isRunning()) { + oscSendCounter++; + if (oscSendCounter >= OSC_SEND_INTERVAL_SAMPLES) { + oscSendCounter = 0; + oscServer->sendOutputs(slewOutputs, NUM_ML_OUTPUTS); + oscServer->sendInputs(lastInputs, NUM_ML_INPUTS); + } + } } // ── Serialization ───────────────────────────────────────────────── @@ -440,6 +514,8 @@ struct MEMLNaut : Module { json_object_set_new(root, "version", json_integer(1)); json_object_set_new(root, "noiseLevel", json_real(noiseLevel)); json_object_set_new(root, "slewMs", json_real(slewMs)); + json_object_set_new(root, "oscEnabled", json_boolean(oscEnabled)); + json_object_set_new(root, "oscPort", json_integer(oscPort)); // Output ranges json_t* outRanges = json_array(); @@ -511,6 +587,16 @@ struct MEMLNaut : Module { if ((j = json_object_get(root, "slewMs"))) slewMs = json_real_value(j); + // OSC + if ((j = json_object_get(root, "oscPort"))) + oscPort = json_integer_value(j); + if ((j = json_object_get(root, "oscEnabled"))) { + if (json_boolean_value(j)) + startOsc(); + else + stopOsc(); + } + // Output ranges json_t* outRanges = json_object_get(root, "outputRangeUnipolar"); if (outRanges) { @@ -800,6 +886,37 @@ struct MEMLNautWidget : ModuleWidget { json_decref(root); })); + + // ── OSC bridge ─────────────────────────────────────────────── + menu->addChild(new MenuSeparator); + menu->addChild(createMenuLabel("OSC Bridge")); + + menu->addChild(createCheckMenuItem( + string::f("Enable OSC server (port %d)", module->oscPort), "", + [=]() { return module->oscEnabled; }, + [=]() { + if (module->oscEnabled) { + module->stopOsc(); + } else { + module->startOsc(); + } + } + )); + + menu->addChild(createSubmenuItem("OSC listen port", string::f("%d", module->oscPort), [=](Menu* childMenu) { + for (int port : {9000, 9001, 9002, 8000, 7000}) { + childMenu->addChild(createCheckMenuItem( + string::f("%d", port), "", + [=]() { return module->oscPort == port; }, + [=]() { + bool wasRunning = module->oscEnabled; + if (wasRunning) module->stopOsc(); + module->oscPort = port; + if (wasRunning) module->startOsc(); + } + )); + } + })); } }; diff --git a/vcv/src/osc_server.hpp b/vcv/src/osc_server.hpp new file mode 100644 index 0000000..6112494 --- /dev/null +++ b/vcv/src/osc_server.hpp @@ -0,0 +1,356 @@ +// osc_server.hpp — Minimal OSC server for MEMLNaut VCV module +// Lightweight header-only implementation using raw UDP sockets. +// Supports a tiny subset of OSC: float and string arguments only. +// No bundles, no timetags, no pattern matching. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 + #ifndef WIN32_LEAN_AND_MEAN + #define WIN32_LEAN_AND_MEAN + #endif + #include + #include + #pragma comment(lib, "ws2_32.lib") + using socket_t = SOCKET; + static constexpr socket_t INVALID_SOCK = INVALID_SOCKET; + #define CLOSE_SOCKET(s) closesocket(s) +#else + #include + #include + #include + #include + #include + #include + using socket_t = int; + static constexpr socket_t INVALID_SOCK = -1; + #define CLOSE_SOCKET(s) ::close(s) +#endif + +namespace memlnaut { + +// ── OSC encoding/decoding helpers ──────────────────────────────────── + +namespace osc { + +// Pad length to next multiple of 4 +inline size_t padded(size_t len) { + return (len + 3) & ~size_t(3); +} + +// Read a null-terminated, 4-byte-padded OSC string from buf at offset. +// Returns the string and advances offset past the padded data. +inline std::string readString(const uint8_t* buf, size_t bufLen, size_t& offset) { + if (offset >= bufLen) return ""; + const char* start = reinterpret_cast(buf + offset); + size_t maxLen = bufLen - offset; + size_t slen = strnlen(start, maxLen); + std::string s(start, slen); + offset += padded(slen + 1); // +1 for null terminator + return s; +} + +// Read a big-endian float32 from buf at offset. +inline float readFloat(const uint8_t* buf, size_t bufLen, size_t& offset) { + if (offset + 4 > bufLen) return 0.f; + uint32_t raw = (uint32_t(buf[offset]) << 24) + | (uint32_t(buf[offset + 1]) << 16) + | (uint32_t(buf[offset + 2]) << 8) + | uint32_t(buf[offset + 3]); + offset += 4; + float f; + std::memcpy(&f, &raw, 4); + return f; +} + +// Read a big-endian int32 from buf at offset. +inline int32_t readInt32(const uint8_t* buf, size_t bufLen, size_t& offset) { + if (offset + 4 > bufLen) return 0; + int32_t val = (int32_t(buf[offset]) << 24) + | (int32_t(buf[offset + 1]) << 16) + | (int32_t(buf[offset + 2]) << 8) + | int32_t(buf[offset + 3]); + offset += 4; + return val; +} + +// Write a null-terminated, 4-byte-padded string into out. +inline void writeString(std::vector& out, const std::string& s) { + size_t start = out.size(); + size_t total = padded(s.size() + 1); + out.resize(start + total, 0); + std::memcpy(out.data() + start, s.c_str(), s.size()); + // Remaining bytes are already zero (null terminator + padding) +} + +// Write a big-endian float32 into out. +inline void writeFloat(std::vector& out, float f) { + uint32_t raw; + std::memcpy(&raw, &f, 4); + out.push_back(uint8_t(raw >> 24)); + out.push_back(uint8_t(raw >> 16)); + out.push_back(uint8_t(raw >> 8)); + out.push_back(uint8_t(raw)); +} + +// Build an OSC message with a float array payload. +// Address: e.g. "/nisps/output" +// Type tag string: ",fff..." (one 'f' per float) +inline std::vector messageFloats(const std::string& address, + const float* values, size_t count) { + std::vector msg; + writeString(msg, address); + // Type tag string: "," + count 'f' chars + std::string tags = ","; + for (size_t i = 0; i < count; i++) tags += 'f'; + writeString(msg, tags); + for (size_t i = 0; i < count; i++) { + writeFloat(msg, values[i]); + } + return msg; +} + +// Build an OSC message with a single string payload. +inline std::vector messageString(const std::string& address, + const std::string& value) { + std::vector msg; + writeString(msg, address); + writeString(msg, ",s"); + writeString(msg, value); + return msg; +} + +} // namespace osc + +// ── OscServer ──────────────────────────────────────────────────────── + +class OscServer { +public: + using StringCallback = std::function; + + OscServer() = default; + ~OscServer() { stop(); } + + // Non-copyable + OscServer(const OscServer&) = delete; + OscServer& operator=(const OscServer&) = delete; + + // Register handlers before starting + void onState(StringCallback cb) { stateCallback_ = std::move(cb); } + void onWeights(StringCallback cb) { weightsCallback_ = std::move(cb); } + + // Set the target address for sending (where the webapp bridge listens). + // Default: 127.0.0.1:9001 + void setSendTarget(const std::string& host, int port) { + std::lock_guard lock(sendMutex_); + sendHost_ = host; + sendPort_ = port; + sendTargetDirty_ = true; + } + + bool start(int listenPort = 9000) { + if (running_.load()) return true; + +#ifdef _WIN32 + WSADATA wsaData; + if (WSAStartup(MAKEWORD(2, 2), &wsaData) != 0) return false; + wsaInit_ = true; +#endif + + recvSock_ = socket(AF_INET, SOCK_DGRAM, 0); + if (recvSock_ == INVALID_SOCK) return false; + + // Allow address reuse + int opt = 1; +#ifdef _WIN32 + setsockopt(recvSock_, SOL_SOCKET, SO_REUSEADDR, + reinterpret_cast(&opt), sizeof(opt)); +#else + setsockopt(recvSock_, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt)); +#endif + + sockaddr_in addr{}; + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = INADDR_ANY; + addr.sin_port = htons(static_cast(listenPort)); + + if (bind(recvSock_, reinterpret_cast(&addr), sizeof(addr)) != 0) { + CLOSE_SOCKET(recvSock_); + recvSock_ = INVALID_SOCK; + return false; + } + + // Set receive timeout so the thread can check shouldStop +#ifdef _WIN32 + DWORD timeout = 200; // ms + setsockopt(recvSock_, SOL_SOCKET, SO_RCVTIMEO, + reinterpret_cast(&timeout), sizeof(timeout)); +#else + struct timeval tv; + tv.tv_sec = 0; + tv.tv_usec = 200000; // 200ms + setsockopt(recvSock_, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); +#endif + + // Create send socket + sendSock_ = socket(AF_INET, SOCK_DGRAM, 0); + if (sendSock_ == INVALID_SOCK) { + CLOSE_SOCKET(recvSock_); + recvSock_ = INVALID_SOCK; + return false; + } + + listenPort_ = listenPort; + shouldStop_.store(false); + running_.store(true); + recvThread_ = std::thread(&OscServer::recvLoop, this); + return true; + } + + void stop() { + if (!running_.load()) return; + shouldStop_.store(true); + if (recvThread_.joinable()) recvThread_.join(); + if (recvSock_ != INVALID_SOCK) { CLOSE_SOCKET(recvSock_); recvSock_ = INVALID_SOCK; } + if (sendSock_ != INVALID_SOCK) { CLOSE_SOCKET(sendSock_); sendSock_ = INVALID_SOCK; } + running_.store(false); + +#ifdef _WIN32 + if (wsaInit_) { WSACleanup(); wsaInit_ = false; } +#endif + } + + bool isRunning() const { return running_.load(); } + int getPort() const { return listenPort_; } + + // ── Send methods ───────────────────────────────────────────────── + + // Send current output values (12 floats) to the webapp bridge + void sendOutputs(const float* values, size_t count) { + auto msg = osc::messageFloats("/nisps/output", values, count); + sendPacket(msg); + } + + // Send current input values (2 floats) + void sendInputs(const float* values, size_t count) { + auto msg = osc::messageFloats("/nisps/input", values, count); + sendPacket(msg); + } + +private: + void recvLoop() { + uint8_t buf[65536]; + while (!shouldStop_.load()) { + sockaddr_in from{}; + socklen_t fromLen = sizeof(from); + ssize_t n = recvfrom(recvSock_, reinterpret_cast(buf), sizeof(buf), 0, + reinterpret_cast(&from), &fromLen); + if (n <= 0) continue; // timeout or error + + // Remember sender for replies + { + std::lock_guard lock(sendMutex_); + lastSender_ = from; + hasLastSender_ = true; + } + + parseMessage(buf, static_cast(n)); + } + } + + void parseMessage(const uint8_t* buf, size_t len) { + size_t offset = 0; + + // Read address + std::string address = osc::readString(buf, len, offset); + if (address.empty() || address[0] != '/') return; + + // Read type tag string + std::string tags = osc::readString(buf, len, offset); + if (tags.empty() || tags[0] != ',') return; + + // Dispatch based on address + if (address == "/nisps/state") { + // Expect a single string argument + if (tags.size() >= 2 && tags[1] == 's') { + std::string payload = osc::readString(buf, len, offset); + if (stateCallback_) stateCallback_(payload); + } + } else if (address == "/nisps/weights") { + // Expect a single string argument (JSON) + if (tags.size() >= 2 && tags[1] == 's') { + std::string payload = osc::readString(buf, len, offset); + if (weightsCallback_) weightsCallback_(payload); + } + } + // Unknown addresses are silently ignored + } + + void sendPacket(const std::vector& packet) { + if (sendSock_ == INVALID_SOCK) return; + + std::lock_guard lock(sendMutex_); + + sockaddr_in target{}; + target.sin_family = AF_INET; + + if (sendTargetDirty_ || !hasExplicitTarget_) { + // Use explicit target if set, otherwise reply to last sender + if (!sendHost_.empty()) { + inet_pton(AF_INET, sendHost_.c_str(), &target.sin_addr); + target.sin_port = htons(static_cast(sendPort_)); + hasExplicitTarget_ = true; + sendTargetDirty_ = false; + sendAddr_ = target; + } else if (hasLastSender_) { + target = lastSender_; + target.sin_port = htons(static_cast(sendPort_)); + sendAddr_ = target; + } else { + return; // nobody to send to + } + } + + sendto(sendSock_, reinterpret_cast(packet.data()), packet.size(), 0, + reinterpret_cast(&sendAddr_), sizeof(sendAddr_)); + } + + // Sockets + socket_t recvSock_ = INVALID_SOCK; + socket_t sendSock_ = INVALID_SOCK; + int listenPort_ = 9000; + + // Thread control + std::thread recvThread_; + std::atomic shouldStop_{false}; + std::atomic running_{false}; + + // Callbacks + StringCallback stateCallback_; + StringCallback weightsCallback_; + + // Send target + std::mutex sendMutex_; + std::string sendHost_ = "127.0.0.1"; + int sendPort_ = 9001; + bool sendTargetDirty_ = false; + bool hasExplicitTarget_ = false; + bool hasLastSender_ = false; + sockaddr_in lastSender_{}; + sockaddr_in sendAddr_{}; + +#ifdef _WIN32 + bool wsaInit_ = false; +#endif +}; + +} // namespace memlnaut diff --git a/vcv/test/smoke_test b/vcv/test/smoke_test new file mode 100755 index 0000000..2244f04 Binary files /dev/null and b/vcv/test/smoke_test differ