diff --git a/manifold/.gitignore b/manifold/.gitignore new file mode 100644 index 0000000..f1b33ac --- /dev/null +++ b/manifold/.gitignore @@ -0,0 +1,6 @@ +# Build output (regenerated by `bun run build` / the deploy webhook) +dist/ + +# Test artifacts +test-results/ +playwright-report/ diff --git a/manifold/index.html b/manifold/index.html new file mode 100644 index 0000000..f7fc26f --- /dev/null +++ b/manifold/index.html @@ -0,0 +1,12 @@ + + + + + + Manifold + + +
+ + + diff --git a/manifold/osc-bridge/bridge.mjs b/manifold/osc-bridge/bridge.mjs new file mode 100644 index 0000000..2439f8e --- /dev/null +++ b/manifold/osc-bridge/bridge.mjs @@ -0,0 +1,275 @@ +#!/usr/bin/env node + +// NISPS <-> OSC Bridge (bidirectional) +// WebSocket server that bridges the browser webapp and OSC-capable software +// (VCV Rack MEMLNaut module, SuperCollider, etc.). +// +// Usage: +// node bridge.mjs # defaults +// node bridge.mjs --osc-host 192.168.1.5 # send to another machine +// node bridge.mjs --osc-port 9000 # target port +// node bridge.mjs --ws-port 8765 # WebSocket listen port +// node bridge.mjs --listen-port 9001 # UDP listen port for incoming OSC +// node bridge.mjs --osc-prefix /nisps # OSC address prefix +// node bridge.mjs --bundle # send OSC bundles +// +// Webapp -> Bridge -> OSC target (param updates, state, weights) +// OSC target -> Bridge -> Webapp (output values, input values) + +import { createSocket } from 'node:dgram'; +import { WebSocketServer } from 'ws'; + +// ---- CLI args ---- +const args = process.argv.slice(2); +function flag(name, fallback) { + const idx = args.indexOf(`--${name}`); + if (idx === -1) return fallback; + return args[idx + 1] ?? fallback; +} +const hasFlag = (name) => args.includes(`--${name}`); + +const WS_PORT = parseInt(flag('ws-port', '8765'), 10); +const OSC_HOST = flag('osc-host', '127.0.0.1'); +const OSC_PORT = parseInt(flag('osc-port', '9000'), 10); +const OSC_PREFIX = flag('osc-prefix', '/nisps'); +const LISTEN_PORT = parseInt(flag('listen-port', '9001'), 10); +const USE_BUNDLES = hasFlag('bundle'); + +// ---- OSC encoding (minimal, no dependencies) ---- + +function oscPadded(len) { + return len + (4 - (len % 4)) % 4; +} + +function oscString(str) { + const len = str.length + 1; // null terminator + const padded = oscPadded(len); + const buf = Buffer.alloc(padded); + buf.write(str, 'ascii'); + return buf; +} + +function oscFloat(val) { + const buf = Buffer.alloc(4); + buf.writeFloatBE(val, 0); + return buf; +} + +function oscMessage(address, value) { + return Buffer.concat([ + oscString(address), + oscString(',f'), + oscFloat(value), + ]); +} + +function oscMessageString(address, value) { + return Buffer.concat([ + oscString(address), + oscString(',s'), + oscString(value), + ]); +} + +function oscBundle(messages) { + const header = oscString('#bundle'); + const timetag = Buffer.alloc(8); + timetag.writeUInt32BE(1, 0); + + const parts = [header, timetag]; + for (const msg of messages) { + const size = Buffer.alloc(4); + size.writeUInt32BE(msg.length, 0); + parts.push(size, msg); + } + return Buffer.concat(parts); +} + +// ---- OSC decoding ---- + +function readOscString(buf, offset) { + let end = offset; + while (end < buf.length && buf[end] !== 0) end++; + const str = buf.toString('ascii', offset, end); + const nextOffset = offset + oscPadded(end - offset + 1); + return [str, nextOffset]; +} + +function readOscFloat(buf, offset) { + if (offset + 4 > buf.length) return [0, offset + 4]; + return [buf.readFloatBE(offset), offset + 4]; +} + +function parseOscMessage(buf) { + if (buf.length < 4) return null; + let offset = 0; + + const [address, off1] = readOscString(buf, offset); + if (!address.startsWith('/')) return null; + offset = off1; + + const [tags, off2] = readOscString(buf, offset); + if (!tags.startsWith(',')) return null; + offset = off2; + + const types = tags.slice(1); + const args = []; + + for (const t of types) { + if (t === 'f') { + const [val, off] = readOscFloat(buf, offset); + args.push(val); + offset = off; + } else if (t === 's') { + const [val, off] = readOscString(buf, offset); + args.push(val); + offset = off; + } + } + + return { address, types, args }; +} + +// ---- UDP sockets ---- + +// Outgoing: sends OSC to target +const udpSend = createSocket('udp4'); + +function sendOSC(address, value) { + const msg = oscMessage(address, value); + udpSend.send(msg, OSC_PORT, OSC_HOST); +} + +function sendOSCString(address, value) { + const msg = oscMessageString(address, value); + udpSend.send(msg, OSC_PORT, OSC_HOST); +} + +function sendOSCBundle(params) { + const messages = params.map(([name, value]) => + oscMessage(`${OSC_PREFIX}/${name}`, value) + ); + const bundle = oscBundle(messages); + udpSend.send(bundle, OSC_PORT, OSC_HOST); +} + +// Incoming: listens for OSC from target +const udpRecv = createSocket('udp4'); +udpRecv.bind(LISTEN_PORT, '0.0.0.0'); + +udpRecv.on('message', (buf, _rinfo) => { + const msg = parseOscMessage(buf); + if (!msg) return; + + const wsMsg = { type: 'osc', address: msg.address }; + + if (msg.address === `${OSC_PREFIX}/output` || msg.address === '/nisps/output') { + wsMsg.type = 'outputs'; + wsMsg.values = msg.args.filter(a => typeof a === 'number'); + } else if (msg.address === `${OSC_PREFIX}/input` || msg.address === '/nisps/input') { + wsMsg.type = 'inputs'; + wsMsg.values = msg.args.filter(a => typeof a === 'number'); + } else { + wsMsg.args = msg.args; + } + + broadcastToWs(JSON.stringify(wsMsg)); +}); + +udpRecv.on('error', (err) => { + console.error('[udp] Listen error:', err.message); +}); + +// ---- WebSocket server ---- +const wss = new WebSocketServer({ port: WS_PORT }); +const wsClients = new Set(); + +function broadcastToWs(data) { + for (const ws of wsClients) { + try { + if (ws.readyState === 1) { // OPEN + ws.send(data); + } + } catch { + // ignore + } + } +} + +wss.on('connection', (ws) => { + wsClients.add(ws); + console.log(`[ws] Client connected (${wsClients.size} total)`); + + ws.send(JSON.stringify({ + type: 'info', + message: `OSC <-> ${OSC_HOST}:${OSC_PORT} (prefix: ${OSC_PREFIX}, listen: ${LISTEN_PORT})`, + })); + + ws.on('message', (raw) => { + try { + const data = JSON.parse(raw); + + // Structured message format: { type, payload } + if (data && typeof data === 'object' && data.type) { + switch (data.type) { + case 'state': + sendOSCString(`${OSC_PREFIX}/state`, JSON.stringify(data.payload)); + return; + case 'weights': + sendOSCString(`${OSC_PREFIX}/weights`, JSON.stringify(data.payload)); + return; + case 'params': + if (Array.isArray(data.payload)) { + if (USE_BUNDLES) { + sendOSCBundle(data.payload); + } else { + for (const [name, value] of data.payload) { + sendOSC(`${OSC_PREFIX}/${name}`, value); + } + } + } + return; + } + } + + // Legacy format: [[paramName, value], ...] + if (Array.isArray(data)) { + if (USE_BUNDLES) { + sendOSCBundle(data); + } else { + for (const [name, value] of data) { + sendOSC(`${OSC_PREFIX}/${name}`, value); + } + } + } + } catch (e) { + console.error('[ws] Bad message:', e.message); + } + }); + + ws.on('close', () => { + wsClients.delete(ws); + console.log(`[ws] Client disconnected (${wsClients.size} remaining)`); + }); +}); + +console.log(` +NISPS <-> OSC Bridge (bidirectional) +──────────────────────────────────── + WebSocket: ws://localhost:${WS_PORT} + OSC target: ${OSC_HOST}:${OSC_PORT} + OSC listen: 0.0.0.0:${LISTEN_PORT} + Prefix: ${OSC_PREFIX} + Mode: ${USE_BUNDLES ? 'bundles' : 'individual messages'} + + Webapp -> VCV: + params: [[name, value], ...] or { type: "params", payload: [...] } + state: { type: "state", payload: } + weights: { type: "weights", payload: } + + VCV -> Webapp: + /nisps/output -> { type: "outputs", values: [...] } + /nisps/input -> { type: "inputs", values: [...] } + + Waiting for connections... +`); diff --git a/manifold/osc-bridge/bridge.ts b/manifold/osc-bridge/bridge.ts new file mode 100644 index 0000000..a7cf565 --- /dev/null +++ b/manifold/osc-bridge/bridge.ts @@ -0,0 +1,350 @@ +#!/usr/bin/env -S deno run --allow-net --unstable-net + +// NISPS <-> OSC Bridge (bidirectional) +// WebSocket server that bridges the browser webapp and OSC-capable software +// (VCV Rack MEMLNaut module, SuperCollider, etc.). +// +// Webapp -> Bridge -> OSC target (param updates, state, weights) +// OSC target -> Bridge -> Webapp (output values, input values) +// +// Run with Deno: +// deno run --allow-net bridge.ts +// +// Options: +// --osc-host 192.168.1.5 Target IP (default: 127.0.0.1) +// --osc-port 9000 Target port (default: 9000 / VCV MEMLNaut) +// --osc-prefix /my Address prefix (default: /nisps) +// --ws-port 8765 WebSocket listen port (default: 8765) +// --listen-port 9001 UDP port for incoming OSC (default: 9001) +// --bundle Send OSC bundles instead of individual messages +// +// OSC address format: +// /nisps/ (webapp -> target) +// /nisps/state (webapp -> target: full JSON state) +// /nisps/weights (webapp -> target: weights JSON) +// /nisps/output (target -> webapp: output float array) +// /nisps/input (target -> webapp: input float array) + +import { parseArgs } from "jsr:@std/cli@1/parse-args"; + +// ---- CLI args ---- +const args = parseArgs(Deno.args, { + string: ["osc-host", "osc-port", "osc-prefix", "ws-port", "listen-port"], + boolean: ["bundle", "help"], + default: { + "osc-host": "127.0.0.1", + "osc-port": "9000", + "osc-prefix": "/nisps", + "ws-port": "8765", + "listen-port": "9001", + "bundle": false, + "help": false, + }, +}); + +if (args.help) { + console.log(` +NISPS <-> OSC Bridge (bidirectional) + +Usage: nisps-osc-bridge [options] + +Options: + --osc-host Target IP address (default: 127.0.0.1) + --osc-port Target UDP port (default: 9000) + --osc-prefix OSC address prefix (default: /nisps) + --ws-port WebSocket listen port (default: 8765) + --listen-port UDP listen port for incoming OSC (default: 9001) + --bundle Send OSC bundles instead of individual messages + --help Show this help +`); + Deno.exit(0); +} + +const WS_PORT = parseInt(args["ws-port"]); +const OSC_HOST = args["osc-host"]; +const OSC_PORT = parseInt(args["osc-port"]); +const OSC_PREFIX = args["osc-prefix"]; +const LISTEN_PORT = parseInt(args["listen-port"]); +const USE_BUNDLES = args.bundle; + +// ---- OSC encoding (zero dependencies) ---- + +function oscPadded(len: number): number { + return len + (4 - (len % 4)) % 4; +} + +function oscString(str: string): Uint8Array { + const encoder = new TextEncoder(); + const strBytes = encoder.encode(str); + const len = strBytes.length + 1; // null terminator + const padded = oscPadded(len); + const buf = new Uint8Array(padded); + buf.set(strBytes); + return buf; +} + +function oscFloat(val: number): Uint8Array { + const buf = new ArrayBuffer(4); + new DataView(buf).setFloat32(0, val, false); // big-endian + return new Uint8Array(buf); +} + +function concat(...arrays: Uint8Array[]): Uint8Array { + const total = arrays.reduce((s, a) => s + a.length, 0); + const result = new Uint8Array(total); + let offset = 0; + for (const a of arrays) { + result.set(a, offset); + offset += a.length; + } + return result; +} + +function oscMessage(address: string, value: number): Uint8Array { + return concat(oscString(address), oscString(",f"), oscFloat(value)); +} + +function oscMessageString(address: string, value: string): Uint8Array { + return concat(oscString(address), oscString(",s"), oscString(value)); +} + +function u32be(val: number): Uint8Array { + const buf = new ArrayBuffer(4); + new DataView(buf).setUint32(0, val, false); + return new Uint8Array(buf); +} + +function oscBundle(messages: Uint8Array[]): Uint8Array { + const header = oscString("#bundle"); + // NTP timestamp: immediately (1 in upper 32 bits) + const timetag = new Uint8Array(8); + new DataView(timetag.buffer).setUint32(0, 1, false); + + const parts: Uint8Array[] = [header, timetag]; + for (const msg of messages) { + parts.push(u32be(msg.length), msg); + } + return concat(...parts); +} + +// ---- OSC decoding ---- + +function readOscString(buf: Uint8Array, offset: number): [string, number] { + let end = offset; + while (end < buf.length && buf[end] !== 0) end++; + const decoder = new TextDecoder(); + const str = decoder.decode(buf.slice(offset, end)); + const nextOffset = offset + oscPadded(end - offset + 1); + return [str, nextOffset]; +} + +function readOscFloat(buf: Uint8Array, offset: number): [number, number] { + if (offset + 4 > buf.length) return [0, offset + 4]; + const view = new DataView(buf.buffer, buf.byteOffset + offset, 4); + return [view.getFloat32(0, false), offset + 4]; +} + +interface ParsedOscMessage { + address: string; + types: string; + args: (number | string)[]; +} + +function parseOscMessage(buf: Uint8Array): ParsedOscMessage | null { + if (buf.length < 4) return null; + let offset = 0; + + const [address, off1] = readOscString(buf, offset); + if (!address.startsWith("/")) return null; + offset = off1; + + const [tags, off2] = readOscString(buf, offset); + if (!tags.startsWith(",")) return null; + offset = off2; + + const types = tags.slice(1); + const args: (number | string)[] = []; + + for (const t of types) { + if (t === "f") { + const [val, off] = readOscFloat(buf, offset); + args.push(val); + offset = off; + } else if (t === "s") { + const [val, off] = readOscString(buf, offset); + args.push(val); + offset = off; + } + // skip unknown types + } + + return { address, types, args }; +} + +// ---- UDP sockets ---- + +// Outgoing: sends OSC to the target (VCV module) +const udpSend = Deno.listenDatagram({ port: 0, transport: "udp", hostname: "0.0.0.0" }); +const oscAddr: Deno.NetAddr = { transport: "udp", hostname: OSC_HOST, port: OSC_PORT }; + +function sendOSC(address: string, value: number): void { + const msg = oscMessage(address, value); + udpSend.send(msg, oscAddr); +} + +function sendOSCString(address: string, value: string): void { + const msg = oscMessageString(address, value); + udpSend.send(msg, oscAddr); +} + +function sendOSCBundle(params: [string, number][]): void { + const messages = params.map(([name, value]) => + oscMessage(`${OSC_PREFIX}/${name}`, value) + ); + const bundle = oscBundle(messages); + udpSend.send(bundle, oscAddr); +} + +// Incoming: listens for OSC from the target (VCV module) +const udpRecv = Deno.listenDatagram({ port: LISTEN_PORT, transport: "udp", hostname: "0.0.0.0" }); + +// ---- WebSocket server ---- +const wsClients: Set = new Set(); + +function broadcastToWs(data: string): void { + for (const ws of wsClients) { + try { + if (ws.readyState === WebSocket.OPEN) { + ws.send(data); + } + } catch { + // ignore send errors + } + } +} + +function handleWs(ws: WebSocket): void { + wsClients.add(ws); + console.log(`[ws] Client connected (${wsClients.size} total)`); + + ws.onopen = () => { + ws.send(JSON.stringify({ + type: "info", + message: `OSC <-> ${OSC_HOST}:${OSC_PORT} (prefix: ${OSC_PREFIX}, listen: ${LISTEN_PORT})`, + })); + }; + + ws.onmessage = (e) => { + try { + const data = JSON.parse(e.data as string); + + // New structured message format: { type, payload } + if (data && typeof data === "object" && data.type) { + switch (data.type) { + case "state": + // Send full state JSON as OSC string to /nisps/state + sendOSCString(`${OSC_PREFIX}/state`, JSON.stringify(data.payload)); + return; + case "weights": + // Send weights JSON as OSC string to /nisps/weights + sendOSCString(`${OSC_PREFIX}/weights`, JSON.stringify(data.payload)); + return; + case "params": + // Legacy batch format embedded in structured message + if (Array.isArray(data.payload)) { + if (USE_BUNDLES) { + sendOSCBundle(data.payload); + } else { + for (const [name, value] of data.payload) { + sendOSC(`${OSC_PREFIX}/${name}`, value); + } + } + } + return; + } + } + + // Legacy format: [[paramName, value], ...] + if (Array.isArray(data)) { + if (USE_BUNDLES) { + sendOSCBundle(data); + } else { + for (const [name, value] of data) { + sendOSC(`${OSC_PREFIX}/${name}`, value); + } + } + } + } catch (err) { + console.error("[ws] Bad message:", (err as Error).message); + } + }; + + ws.onclose = () => { + wsClients.delete(ws); + console.log(`[ws] Client disconnected (${wsClients.size} remaining)`); + }; +} + +// ---- UDP receive loop (OSC from VCV -> relay to WebSocket clients) ---- + +async function udpReceiveLoop(): Promise { + for await (const [data, _addr] of udpRecv) { + const msg = parseOscMessage(data); + if (!msg) continue; + + // Relay parsed OSC messages to all connected WebSocket clients + const wsMsg: Record = { type: "osc", address: msg.address }; + + if (msg.address === `${OSC_PREFIX}/output` || msg.address === "/nisps/output") { + // Float array of outputs + wsMsg.type = "outputs"; + wsMsg.values = msg.args.filter((a): a is number => typeof a === "number"); + } else if (msg.address === `${OSC_PREFIX}/input` || msg.address === "/nisps/input") { + // Float array of inputs + wsMsg.type = "inputs"; + wsMsg.values = msg.args.filter((a): a is number => typeof a === "number"); + } else { + // Generic OSC message + wsMsg.args = msg.args; + } + + broadcastToWs(JSON.stringify(wsMsg)); + } +} + +// Start UDP receive loop +udpReceiveLoop().catch((err) => { + console.error("[udp] Receive loop error:", err); +}); + +// Start WebSocket server +Deno.serve({ port: WS_PORT }, (req) => { + const upgrade = req.headers.get("upgrade") || ""; + if (upgrade.toLowerCase() !== "websocket") { + return new Response("NISPS OSC Bridge — connect via WebSocket", { status: 200 }); + } + const { socket, response } = Deno.upgradeWebSocket(req); + handleWs(socket); + return response; +}); + +console.log(` +NISPS <-> OSC Bridge (bidirectional) +──────────────────────────────────── + WebSocket: ws://localhost:${WS_PORT} + OSC target: ${OSC_HOST}:${OSC_PORT} + OSC listen: 0.0.0.0:${LISTEN_PORT} + Prefix: ${OSC_PREFIX} + Mode: ${USE_BUNDLES ? "bundles" : "individual messages"} + + Webapp -> VCV: + params: [[name, value], ...] or { type: "params", payload: [...] } + state: { type: "state", payload: } + weights: { type: "weights", payload: } + + VCV -> Webapp: + /nisps/output -> { type: "outputs", values: [...] } + /nisps/input -> { type: "inputs", values: [...] } + + Waiting for connections... +`); diff --git a/manifold/osc-bridge/compile.sh b/manifold/osc-bridge/compile.sh new file mode 100755 index 0000000..3c37b97 --- /dev/null +++ b/manifold/osc-bridge/compile.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Compile NISPS OSC Bridge for all platforms +# Requires: deno 2.x +# Outputs go to dist/ +# +# Note: macOS cross-compilation from Linux has a known Deno bug. +# macOS binaries must be built on macOS (or via GitHub Actions CI). + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +DIST="$SCRIPT_DIR/dist" +SRC="$SCRIPT_DIR/bridge.ts" +NAME="nisps-osc-bridge" + +mkdir -p "$DIST" + +# Detect host OS for cross-compile compatibility +HOST_OS="$(uname -s)" + +TARGETS=( + "x86_64-unknown-linux-gnu:linux-x86_64" + "aarch64-unknown-linux-gnu:linux-arm64" + "x86_64-apple-darwin:macos-x86_64" + "aarch64-apple-darwin:macos-arm64" + "x86_64-pc-windows-msvc:windows-x86_64" +) + +FAILED=() + +for entry in "${TARGETS[@]}"; do + target="${entry%%:*}" + suffix="${entry##*:}" + + outname="$NAME-$suffix" + if [[ "$target" == *windows* ]]; then + outname="$outname.exe" + fi + + echo "Compiling $outname ($target)..." + if deno compile \ + --allow-net \ + --unstable-net \ + --target "$target" \ + --output "$DIST/$outname" \ + "$SRC" 2>&1; then + echo " OK" + else + echo " FAILED (skipping — cross-compile to this target may not work on $HOST_OS)" + FAILED+=("$outname") + fi + echo "" +done + +echo "Binaries in $DIST/:" +ls -lh "$DIST/" 2>/dev/null || echo " (none)" + +if [[ ${#FAILED[@]} -gt 0 ]]; then + echo "" + echo "Failed targets: ${FAILED[*]}" + echo "These may need to be built natively or via CI." +fi diff --git a/manifold/package.json b/manifold/package.json new file mode 100644 index 0000000..4218bec --- /dev/null +++ b/manifold/package.json @@ -0,0 +1,28 @@ +{ + "name": "manifold", + "private": true, + "version": "0.0.0", + "type": "module", + "description": "Manifold — convertible-mode React front-end on the real NISPS ML+audio engine", + "scripts": { + "dev": "vite", + "build": "tsc --noEmit && vite build", + "preview": "vite preview --port 4273", + "typecheck": "tsc --noEmit", + "test:e2e": "playwright test", + "test:e2e:headed": "playwright test --headed" + }, + "dependencies": { + "react": "^18.3.1", + "react-dom": "^18.3.1" + }, + "devDependencies": { + "@playwright/test": "^1.48.0", + "@types/node": "^22.7.0", + "@types/react": "^18.3.11", + "@types/react-dom": "^18.3.1", + "@vitejs/plugin-react": "^4.3.2", + "typescript": "^5.5.4", + "vite": "^5.4.10" + } +} diff --git a/manifold/playwright.config.ts b/manifold/playwright.config.ts new file mode 100644 index 0000000..2119261 --- /dev/null +++ b/manifold/playwright.config.ts @@ -0,0 +1,26 @@ +import { defineConfig } from '@playwright/test'; + +/** + * Playwright config for the Manifold app. The webServer runs `vite preview` + * against the production build (dist/), which honours the COOP/COEP headers the + * AudioWorklet + WASM bridge need. Run `bun run build` first. + */ +export default defineConfig({ + testDir: './tests/e2e', + timeout: 30_000, + expect: { timeout: 10_000 }, + use: { + baseURL: 'http://localhost:4273', + headless: true, + ignoreHTTPSErrors: true, + }, + webServer: { + command: 'bun run preview', + cwd: '.', + url: 'http://localhost:4273', + reuseExistingServer: !process.env.CI, + timeout: 30_000, + }, + projects: [{ name: 'chromium', use: { browserName: 'chromium' } }], + reporter: [['list']], +}); diff --git a/manifold/public/nisps.js b/manifold/public/nisps.js new file mode 100644 index 0000000..74c07fe --- /dev/null +++ b/manifold/public/nisps.js @@ -0,0 +1,19 @@ + +var createNispsModule = (() => { + var _scriptName = typeof document != 'undefined' ? document.currentScript?.src : undefined; + + return ( +function(moduleArg = {}) { + var moduleRtn; + +var Module=moduleArg;var readyPromiseResolve,readyPromiseReject;var readyPromise=new Promise((resolve,reject)=>{readyPromiseResolve=resolve;readyPromiseReject=reject});var ENVIRONMENT_IS_WEB=typeof window=="object";var ENVIRONMENT_IS_WORKER=typeof importScripts=="function";var ENVIRONMENT_IS_NODE=typeof process=="object"&&typeof process.versions=="object"&&typeof process.versions.node=="string"&&process.type!="renderer";var moduleOverrides=Object.assign({},Module);var arguments_=[];var thisProgram="./this.program";var scriptDirectory="";function locateFile(path){if(Module["locateFile"]){return Module["locateFile"](path,scriptDirectory)}return scriptDirectory+path}var readAsync,readBinary;if(ENVIRONMENT_IS_WEB||ENVIRONMENT_IS_WORKER){if(ENVIRONMENT_IS_WORKER){scriptDirectory=self.location.href}else if(typeof document!="undefined"&&document.currentScript){scriptDirectory=document.currentScript.src}if(_scriptName){scriptDirectory=_scriptName}if(scriptDirectory.startsWith("blob:")){scriptDirectory=""}else{scriptDirectory=scriptDirectory.substr(0,scriptDirectory.replace(/[?#].*/,"").lastIndexOf("/")+1)}{if(ENVIRONMENT_IS_WORKER){readBinary=url=>{var xhr=new XMLHttpRequest;xhr.open("GET",url,false);xhr.responseType="arraybuffer";xhr.send(null);return new Uint8Array(xhr.response)}}readAsync=url=>fetch(url,{credentials:"same-origin"}).then(response=>{if(response.ok){return response.arrayBuffer()}return Promise.reject(new Error(response.status+" : "+response.url))})}}else{}var out=Module["print"]||console.log.bind(console);var err=Module["printErr"]||console.error.bind(console);Object.assign(Module,moduleOverrides);moduleOverrides=null;if(Module["arguments"])arguments_=Module["arguments"];if(Module["thisProgram"])thisProgram=Module["thisProgram"];var wasmBinary=Module["wasmBinary"];var wasmMemory;var ABORT=false;var HEAP8,HEAPU8,HEAP16,HEAPU16,HEAP32,HEAPU32,HEAPF32,HEAPF64;function updateMemoryViews(){var b=wasmMemory.buffer;Module["HEAP8"]=HEAP8=new Int8Array(b);Module["HEAP16"]=HEAP16=new Int16Array(b);Module["HEAPU8"]=HEAPU8=new Uint8Array(b);Module["HEAPU16"]=HEAPU16=new Uint16Array(b);Module["HEAP32"]=HEAP32=new Int32Array(b);Module["HEAPU32"]=HEAPU32=new Uint32Array(b);Module["HEAPF32"]=HEAPF32=new Float32Array(b);Module["HEAPF64"]=HEAPF64=new Float64Array(b)}var __ATPRERUN__=[];var __ATINIT__=[];var __ATPOSTRUN__=[];var runtimeInitialized=false;function preRun(){var preRuns=Module["preRun"];if(preRuns){if(typeof preRuns=="function")preRuns=[preRuns];preRuns.forEach(addOnPreRun)}callRuntimeCallbacks(__ATPRERUN__)}function initRuntime(){runtimeInitialized=true;callRuntimeCallbacks(__ATINIT__)}function postRun(){var postRuns=Module["postRun"];if(postRuns){if(typeof postRuns=="function")postRuns=[postRuns];postRuns.forEach(addOnPostRun)}callRuntimeCallbacks(__ATPOSTRUN__)}function addOnPreRun(cb){__ATPRERUN__.unshift(cb)}function addOnInit(cb){__ATINIT__.unshift(cb)}function addOnPostRun(cb){__ATPOSTRUN__.unshift(cb)}var runDependencies=0;var runDependencyWatcher=null;var dependenciesFulfilled=null;function addRunDependency(id){runDependencies++;Module["monitorRunDependencies"]?.(runDependencies)}function removeRunDependency(id){runDependencies--;Module["monitorRunDependencies"]?.(runDependencies);if(runDependencies==0){if(runDependencyWatcher!==null){clearInterval(runDependencyWatcher);runDependencyWatcher=null}if(dependenciesFulfilled){var callback=dependenciesFulfilled;dependenciesFulfilled=null;callback()}}}function abort(what){Module["onAbort"]?.(what);what="Aborted("+what+")";err(what);ABORT=true;what+=". Build with -sASSERTIONS for more info.";var e=new WebAssembly.RuntimeError(what);readyPromiseReject(e);throw e}var dataURIPrefix="data:application/octet-stream;base64,";var isDataURI=filename=>filename.startsWith(dataURIPrefix);function findWasmBinary(){var f="nisps.wasm";if(!isDataURI(f)){return locateFile(f)}return f}var wasmBinaryFile;function getBinarySync(file){if(file==wasmBinaryFile&&wasmBinary){return new Uint8Array(wasmBinary)}if(readBinary){return readBinary(file)}throw"both async and sync fetching of the wasm failed"}function getBinaryPromise(binaryFile){if(!wasmBinary){return readAsync(binaryFile).then(response=>new Uint8Array(response),()=>getBinarySync(binaryFile))}return Promise.resolve().then(()=>getBinarySync(binaryFile))}function instantiateArrayBuffer(binaryFile,imports,receiver){return getBinaryPromise(binaryFile).then(binary=>WebAssembly.instantiate(binary,imports)).then(receiver,reason=>{err(`failed to asynchronously prepare wasm: ${reason}`);abort(reason)})}function instantiateAsync(binary,binaryFile,imports,callback){if(!binary&&typeof WebAssembly.instantiateStreaming=="function"&&!isDataURI(binaryFile)&&typeof fetch=="function"){return fetch(binaryFile,{credentials:"same-origin"}).then(response=>{var result=WebAssembly.instantiateStreaming(response,imports);return result.then(callback,function(reason){err(`wasm streaming compile failed: ${reason}`);err("falling back to ArrayBuffer instantiation");return instantiateArrayBuffer(binaryFile,imports,callback)})})}return instantiateArrayBuffer(binaryFile,imports,callback)}function getWasmImports(){return{a:wasmImports}}function createWasm(){var info=getWasmImports();function receiveInstance(instance,module){wasmExports=instance.exports;wasmMemory=wasmExports["d"];updateMemoryViews();addOnInit(wasmExports["e"]);removeRunDependency("wasm-instantiate");return wasmExports}addRunDependency("wasm-instantiate");function receiveInstantiationResult(result){receiveInstance(result["instance"])}if(Module["instantiateWasm"]){try{return Module["instantiateWasm"](info,receiveInstance)}catch(e){err(`Module.instantiateWasm callback failed with error: ${e}`);readyPromiseReject(e)}}wasmBinaryFile??=findWasmBinary();instantiateAsync(wasmBinary,wasmBinaryFile,info,receiveInstantiationResult).catch(readyPromiseReject);return{}}var callRuntimeCallbacks=callbacks=>{callbacks.forEach(f=>f(Module))};var noExitRuntime=Module["noExitRuntime"]||true;var stackRestore=val=>__emscripten_stack_restore(val);var stackSave=()=>_emscripten_stack_get_current();var __abort_js=()=>{abort("")};var __emscripten_memcpy_js=(dest,src,num)=>HEAPU8.copyWithin(dest,src,src+num);var getHeapMax=()=>2147483648;var alignMemory=(size,alignment)=>Math.ceil(size/alignment)*alignment;var growMemory=size=>{var b=wasmMemory.buffer;var pages=(size-b.byteLength+65535)/65536|0;try{wasmMemory.grow(pages);updateMemoryViews();return 1}catch(e){}};var _emscripten_resize_heap=requestedSize=>{var oldSize=HEAPU8.length;requestedSize>>>=0;var maxHeapSize=getHeapMax();if(requestedSize>maxHeapSize){return false}for(var cutDown=1;cutDown<=4;cutDown*=2){var overGrownHeapSize=oldSize*(1+.2/cutDown);overGrownHeapSize=Math.min(overGrownHeapSize,requestedSize+100663296);var newSize=Math.min(maxHeapSize,alignMemory(Math.max(requestedSize,overGrownHeapSize),65536));var replacement=growMemory(newSize);if(replacement){return true}}return false};var getCFunc=ident=>{var func=Module["_"+ident];return func};var writeArrayToMemory=(array,buffer)=>{HEAP8.set(array,buffer)};var lengthBytesUTF8=str=>{var len=0;for(var i=0;i=55296&&c<=57343){len+=4;++i}else{len+=3}}return len};var stringToUTF8Array=(str,heap,outIdx,maxBytesToWrite)=>{if(!(maxBytesToWrite>0))return 0;var startIdx=outIdx;var endIdx=outIdx+maxBytesToWrite-1;for(var i=0;i=55296&&u<=57343){var u1=str.charCodeAt(++i);u=65536+((u&1023)<<10)|u1&1023}if(u<=127){if(outIdx>=endIdx)break;heap[outIdx++]=u}else if(u<=2047){if(outIdx+1>=endIdx)break;heap[outIdx++]=192|u>>6;heap[outIdx++]=128|u&63}else if(u<=65535){if(outIdx+2>=endIdx)break;heap[outIdx++]=224|u>>12;heap[outIdx++]=128|u>>6&63;heap[outIdx++]=128|u&63}else{if(outIdx+3>=endIdx)break;heap[outIdx++]=240|u>>18;heap[outIdx++]=128|u>>12&63;heap[outIdx++]=128|u>>6&63;heap[outIdx++]=128|u&63}}heap[outIdx]=0;return outIdx-startIdx};var stringToUTF8=(str,outPtr,maxBytesToWrite)=>stringToUTF8Array(str,HEAPU8,outPtr,maxBytesToWrite);var stackAlloc=sz=>__emscripten_stack_alloc(sz);var stringToUTF8OnStack=str=>{var size=lengthBytesUTF8(str)+1;var ret=stackAlloc(size);stringToUTF8(str,ret,size);return ret};var UTF8Decoder=typeof TextDecoder!="undefined"?new TextDecoder:undefined;var UTF8ArrayToString=(heapOrArray,idx=0,maxBytesToRead=NaN)=>{var endIdx=idx+maxBytesToRead;var endPtr=idx;while(heapOrArray[endPtr]&&!(endPtr>=endIdx))++endPtr;if(endPtr-idx>16&&heapOrArray.buffer&&UTF8Decoder){return UTF8Decoder.decode(heapOrArray.subarray(idx,endPtr))}var str="";while(idx>10,56320|ch&1023)}}return str};var UTF8ToString=(ptr,maxBytesToRead)=>ptr?UTF8ArrayToString(HEAPU8,ptr,maxBytesToRead):"";var ccall=(ident,returnType,argTypes,args,opts)=>{var toC={string:str=>{var ret=0;if(str!==null&&str!==undefined&&str!==0){ret=stringToUTF8OnStack(str)}return ret},array:arr=>{var ret=stackAlloc(arr.length);writeArrayToMemory(arr,ret);return ret}};function convertReturnValue(ret){if(returnType==="string"){return UTF8ToString(ret)}if(returnType==="boolean")return Boolean(ret);return ret}var func=getCFunc(ident);var cArgs=[];var stack=0;if(args){for(var i=0;i{var numericArgs=!argTypes||argTypes.every(type=>type==="number"||type==="boolean");var numericRet=returnType!=="string";if(numericRet&&numericArgs&&!opts){return getCFunc(ident)}return(...args)=>ccall(ident,returnType,argTypes,args,opts)};var wasmImports={a:__abort_js,c:__emscripten_memcpy_js,b:_emscripten_resize_heap};var wasmExports=createWasm();var ___wasm_call_ctors=()=>(___wasm_call_ctors=wasmExports["e"])();var _nisps_ml_create=Module["_nisps_ml_create"]=(a0,a1,a2,a3,a4)=>(_nisps_ml_create=Module["_nisps_ml_create"]=wasmExports["f"])(a0,a1,a2,a3,a4);var _nisps_ml_destroy=Module["_nisps_ml_destroy"]=a0=>(_nisps_ml_destroy=Module["_nisps_ml_destroy"]=wasmExports["g"])(a0);var _nisps_ml_set_input=Module["_nisps_ml_set_input"]=(a0,a1,a2)=>(_nisps_ml_set_input=Module["_nisps_ml_set_input"]=wasmExports["h"])(a0,a1,a2);var _nisps_ml_process=Module["_nisps_ml_process"]=a0=>(_nisps_ml_process=Module["_nisps_ml_process"]=wasmExports["i"])(a0);var _nisps_ml_outputs=Module["_nisps_ml_outputs"]=a0=>(_nisps_ml_outputs=Module["_nisps_ml_outputs"]=wasmExports["j"])(a0);var _nisps_ml_infer_batch=Module["_nisps_ml_infer_batch"]=(a0,a1,a2,a3)=>(_nisps_ml_infer_batch=Module["_nisps_ml_infer_batch"]=wasmExports["k"])(a0,a1,a2,a3);var _nisps_ml_add_example=Module["_nisps_ml_add_example"]=(a0,a1,a2)=>(_nisps_ml_add_example=Module["_nisps_ml_add_example"]=wasmExports["l"])(a0,a1,a2);var _nisps_ml_train=Module["_nisps_ml_train"]=(a0,a1,a2,a3,a4)=>(_nisps_ml_train=Module["_nisps_ml_train"]=wasmExports["m"])(a0,a1,a2,a3,a4);var _nisps_ml_eval_loss=Module["_nisps_ml_eval_loss"]=a0=>(_nisps_ml_eval_loss=Module["_nisps_ml_eval_loss"]=wasmExports["n"])(a0);var _nisps_ml_weight_count=Module["_nisps_ml_weight_count"]=a0=>(_nisps_ml_weight_count=Module["_nisps_ml_weight_count"]=wasmExports["o"])(a0);var _nisps_ml_get_weights=Module["_nisps_ml_get_weights"]=(a0,a1)=>(_nisps_ml_get_weights=Module["_nisps_ml_get_weights"]=wasmExports["p"])(a0,a1);var _nisps_ml_set_weights=Module["_nisps_ml_set_weights"]=(a0,a1)=>(_nisps_ml_set_weights=Module["_nisps_ml_set_weights"]=wasmExports["q"])(a0,a1);var _nisps_ml_draw_weights=Module["_nisps_ml_draw_weights"]=(a0,a1)=>(_nisps_ml_draw_weights=Module["_nisps_ml_draw_weights"]=wasmExports["r"])(a0,a1);var _nisps_ml_move_weights=Module["_nisps_ml_move_weights"]=(a0,a1,a2,a3)=>(_nisps_ml_move_weights=Module["_nisps_ml_move_weights"]=wasmExports["s"])(a0,a1,a2,a3);var _nisps_ml_feedback_set_mode=Module["_nisps_ml_feedback_set_mode"]=(a0,a1)=>(_nisps_ml_feedback_set_mode=Module["_nisps_ml_feedback_set_mode"]=wasmExports["t"])(a0,a1);var _nisps_ml_feedback_get_mode=Module["_nisps_ml_feedback_get_mode"]=a0=>(_nisps_ml_feedback_get_mode=Module["_nisps_ml_feedback_get_mode"]=wasmExports["u"])(a0);var _nisps_ml_feedback_exploring=Module["_nisps_ml_feedback_exploring"]=a0=>(_nisps_ml_feedback_exploring=Module["_nisps_ml_feedback_exploring"]=wasmExports["v"])(a0);var _nisps_ml_feedback_learning_paused=Module["_nisps_ml_feedback_learning_paused"]=a0=>(_nisps_ml_feedback_learning_paused=Module["_nisps_ml_feedback_learning_paused"]=wasmExports["w"])(a0);var _nisps_ml_feedback_set_focus=Module["_nisps_ml_feedback_set_focus"]=(a0,a1,a2)=>(_nisps_ml_feedback_set_focus=Module["_nisps_ml_feedback_set_focus"]=wasmExports["x"])(a0,a1,a2);var _nisps_ml_feedback_down=Module["_nisps_ml_feedback_down"]=(a0,a1,a2,a3,a4)=>(_nisps_ml_feedback_down=Module["_nisps_ml_feedback_down"]=wasmExports["y"])(a0,a1,a2,a3,a4);var _nisps_ml_feedback_up=Module["_nisps_ml_feedback_up"]=a0=>(_nisps_ml_feedback_up=Module["_nisps_ml_feedback_up"]=wasmExports["z"])(a0);var _nisps_ml_feedback_drag=Module["_nisps_ml_feedback_drag"]=a0=>(_nisps_ml_feedback_drag=Module["_nisps_ml_feedback_drag"]=wasmExports["A"])(a0);var _nisps_ml_feedback_static_output=Module["_nisps_ml_feedback_static_output"]=(a0,a1)=>(_nisps_ml_feedback_static_output=Module["_nisps_ml_feedback_static_output"]=wasmExports["B"])(a0,a1);var _nisps_ml_get_layer_stats=Module["_nisps_ml_get_layer_stats"]=(a0,a1)=>(_nisps_ml_get_layer_stats=Module["_nisps_ml_get_layer_stats"]=wasmExports["C"])(a0,a1);var _nisps_ml_example_count=Module["_nisps_ml_example_count"]=a0=>(_nisps_ml_example_count=Module["_nisps_ml_example_count"]=wasmExports["D"])(a0);var _nisps_ml_clear_examples=Module["_nisps_ml_clear_examples"]=a0=>(_nisps_ml_clear_examples=Module["_nisps_ml_clear_examples"]=wasmExports["E"])(a0);var _nisps_ml_reset=Module["_nisps_ml_reset"]=a0=>(_nisps_ml_reset=Module["_nisps_ml_reset"]=wasmExports["F"])(a0);var _nisps_ml_describe=Module["_nisps_ml_describe"]=a0=>(_nisps_ml_describe=Module["_nisps_ml_describe"]=wasmExports["G"])(a0);var _nisps_engine_create=Module["_nisps_engine_create"]=(a0,a1)=>(_nisps_engine_create=Module["_nisps_engine_create"]=wasmExports["H"])(a0,a1);var _nisps_engine_destroy=Module["_nisps_engine_destroy"]=a0=>(_nisps_engine_destroy=Module["_nisps_engine_destroy"]=wasmExports["I"])(a0);var _nisps_engine_set_params=Module["_nisps_engine_set_params"]=(a0,a1,a2)=>(_nisps_engine_set_params=Module["_nisps_engine_set_params"]=wasmExports["J"])(a0,a1,a2);var _nisps_engine_process_block=Module["_nisps_engine_process_block"]=(a0,a1,a2,a3,a4,a5)=>(_nisps_engine_process_block=Module["_nisps_engine_process_block"]=wasmExports["K"])(a0,a1,a2,a3,a4,a5);var _malloc=Module["_malloc"]=a0=>(_malloc=Module["_malloc"]=wasmExports["M"])(a0);var _free=Module["_free"]=a0=>(_free=Module["_free"]=wasmExports["N"])(a0);var __emscripten_stack_restore=a0=>(__emscripten_stack_restore=wasmExports["O"])(a0);var __emscripten_stack_alloc=a0=>(__emscripten_stack_alloc=wasmExports["P"])(a0);var _emscripten_stack_get_current=()=>(_emscripten_stack_get_current=wasmExports["Q"])();Module["ccall"]=ccall;Module["cwrap"]=cwrap;var calledRun;var calledPrerun;dependenciesFulfilled=function runCaller(){if(!calledRun)run();if(!calledRun)dependenciesFulfilled=runCaller};function run(){if(runDependencies>0){return}if(!calledPrerun){calledPrerun=1;preRun();if(runDependencies>0){return}}function doRun(){if(calledRun)return;calledRun=1;Module["calledRun"]=1;if(ABORT)return;initRuntime();readyPromiseResolve(Module);Module["onRuntimeInitialized"]?.();postRun()}if(Module["setStatus"]){Module["setStatus"]("Running...");setTimeout(()=>{setTimeout(()=>Module["setStatus"](""),1);doRun()},1)}else{doRun()}}if(Module["preInit"]){if(typeof Module["preInit"]=="function")Module["preInit"]=[Module["preInit"]];while(Module["preInit"].length>0){Module["preInit"].pop()()}}run();moduleRtn=readyPromise; + + + return moduleRtn; +} +); +})(); +if (typeof exports === 'object' && typeof module === 'object') + module.exports = createNispsModule; +else if (typeof define === 'function' && define['amd']) + define([], () => createNispsModule); diff --git a/manifold/public/nisps.wasm b/manifold/public/nisps.wasm new file mode 100755 index 0000000..02da865 Binary files /dev/null and b/manifold/public/nisps.wasm differ diff --git a/manifold/src/App.tsx b/manifold/src/App.tsx new file mode 100644 index 0000000..f8cb9fe --- /dev/null +++ b/manifold/src/App.tsx @@ -0,0 +1,60 @@ +/** + * Manifold — app root. The convertible Console (ConsoleApp) wired to the real + * engine, mounted under EngineProvider. Defaults to the hero `focus="composite"` + * (the convertible centerpiece). The `?debug=1` probe is installed once the + * engine is live. + */ + +import { useEffect } from 'react'; +import { EngineProvider } from './engine/EngineProvider'; +import { useEngine } from './engine/useEngine'; +import { installDebugProbe } from './debug/probe'; +import { ConsoleApp } from './console'; + +function Loading() { + return ( +
+ + Manifold + + loading engine… +
+ ); +} + +/** Installs the debug probe once the engine is in context. */ +function ProbeInstaller() { + const engine = useEngine(); + useEffect(() => { + if (engine) installDebugProbe(engine); + }, [engine]); + return null; +} + +export function App() { + return ( + }> + + + + ); +} diff --git a/manifold/src/backends/README.md b/manifold/src/backends/README.md new file mode 100644 index 0000000..48af9c5 --- /dev/null +++ b/manifold/src/backends/README.md @@ -0,0 +1,47 @@ +# Output Backends (`manifold/src/backends/`) + +Real output transports for the Manifold app. Exactly one backend is *active* at a +time, chosen by the dock **Mode** (Particle / MIDI / OSC / Built-in Synth / +Editor → `BackendId`). The `BackendManager` consumes the engine spine and +forwards each routed output vector to the active backend's `send()`. + +See `docs/redesign/backends-spec.md` for the authoritative design. + +## Files + +| File | Role | +|---|---| +| `backend.ts` | The `OutputBackend` interface + `BackendContext` / `OutputMapping` / `BackendStatus`. | +| `mapping.ts` | Universal per-output baseline mapping (`applyCurve`, `mapOutput`) — shared by all backends, input-clamped. | +| `manager.ts` | `BackendManager` — single spine consumer; switches/teardowns backends; **gates synth audio**. | +| `midi-backend.ts` | `WebMidiBackend` — real Web MIDI CC out (per-output CC#/channel/range/name, throttled + dead-zone). | +| `osc-client.ts` | `NispsOscClient` — WS transport to the Deno OSC bridge (JSON protocol, auto-reconnect). | +| `osc-backend.ts` | `OscBridgeBackend` — OSC out over WS; per-output address path + physical range. | +| `passthrough-backend.ts` | No-op sink for synth (plays in-engine) / particles (rAF consumer) / editor. | +| `presets.ts` | Named per-backend output-config presets (localStorage, per-backend namespace). | +| `useBackendManager.ts` | Thin React binding: builds `BackendContext` from the store, switches Mode, surfaces status. | + +## Audio gating + +The Built-in Synth plays **inside** the engine (`EngineHost` pushes params to the +worklet on every spine tick). So the manager mutes audio on every non-synth Mode +via `engine.audio.setMuted(true)` and unmutes on the synth Mode. This is the +documented gate — cleanly suppressing the worklet push would need an engine +change that this workstream doesn't make. + +## OSC bridge — the process must be running + +Browsers cannot send UDP, so the OSC backend connects over WebSocket to the Deno +bridge, which encodes OSC and forwards over UDP. **Start the bridge locally** or +the OSC backend shows "bridge not running" (it auto-reconnects): + +```bash +cd manifold/osc-bridge +deno run --allow-net bridge.ts +# --osc-host 127.0.0.1 --osc-port 9000 --ws-port 8765 --listen-port 9001 +# or, without Deno: +node bridge.mjs +``` + +Default bridge URL: `ws://localhost:8765` (configurable in the OSC config panel). +WS protocol (browser → bridge): `{ type:'params', payload:[[path,value],…] }`. diff --git a/manifold/src/backends/backend.ts b/manifold/src/backends/backend.ts new file mode 100644 index 0000000..d8eb38f --- /dev/null +++ b/manifold/src/backends/backend.ts @@ -0,0 +1,79 @@ +/** + * OutputBackend — the adapter interface every output sink implements + * (backends-spec §1). Exactly one backend is "active" at a time; the + * BackendManager (manager.ts) consumes the engine's reactive spine and forwards + * each routed output vector to the active backend's `send()`. + * + * Concrete backends are framework-neutral (NO React). The manager is the single + * consumer of `engine.subscribe()` + `engine.routedOutput()`; backends never + * touch the engine — they receive a `Float32Array` and push it to their sink + * (WebMIDI port, OSC-over-WS bridge, particle visualiser, synth worklet). + * + * British spelling in product copy; the synth is the "Built-in Synth", never + * "C15". + */ +import type { BackendId } from '../dock/output-state'; + +/** Per-output baseline mapping (backends-spec §3). One per output dim. */ +export interface OutputMapping { + /** Tri-state — 'off' excludes the output entirely, 'fixed' pins it. */ + state: 'off' | 'fixed' | 'live'; + /** Downstream silence — still computed, but not emitted to the sink. */ + muted: boolean; + /** Baseline range floor (normalised 0..1 maps here). */ + min: number; + /** Baseline range ceil. */ + max: number; + /** 0..1, 0.5 = linear. */ + curve: number; + /** Held value when state === 'fixed'. */ + fixedValue: number; +} + +/** What a backend needs to know about the active mode/output set. */ +export interface BackendContext { + modeId: string; + /** Model output dims in use (≤ 126). */ + outputCount: number; + /** Per-output baseline mapping, length === outputCount. */ + mappings: OutputMapping[]; + /** Per-output user-facing names (for OSC paths / MIDI names / labels). */ + names: string[]; +} + +/** Connection / readiness status surfaced to the dock. */ +export interface BackendStatus { + /** Coarse state for status dots. */ + state: 'idle' | 'connecting' | 'ready' | 'error' | 'unavailable'; + /** Human-readable one-liner (British spelling). */ + message: string; +} + +export interface OutputBackend { + readonly id: BackendId; + + /** Probe — WebMIDI / WebSocket / Canvas availability. */ + isAvailable(): boolean; + + /** Becomes active. Resolve only when ready to receive send(). */ + start(ctx: BackendContext): Promise; + + /** + * Hot per-frame path. `routed` is the post-pipeline Float32Array (0..1), + * length === ctx.outputCount. MUST NOT allocate; MUST NOT mutate `routed`. + * Throttling / dead-zone live INSIDE each backend. + */ + send(routed: Float32Array): void; + + /** Switching away / unmount. Release WS / MIDI / threads. */ + teardown(): Promise; + + /** Latest status (polled by the dock hook). */ + status(): BackendStatus; + + /** Apply a fresh BackendContext (mappings / names changed) without a restart. */ + setContext?(ctx: BackendContext): void; + + /** Subscribe to status changes (returns an unsubscribe). */ + onStatusChange?(cb: (s: BackendStatus) => void): () => void; +} diff --git a/manifold/src/backends/index.ts b/manifold/src/backends/index.ts new file mode 100644 index 0000000..a0bea84 --- /dev/null +++ b/manifold/src/backends/index.ts @@ -0,0 +1,38 @@ +/** + * Output backends barrel (backends-spec). Real MIDI / OSC transports + the + * BackendManager that consumes the engine spine and forwards routed outputs to + * the active backend. Framework-neutral core + a thin React hook. + */ +export type { + OutputBackend, + BackendContext, + BackendStatus, + OutputMapping, +} from './backend'; +export { applyCurve, mapOutput, isSilent, clamp01 } from './mapping'; +export { BackendManager } from './manager'; +export type { ManagerEngine } from './manager'; +export { WebMidiBackend } from './midi-backend'; +export type { MidiBackendConfig } from './midi-backend'; +export { OscBridgeBackend } from './osc-backend'; +export type { OscBackendConfig } from './osc-backend'; +export { NispsOscClient } from './osc-client'; +export { PassthroughBackend } from './passthrough-backend'; +export { + useBackendManager, +} from './useBackendManager'; +export type { + UseBackendManager, + MidiSettings, + OscSettings, +} from './useBackendManager'; +export { + listPresets, + savePreset, + getPreset, + deletePreset, + renamePreset, + applyPreset, + rowsFromParams, +} from './presets'; +export type { OutputPreset, OutputPresetRow } from './presets'; diff --git a/manifold/src/backends/manager.ts b/manifold/src/backends/manager.ts new file mode 100644 index 0000000..4afe0c2 --- /dev/null +++ b/manifold/src/backends/manager.ts @@ -0,0 +1,144 @@ +/** + * BackendManager — the single consumer of the engine spine that forwards each + * routed output vector to the ACTIVE output backend (backends-spec §0–§1). + * + * It is a CONSUMER of the engine, exactly like the stage / dock: it + * `engine.subscribe()`s and reads `engine.routedOutput()` on each bump, then + * calls `activeBackend.send(routed)`. It NEVER modifies the engine. + * + * Audio gating: the Built-in Synth plays inside the engine (EngineHost pushes + * params to the worklet on every spine tick — see engine-api.ts `send`). So + * selecting MIDI / OSC / Particle would ALSO blast the synth. The manager gates + * this with `engine.audio.setMuted(true)` on every non-synth mode and + * `setMuted(false)` on the synth mode. (Cleanly gating the worklet push without + * an engine change isn't possible — muting is the documented approach, + * backends-spec §1 / task constraint.) + * + * Framework-neutral: no React. A thin hook (useBackendManager.ts) exposes status. + */ +import type { BackendContext, BackendStatus, OutputBackend } from './backend'; +import type { BackendId } from '../dock/output-state'; +import { WebMidiBackend } from './midi-backend'; +import { OscBridgeBackend } from './osc-backend'; +import { PassthroughBackend } from './passthrough-backend'; + +/** The slice of EngineApi the manager depends on (keeps it decoupled/testable). */ +export interface ManagerEngine { + subscribe(cb: () => void): () => void; + routedOutput(): Float32Array | null; + audio: { setMuted(muted: boolean): void }; +} + +export class BackendManager { + private engine: ManagerEngine; + private backends: Map; + private active: OutputBackend | null = null; + private activeId: BackendId | null = null; + private ctx: BackendContext | null = null; + private unsub: (() => void) | null = null; + private switching = false; + + private statusListeners = new Set<(id: BackendId, s: BackendStatus) => void>(); + private offBackendStatus: (() => void) | null = null; + + constructor(engine: ManagerEngine, backends?: Partial>) { + this.engine = engine; + this.backends = new Map([ + ['midi', backends?.midi ?? new WebMidiBackend()], + ['osc', backends?.osc ?? new OscBridgeBackend()], + ['synth', backends?.synth ?? new PassthroughBackend('synth', 'Built-in Synth — audio plays in the engine')], + ['particles', backends?.particles ?? new PassthroughBackend('particles', 'Particle visualiser')], + ['cvgate', backends?.cvgate ?? new PassthroughBackend('cvgate', 'CV / gate (via VCV bridge)')], + ['vcv', backends?.vcv ?? new PassthroughBackend('vcv', 'VCV bridge')], + ]); + + // Single subscription to the spine: forward routed → active backend. + this.unsub = this.engine.subscribe(() => { + if (!this.active) return; + const routed = this.engine.routedOutput(); + if (routed) this.active.send(routed); + }); + } + + /** Typed handle to a concrete backend (for the dock's per-backend config). */ + midi(): WebMidiBackend | null { + const b = this.backends.get('midi'); + return b instanceof WebMidiBackend ? b : null; + } + + osc(): OscBridgeBackend | null { + const b = this.backends.get('osc'); + return b instanceof OscBridgeBackend ? b : null; + } + + get(id: BackendId): OutputBackend | undefined { + return this.backends.get(id); + } + + getActiveId(): BackendId | null { + return this.activeId; + } + + /** Provide / refresh the BackendContext (mappings + names) for the active set. */ + setContext(ctx: BackendContext): void { + this.ctx = ctx; + this.active?.setContext?.(ctx); + } + + /** + * Switch the active backend. Tears down the old, starts the new, and applies + * the synth audio gate. Idempotent for the same id. + */ + async setActive(id: BackendId): Promise { + if (this.activeId === id || this.switching) return; + this.switching = true; + try { + // Gate audio: only the synth mode drives sound. + this.engine.audio.setMuted(id !== 'synth'); + + const next = this.backends.get(id); + if (!next) { + this.switching = false; + return; + } + if (this.active) { + this.offBackendStatus?.(); + this.offBackendStatus = null; + await this.active.teardown(); + } + this.active = next; + this.activeId = id; + this.offBackendStatus = next.onStatusChange?.((s) => this.emitStatus(id, s)) ?? null; + if (this.ctx) { + next.setContext?.(this.ctx); + await next.start(this.ctx); + } + this.emitStatus(id, next.status()); + } finally { + this.switching = false; + } + } + + status(id?: BackendId): BackendStatus { + const b = id ? this.backends.get(id) : this.active; + return b?.status() ?? { state: 'idle', message: 'idle' }; + } + + onStatusChange(cb: (id: BackendId, s: BackendStatus) => void): () => void { + this.statusListeners.add(cb); + return () => this.statusListeners.delete(cb); + } + + dispose(): void { + this.unsub?.(); + this.unsub = null; + this.offBackendStatus?.(); + if (this.active) void this.active.teardown(); + this.active = null; + this.activeId = null; + } + + private emitStatus(id: BackendId, s: BackendStatus): void { + for (const cb of this.statusListeners) cb(id, s); + } +} diff --git a/manifold/src/backends/mapping.ts b/manifold/src/backends/mapping.ts new file mode 100644 index 0000000..c9e6444 --- /dev/null +++ b/manifold/src/backends/mapping.ts @@ -0,0 +1,37 @@ +/** + * Universal per-output baseline mapping (backends-spec §3). Every backend shares + * ONE mapping from a normalised model output v ∈ [0,1] to a sink value, so + * behaviour is identical across sinks and the override UI is backend-agnostic. + * + * `applyCurve` clamps the input to [0,1] BEFORE the pow (matching the deployed + * `param-map.js:287` and the verification correction in backends-spec §"minor"). + */ +import type { OutputMapping } from './backend'; + +export function clamp01(v: number): number { + return v < 0 ? 0 : v > 1 ? 1 : v; +} + +/** 0.5 = linear; <0.5 ease-in, >0.5 ease-out. Input clamped to [0,1] first. */ +export function applyCurve(v: number, c: number): number { + const x = clamp01(v); + if (c === 0.5) return x; + return Math.pow(x, Math.pow(2, 4 * (c - 0.5))); +} + +/** + * Per-output baseline: curve, then scale into [min,max]. Honours freeze (fixed) + * but NOT mute (mute is a sink-level skip, handled per backend so the value is + * still computed/visible). Returns a value in [min,max]. + */ +export function mapOutput(v: number, p: OutputMapping): number { + if (p.state === 'fixed') { + return p.min + clamp01(p.fixedValue) * (p.max - p.min); + } + return p.min + applyCurve(v, p.curve) * (p.max - p.min); +} + +/** True when this output must NOT be emitted to the sink (off or muted). */ +export function isSilent(p: OutputMapping): boolean { + return p.state === 'off' || p.muted; +} diff --git a/manifold/src/backends/midi-backend.ts b/manifold/src/backends/midi-backend.ts new file mode 100644 index 0000000..9a9bd38 --- /dev/null +++ b/manifold/src/backends/midi-backend.ts @@ -0,0 +1,176 @@ +/** + * WebMidiBackend — real Web MIDI CC output (backends-spec §2.3). + * + * Per non-silent output: map (0..1) → baseline (min/max/curve) → round(×127) → + * send `[0xB0|(ch-1), cc, value]` on the configured CC#/channel. Throttled to + * ~50ms (≈20 Hz) with a per-CC dead-zone (Δ ≥ 1) so we never flood the port. + * + * The per-output MIDI spec (cc/channel/name/range) lives on the shared MFParam + * store (output-state.ts MidiCcSpec) and arrives via BackendContext.mappings + + * a parallel `midiSpecs` array set through {@link setMidiConfig}. The port is + * picked in the Outputs panel and applied via {@link selectOutput}. + * + * No per-frame allocation: the 3-byte message array is reused; `lastSent` + * tracks per-CC values for the dead-zone. + */ +import type { BackendContext, BackendStatus, OutputBackend } from './backend'; +import { isSilent, mapOutput } from './mapping'; +import type { MidiCcSpec } from '../dock/output-state'; + +const SEND_INTERVAL_MS = 50; +const DEAD_ZONE = 1; + +export interface MidiBackendConfig { + /** Selected output port id (null = none). */ + outputId: string | null; + /** How many of the outputs are mapped to CCs (subset of outputCount). */ + ccCount: number; +} + +export class WebMidiBackend implements OutputBackend { + readonly id = 'midi' as const; + + private access: MIDIAccess | null = null; + private output: MIDIOutput | null = null; + private outputId: string | null = null; + private ccCount = 0; + + private ctx: BackendContext | null = null; + /** Per-output MIDI specs, index-aligned with ctx.mappings. */ + private specs: MidiCcSpec[] = []; + + private lastSent = new Int16Array(0); // per-output last value, -1 = unsent + private msg: number[] = [0, 0, 0]; // reused 3-byte buffer + private lastSendMs = 0; + + private statusState: BackendStatus = { state: 'idle', message: 'MIDI idle' }; + private statusListeners = new Set<(s: BackendStatus) => void>(); + + isAvailable(): boolean { + return typeof navigator !== 'undefined' && typeof navigator.requestMIDIAccess === 'function'; + } + + async start(ctx: BackendContext): Promise { + this.ctx = ctx; + this.lastSent = new Int16Array(ctx.outputCount).fill(-1); + if (!this.isAvailable()) { + this.setStatus({ state: 'unavailable', message: 'Web MIDI not supported in this browser' }); + return; + } + this.setStatus({ state: 'connecting', message: 'Requesting MIDI access…' }); + try { + this.access = await navigator.requestMIDIAccess!({ sysex: false }); + this.access.onstatechange = () => this.refreshPort(); + this.refreshPort(); + if (this.output) { + this.setStatus({ state: 'ready', message: `MIDI → ${this.output.name ?? 'output'}` }); + } else { + this.setStatus({ state: 'ready', message: 'MIDI ready — pick an output port' }); + } + } catch (err) { + this.setStatus({ state: 'error', message: `MIDI access denied: ${(err as Error).message}` }); + } + } + + setContext(ctx: BackendContext): void { + this.ctx = ctx; + if (this.lastSent.length !== ctx.outputCount) { + this.lastSent = new Int16Array(ctx.outputCount).fill(-1); + } + } + + /** Update the per-output MIDI specs + how many CCs are mapped + the port. */ + setMidiConfig(specs: MidiCcSpec[], cfg: MidiBackendConfig): void { + this.specs = specs; + this.ccCount = cfg.ccCount; + if (cfg.outputId !== this.outputId) { + this.outputId = cfg.outputId; + this.refreshPort(); + } + this.lastSent.fill(-1); // CC map changed → re-send next frame + } + + /** Enumerate the available MIDI output ports. */ + listOutputs(): { id: string; name: string }[] { + if (!this.access) return []; + const out: { id: string; name: string }[] = []; + this.access.outputs.forEach((o, id) => out.push({ id, name: o.name ?? `MIDI Output ${id}` })); + return out; + } + + selectOutput(outputId: string | null): void { + this.outputId = outputId; + this.lastSent.fill(-1); + this.refreshPort(); + } + + private refreshPort(): void { + if (!this.access) { + this.output = null; + return; + } + if (this.outputId) { + this.output = this.access.outputs.get(this.outputId) ?? null; + } else { + this.output = null; + } + if (this.output) { + this.setStatus({ state: 'ready', message: `MIDI → ${this.output.name ?? 'output'}` }); + } + } + + send(routed: Float32Array): void { + const out = this.output; + const ctx = this.ctx; + if (!out || !ctx) return; + + const now = typeof performance !== 'undefined' ? performance.now() : Date.now(); + if (now - this.lastSendMs < SEND_INTERVAL_MS) return; + this.lastSendMs = now; + + const n = Math.min(this.ccCount, routed.length, ctx.mappings.length, this.specs.length); + for (let i = 0; i < n; i++) { + const m = ctx.mappings[i]; + if (isSilent(m)) continue; + const spec = this.specs[i]; + if (!spec) continue; + // Baseline maps into [min,max] (already 0..1 here), then scale to 7-bit. + const mapped = mapOutput(routed[i], m); + const val = Math.max(0, Math.min(127, Math.round(mapped * 127))); + const prev = this.lastSent[i]; + if (prev >= 0 && Math.abs(val - prev) < DEAD_ZONE) continue; + this.lastSent[i] = val; + const ch = Math.max(0, Math.min(15, (spec.channel - 1) | 0)); + const cc = Math.max(0, Math.min(127, spec.cc | 0)); + this.msg[0] = 0xb0 | ch; + this.msg[1] = cc; + this.msg[2] = val; + try { + out.send(this.msg); + } catch { + /* port unplugged mid-send — refresh on next statechange */ + } + } + } + + async teardown(): Promise { + if (this.access) this.access.onstatechange = null; + this.access = null; + this.output = null; + this.setStatus({ state: 'idle', message: 'MIDI idle' }); + } + + status(): BackendStatus { + return this.statusState; + } + + onStatusChange(cb: (s: BackendStatus) => void): () => void { + this.statusListeners.add(cb); + return () => this.statusListeners.delete(cb); + } + + private setStatus(s: BackendStatus): void { + this.statusState = s; + for (const cb of this.statusListeners) cb(s); + } +} diff --git a/manifold/src/backends/osc-backend.ts b/manifold/src/backends/osc-backend.ts new file mode 100644 index 0000000..290cfa6 --- /dev/null +++ b/manifold/src/backends/osc-backend.ts @@ -0,0 +1,147 @@ +/** + * OscBridgeBackend — OSC output over the Deno WebSocket↔UDP bridge + * (backends-spec §2.4). Browsers can't send UDP, so we connect to the bridge + * (default ws://localhost:8765) and post `{ type:'params', payload:[[path, + * value], ...] }`; the bridge encodes OSC and forwards to the target. + * + * Per output: a configurable OSC address path + a physical range. The 0..1 + * model output is baseline-mapped (min/max/curve) then linearly scaled into the + * per-output physical [rangeMin,rangeMax] (unless "send raw 0..1" is toggled). + * + * The bridge process must be running locally; the client auto-reconnects and we + * surface "bridge not running" until the WS connects. Throttled to ~50ms with a + * per-output dead-zone (matching the deployed osc-output.js). + */ +import type { BackendContext, BackendStatus, OutputBackend } from './backend'; +import { isSilent, mapOutput } from './mapping'; +import { NispsOscClient } from './osc-client'; +import type { OscSpec } from '../dock/output-state'; + +const SEND_INTERVAL_MS = 50; +const DEAD_ZONE = 0.002; // on the normalised value, pre physical-scale + +export interface OscBackendConfig { + /** Bridge WebSocket URL. */ + url: string; + /** Send raw normalised 0..1 instead of the physical range. */ + sendRaw: boolean; +} + +export class OscBridgeBackend implements OutputBackend { + readonly id = 'osc' as const; + + private client = new NispsOscClient(); + private ctx: BackendContext | null = null; + private specs: OscSpec[] = []; + private sendRaw = false; + + private lastSent: Float32Array = new Float32Array(0); // last normalised value + private batch: Array<[string, number]> = []; // reused outer; entries reused + private lastSendMs = 0; + + private statusState: BackendStatus = { state: 'idle', message: 'OSC idle' }; + private statusListeners = new Set<(s: BackendStatus) => void>(); + private offConn: (() => void) | null = null; + private offInfo: (() => void) | null = null; + + isAvailable(): boolean { + return typeof WebSocket !== 'undefined'; + } + + async start(ctx: BackendContext): Promise { + this.ctx = ctx; + this.lastSent = new Float32Array(ctx.outputCount).fill(-1); + if (!this.isAvailable()) { + this.setStatus({ state: 'unavailable', message: 'WebSocket not available' }); + return; + } + this.offConn = this.client.onConnectionChange((connected) => { + this.setStatus( + connected + ? { state: 'ready', message: `OSC bridge connected (${this.client.url})` } + : { state: 'error', message: `OSC bridge not running — start it (${this.client.url})` }, + ); + }); + this.offInfo = this.client.onInfo((m) => { + if (this.client.connected) this.setStatus({ state: 'ready', message: m }); + }); + this.setStatus({ state: 'connecting', message: `Connecting to OSC bridge (${this.client.url})…` }); + // Reject is non-fatal — the client keeps reconnecting in the background. + this.client.connect({ reconnect: true }).catch(() => { + this.setStatus({ state: 'error', message: `OSC bridge not running — start it (${this.client.url})` }); + }); + } + + setContext(ctx: BackendContext): void { + this.ctx = ctx; + if (this.lastSent.length !== ctx.outputCount) { + this.lastSent = new Float32Array(ctx.outputCount).fill(-1); + } + } + + /** Update per-output OSC specs + bridge URL/raw toggle. */ + setOscConfig(specs: OscSpec[], cfg: OscBackendConfig): void { + this.specs = specs; + this.sendRaw = cfg.sendRaw; + if (cfg.url !== this.client.url) { + this.client.setUrl(cfg.url); + if (this.isAvailable()) { + this.setStatus({ state: 'connecting', message: `Connecting to OSC bridge (${cfg.url})…` }); + this.client.connect({ reconnect: true }).catch(() => { + this.setStatus({ state: 'error', message: `OSC bridge not running — start it (${cfg.url})` }); + }); + } + } + this.lastSent.fill(-1); + } + + send(routed: Float32Array): void { + const ctx = this.ctx; + if (!ctx || !this.client.connected) return; + + const now = typeof performance !== 'undefined' ? performance.now() : Date.now(); + if (now - this.lastSendMs < SEND_INTERVAL_MS) return; + this.lastSendMs = now; + + const n = Math.min(routed.length, ctx.mappings.length, this.specs.length); + this.batch.length = 0; + for (let i = 0; i < n; i++) { + const m = ctx.mappings[i]; + if (isSilent(m)) continue; + const spec = this.specs[i]; + if (!spec || !spec.path) continue; + const mapped = mapOutput(routed[i], m); // 0..1 in [min,max] + const prev = this.lastSent[i]; + if (prev >= 0 && Math.abs(mapped - prev) < DEAD_ZONE) continue; + this.lastSent[i] = mapped; + const value = this.sendRaw + ? mapped + : spec.rangeMin + mapped * (spec.rangeMax - spec.rangeMin); + this.batch.push([spec.path, value]); + } + if (this.batch.length) this.client.sendParams(this.batch); + } + + async teardown(): Promise { + this.offConn?.(); + this.offInfo?.(); + this.offConn = null; + this.offInfo = null; + this.client.disconnect(); + this.setStatus({ state: 'idle', message: 'OSC idle' }); + } + + status(): BackendStatus { + return this.statusState; + } + + onStatusChange(cb: (s: BackendStatus) => void): () => void { + this.statusListeners.add(cb); + return () => this.statusListeners.delete(cb); + } + + private setStatus(s: BackendStatus): void { + this.statusState = s; + for (const cb of this.statusListeners) cb(s); + } +} diff --git a/manifold/src/backends/osc-client.ts b/manifold/src/backends/osc-client.ts new file mode 100644 index 0000000..ac43f98 --- /dev/null +++ b/manifold/src/backends/osc-client.ts @@ -0,0 +1,193 @@ +/** + * NispsOscClient — WebSocket transport to the Deno OSC bridge (osc-bridge/ + * bridge.ts). Ported cleanly to TS from the deployed + * `js/nisps/osc-client.js`; the bridge does the actual OSC encode/UDP send, so + * this client speaks the bridge's JSON protocol: + * + * browser → bridge: + * { type: 'params', payload: [[name, value], ...] } (per-param floats) + * { type: 'state', payload: } (full state) + * { type: 'weights',payload: } (weights only) + * bridge → browser: + * { type: 'outputs', values: [...] } + * { type: 'inputs', values: [...] } + * { type: 'info', message: '...' } + * + * Auto-reconnect with backoff; the bridge process must be running locally + * (default ws://localhost:8765). + */ + +export type OscParamBatch = ReadonlyArray; + +export class NispsOscClient { + private wsUrl: string; + private ws: WebSocket | null = null; + private connected_ = false; + private reconnect = false; + private reconnectDelay = 1000; + private reconnectTimer: ReturnType | null = null; + + private outputsCbs: Array<(v: number[]) => void> = []; + private inputsCbs: Array<(v: number[]) => void> = []; + private infoCbs: Array<(m: string) => void> = []; + private stateCbs: Array<(connected: boolean) => void> = []; + + constructor(wsUrl = 'ws://localhost:8765') { + this.wsUrl = wsUrl; + } + + get connected(): boolean { + return this.connected_; + } + + get url(): string { + return this.wsUrl; + } + + setUrl(url: string): void { + if (this.connected_) this.disconnect(); + this.wsUrl = url; + } + + connect({ reconnect = true } = {}): Promise { + this.reconnect = reconnect; + return new Promise((resolve, reject) => { + if (this.connected_ && this.ws) { + resolve(); + return; + } + if (typeof WebSocket === 'undefined') { + reject(new Error('WebSocket not available')); + return; + } + let ws: WebSocket; + try { + ws = new WebSocket(this.wsUrl); + } catch (err) { + reject(err as Error); + return; + } + this.ws = ws; + ws.onopen = () => { + this.connected_ = true; + this.reconnectDelay = 1000; + this.emitState(); + resolve(); + }; + ws.onclose = () => { + const wasConnected = this.connected_; + this.connected_ = false; + this.ws = null; + this.emitState(); + if (this.reconnect) this.scheduleReconnect(); + if (!wasConnected) reject(new Error('WebSocket closed before connecting')); + }; + ws.onerror = () => { + /* surfaced via onclose */ + }; + ws.onmessage = (e) => this.handleMessage(e.data as string); + }); + } + + disconnect(): void { + this.reconnect = false; + if (this.reconnectTimer) { + clearTimeout(this.reconnectTimer); + this.reconnectTimer = null; + } + if (this.ws) { + this.ws.onclose = null; + try { + this.ws.close(); + } catch { + /* ignore */ + } + this.ws = null; + } + this.connected_ = false; + this.emitState(); + } + + // ── Send ─────────────────────────────────────────────────────────── + sendParams(params: OscParamBatch): void { + this.send({ type: 'params', payload: params }); + } + + sendState(stateJson: object | string): void { + const payload = typeof stateJson === 'string' ? JSON.parse(stateJson) : stateJson; + this.send({ type: 'state', payload }); + } + + sendWeights(weightsObj: object | string): void { + const payload = typeof weightsObj === 'string' ? JSON.parse(weightsObj) : weightsObj; + this.send({ type: 'weights', payload }); + } + + // ── Receive ──────────────────────────────────────────────────────── + onOutputsReceived(cb: (v: number[]) => void): () => void { + this.outputsCbs.push(cb); + return () => this.removeCb(this.outputsCbs, cb); + } + + onInputsReceived(cb: (v: number[]) => void): () => void { + this.inputsCbs.push(cb); + return () => this.removeCb(this.inputsCbs, cb); + } + + onInfo(cb: (m: string) => void): () => void { + this.infoCbs.push(cb); + return () => this.removeCb(this.infoCbs, cb); + } + + onConnectionChange(cb: (connected: boolean) => void): () => void { + this.stateCbs.push(cb); + return () => this.removeCb(this.stateCbs, cb); + } + + // ── Internal ─────────────────────────────────────────────────────── + private send(data: unknown): void { + if (!this.ws || this.ws.readyState !== WebSocket.OPEN) return; + this.ws.send(JSON.stringify(data)); + } + + private handleMessage(raw: string): void { + let msg: { type?: string; values?: number[]; message?: string }; + try { + msg = JSON.parse(raw); + } catch { + return; + } + switch (msg.type) { + case 'outputs': + if (msg.values) for (const cb of this.outputsCbs) cb(msg.values); + break; + case 'inputs': + if (msg.values) for (const cb of this.inputsCbs) cb(msg.values); + break; + case 'info': + if (msg.message) for (const cb of this.infoCbs) cb(msg.message); + break; + } + } + + private scheduleReconnect(): void { + if (this.reconnectTimer) return; + this.reconnectTimer = setTimeout(() => { + this.reconnectTimer = null; + if (!this.connected_ && this.reconnect) { + this.connect({ reconnect: true }).catch(() => { + this.reconnectDelay = Math.min(this.reconnectDelay * 1.5, 30000); + }); + } + }, this.reconnectDelay); + } + + private emitState(): void { + for (const cb of this.stateCbs) cb(this.connected_); + } + + private removeCb(arr: T[], cb: T): void { + const i = arr.indexOf(cb); + if (i >= 0) arr.splice(i, 1); + } +} diff --git a/manifold/src/backends/passthrough-backend.ts b/manifold/src/backends/passthrough-backend.ts new file mode 100644 index 0000000..c6d0a56 --- /dev/null +++ b/manifold/src/backends/passthrough-backend.ts @@ -0,0 +1,45 @@ +/** + * PassthroughBackend — the no-op sink for backends whose output is consumed + * elsewhere in the app rather than transported by the BackendManager: + * + * - 'synth' : the Built-in Synth plays INSIDE the engine (EngineHost → + * worklet). The manager only gates audio (mute on non-synth + * modes) — it does NOT re-send params, the engine already does. + * - 'particles': the FlowFieldVisualiser reads engine.getOutputs() in its own + * rAF loop (a separate consumer of the spine). + * - 'editor' : the MEMLNaut serial Editor is not an output sink. + * + * `send()` is intentionally empty. The manager applies the synth audio gate. + */ +import type { BackendContext, BackendStatus, OutputBackend } from './backend'; +import type { BackendId } from '../dock/output-state'; + +export class PassthroughBackend implements OutputBackend { + readonly id: BackendId; + private message: string; + + constructor(id: BackendId, message: string) { + this.id = id; + this.message = message; + } + + isAvailable(): boolean { + return true; + } + + async start(_ctx: BackendContext): Promise { + /* nothing to start */ + } + + send(_routed: Float32Array): void { + /* output consumed elsewhere (engine worklet / particle rAF / serial) */ + } + + async teardown(): Promise { + /* nothing to release */ + } + + status(): BackendStatus { + return { state: 'ready', message: this.message }; + } +} diff --git a/manifold/src/backends/presets.ts b/manifold/src/backends/presets.ts new file mode 100644 index 0000000..3a88275 --- /dev/null +++ b/manifold/src/backends/presets.ts @@ -0,0 +1,162 @@ +/** + * Named output presets (backends-spec §5). The per-backend output configuration + * — the whole set of per-output specs (baseline + backend-specific) plus + * backend-level settings — is saveable/restorable as NAMED presets, persisted + * to localStorage and keyed PER BACKEND (a MIDI preset and an OSC preset live in + * separate namespaces). + * + * A preset stores a slice of each output's MFParam (the editable output config) + * + a free-form `settings` blob for backend-level fields (MIDI port/ccCount, + * OSC bridge URL/sendRaw). Restoring applies the rows back to the live store. + */ +import type { MFParam } from '../console/model'; +import type { BackendId } from '../dock/output-state'; + +/** The per-output config a preset captures (a slice of MFParam). */ +export interface OutputPresetRow { + name: string; + status: MFParam['status']; + muted?: boolean; + armed?: boolean; + min: number; + max: number; + curve: number; + val: number; + midi?: MFParam['midi']; + osc?: MFParam['osc']; + vcv?: MFParam['vcv']; +} + +export interface OutputPreset { + name: string; + backend: BackendId; + rows: OutputPresetRow[]; + /** Backend-level settings (MIDI: { outputId, ccCount }; OSC: { url, sendRaw }). */ + settings?: Record; + savedAt: number; +} + +const KEY_PREFIX = 'manifold-output-presets'; + +function storageKey(backend: BackendId): string { + return `${KEY_PREFIX}:${backend}`; +} + +function read(backend: BackendId): OutputPreset[] { + if (typeof localStorage === 'undefined') return []; + try { + const raw = localStorage.getItem(storageKey(backend)); + if (!raw) return []; + const parsed = JSON.parse(raw); + return Array.isArray(parsed) ? (parsed as OutputPreset[]) : []; + } catch { + return []; + } +} + +function write(backend: BackendId, presets: OutputPreset[]): void { + if (typeof localStorage === 'undefined') return; + try { + localStorage.setItem(storageKey(backend), JSON.stringify(presets)); + } catch { + /* quota / disabled — non-fatal */ + } +} + +/** List preset names for a backend (most-recently-saved first). */ +export function listPresets(backend: BackendId): OutputPreset[] { + return read(backend).sort((a, b) => b.savedAt - a.savedAt); +} + +/** Project the live params into preset rows. */ +export function rowsFromParams(params: MFParam[]): OutputPresetRow[] { + return params.map((p) => ({ + name: p.name, + status: p.status, + muted: p.muted, + armed: p.armed, + min: p.min, + max: p.max, + curve: p.curve, + val: p.val, + midi: p.midi, + osc: p.osc, + vcv: p.vcv, + })); +} + +/** Save (or overwrite by name) a preset for a backend. */ +export function savePreset( + backend: BackendId, + name: string, + params: MFParam[], + settings?: Record, +): OutputPreset { + const trimmed = name.trim(); + const preset: OutputPreset = { + name: trimmed, + backend, + rows: rowsFromParams(params), + settings, + savedAt: Date.now(), + }; + const all = read(backend).filter((p) => p.name !== trimmed); + all.push(preset); + write(backend, all); + return preset; +} + +/** Look up a preset by name. */ +export function getPreset(backend: BackendId, name: string): OutputPreset | null { + return read(backend).find((p) => p.name === name) ?? null; +} + +/** Delete a preset by name. */ +export function deletePreset(backend: BackendId, name: string): void { + write( + backend, + read(backend).filter((p) => p.name !== name), + ); +} + +/** Rename a preset (no-op if the new name collides or the old is missing). */ +export function renamePreset(backend: BackendId, from: string, to: string): boolean { + const trimmed = to.trim(); + if (!trimmed) return false; + const all = read(backend); + if (all.some((p) => p.name === trimmed)) return false; + const target = all.find((p) => p.name === from); + if (!target) return false; + target.name = trimmed; + write(backend, all); + return true; +} + +/** + * Apply a preset's rows back onto the live params, by INDEX (rows are + * index-aligned with the output set). Returns a new MFParam[] (immutable update + * for React state). Rows beyond the live param count are ignored; missing rows + * leave that param untouched. + */ +export function applyPreset(params: MFParam[], preset: OutputPreset): MFParam[] { + return params.map((p, i) => { + const r = preset.rows[i]; + if (!r) return p; + return { + ...p, + // name is preset-driven for MIDI/OSC where the user renames outputs; + // keep the live name if the preset row didn't carry one. + name: r.name ?? p.name, + status: r.status ?? p.status, + muted: r.muted ?? p.muted, + armed: r.armed ?? p.armed, + min: r.min ?? p.min, + max: r.max ?? p.max, + curve: r.curve ?? p.curve, + val: r.val ?? p.val, + midi: r.midi ?? p.midi, + osc: r.osc ?? p.osc, + vcv: r.vcv ?? p.vcv, + }; + }); +} diff --git a/manifold/src/backends/useBackendManager.ts b/manifold/src/backends/useBackendManager.ts new file mode 100644 index 0000000..875b37a --- /dev/null +++ b/manifold/src/backends/useBackendManager.ts @@ -0,0 +1,136 @@ +/** + * useBackendManager — the thin React binding over the framework-neutral + * {@link BackendManager}. It: + * + * - creates ONE BackendManager per engine (the manager subscribes to the + * spine and forwards routed → active backend); + * - rebuilds the BackendContext (per-output baseline mappings + names) from + * the live MFParam store whenever it changes, and pushes it to the manager; + * - switches the active backend when the dock Mode (→ BackendId) changes + * (which also applies the synth audio mute gate); + * - pushes the per-backend config (MIDI port/cc map; OSC bridge/specs) down; + * - surfaces the active backend's status for the Outputs panel. + * + * Returns the live manager + status so the Outputs drawer can render + * specialised, editable per-backend config. + */ +import { useEffect, useMemo, useRef, useState } from 'react'; +import type { EngineApi } from '../engine'; +import type { MFParam } from '../console/model'; +import type { BackendContext, BackendStatus, OutputMapping } from './backend'; +import type { BackendId, MidiCcSpec, OscSpec } from '../dock/output-state'; +import { defaultMidiSpec, defaultOscSpec } from '../dock/output-state'; +import { BackendManager } from './manager'; + +export interface MidiSettings { + outputId: string | null; + ccCount: number; +} + +export interface OscSettings { + url: string; + sendRaw: boolean; +} + +function toMapping(p: MFParam): OutputMapping { + return { + state: p.status, + muted: p.muted ?? false, + min: p.min, + max: p.max, + curve: p.curve, + fixedValue: p.val, + }; +} + +export interface UseBackendManager { + manager: BackendManager | null; + status: BackendStatus; + /** Available MIDI output ports (refreshes when MIDI starts/hot-plugs). */ + midiPorts: { id: string; name: string }[]; + refreshMidiPorts: () => void; +} + +export function useBackendManager( + engine: EngineApi | null, + backendId: BackendId, + modeId: string, + params: MFParam[], + midiSettings: MidiSettings, + oscSettings: OscSettings, +): UseBackendManager { + const managerRef = useRef(null); + const [status, setStatus] = useState({ state: 'idle', message: 'idle' }); + const [midiPorts, setMidiPorts] = useState<{ id: string; name: string }[]>([]); + + // One manager per engine. + if (engine && !managerRef.current) { + managerRef.current = new BackendManager(engine); + } + const manager = managerRef.current; + + useEffect(() => { + return () => { + managerRef.current?.dispose(); + managerRef.current = null; + }; + }, []); + + // Status wiring. + useEffect(() => { + if (!manager) return; + setStatus(manager.status()); + return manager.onStatusChange((id, s) => { + if (id === manager.getActiveId()) setStatus(s); + }); + }, [manager]); + + // Build the BackendContext from the live params. + const ctx: BackendContext = useMemo( + () => ({ + modeId, + outputCount: params.length, + mappings: params.map(toMapping), + names: params.map((p) => p.name), + }), + [modeId, params], + ); + + useEffect(() => { + manager?.setContext(ctx); + }, [manager, ctx]); + + // Switch the active backend on Mode change (applies the audio gate). + useEffect(() => { + if (!manager) return; + manager.setContext(ctx); + void manager.setActive(backendId); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [manager, backendId]); + + // Push per-output MIDI config + port/ccCount whenever they change. + const refreshMidiPorts = () => { + const midi = manager?.midi(); + if (midi) setMidiPorts(midi.listOutputs()); + }; + useEffect(() => { + const midi = manager?.midi(); + if (!midi) return; + const specs: MidiCcSpec[] = params.map((p, i) => p.midi ?? defaultMidiSpec(i)); + midi.setMidiConfig(specs, { + outputId: midiSettings.outputId, + ccCount: midiSettings.ccCount, + }); + setMidiPorts(midi.listOutputs()); + }, [manager, params, midiSettings.outputId, midiSettings.ccCount, status.state]); + + // Push per-output OSC config + bridge URL/raw whenever they change. + useEffect(() => { + const osc = manager?.osc(); + if (!osc) return; + const specs: OscSpec[] = params.map((p) => p.osc ?? defaultOscSpec(p.name)); + osc.setOscConfig(specs, { url: oscSettings.url, sendRaw: oscSettings.sendRaw }); + }, [manager, params, oscSettings.url, oscSettings.sendRaw]); + + return { manager, status, midiPorts, refreshMidiPorts }; +} diff --git a/manifold/src/backends/webmidi.d.ts b/manifold/src/backends/webmidi.d.ts new file mode 100644 index 0000000..d2b3411 --- /dev/null +++ b/manifold/src/backends/webmidi.d.ts @@ -0,0 +1,32 @@ +/** + * Minimal Web MIDI API type declarations (the `@types/webmidi` package is not a + * dependency). Covers only the surface midi-backend.ts uses: requestMIDIAccess, + * the outputs map, and MIDIOutput.send. Feature-detected at runtime. + */ + +interface MIDIOutput { + readonly id: string; + readonly name?: string | null; + send(data: number[] | Uint8Array, timestamp?: number): void; +} + +interface MIDIOutputMap { + forEach(cb: (value: MIDIOutput, key: string, map: MIDIOutputMap) => void): void; + get(id: string): MIDIOutput | undefined; + readonly size: number; + [Symbol.iterator](): IterableIterator<[string, MIDIOutput]>; +} + +interface MIDIAccess { + readonly outputs: MIDIOutputMap; + onstatechange: ((this: MIDIAccess, ev: Event) => void) | null; +} + +interface MIDIOptions { + sysex?: boolean; + software?: boolean; +} + +interface Navigator { + requestMIDIAccess?(options?: MIDIOptions): Promise; +} diff --git a/manifold/src/console/CompositeStage.tsx b/manifold/src/console/CompositeStage.tsx new file mode 100644 index 0000000..5dd192e --- /dev/null +++ b/manifold/src/console/CompositeStage.tsx @@ -0,0 +1,623 @@ +/** + * CompositeStage — THE convertible centerpiece. ONE continuous view that becomes + * inputs-first, outputs-first, or 50/50 by dragging a single divider. No discrete + * modes — the layout is a single ratio `split` ∈ [0,1] (the input's share of the + * width): + * split → 1 inputs-first (output demotes to a slim readout list, then a minimap) + * split = 0.5 dual / 50-50 + * split → 0 outputs-first (input demotes to a pad, then a minimap) + * Pull the seam all the way to an edge and the small side snaps shut, popping out + * as a draggable corner minimap. Each panel chooses its representation from its + * MEASURED width, so it never becomes a useless sliver — it demotes. Handle snaps + * to 0.14·0.33·0.5·0.66·0.86 with light magnetism; presets tween the ratio. + * + * `split` is persisted (localStorage 'mf-composite-split') by ConsoleApp; the + * minimap corners are persisted here ('mf-mm-incorner' / 'mf-mm-outcorner'). + * + * Ported faithfully from the window-global `CompositeStage.jsx`. + */ +import { useEffect, useRef, useState } from 'react'; +import type { CSSProperties, PointerEvent as ReactPointerEvent } from 'react'; +import { VirtualJoystick, XYPad } from '../primitives'; +import { Manifold } from './Manifold'; +import { OutputStage } from './OutputStage'; +import { MiniMeters } from './shared-ui'; +import type { MFMode, MFParam } from './model'; +import type { FeedbackMarker, Pin } from './types'; + +const SNAPS = [0.14, 0.33, 0.5, 0.66, 0.86]; +const MAGNET = 0.026; +const SHUT = 0.1; + +const GC: Record = { + formant: '--accent', + pitch: '--accent-2', + amp: '--good', + filter: '--warn', + fx: '--info', + mod: '--accent-3', +}; + +const CORNERS: Record = { + tl: { top: 62, left: 14 }, + tr: { top: 62, right: 14 }, + bl: { bottom: 14, left: 14 }, + br: { bottom: 14, right: 14 }, +}; + +export interface CompositeStageProps { + split: number; + onSplit: (s: number) => void; + mode: MFMode; + pos: [number, number]; + onMove: (x: number, y: number) => void; + noiseCap: number; + pins: Pin[]; + markers?: FeedbackMarker[]; + variant?: 'rectangular' | 'circular'; + follow: boolean; + onLongPress: (p: [number, number]) => void; + params: MFParam[]; + values: number[]; + onChange: (i: number, patch: Partial) => void; +} + +export function CompositeStage({ + split, + onSplit, + mode, + pos, + onMove, + noiseCap, + pins, + markers = [], + variant = 'rectangular', + follow, + onLongPress, + params, + values, + onChange, +}: CompositeStageProps) { + const ref = useRef(null); + const [size, setSize] = useState({ w: 0, h: 0 }); + const drag = useRef(false); + const splitRef = useRef(split); + splitRef.current = split; + + const [inCorner, setInCorner] = useState( + () => localStorage.getItem('mf-mm-incorner') || 'tl', + ); + const [outCorner, setOutCorner] = useState( + () => localStorage.getItem('mf-mm-outcorner') || 'tl', + ); + useEffect(() => { + localStorage.setItem('mf-mm-incorner', inCorner); + }, [inCorner]); + useEffect(() => { + localStorage.setItem('mf-mm-outcorner', outCorner); + }, [outCorner]); + + useEffect(() => { + const el = ref.current; + if (!el) return; + const read = () => { + if (el.clientWidth) setSize({ w: el.clientWidth, h: el.clientHeight }); + }; + const ro = new ResizeObserver(read); + ro.observe(el); + read(); + let t: ReturnType | null = null; + let k = 0; + const kick = () => { + read(); + if (!el.clientWidth && k++ < 80) t = setTimeout(kick, 40); + }; + kick(); + return () => { + ro.disconnect(); + if (t) clearTimeout(t); + }; + }, []); + + const setFromClientX = (clientX: number) => { + const el = ref.current; + if (!el) return; + const r = el.getBoundingClientRect(); + let f = (clientX - r.left) / r.width; + f = Math.max(0, Math.min(1, f)); + if (f < SHUT) f = 0; + else if (f > 1 - SHUT) f = 1; + else + for (const s of SNAPS) + if (Math.abs(f - s) < MAGNET) { + f = s; + break; + } + onSplit(f); + }; + const down = (e: ReactPointerEvent) => { + drag.current = true; + e.currentTarget.setPointerCapture?.(e.pointerId); + setFromClientX(e.clientX); + }; + const move = (e: ReactPointerEvent) => { + if (drag.current) setFromClientX(e.clientX); + }; + const up = (e: ReactPointerEvent) => { + drag.current = false; + e.currentTarget.releasePointerCapture?.(e.pointerId); + }; + + const tweenTo = (target: number) => { + const start = splitRef.current; + const t0 = performance.now(); + const dur = 300; + const ease = (p: number) => 1 - Math.pow(1 - p, 3); + const step = () => { + const p = Math.min(1, (performance.now() - t0) / dur); + onSplit(start + (target - start) * ease(p)); + if (p < 1) requestAnimationFrame(step); + }; + requestAnimationFrame(step); + }; + + const { w, h } = size; + const knownW = w > 0; + const collapsed = split <= 0.001 ? 'in' : split >= 0.999 ? 'out' : null; + const wIn = w * split; + const wOut = w * (1 - split); + const inTier = !knownW || wIn >= 300 ? 'full' : 'pad'; + const outTier = !knownW || wOut >= 230 ? 'field' : 'list'; + + // The input-map Setting overrides the mode's declared input shape. + const isJoy = variant === 'circular'; + const renderPad = (padSize: number) => + isJoy ? ( + onMove(x, y)} /> + ) : ( + onMove(x, y)} showGrid /> + ); + + const tag = (text: string, side: 'left' | 'right') => ( +
+ {text} +
+ ); + + // ---- INPUT panel content by tier (non-collapsed) ---- + const renderInput = () => { + if (inTier === 'full') { + return ( + + ); + } + const s = Math.max(88, Math.min(wIn - 28, h - 88)); + return ( +
+ {renderPad(s)} + + {pos[0].toFixed(2)}, {pos[1].toFixed(2)} + +
+ ); + }; + + // ---- OUTPUT panel content by tier (non-collapsed) ---- + const renderOutput = () => { + if (outTier === 'field') { + return ( + + ); + } + return ( +
+ {params.map((p, i) => { + const eff = values[i] ?? 0; + const gc = `var(${GC[p.group] || '--accent'})`; + const dim = p.status === 'off'; + const set = (cx: number, el: HTMLDivElement) => { + const r = el.getBoundingClientRect(); + onChange(i, { val: Math.max(0, Math.min(1, (cx - r.left) / r.width)) }); + }; + return ( +
+
+ + {p.name} + + + {eff.toFixed(2)} + +
+
{ + e.currentTarget.setPointerCapture?.(e.pointerId); + (e.currentTarget as HTMLDivElement & { _d?: boolean })._d = true; + set(e.clientX, e.currentTarget); + }} + onPointerMove={(e) => { + if ((e.currentTarget as HTMLDivElement & { _d?: boolean })._d) + set(e.clientX, e.currentTarget); + }} + onPointerUp={(e) => { + (e.currentTarget as HTMLDivElement & { _d?: boolean })._d = false; + e.currentTarget.releasePointerCapture?.(e.pointerId); + }} + style={{ + position: 'relative', + height: 6, + background: 'var(--bg-1)', + border: '1px solid var(--line)', + borderRadius: 999, + cursor: 'ew-resize', + touchAction: 'none', + }} + > +
+
+
+
+ ); + })} +
+ ); + }; + + // ---- minimap (the collapsed side as a draggable corner rectangle) ---- + const [mmPos, setMmPos] = useState<{ x: number; y: number; side: 'in' | 'out' } | null>(null); + const mmRef = useRef<{ + side: 'in' | 'out' | null; + dx?: number; + dy?: number; + cw?: number; + ch?: number; + }>({ side: null }); + const mmDown = (side: 'in' | 'out') => (e: ReactPointerEvent) => { + const card = (e.currentTarget as HTMLElement).closest('[data-mm]') as HTMLElement | null; + if (!card || !ref.current) return; + e.currentTarget.setPointerCapture?.(e.pointerId); + const r = ref.current.getBoundingClientRect(); + const cr = card.getBoundingClientRect(); + mmRef.current = { + side, + dx: e.clientX - cr.left, + dy: e.clientY - cr.top, + cw: cr.width, + ch: cr.height, + }; + setMmPos({ x: cr.left - r.left, y: cr.top - r.top, side }); + }; + const mmMove = (e: ReactPointerEvent) => { + const m = mmRef.current; + if (!m.side || !ref.current) return; + const r = ref.current.getBoundingClientRect(); + const x = Math.max(8, Math.min(r.width - (m.cw ?? 0) - 8, e.clientX - r.left - (m.dx ?? 0))); + const y = Math.max(8, Math.min(r.height - (m.ch ?? 0) - 8, e.clientY - r.top - (m.dy ?? 0))); + setMmPos({ x, y, side: m.side }); + }; + const mmUp = (e: ReactPointerEvent) => { + const m = mmRef.current; + if (!m.side || !ref.current) return; + const r = ref.current.getBoundingClientRect(); + const p = mmPos || { x: 0, y: 0 }; + const corner = + (p.y + (m.ch ?? 0) / 2 < r.height / 2 ? 't' : 'b') + + (p.x + (m.cw ?? 0) / 2 < r.width / 2 ? 'l' : 'r'); + (m.side === 'in' ? setInCorner : setOutCorner)(corner); + mmRef.current = { side: null }; + setMmPos(null); + e.currentTarget.releasePointerCapture?.(e.pointerId); + }; + + const miniCard = (side: 'in' | 'out', corner: string, body: JSX.Element) => { + const dragging = mmPos && mmPos.side === side; + const place: CSSProperties = dragging + ? { left: mmPos.x, top: mmPos.y } + : CORNERS[corner]; + const restore = () => tweenTo(0.5); + return ( +
+
+ + ⠿ {side === 'in' ? 'INPUT' : 'OUTPUT'} + + +
+ {body} +
+ ); + }; + const miniInput = () => miniCard('in', inCorner, renderPad(118)); + const miniOutput = () => + miniCard( + 'out', + outCorner, +
+ +
, + ); + + // ---- divider geometry ---- + const presetActive = + Math.abs(split - 0.86) < 0.04 + ? 'in' + : Math.abs(split - 0.14) < 0.04 + ? 'out' + : Math.abs(split - 0.5) < 0.04 + ? 'dual' + : null; + const handleLeft = split <= 0 ? '0%' : split >= 1 ? '100%' : `${split * 100}%`; + const handleMargin = split <= 0 ? 0 : split >= 1 ? -18 : -9; + + return ( +
+ {collapsed === 'in' ? ( + <> +
+ {renderOutput()} +
+ {miniInput()} + + ) : collapsed === 'out' ? ( + <> +
+ {tag('INPUT', 'left')} + {renderInput()} +
+ {miniOutput()} + + ) : ( + <> +
+ {tag('INPUT', 'left')} + {renderInput()} +
+
+ {renderOutput()} +
+ {SNAPS.map((s) => ( +
+ ))} + + )} + + {/* the divider handle — at the seam, or an edge tab when collapsed */} +
+ tweenTo(presetActive === 'dual' ? 0.86 : presetActive === 'in' ? 0.14 : 0.5) + } + title={collapsed ? 'pull to reveal' : 'drag to rebalance · double-click to cycle'} + style={{ + position: 'absolute', + top: 0, + bottom: 0, + left: handleLeft, + width: 18, + marginLeft: handleMargin, + zIndex: 26, + cursor: 'col-resize', + touchAction: 'none', + display: 'flex', + alignItems: 'center', + justifyContent: 'center', + }} + > +
+
+ {[0, 1, 2].map((k) => ( + + ))} +
+
+ +
+ ); +} diff --git a/manifold/src/console/ConsoleApp.tsx b/manifold/src/console/ConsoleApp.tsx new file mode 100644 index 0000000..ef3b4cb --- /dev/null +++ b/manifold/src/console/ConsoleApp.tsx @@ -0,0 +1,858 @@ +/** + * ConsoleApp — the reactive spine + layout for the convertible Console, wired to + * the REAL engine. + * + * What changed vs the window-global `ConsoleApp.jsx`: + * - The pseudo-inference (`MF_infer`, sin/cos) is GONE. The `values` every + * consumer reads now come from `engine.getOutputs()`, mapped onto the mode's + * params by `shapeValues` (status/min/max/curve applied here). Pad/joystick + * motion drives `engine.setInput(x,y)`; we subscribe to engine changes via + * `useEngineVersion` and re-derive `values` imperatively on render. + * - Verdicts wire to the engine: commit → feedback.thumbsUp(); perturb → + * feedback.thumbsDown(); reroll → randomise(); each followed by process(). + * - The default feedback mode is "Explore and place" → randomise_mlp (set on + * mount; per docs/redesign/rl-feedback-design.md). + * - AltitudeNav switches `focus` via React state (in|split|out|composite), not + * by navigating to separate HTML files. + * - `c15` is labelled "Powerful Synth Engine" (in model.ts) — "C15" never shows. + * + * UI-only state (params status/min/max/curve, snapshots, A/B seed, axes, + * noiseCap, health/rev visuals) is preserved as faithful local React state. + */ +import { useEffect, useMemo, useRef, useState } from 'react'; +import type { CSSProperties } from 'react'; +import { useEngine, useEngineVersion } from '../engine'; +import { MF_MODES, modeEngineId, seededGradient, shapeValues } from './model'; +import type { MFParam } from './model'; +import { CompositeStage } from './CompositeStage'; +import { SplitStage } from './SplitStage'; +import { OutputStage } from './OutputStage'; +import { InputMini } from './InputMini'; +import { Manifold } from './Manifold'; +import { ReadoutStrip } from './ReadoutStrip'; +import { VerdictCluster } from './VerdictCluster'; +import { Dock } from './Dock'; +import type { + Axes, + ConsoleCtx, + DrawerDepth, + DrawerKey, + FeedbackMarker, + FeedbackModeUI, + Focus, + OutputMode, + Pin, + Snapshot, + SoloMode, +} from './types'; +import type { BackendId } from '../dock/output-state'; +import { buildArmMask } from '../dock/output-state'; +import { FeedbackController, type ProtoFeedbackMode } from '../feedback'; +import { DEFAULT_OUTPUT_MODE, OUTPUT_MODES, outputModeDescriptor } from './output-mode'; +import { useSettings, resolveInputMap } from '../settings/settings-store'; +import { useBackendManager } from '../backends'; + +let SNAP_ID = 0; + +/** Small pill-button style for the exploring-scratchpad banner controls. */ +function pillBtn(color: string): CSSProperties { + return { + fontFamily: 'var(--font-mono)', + fontSize: 'var(--fs-xs)', + padding: '3px 10px', + borderRadius: 'var(--r-pill)', + border: `1px solid ${color}`, + background: 'transparent', + color, + cursor: 'pointer', + }; +} + +export interface ConsoleAppProps { + focus?: Focus; +} + +export function ConsoleApp({ focus: initialFocus = 'composite' }: ConsoleAppProps) { + const engine = useEngine(); + const version = useEngineVersion(engine); + const { settings } = useSettings(); + + const [focus, setFocus] = useState(initialFocus); + const [modeId, setModeId] = useState('paf_synth'); + const mode = MF_MODES.find((m) => m.id === modeId) ?? MF_MODES[0]; + const [params, setParams] = useState(() => mode.params.map((p) => ({ ...p }))); + const [pos, setPos] = useState<[number, number]>([0.5, 0.5]); + + // A/B seed-snapshot model (kept as visual parity; A holds a remembered weight + // snapshot conceptually — here we mirror the JSX's seed-based preview marker). + const [seed, setSeed] = useState(0.4); + const [axes, setAxes] = useState({ boldness: 0.55, memory: 0.4, precision: 0.5 }); + const [preset, setPreset] = useState('Sculpt'); + const [noiseCap, setNoiseCap] = useState(0.12); + const [examples, setExamples] = useState(0); + const [addingExample, setAddingExample] = useState(false); + const [loss, setLoss] = useState([]); + const [busy, setBusy] = useState(false); + const [snapshots, setSnapshots] = useState([]); + const [ab, setAB] = useState<'A' | 'B'>('B'); + const [, setHoldingA] = useState(false); + const aRef = useRef<{ seed: number } | null>(null); + const [spread, setSpread] = useState(false); + const [tame, setTame] = useState(0.85); + const [health, setHealth] = useState(0.8); + const [rev, setRev] = useState(1); + const [active, setActive] = useState('learn'); + const [depth, setDepth] = useState('peek'); + + // Learning-behaviour store (dock-spec §1; rl-feedback-design). Default + // feedback mode = "Explore and place"; default solo = "Mask gradients". + const [feedbackMode, setFeedbackModeState] = useState('explore-and-place'); + const [soloMode, setSoloMode] = useState('mask-gradients'); + const [exploring, setExploring] = useState(false); + const [learningPaused, setLearningPaused] = useState(false); + // Explore-and-place scratchpad session state (workstream B; rl-feedback §2.2). + const [picking, setPicking] = useState(false); + const [anchorCount, setAnchorCount] = useState(0); + const [undoDepth, setUndoDepth] = useState(0); + const [learningRate, setLearningRate] = useState(0.00001); + const [decay, setDecay] = useState(0.97); + const [spreadLevel, setSpreadLevel] = useState(0.6); + // Active output MODE (TOP dock selector) — default Particle System. The dock + // backend + audio backend derive from this. + const [outputMode, setOutputModeState] = useState(DEFAULT_OUTPUT_MODE); + const outputBackend: BackendId = outputModeDescriptor(outputMode).backend; + // Per-backend transport settings (backends-spec §2.3/§2.4). Persisted via the + // named-preset system; these are the live working values. + const [midiOutputId, setMidiOutputId] = useState(null); + const [midiCcCount, setMidiCcCount] = useState(8); + const [oscUrl, setOscUrl] = useState('ws://localhost:8765'); + const [oscSendRaw, setOscSendRaw] = useState(false); + // Feedback markers plotted on the 2D map (both polarities; session-scoped). + const [markers, setMarkers] = useState([]); + const [volume, setVolume] = useState(0.8); + const [bpm, setBpm] = useState(120); + const [audioStarted, setAudioStarted] = useState(false); + const [follow, setFollow] = useState(false); + const [split, setSplit] = useState(() => { + const v = parseFloat(localStorage.getItem('mf-composite-split') ?? ''); + return Number.isFinite(v) ? Math.max(0, Math.min(1, v)) : 0.5; + }); + useEffect(() => { + localStorage.setItem('mf-composite-split', String(split)); + }, [split]); + const [stripPinned, setStripPinned] = useState(true); + const [firstSession, setFirstSession] = useState(true); + const [pins, setPins] = useState([]); + + // The framework-neutral learning-engine controller (workstream B). Owns BOTH + // feedback modes' BEHAVIOUR on the existing engine primitives. The dock owns + // the mode/solo SELECTOR UI; this controller implements what those selectors + // mean. One instance per engine, created once the engine resolves. + const controllerRef = useRef(null); + if (engine && !controllerRef.current) { + controllerRef.current = new FeedbackController(engine, { + seed: 0xfeedbacc, + spread: 0.6, + }); + } + + // Pull the controller's observable state into React after any action. + const syncController = () => { + const c = controllerRef.current; + if (!c) return; + const s = c.getState(); + setExploring(s.exploring); + setLearningPaused(s.exploring); // Mode-2 pauses learning while scratchpad-ing + setPicking(s.picking); + setAnchorCount(s.anchorCount); + setUndoDepth(s.undoDepth); + }; + + const setFeedbackMode = (m: FeedbackModeUI) => { + setFeedbackModeState(m); + controllerRef.current?.setMode(m as ProtoFeedbackMode); + syncController(); + }; + + // Push the active UI feedback mode to the controller on mount + on change. + useEffect(() => { + controllerRef.current?.setMode(feedbackMode as ProtoFeedbackMode); + syncController(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [engine, feedbackMode]); + + // Push the selected solo-mode + the arm mask into the controller whenever they + // change (dock-spec §1.2). The controller RESPECTS the arm mask at the example + // level in BOTH modes and forwards it to engine.feedback.setFocus. + // TODO(rl-feedback-design §3): soloMode (mask-gradients / zero-loss / + // dont-care) selects HOW the mask is applied during training; the C API only + // exposes set_focus today, so the controller approximates it at the example + // level — the true gradient column-freeze (`train_masked`) is the C++ step. + useEffect(() => { + controllerRef.current?.setSoloMode(soloMode); + controllerRef.current?.setArmMask(buildArmMask(params)); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [engine, params, soloMode]); + + // Keep the audio backend pointed at the current mode (audio itself is gated + // behind a user gesture — see startAudio below). + useEffect(() => { + if (engine) engine.audio.setBackend(modeEngineId(modeId) as Parameters[0]); + }, [engine, modeId]); + + // reset transient state on mode switch + useEffect(() => { + setParams(mode.params.map((p) => ({ ...p }))); + setPos([0.5, 0.5]); + setExamples(0); + setLoss([]); + setSnapshots([]); + setSeed(0.4); + setFollow(false); + setPins([]); + setMarkers([]); + setActive('learn'); + setDepth('peek'); + if (engine) engine.setInput(0.5, 0.5); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [modeId]); + + // Select the active output MODE (TOP dock selector). For the audio (synth) + // mode we keep the audio backend pointed at the current instrument mode (the + // existing effect handles engine.audio.setBackend). The BackendManager (via + // useBackendManager, below) reacts to the derived BackendId: it switches the + // active output backend, gates synth audio (mutes on non-synth modes), and + // drives the real MIDI / OSC transports. Particle reads the spine in its own + // rAF loop; the Editor serial protocol remains its own panel. + const setOutputMode = (m: OutputMode) => setOutputModeState(m); + + // Drive a pad/joystick/manifold move through the real engine, then mirror the + // raw position into React state for readouts. + const onMove = (x: number, y: number) => { + engine?.setInput(x, y); + setPos([x, y]); + }; + + // PICK-LOCATION: when "place" is pending, the next manifold pointer-down picks + // the anchor location → commit the positive anchor there (rl-feedback §2.2 §3). + // The Manifold calls this on pointer-down while `picking` is true; it moves the + // scratchpad input there, captures the output, and stores the anchor. + const onPickLocation = (x: number, y: number) => { + const c = controllerRef.current; + if (!c || !c.isPicking()) return; + c.placeCommit(x, y); + setPos([x, y]); + pushSnap('anchor'); + pushMarker([x, y], 'positive'); + syncController(); + setRev((r) => r + 1); + }; + + // Output backend transport (backends-spec). The manager consumes the engine + // spine and forwards routed outputs to the active backend; switching Mode + // tears down the old backend, starts the new one, and gates synth audio + // (mute on non-synth modes). MIDI/OSC config + names ride the shared params. + const { status: backendStatus, midiPorts, refreshMidiPorts } = useBackendManager( + engine, + outputBackend, + modeId, + params, + { outputId: midiOutputId, ccCount: midiCcCount }, + { url: oscUrl, sendRaw: oscSendRaw }, + ); + + // values come from the REAL engine output, shaped per-param. Recomputed when + // the engine version bumps (new inference / weights) or params change. + const values = useMemo( + () => shapeValues(params, engine?.getOutputs() ?? null), + // version drives re-read of the live (reused) output buffer. + // eslint-disable-next-line react-hooks/exhaustive-deps + [params, version, engine], + ); + + const gradient = useMemo(() => seededGradient(rev), [rev]); + + const pushSnap = (tag: string) => + setSnapshots((s) => [...s, { id: ++SNAP_ID, tag, noise: noiseCap, seed }].slice(-50)); + + /** Plot a feedback marker at the input location it was given (session-scoped). */ + const pushMarker = (at: [number, number], polarity: 'positive' | 'negative') => + setMarkers((m) => [...m, { x: at[0], y: at[1], polarity }].slice(-200)); + + // ------------------------------------------------------------------- + // Verdict actions — routed per active feedback mode (rl-feedback-design §0). + // + // Explore & place (Mode 2, default): thumbs-DOWN = enter/cancel explore; + // thumbs-UP = place (when exploring) / commit. The scratchpad is never + // trained — it only generates candidate sounds to audition. + // Geometric dislike (Mode 1): thumbs-DOWN = dislike (push away); + // thumbs-UP = like + train. + // + // VerdictCluster reads `feedbackMode` from ctx and relabels itself; the same + // onCommit / onPerturb handlers below dispatch on the active mode. + // ------------------------------------------------------------------- + + /** thumbs-UP. */ + const commit = () => { + const c = controllerRef.current; + setFirstSession(false); + setBusy(true); + if (feedbackMode === 'explore-and-place') { + if (c?.getState().exploring) { + // Place the current candidate → next manifold tap chooses the location. + pushSnap('place'); + c.place(); + } else { + // Not exploring: a plain positive reinforcement of the current mapping. + pushSnap('commit +'); + c?.like(pos, engine?.getOutputs() ?? new Float32Array(0)); + pushMarker(pos, 'positive'); + } + } else { + // Geometric dislike: thumbs-up = like + train. + pushSnap('like +'); + c?.like(pos, engine?.getOutputs() ?? new Float32Array(0)); + pushMarker(pos, 'positive'); + } + syncController(); + setNoiseCap((n) => Math.max(0.02, n * 0.7)); + setHealth((h) => Math.min(1, h + 0.08)); + setRev((r) => r + 1); + const l = engine?.evalLoss(); + setLoss((prev) => + [...prev, Number.isFinite(l) ? (l as number) : prev.length ? prev[prev.length - 1] : 0.5].slice(-120), + ); + setBusy(false); + }; + + /** thumbs-DOWN. */ + const perturb = () => { + const c = controllerRef.current; + setFirstSession(false); + if (feedbackMode === 'explore-and-place') { + // Enter the scratchpad (or, if already exploring, cancel back to the real + // net). NEVER a dislike — Mode 2 is positive-only. + if (c?.getState().exploring) { + pushSnap('cancel explore'); + c.cancel(); + } else { + pushSnap('explore'); + c?.enterExplore(); + } + } else { + // Geometric dislike: push the current mapping away from this sound. + pushSnap('dislike −'); + c?.dislike(pos, engine?.getOutputs() ?? new Float32Array(0), noiseCap, spread ? 1 : 0.6); + pushMarker(pos, 'negative'); + setSeed((s) => s + (Math.random() - 0.5) * (noiseCap * 4 + 0.3)); + setNoiseCap((n) => Math.min(0.5, n + 0.06)); + setHealth((h) => Math.max(0.1, h - 0.06)); + } + syncController(); + setRev((r) => r + 1); + }; + + /** Long-press perturb / explicit re-roll. */ + const reroll = () => { + const c = controllerRef.current; + setFirstSession(false); + pushSnap('re-roll'); + if (feedbackMode === 'explore-and-place' && c?.getState().exploring) { + // Re-roll the scratchpad net (undoable) without leaving the session. + c.reroll(); + } else { + // Outside a scratchpad session a re-roll randomises the real net directly. + engine?.randomise(spread ? 1 : 0.6); + } + syncController(); + setSeed(Math.random() * 6); + setNoiseCap(0.4); + setHealth(0.5); + setRev((r) => r + 1); + }; + + /** + * Undo. While exploring (Mode 2) this pops the scratchpad undo ring (reroll / + * nudge). Otherwise it falls back to the UI snapshot stack (visual A/B seed). + */ + const undo = () => { + const c = controllerRef.current; + if (feedbackMode === 'explore-and-place' && c?.getState().exploring) { + c.undo(); + syncController(); + setRev((r) => r + 1); + return; + } + setSnapshots((s) => { + if (!s.length) return s; + const last = s[s.length - 1]; + setSeed(last.seed); + setNoiseCap(last.noise); + setRev((r) => r + 1); + return s.slice(0, -1); + }); + }; + + // ---- Explore-and-place scratchpad ops surfaced to the dock + cluster ---- + const onExplore = () => { + controllerRef.current?.enterExplore(); + syncController(); + setRev((r) => r + 1); + }; + const onScratchReroll = () => { + controllerRef.current?.reroll(); + syncController(); + setRev((r) => r + 1); + }; + const onScratchNudge = () => { + controllerRef.current?.nudge(); + syncController(); + setRev((r) => r + 1); + }; + const onScratchUndo = () => { + controllerRef.current?.undo(); + syncController(); + setRev((r) => r + 1); + }; + const onPlace = () => { + controllerRef.current?.place(); + syncController(); + }; + const onFinalise = () => { + setBusy(true); + controllerRef.current?.finalise(); + syncController(); + setRev((r) => r + 1); + setBusy(false); + }; + const onCancelExplore = () => { + controllerRef.current?.cancel(); + syncController(); + setRev((r) => r + 1); + }; + const train = () => { + setBusy(true); + const l = engine?.train(); + engine?.process(); + setLoss((p) => + [...p, Number.isFinite(l) ? (l as number) : p.length ? p[p.length - 1] * 0.82 : 0.5].slice(-120), + ); + setBusy(false); + }; + const addExample = () => { + if (!addingExample) { + setAddingExample(true); + return; + } + setAddingExample(false); + // Snapshot the current input → current (shaped) output as a training example. + engine?.addExample([pos[0], pos[1]], Array.from(values)); + setExamples((e) => e + 1); + pushSnap('example'); + train(); + }; + + const setParam = (i: number, patch: Partial) => + setParams((ps) => ps.map((p, j) => (j === i ? { ...p, ...patch } : p))); + const cycleStatus = (i: number) => + setParams((ps) => + ps.map((p, j) => + j === i + ? { ...p, status: ({ off: 'fixed', fixed: 'live', live: 'off' } as const)[p.status] } + : p, + ), + ); + + const toggleAB = () => { + if (ab === 'B') { + aRef.current = { seed }; + setAB('A'); + } else { + if (aRef.current) setSeed(aRef.current.seed); + setAB('B'); + } + }; + + // keyboard accelerators + useEffect(() => { + const onKey = (e: KeyboardEvent) => { + if ((e.target as HTMLElement | null)?.tagName === 'INPUT') return; + const map: Record = { + '1': 'learn', + '2': 'inputs', + '3': 'route', + '4': 'settings', + '5': 'help', + }; + if (map[e.key]) { + setActive((a) => (a === map[e.key] ? null : map[e.key])); + setDepth('peek'); + } else if (e.key === '\\') setDepth((d) => (d === 'full' ? 'peek' : 'full')); + else if (focus === 'composite' && e.key === '[') { + e.preventDefault(); + setSplit((s) => Math.max(0, s - 0.04)); + } else if (focus === 'composite' && e.key === ']') { + e.preventDefault(); + setSplit((s) => Math.min(1, s + 0.04)); + } else if (focus === 'composite' && (e.key === '=' || e.key === '0')) { + e.preventDefault(); + setSplit(0.5); + } else if (e.key === ' ' || e.key === 'ArrowUp') { + e.preventDefault(); + commit(); + } else if (e.key === 'ArrowDown') { + e.preventDefault(); + perturb(); + } else if (e.key.toLowerCase() === 'z') undo(); + }; + window.addEventListener('keydown', onKey); + return () => window.removeEventListener('keydown', onKey); + }); + + const onToggleAudio = () => { + if (!engine) return; + if (engine.audio.isStarted) { + void engine.audio.stop(); + setAudioStarted(false); + } else { + void engine.audio.start().then(() => setAudioStarted(true)); + } + }; + + const ctx: ConsoleCtx = { + modes: MF_MODES, + modeId, + setModeId, + mode, + axes, + setAxis: (k, v) => setAxes((s) => ({ ...s, [k]: v })), + preset, + setPreset, + offsetActive: preset !== 'Sculpt', + datasetCount: examples, + loss, + busy, + addingExample, + onAddExample: addExample, + onTrain: train, + onClear: () => { + engine?.clearExamples(); + setExamples(0); + setLoss([]); + setMarkers([]); + }, + snapshots, + onJump: (id) => { + const s = snapshots.find((x) => x.id === id); + if (s) { + setSeed(s.seed); + setNoiseCap(s.noise); + setRev((r) => r + 1); + } + }, + params, + cycleStatus, + setParam, + outputBackend, + setOutputBackend: (b) => { + // Map a backend id back onto the active Mode (the Mode is the source of + // truth; the Outputs drawer drives it via setOutputMode). + const m = OUTPUT_MODES.find((om) => om.backend === b); + if (m) setOutputMode(m.id); + }, + outputMode, + setOutputMode, + // ---- output backend transport ---- + backendStatus, + midiPorts, + refreshMidiPorts, + midiOutputId, + setMidiOutputId, + midiCcCount, + setMidiCcCount, + oscUrl, + setOscUrl, + oscSendRaw, + setOscSendRaw, + setParams: (next: MFParam[]) => setParams(next), + markers, + health, + gradient: gradient.norms, + gradientStatus: gradient.status, + weightsRevision: rev, + spread, + setSpread, + tame, + setTame, + noiseCap, + setNoiseCap, + // learning-behaviour + feedbackMode, + setFeedbackMode, + soloMode, + setSoloMode, + exploring, + learningPaused, + armedCount: params.filter((p) => p.armed).length, + clearArmed: () => setParams((ps) => ps.map((p) => (p.armed ? { ...p, armed: false } : p))), + learningRate, + setLearningRate, + decay, + setDecay, + spreadLevel, + setSpreadLevel, + // synth + audioStarted, + onToggleAudio, + volume, + setVolume, + bpm, + setBpm, + // explore-and-place scratchpad session (workstream B) + picking, + anchorCount, + undoDepth, + onExplore, + onScratchReroll, + onScratchNudge, + onPlace, + onScratchUndo, + onFinalise, + onCancelExplore, + }; + + // Resolve the effective input-map shape from Settings + the mode's declared input. + const inputMapVariant = resolveInputMap(settings.inputMap, mode.input); + + const healthColor = + health > 0.66 ? 'rgba(107,194,107,' : health > 0.33 ? 'rgba(245,196,94,' : 'rgba(255,68,102,'; + const addPin = (p: [number, number]) => + setPins((ps) => [...ps, { x: p[0], y: p[1], color: 'rgba(255,106,0,0.16)' }]); + + return ( +
+ + + {/* ambient health glow at the screen edge */} +
+ + {/* stage = manifold area (left of dock) */} +
+ {focus === 'in' && (stripPinned || mode.cls !== 'Synth') && ( +
+ setStripPinned((p) => !p)} + /> +
+ )} + +
+ {focus === 'composite' ? ( + + ) : focus === 'split' ? ( + + ) : focus === 'out' ? ( + <> + + + + ) : ( + + )} + + {/* corner overlay */} +
+ MEMLNaut +
+ + 0 : snapshots.length > 0} + ab={ab} + onToggleAB={toggleAB} + onHoldA={setHoldingA} + firstSession={firstSession} + feedbackMode={feedbackMode} + exploring={exploring} + picking={picking} + /> + + {/* Exploring-scratchpad banner (workstream B; rl-feedback §2.2 §7). */} + {exploring && ( +
+ + {picking ? 'tap the manifold to place' : 'exploring (scratchpad)'} + + + {anchorCount} anchor{anchorCount === 1 ? '' : 's'} placed · undo {undoDepth} + + + + + +
+ )} + + {/* Global PICK-LOCATION capture overlay: works in any focus (composite/ + split stages don't expose picking). The directly-rendered Manifold + (focus==='in') also handles picks + draws the reticle; this overlay + guarantees the place→pick loop is reachable everywhere. */} + {picking && focus !== 'in' && ( +
{ + const r = e.currentTarget.getBoundingClientRect(); + const x = Math.max(0, Math.min(1, (e.clientX - r.left) / r.width)); + const y = Math.max(0, Math.min(1, 1 - (e.clientY - r.top) / r.height)); + onPickLocation(x, y); + }} + style={{ + position: 'absolute', + inset: 0, + zIndex: 35, + cursor: 'cell', + background: 'rgba(0,204,255,0.04)', + }} + /> + )} +
+
+ + +
+ ); +} diff --git a/manifold/src/console/CurvePad.tsx b/manifold/src/console/CurvePad.tsx new file mode 100644 index 0000000..ca87f92 --- /dev/null +++ b/manifold/src/console/CurvePad.tsx @@ -0,0 +1,121 @@ +/** + * CurvePad — a square response-curve plot. Vertical drag reshapes the curve. + * `curve` is 0..1 where ~0.43 reads as linear; mirrors the engine's applyCurve. + * Ported from `CurvePad.jsx`. + */ +import { useEffect, useRef } from 'react'; +import type { PointerEvent as ReactPointerEvent } from 'react'; + +export interface CurvePadProps { + curve?: number; + onChange?: (c: number) => void; + size?: number; +} + +export function CurvePad({ curve = 0.5, onChange, size = 116 }: CurvePadProps) { + const ref = useRef(null); + const drag = useRef({ active: false, startY: 0, startC: 0 }); + + useEffect(() => { + const cv = ref.current; + if (!cv) return; + const dpr = window.devicePixelRatio || 1; + cv.width = size * dpr; + cv.height = size * dpr; + const ctx = cv.getContext('2d'); + if (!ctx) return; + ctx.setTransform(dpr, 0, 0, dpr, 0, 0); + ctx.clearRect(0, 0, size, size); + ctx.strokeStyle = 'rgba(255,255,255,0.06)'; + ctx.lineWidth = 1; + ctx.strokeRect(0.5, 0.5, size - 1, size - 1); + for (const t of [0.25, 0.5, 0.75]) { + ctx.beginPath(); + ctx.moveTo(t * size, 0); + ctx.lineTo(t * size, size); + ctx.stroke(); + ctx.beginPath(); + ctx.moveTo(0, t * size); + ctx.lineTo(size, t * size); + ctx.stroke(); + } + ctx.strokeStyle = 'rgba(255,255,255,0.10)'; + ctx.setLineDash([3, 3]); + ctx.beginPath(); + ctx.moveTo(0, size); + ctx.lineTo(size, 0); + ctx.stroke(); + ctx.setLineDash([]); + const css = getComputedStyle(cv); + const accent = css.getPropertyValue('--accent').trim() || '#ff6a00'; + const e = 0.25 + curve * 1.75; + ctx.strokeStyle = accent; + ctx.lineWidth = 2; + ctx.beginPath(); + const pad = 3; + for (let p = 0; p <= 80; p++) { + const xv = p / 80; + const yv = Math.pow(xv, e); + const px = pad + xv * (size - 2 * pad); + const py = size - pad - yv * (size - 2 * pad); + if (p === 0) ctx.moveTo(px, py); + else ctx.lineTo(px, py); + } + ctx.stroke(); + }, [curve, size]); + + const down = (e: ReactPointerEvent) => { + e.currentTarget.setPointerCapture?.(e.pointerId); + drag.current = { active: true, startY: e.clientY, startC: curve }; + }; + const move = (e: ReactPointerEvent) => { + const d = drag.current; + if (!d.active) return; + const dc = (e.clientY - d.startY) / size; + onChange?.(Math.max(0, Math.min(1, d.startC + dc))); + }; + const up = (e: ReactPointerEvent) => { + drag.current.active = false; + e.currentTarget.releasePointerCapture?.(e.pointerId); + }; + + return ( +
+
+ + curve + + + {(0.25 + curve * 1.75).toFixed(2)} + +
+ +
+ ); +} diff --git a/manifold/src/console/Dock.tsx b/manifold/src/console/Dock.tsx new file mode 100644 index 0000000..4f46ea7 --- /dev/null +++ b/manifold/src/console/Dock.tsx @@ -0,0 +1,374 @@ +/** + * The Console dock — right-edge 48px rail + one mutually-exclusive drawer with + * three depth states (Peek 320 / Expand 520 / Full modal). + * + * RESTRUCTURED (operator dock restructure): + * - TOP, pinned: the **Mode** selector — the active OUTPUT BACKEND/target + * (Particle System / MIDI / OSC / Built-in Synth / MEMLNaut Editor). Opens a + * popover listing the five with the current marked. NEVER shows "C15". + * - BELOW: the five drawer icons (Learning, Inputs, Outputs, Settings, Help), + * **vertically centred in the remaining rail space** like a macOS dock — the + * Mode button is a fixed top element; the drawer group is centred in the + * leftover height (flex column with the group in a flex:1 centred wrapper). + * + * Icons are monochrome inline-SVG (icons.tsx), currentColor-driven: focused = + * --accent (orange), unfocused = the Settings unfocused colour. When monochrome + * is OFF the prior glyphs are used. + */ +import { useState } from 'react'; +import type { CSSProperties } from 'react'; +import { DRAWERS } from './Drawers'; +import type { ConsoleCtx, DrawerDepth, DrawerKey, OutputMode } from './types'; +import { OUTPUT_MODES } from './output-mode'; +import { useSettings, unfocusedIconCss } from '../settings/settings-store'; +import { + ModeIcon, + ParticleIcon, + MidiIcon, + OscIcon, + SynthIcon, + EditorIcon, + CloseIcon, + ExpandIcon, + GLYPH_FALLBACK, +} from './icons'; +import type { IconProps } from './icons'; + +const dockMini: CSSProperties = { + width: 26, + height: 24, + borderRadius: 'var(--r-1)', + border: '1px solid var(--line)', + background: 'var(--bg-2)', + color: 'var(--fg-mute)', + cursor: 'pointer', + fontSize: 11, + lineHeight: 1, + display: 'inline-flex', + alignItems: 'center', + justifyContent: 'center', +}; + +/** Per-Mode icon + fallback glyph. */ +const MODE_ICON: Record JSX.Element; glyph: string }> = { + particles: { Icon: ParticleIcon, glyph: GLYPH_FALLBACK.particles }, + midi: { Icon: MidiIcon, glyph: GLYPH_FALLBACK.midi }, + osc: { Icon: OscIcon, glyph: GLYPH_FALLBACK.osc }, + synth: { Icon: SynthIcon, glyph: GLYPH_FALLBACK.synth }, + editor: { Icon: EditorIcon, glyph: GLYPH_FALLBACK.editor }, +}; + +function ModeSelector({ + outputMode, + setOutputMode, + mono, +}: { + outputMode: OutputMode; + setOutputMode: (m: OutputMode) => void; + mono: boolean; +}) { + const [open, setOpen] = useState(false); + const active = OUTPUT_MODES.find((m) => m.id === outputMode) ?? OUTPUT_MODES[0]; + return ( +
+ + {open && ( +
+
+ Mode · output target +
+ {OUTPUT_MODES.map((m) => { + const on = m.id === outputMode; + const { Icon, glyph } = MODE_ICON[m.id]; + return ( + + ); + })} +
+ )} +
+ ); +} + +export interface DockProps { + ctx: ConsoleCtx; + active: DrawerKey | null; + setActive: (k: DrawerKey | null) => void; + depth: DrawerDepth; + setDepth: (d: DrawerDepth) => void; +} + +const ORDER: DrawerKey[] = ['learn', 'inputs', 'route', 'settings', 'help']; + +export function Dock({ ctx, active, setActive, depth, setDepth }: DockProps) { + const { settings } = useSettings(); + const mono = settings.monochromeIcons; + const restColour = unfocusedIconCss(settings.unfocusedIconColour); + + const iconBtn = (key: DrawerKey) => { + const s = DRAWERS[key]; + const on = active === key; + return ( + + ); + }; + + const width = depth === 'expand' ? 520 : 320; + const full = depth === 'full'; + const section = active ? DRAWERS[active] : null; + + return ( + <> + {section && ( + + )} + + + + ); +} diff --git a/manifold/src/console/Drawers.tsx b/manifold/src/console/Drawers.tsx new file mode 100644 index 0000000..9c583fc --- /dev/null +++ b/manifold/src/console/Drawers.tsx @@ -0,0 +1,552 @@ +/** + * Console — the FIVE real dock drawers (operator dock restructure). Each renderer + * takes (ctx, depth); what shows is gated by depth (peek | expand | full): + * + * learn — Learning : feedback-mode selector, solo/arm chooser, live params + * inputs — Inputs : input source + * route — Outputs : per-output control matrix for the ACTIVE mode/backend, + * a Mode-specific config section, and (Editor mode) the + * MEMLNaut serial panel. The old separate "Synth" and + * "Particle/Visual" drawers are REMOVED — their config now + * lives here under the active Mode (TOP dock selector). + * settings — Settings : icon style + input-map shape (settings-store) + * help — Help : keymap + the loop explanation + * + * The TOP dock selector ("Mode") chooses the active OUTPUT backend/target; this + * drawer renders whatever that backend needs. + * + * Engine wiring: the feedback-mode pill → controller.setMode; the arm flags → + * engine.feedback.setFocus; per-output rows write the shared MFParam store. + * Where engine support does not yet exist the UI + state are wired and a TODO + * references the relevant spec — no faked engine behaviour. + */ +import type { ReactNode } from 'react'; +import { Badge, Button, PillToggle, Slider, Switch } from '../primitives'; +import type { ConsoleCtx, DrawerDepth, DrawerKey, FeedbackModeUI, SoloMode } from './types'; +import { OutputControlRow } from '../dock/OutputControlRow'; +import { BackendAdvanced } from '../dock/BackendAdvanced'; +import { OutputsBackendConfig, BackendStatusChip } from '../dock/OutputsBackendConfig'; +import { BACKENDS } from '../dock/output-state'; +import { shapeValues } from './model'; +import { outputModeDescriptor } from './output-mode'; +import { useSettings, unfocusedIconCss } from '../settings/settings-store'; +import type { UnfocusedIconColour, InputMapMode } from '../settings/settings-store'; +import { EditorPanel } from '../serial/EditorPanel'; +import { + LearningIcon, + InputsIcon, + OutputsIcon, + SettingsIcon, + HelpIcon, +} from './icons'; + +function Chip({ children, tone }: { children: ReactNode; tone?: string }) { + return ( + + {children} + + ); +} +function SectionLabel({ children }: { children: ReactNode }) { + return ( +
+ {children} +
+ ); +} + +/** A 2+ segment selector pill that drives a typed value. */ +function Segmented({ + value, + onChange, + options, +}: { + value: T; + onChange: (v: T) => void; + options: { value: T; label: string }[]; +}) { + return ( +
+ {options.map((o) => { + const on = value === o.value; + return ( + + ); + })} +
+ ); +} + +// =========================================================================== +// 1. LEARNING-BEHAVIOUR (dock-spec §1; rl-feedback-design) +// =========================================================================== + +const FEEDBACK_OPTS: { value: FeedbackModeUI; label: string }[] = [ + { value: 'geometric-dislike', label: 'Push away' }, + { value: 'explore-and-place', label: 'Explore & place' }, +]; +const FEEDBACK_DESC: Record = { + 'geometric-dislike': + 'Down carves the current sound away from what you like — directed repulsion (Mode 1).', + 'explore-and-place': + 'Down re-rolls the whole net into a scratchpad you audition; + places a liked sound (Mode 2).', +}; +const SOLO_OPTS: { value: SoloMode; label: string }[] = [ + { value: 'mask-gradients', label: 'Mask gradients' }, + { value: 'zero-loss', label: 'Zero loss' }, + { value: 'dont-care', label: "Don't-care mask" }, +]; +const SOLO_DESC: Record = { + 'mask-gradients': 'Column-freeze (default) — only the armed output moves; the rest stay bit-identical.', + 'zero-loss': 'Expressive, but armed and unarmed outputs share hidden weights, so others can drift.', + 'dont-care': 'Each example stores a per-output mask so stale labels never pull unarmed outputs.', +}; + +function LearningDrawer(ctx: ConsoleCtx, depth: DrawerDepth) { + return ( + <> +
+ {FEEDBACK_OPTS.find((o) => o.value === ctx.feedbackMode)?.label} + arm: {ctx.armedCount ? `${ctx.armedCount} output${ctx.armedCount > 1 ? 's' : ''}` : 'all'} + {ctx.exploring && exploring…} + {ctx.learningPaused && learning paused} +
+ + Down action · feedback mode + + {depth !== 'peek' && ( +

+ {FEEDBACK_DESC[ctx.feedbackMode]} +

+ )} + + {depth !== 'peek' && ( + <> + Solo / arm scope +
+ + + {ctx.armedCount + ? `${ctx.armedCount} armed — arm with the S button on each output row` + : 'every live output learns'} + +
+ Solo behaviour + +

+ {SOLO_DESC[ctx.soloMode]} Solo only freezes the rest as far as a shared network allows. +

+ + Live training params + + + + v.toExponential(1)} + /> + + + )} + + {depth === 'full' && ( + <> + Feedback lab +

+ State machine: idle → exploring → commit / cancel. While Explore & place is exploring, + training is paused and the joystick auditions a random scratchpad net; + commits a placed + anchor and restores the real net. +

+ + {/* TODO(dock-spec §1.3): real LossPlot / WeightHealth / LayerStats / GradientFlow need + nisps_ml_loss_history plumbed through the C API. Diagnostics suite deferred. */} +

+ Loss plot · weight-health · layer-stats · gradient-flow land here once the loss-history C API + is plumbed (dock-spec §1.3). +

+ + )} + + ); +} + +// =========================================================================== +// 2. INPUTS (dock-spec §2 — workstream F territory, referenced) +// =========================================================================== + +function InputsDrawer(ctx: ConsoleCtx, depth: DrawerDepth) { + const src = + ctx.mode.input === 'joystick' ? 'Joystick' : ctx.mode.input === 'audio_in' ? 'Mic (1-input)' : 'XY pad'; + return ( + <> +
+ {src} + 2 inputs +
+ Source + { + /* TODO(workstream F, inputs-spec): MIDI / gamepad / hands sources land here; the + input source is currently fixed by the mode. engine.setInput(x,y) is the only door. */ + }} + options={[ + { value: 'xy', label: 'XY pad' }, + { value: 'joystick', label: 'Joystick' }, + { value: 'audio_in', label: 'Mic' }, + ]} + /> + {depth !== 'peek' && ( +

+ v1 browser is fixed-2-input ({`MLP<2,…>`}). The modular N×M input matrix, per-axis pipeline + (deadzone → zoom → curve → smoothing → momentum) and MIDI / gamepad sources are owned by + workstream F (inputs-spec); this drawer fixes the dock shape only. +

+ )} + + ); +} + +// =========================================================================== +// 3. OUTPUTS / ROUTING (dock-spec §3, §4) +// =========================================================================== + +const VISUAL_NAMES = [ + 'Flow', 'Scale', 'Speed', 'Hue', 'Spread', 'Size', 'Trail', 'Turb', 'Attract', 'Radius', + 'DispRate', 'DispAmt', 'Lifetime', 'Respawn', 'Advection', 'Inertia', 'Drag', 'Repulse', + 'RepCnt', 'RepRate', +]; + +/** + * Per-Mode config section shown ABOVE the per-output rows. The synth Mode shows + * transport + tempo; the particle Mode names the outputs; MIDI/OSC/Editor show + * their own affordances. Replaces the removed Synth + Visual drawers. + */ +function ModeConfig(ctx: ConsoleCtx, depth: DrawerDepth) { + switch (ctx.outputMode) { + case 'synth': + return ( + <> + Transport +
+ + audio starts on the play gesture +
+ + {depth !== 'peek' && ( + <> + Tempo + +

+ The active engine follows the selected mode ({ctx.mode.label}). + {/* TODO(dock-spec §5): arpeggiator + tiered synth presets + the + 18-section group-override matrix are workstream E. */} +

+ + )} + + ); + case 'editor': + return ( + <> + MEMLNaut · USB serial + + + ); + case 'particles': + return depth !== 'peek' ? ( +

+ Flow-field visualiser driven by the first {Math.min(20, ctx.params.length)} outputs (no audio). + {/* TODO(backends-spec §4): port FlowFieldVisualizer + visual preset chips. */} +

+ ) : null; + case 'midi': + // The full MIDI config (port picker, CC count, per-output CC/channel/name) + // + preset bar render via OutputsBackendConfig in RoutingDrawer below. + return depth !== 'peek' ? ( +

+ Each output sends a real Web MIDI CC. Pick a port and set CC# / channel per output. +

+ ) : null; + case 'osc': + return depth !== 'peek' ? ( +

+ Each output sends to an OSC path with a physical range, over the WebSocket bridge. +

+ ) : null; + default: + return null; + } +} + +function RoutingDrawer(ctx: ConsoleCtx, depth: DrawerDepth) { + const values = shapeValues(ctx.params, null); // bar uses shaped held/live value snapshot + const counts = ctx.params.reduce>((a, p) => { + a[p.status] = (a[p.status] || 0) + 1; + return a; + }, {}); + const mutedN = ctx.params.filter((p) => p.muted).length; + const modeDesc = outputModeDescriptor(ctx.outputMode); + const backend = BACKENDS.find((b) => b.id === modeDesc.backend) ?? BACKENDS[0]; + // The particle Mode names its outputs; otherwise use the param names. + const nameFor = (idx: number, fallback: string) => + ctx.outputMode === 'particles' ? VISUAL_NAMES[idx] ?? fallback : fallback; + + if (depth === 'full') { + return ( + <> +
+ {modeDesc.label} + +
+ {ModeConfig(ctx, depth)} + {/* Specialised, editable per-backend config (MIDI/OSC) + named-preset bar. */} + + Advanced · {modeDesc.label} + + + ); + } + + const rows = depth === 'peek' ? ctx.params.slice(0, 6) : ctx.params; + return ( + <> +
+ {modeDesc.label} + live {counts.live || 0} + fixed {counts.fixed || 0} + off {counts.off || 0} + muted {mutedN} + +
+ {ModeConfig(ctx, depth)} + {/* Specialised per-backend config + named-preset bar (MIDI/OSC); hidden at peek. */} + {depth !== 'peek' && } + Outputs · M mute · S arm · off/fixed/live +
+ {rows.map((p) => { + const i = ctx.params.indexOf(p); + const labelled = { ...p, name: nameFor(i, p.name) }; + return ( + ctx.setParam(i, patch)} + showCurve={depth !== 'peek'} + /> + ); + })} +
+ {depth === 'peek' && ctx.params.length > 6 && ( + +{ctx.params.length - 6} more — expand to edit + )} + + ); +} + +// =========================================================================== +// 4. SETTINGS (operator dock restructure — settings-store) +// =========================================================================== + +const ICON_COLOUR_OPTS: { value: UnfocusedIconColour; label: string }[] = [ + { value: 'off-white', label: 'Off-white' }, + { value: 'white', label: 'White' }, + { value: 'orange', label: 'Orange' }, +]; +const INPUT_MAP_OPTS: { value: InputMapMode; label: string }[] = [ + { value: 'follow-mode', label: 'Follow mode' }, + { value: 'rectangular', label: 'Rectangular' }, + { value: 'circular', label: 'Circular' }, +]; + +function SettingsDrawer({ depth }: { ctx: ConsoleCtx; depth: DrawerDepth }) { + const { settings, set } = useSettings(); + return ( + <> + Icons + set('monochromeIcons', v)} + label="Monochrome icons" + /> + {depth !== 'peek' && ( + <> +
Unfocused icon colour
+ set('unfocusedIconColour', v as UnfocusedIconColour)} + options={ICON_COLOUR_OPTS} + /> +

+ Focused / active icons are always accent orange. This sets the resting colour of unfocused + icons (preview below). +

+
+ + + + + + +
+ + )} + + Input map + set('inputMap', v as InputMapMode)} + options={INPUT_MAP_OPTS} + /> + {depth !== 'peek' && ( +

+ The 2D input surface: a rectangular XY map or a circular joystick-style disc. "Follow mode" + uses the active mode's declared input (joystick → circular, else rectangular). +

+ )} + + Chrome + set('cornerRadius', Math.round(v))} + /> + {depth !== 'peek' && ( +

+ Roundness of buttons, control rows, dock icons and panels. Pills and the circular verdict + buttons are intentionally exempt. Default 2px. +

+ )} + + ); +} + +// =========================================================================== +// 5. HELP +// =========================================================================== + +const KEYS: [string, string][] = [ + ['1–5', 'open drawers'], + ['\\', 'full depth'], + ['space / ↑', 'commit +'], + ['↓', 'perturb / down −'], + ['z', 'undo'], + ['[ ] =', 'split (composite)'], +]; +function HelpDrawer() { + return ( + <> + Keyboard +
+ {KEYS.map(([k, v]) => ( +
+ + {k} + + {v} +
+ ))} +
+ The loop +

+ Drag the manifold to explore. Hear something good → + to keep it. Wrong → − to push away or + re-roll (set the behaviour in the Learning drawer). Went too far → undo. The dock reveals + exactly as much machinery as you reach for. +

+ + ); +} + +export interface DrawerSection { + /** Monochrome inline-SVG icon (currentColor-driven by the dock button). */ + icon: ReactNode; + /** Prior colour-emoji glyph, used when monochrome icons are OFF. */ + glyph: string; + label: string; + render: (ctx: ConsoleCtx, depth: DrawerDepth) => ReactNode; +} + +export const DRAWERS: Record = { + learn: { icon: , glyph: '🧠', label: 'Learning', render: LearningDrawer }, + inputs: { icon: , glyph: '🎚', label: 'Inputs', render: InputsDrawer }, + route: { icon: , glyph: '🔀', label: 'Outputs', render: RoutingDrawer }, + settings: { icon: , glyph: '⚙', label: 'Settings', render: (c, d) => }, + help: { icon: , glyph: '?', label: 'Help', render: HelpDrawer }, +}; diff --git a/manifold/src/console/InputMini.tsx b/manifold/src/console/InputMini.tsx new file mode 100644 index 0000000..67c6b36 --- /dev/null +++ b/manifold/src/console/InputMini.tsx @@ -0,0 +1,81 @@ +/** + * InputMini — the input demoted to a compact secondary control for output-first + * views. Ported from `InputMini.jsx`. + */ +import { VirtualJoystick, XYPad } from '../primitives'; +import type { MFMode } from './model'; + +export interface InputMiniProps { + mode: MFMode; + pos: [number, number]; + onMove: (x: number, y: number) => void; + noiseCap?: number; + size?: number; + corner?: 'bottom-left' | 'bottom-right' | 'top-left'; + /** Input-map shape override (Settings). Falls back to the mode's input. */ + variant?: 'rectangular' | 'circular'; +} + +export function InputMini({ + mode, + pos, + onMove, + size = 132, + corner = 'bottom-left', + variant, +}: InputMiniProps) { + const circular = variant ? variant === 'circular' : mode.input === 'joystick'; + const place = { + 'bottom-left': { bottom: 14, left: 14 }, + 'bottom-right': { bottom: 14, right: 14 }, + 'top-left': { top: 14, left: 14 }, + }[corner]; + return ( +
+
+ + input · {circular ? 'joy' : 'xy'} + + + {pos[0].toFixed(2)},{pos[1].toFixed(2)} + +
+ {circular ? ( + onMove(x, y)} /> + ) : ( + onMove(x, y)} showGrid /> + )} +
+ ); +} diff --git a/manifold/src/console/Manifold.tsx b/manifold/src/console/Manifold.tsx new file mode 100644 index 0000000..80f2c83 --- /dev/null +++ b/manifold/src/console/Manifold.tsx @@ -0,0 +1,401 @@ +/** + * The Manifold — full-bleed input surface + joy-map visualisation. The canvas + * bypasses React: it runs its own rAF loop and reads pos / noiseCap / pins + * imperatively from a ref (`stateRef`), never per-frame React state. Pointer + * input calls `onMove(x,y)` which (in ConsoleApp) drives `engine.setInput`. + * + * Ported faithfully from the window-global `Manifold.jsx`. + */ +import { useEffect, useRef } from 'react'; +import type { PointerEvent as ReactPointerEvent } from 'react'; +import type { FeedbackMarker, Pin } from './types'; + +export interface ManifoldProps { + pos: [number, number]; + onMove: (x: number, y: number) => void; + noiseCap?: number; + pins?: Pin[]; + /** Feedback markers (both polarities) plotted at their input location. */ + markers?: FeedbackMarker[]; + /** Input-surface shape — rectangular XY map or circular joystick-style disc. */ + variant?: 'rectangular' | 'circular'; + frozen?: boolean; + follow?: boolean; + onLongPress?: (pos: [number, number]) => void; + /** + * PICK-LOCATION (Explore & place, rl-feedback §2.2 §3). While true, the next + * pointer-down chooses the anchor location and calls {@link onPickLocation} + * instead of the normal pan/drive — a transient marker is shown. + */ + picking?: boolean; + /** Called with the chosen [0,1]² location when a pick lands. */ + onPickLocation?: (x: number, y: number) => void; +} + +export function Manifold({ + pos, + onMove, + noiseCap = 0.1, + pins = [], + markers = [], + variant = 'rectangular', + frozen = false, + follow = false, + onLongPress, + picking = false, + onPickLocation, +}: ManifoldProps) { + const wrapRef = useRef(null); + const canvasRef = useRef(null); + const stateRef = useRef({ pos, noiseCap, pins, markers, variant, frozen, follow, picking }); + const trailRef = useRef<{ x: number; y: number; t: number }[]>([]); + const draggingRef = useRef(false); + const driftRef = useRef({ vx: 0.0011, vy: 0.0008 }); + const lpTimer = useRef | null>(null); + // Transient "just placed" marker location (manifold space), for a brief flash. + const placedRef = useRef<{ x: number; y: number; t: number } | null>(null); + + stateRef.current = { pos, noiseCap, pins, markers, variant, frozen, follow, picking }; + + // push trail point whenever pos changes + useEffect(() => { + trailRef.current.push({ x: pos[0], y: pos[1], t: performance.now() }); + if (trailRef.current.length > 240) trailRef.current.shift(); + }, [pos[0], pos[1]]); + + /** + * Map a pointer event to a normalised [0,1]² position. In the circular + * variant the position is clamped to the inscribed disc (knob stays on/inside + * the boundary), keeping the [0,1]² normalisation consistent across both. + */ + const posFromEvent = (e: ReactPointerEvent): [number, number] | null => { + const el = wrapRef.current; + if (!el) return null; + const r = el.getBoundingClientRect(); + let x = Math.max(0, Math.min(1, (e.clientX - r.left) / r.width)); + let y = Math.max(0, Math.min(1, 1 - (e.clientY - r.top) / r.height)); + if (stateRef.current.variant === 'circular') { + // Clamp to the unit disc centred at (0.5, 0.5). + const dx = x - 0.5; + const dy = y - 0.5; + const d = Math.hypot(dx, dy); + if (d > 0.5) { + x = 0.5 + (dx / d) * 0.5; + y = 0.5 + (dy / d) * 0.5; + } + } + return [x, y]; + }; + + const setFromEvent = (e: ReactPointerEvent) => { + const p = posFromEvent(e); + if (p) onMove(p[0], p[1]); + }; + + const down = (e: ReactPointerEvent) => { + if (stateRef.current.frozen) return; + // PICK-LOCATION: when placing, this pointer-down picks the anchor location + // (rl-feedback §2.2 §3) and does NOT start a pan/drive drag. + if (stateRef.current.picking) { + const p = posFromEvent(e); + if (!p) return; + placedRef.current = { x: p[0], y: p[1], t: performance.now() }; + onPickLocation?.(p[0], p[1]); + return; + } + e.currentTarget.setPointerCapture?.(e.pointerId); + draggingRef.current = true; + setFromEvent(e); + if (lpTimer.current) clearTimeout(lpTimer.current); + lpTimer.current = setTimeout(() => { + onLongPress?.(stateRef.current.pos); + }, 600); + }; + const move = (e: ReactPointerEvent) => { + if (draggingRef.current) { + setFromEvent(e); + if (lpTimer.current) clearTimeout(lpTimer.current); + } + }; + const up = (e: ReactPointerEvent) => { + draggingRef.current = false; + e.currentTarget.releasePointerCapture?.(e.pointerId); + if (lpTimer.current) clearTimeout(lpTimer.current); + }; + + useEffect(() => { + const canvas = canvasRef.current; + const wrap = wrapRef.current; + if (!canvas || !wrap) return; + const ctx = canvas.getContext('2d'); + if (!ctx) return; + let raf = 0; + const css = getComputedStyle(document.documentElement); + const C = (n: string, f: string) => css.getPropertyValue(n).trim() || f; + const accent = C('--accent', '#ff6a00'); + const cyan = C('--accent-2', '#00ccff'); + + const dpr = window.devicePixelRatio || 1; + let lastW = -1; + let lastH = -1; + const ensureSize = (W: number, H: number) => { + if (W === lastW && H === lastH) return; + canvas.width = W * dpr; + canvas.height = H * dpr; + canvas.style.width = W + 'px'; + canvas.style.height = H + 'px'; + ctx.setTransform(dpr, 0, 0, dpr, 0, 0); + lastW = W; + lastH = H; + }; + + const draw = () => { + const W = wrap.clientWidth; + const H = wrap.clientHeight; + if (W === 0 || H === 0) return; + ensureSize(W, H); + const { + pos: p, + noiseCap: nc, + pins: pn, + markers: mk, + variant: vr, + frozen: fz, + follow: fl, + picking: pk, + } = stateRef.current; + const now = performance.now(); + const circular = vr === 'circular'; + // Disc geometry (inscribed circle centred in the surface). + const cx = W / 2; + const cy = H / 2; + const radius = Math.min(W, H) / 2 - 2; + + if (fl && !draggingRef.current && !fz) { + let [x, y] = p; + const d = driftRef.current; + x += d.vx; + y += d.vy; + if (x < 0.05 || x > 0.95) d.vx *= -1; + if (y < 0.05 || y > 0.95) d.vy *= -1; + x = Math.max(0.05, Math.min(0.95, x)); + y = Math.max(0.05, Math.min(0.95, y)); + onMove(x, y); + } + + ctx.clearRect(0, 0, W, H); + + // In the circular variant, clip everything (grid, trail, marks) to the disc. + if (circular) { + ctx.save(); + ctx.beginPath(); + ctx.arc(cx, cy, radius, 0, Math.PI * 2); + ctx.clip(); + } + + const minor = 32; + const major = 8; + ctx.lineWidth = 1; + for (let i = 0; i <= minor; i++) { + const t = i / minor; + const isMajor = i % (minor / major) === 0; + ctx.strokeStyle = isMajor ? 'rgba(255,255,255,0.06)' : 'rgba(255,255,255,0.022)'; + ctx.beginPath(); + ctx.moveTo(t * W, 0); + ctx.lineTo(t * W, H); + ctx.stroke(); + ctx.beginPath(); + ctx.moveTo(0, t * H); + ctx.lineTo(W, t * H); + ctx.stroke(); + } + ctx.strokeStyle = 'rgba(255,255,255,0.05)'; + ctx.beginPath(); + ctx.moveTo(W / 2, 0); + ctx.lineTo(W / 2, H); + ctx.moveTo(0, H / 2); + ctx.lineTo(W, H / 2); + ctx.stroke(); + + // Circular radial guide rings (joystick-style) inside the clip. + if (circular) { + ctx.strokeStyle = 'rgba(255,255,255,0.05)'; + ctx.lineWidth = 1; + for (const f of [0.33, 0.66]) { + ctx.beginPath(); + ctx.arc(cx, cy, radius * f, 0, Math.PI * 2); + ctx.stroke(); + } + } + + const px = p[0] * W; + const py = (1 - p[1]) * H; + + for (const pin of pn) { + const ppx = pin.x * W; + const ppy = (1 - pin.y) * H; + ctx.fillStyle = pin.color || 'rgba(255,106,0,0.18)'; + ctx.beginPath(); + ctx.arc(ppx, ppy, 34, 0, Math.PI * 2); + ctx.fill(); + ctx.strokeStyle = 'rgba(255,255,255,0.18)'; + ctx.lineWidth = 1; + ctx.beginPath(); + ctx.arc(ppx, ppy, 34, 0, Math.PI * 2); + ctx.stroke(); + } + + // Feedback markers: positive = filled accent dot, negative = open red + // ring. Plotted at the input location each verdict was given (session). + for (const m of mk) { + const mx = m.x * W; + const my = (1 - m.y) * H; + if (m.polarity === 'positive') { + ctx.fillStyle = 'rgba(255,106,0,0.9)'; + ctx.beginPath(); + ctx.arc(mx, my, 5, 0, Math.PI * 2); + ctx.fill(); + ctx.strokeStyle = 'rgba(255,106,0,0.35)'; + ctx.lineWidth = 1.5; + ctx.beginPath(); + ctx.arc(mx, my, 8.5, 0, Math.PI * 2); + ctx.stroke(); + } else { + ctx.strokeStyle = 'rgba(255,68,102,0.9)'; + ctx.lineWidth = 2; + ctx.beginPath(); + ctx.arc(mx, my, 6.5, 0, Math.PI * 2); + ctx.stroke(); + } + } + + const LIFE = 5000; + const pts = trailRef.current; + ctx.lineWidth = 2; + for (let i = 1; i < pts.length; i++) { + const a = pts[i - 1]; + const b = pts[i]; + const age = now - b.t; + if (age > LIFE) continue; + const alpha = (1 - age / LIFE) * 0.5; + ctx.strokeStyle = `rgba(0,204,255,${alpha})`; + ctx.beginPath(); + ctx.moveTo(a.x * W, (1 - a.y) * H); + ctx.lineTo(b.x * W, (1 - b.y) * H); + ctx.stroke(); + } + + if (nc > 0.001) { + const breathe = 1 + Math.sin(now / 600) * 0.06; + const rCap = nc * Math.min(W, H) * 0.5 * breathe; + const rCur = rCap * 0.55; + ctx.setLineDash([4, 5]); + ctx.strokeStyle = 'rgba(255,106,0,0.4)'; + ctx.lineWidth = 1.5; + ctx.beginPath(); + ctx.arc(px, py, rCap, 0, Math.PI * 2); + ctx.stroke(); + ctx.strokeStyle = 'rgba(255,106,0,0.7)'; + ctx.beginPath(); + ctx.arc(px, py, rCur, 0, Math.PI * 2); + ctx.stroke(); + ctx.setLineDash([]); + } + + ctx.shadowColor = fz ? cyan : accent; + ctx.shadowBlur = 18; + ctx.fillStyle = fz ? cyan : accent; + ctx.beginPath(); + ctx.arc(px, py, 9, 0, Math.PI * 2); + ctx.fill(); + ctx.shadowBlur = 0; + ctx.fillStyle = '#0d0d0d'; + ctx.beginPath(); + ctx.arc(px, py, 3, 0, Math.PI * 2); + ctx.fill(); + + // PICK-LOCATION affordance: a pulsing dashed reticle prompting the tap. + if (pk) { + const pulse = 0.5 + Math.sin(now / 320) * 0.5; + ctx.setLineDash([6, 6]); + ctx.strokeStyle = `rgba(0,204,255,${0.45 + pulse * 0.45})`; + ctx.lineWidth = 2; + ctx.beginPath(); + ctx.arc(px, py, 22 + pulse * 6, 0, Math.PI * 2); + ctx.stroke(); + ctx.setLineDash([]); + } + + // Transient "just placed" anchor flash (~900ms). + const placed = placedRef.current; + if (placed) { + const age = now - placed.t; + if (age > 900) { + placedRef.current = null; + } else { + const a = 1 - age / 900; + const mx = placed.x * W; + const my = (1 - placed.y) * H; + ctx.strokeStyle = `rgba(0,204,255,${a})`; + ctx.lineWidth = 2; + ctx.beginPath(); + ctx.arc(mx, my, 10 + (1 - a) * 28, 0, Math.PI * 2); + ctx.stroke(); + ctx.fillStyle = `rgba(0,204,255,${a * 0.4})`; + ctx.beginPath(); + ctx.arc(mx, my, 5, 0, Math.PI * 2); + ctx.fill(); + } + } + + // Restore the disc clip + draw the crisp circular boundary on the edge. + if (circular) { + ctx.restore(); + ctx.strokeStyle = 'rgba(255,255,255,0.14)'; + ctx.lineWidth = 1.5; + ctx.beginPath(); + ctx.arc(cx, cy, radius, 0, Math.PI * 2); + ctx.stroke(); + } + }; + + const loop = () => { + draw(); + raf = requestAnimationFrame(loop); + }; + const ro = new ResizeObserver(() => draw()); + ro.observe(wrap); + let timer: ReturnType | null = null; + let kicks = 0; + const kick = () => { + draw(); + if (lastW <= 0 && kicks++ < 80) timer = setTimeout(kick, 40); + }; + kick(); + raf = requestAnimationFrame(loop); + return () => { + cancelAnimationFrame(raf); + ro.disconnect(); + if (timer) clearTimeout(timer); + }; + }, []); + + return ( +
+ +
+ ); +} diff --git a/manifold/src/console/OutputEditor.tsx b/manifold/src/console/OutputEditor.tsx new file mode 100644 index 0000000..4c0efa0 --- /dev/null +++ b/manifold/src/console/OutputEditor.tsx @@ -0,0 +1,148 @@ +/** + * OutputEditor — the per-output menu (state · min · max · static value · curve). + * Opens on hover over an output column/cell. Ported from `OutputEditor.jsx`. + */ +import type { CSSProperties, ReactNode } from 'react'; +import { CurvePad } from './CurvePad'; +import type { MFParam, ParamStatus } from './model'; + +const OE_STATUS: { v: ParamStatus; label: string; color: string }[] = [ + { v: 'off', label: 'Off', color: 'var(--fg-dim)' }, + { v: 'fixed', label: 'Fixed', color: 'var(--accent-2)' }, + { v: 'live', label: 'Live', color: 'var(--accent)' }, +]; + +function MiniSlider({ + label, + value, + onChange, + disabled, +}: { + label: ReactNode; + value: number; + onChange: (v: number) => void; + disabled?: boolean; +}) { + return ( + + ); +} + +export interface OutputEditorProps { + param: MFParam; + onChange: (patch: Partial) => void; + onHold: () => void; + onLeave: () => void; + place: CSSProperties; +} + +export function OutputEditor({ param, onChange, onHold, onLeave, place }: OutputEditorProps) { + const isLive = param.status === 'live'; + return ( +
+
+ {param.name} + {param.group} +
+ +
+ {OE_STATUS.map((s) => { + const on = param.status === s.v; + return ( + + ); + })} +
+ + onChange({ min: v })} /> + onChange({ max: v })} /> + onChange({ val: v })} + disabled={isLive} + /> + onChange({ curve: c })} size={170} /> + + drag a bar to set value · ⌥/alt-click cycles state + +
+ ); +} diff --git a/manifold/src/console/OutputStage.tsx b/manifold/src/console/OutputStage.tsx new file mode 100644 index 0000000..11a71c4 --- /dev/null +++ b/manifold/src/console/OutputStage.tsx @@ -0,0 +1,252 @@ +/** + * OutputStage — the OUTPUT as the hero surface. A full-bleed field of parameter + * columns. Drag/click a bar sets value; ⌥/alt-click cycles state; hover opens + * the OutputEditor. Ported from `OutputStage.jsx`. + */ +import { useRef, useState } from 'react'; +import type { PointerEvent as ReactPointerEvent } from 'react'; +import type { MFParam, ParamStatus } from './model'; +import { OutputEditor } from './OutputEditor'; + +const OUT_GROUP_COLOR: Record = { + formant: '--accent', + pitch: '--accent-2', + amp: '--good', + filter: '--warn', + fx: '--info', + mod: '--accent-3', +}; +const OUT_NEXT: Record = { off: 'fixed', fixed: 'live', live: 'off' }; + +export interface OutputStageProps { + params: MFParam[]; + values: number[]; + onChange: (i: number, patch: Partial) => void; + compact?: boolean; +} + +export function OutputStage({ params, values, onChange, compact = false }: OutputStageProps) { + const [open, setOpen] = useState(null); + const timers = useRef<{ open: ReturnType | null; close: ReturnType | null }>({ + open: null, + close: null, + }); + const drag = useRef<{ i: number; moved: boolean; startY: number; el: HTMLDivElement | null; alt: boolean }>({ + i: -1, + moved: false, + startY: 0, + el: null, + alt: false, + }); + + const scheduleOpen = (i: number) => { + if (timers.current.close) clearTimeout(timers.current.close); + if (timers.current.open) clearTimeout(timers.current.open); + timers.current.open = setTimeout(() => setOpen(i), 110); + }; + const scheduleClose = () => { + if (timers.current.open) clearTimeout(timers.current.open); + if (timers.current.close) clearTimeout(timers.current.close); + timers.current.close = setTimeout(() => setOpen(null), 280); + }; + const hold = () => { + if (timers.current.close) clearTimeout(timers.current.close); + }; + + const valFromEvent = (el: HTMLDivElement, clientY: number) => { + const r = el.getBoundingClientRect(); + return Math.max(0, Math.min(1, 1 - (clientY - r.top) / r.height)); + }; + const down = (e: ReactPointerEvent, i: number) => { + e.currentTarget.setPointerCapture?.(e.pointerId); + drag.current = { + i, + moved: false, + startY: e.clientY, + el: e.currentTarget, + alt: e.altKey || e.metaKey, + }; + }; + const move = (e: ReactPointerEvent, i: number) => { + const d = drag.current; + if (d.i !== i) return; + if (Math.abs(e.clientY - d.startY) > 3) d.moved = true; + if (d.moved && !d.alt && d.el) onChange(i, { val: valFromEvent(d.el, e.clientY) }); + }; + const up = (e: ReactPointerEvent, i: number) => { + const d = drag.current; + if (d.i !== i) return; + if (d.alt && !d.moved) onChange(i, { status: OUT_NEXT[params[i].status] || 'live' }); + else if (!d.moved && d.el) onChange(i, { val: valFromEvent(d.el, e.clientY) }); + drag.current = { i: -1, moved: false, startY: 0, el: null, alt: false }; + }; + + return ( +
+ {params.map((p, i) => { + const eff = values[i] ?? 0; + const gc = `var(${OUT_GROUP_COLOR[p.group] || '--accent'})`; + const dim = p.status === 'off'; + const placeRight = i > params.length - 4; + return ( +
+
scheduleOpen(i)} style={{ cursor: 'help' }}> +
+ {p.name} +
+
+ {eff.toFixed(2)} +
+
+
down(e, i)} + onPointerMove={(e) => move(e, i)} + onPointerUp={(e) => up(e, i)} + onPointerCancel={(e) => up(e, i)} + style={{ + position: 'relative', + flex: 1, + background: 'var(--bg-1)', + border: '1px solid var(--line)', + borderRadius: compact ? 2 : 'var(--r-1)', + overflow: 'hidden', + cursor: 'ns-resize', + opacity: dim ? 0.55 : 1, + touchAction: 'none', + }} + > + {[0.25, 0.5, 0.75].map((t) => ( +
+ ))} +
+
+ {p.status === 'live' && ( +
+ )} +
+ {p.status === 'fixed' ? '⊟' : p.status === 'off' ? '∅' : ''} +
+
+ {open === i && !compact && ( + onChange(i, patch)} + onHold={hold} + onLeave={scheduleClose} + place={{ top: 38, [placeRight ? 'right' : 'left']: 0 }} + /> + )} + {open === i && compact && ( + onChange(i, patch)} + onHold={hold} + onLeave={scheduleClose} + place={{ top: 26, [placeRight ? 'right' : 'left']: 0 }} + /> + )} +
+ ); + })} +
+ ); +} diff --git a/manifold/src/console/ReadoutStrip.tsx b/manifold/src/console/ReadoutStrip.tsx new file mode 100644 index 0000000..bc16239 --- /dev/null +++ b/manifold/src/console/ReadoutStrip.tsx @@ -0,0 +1,219 @@ +/** + * ReadoutStrip — the output heatmap as a thin top control strip. Same model as + * OutputStage. Ported from `ReadoutStrip.jsx`. + */ +import { useRef, useState } from 'react'; +import type { PointerEvent as ReactPointerEvent } from 'react'; +import type { MFParam, ParamStatus } from './model'; +import { OutputEditor } from './OutputEditor'; + +const RS_GROUP_COLOR: Record = { + formant: '--accent', + pitch: '--accent-2', + amp: '--good', + filter: '--warn', + fx: '--info', + mod: '--accent-3', +}; +const RS_NEXT: Record = { off: 'fixed', fixed: 'live', live: 'off' }; + +export interface ReadoutStripProps { + params: MFParam[]; + values: number[]; + onChange: (i: number, patch: Partial) => void; + pinned: boolean; + onTogglePin: () => void; +} + +export function ReadoutStrip({ params, values, onChange, pinned, onTogglePin }: ReadoutStripProps) { + const [open, setOpen] = useState(null); + const timers = useRef<{ open: ReturnType | null; close: ReturnType | null }>({ + open: null, + close: null, + }); + const drag = useRef<{ i: number; moved: boolean; startY: number; el: HTMLDivElement | null; alt: boolean }>({ + i: -1, + moved: false, + startY: 0, + el: null, + alt: false, + }); + + const scheduleOpen = (i: number) => { + if (timers.current.close) clearTimeout(timers.current.close); + if (timers.current.open) clearTimeout(timers.current.open); + timers.current.open = setTimeout(() => setOpen(i), 110); + }; + const scheduleClose = () => { + if (timers.current.open) clearTimeout(timers.current.open); + if (timers.current.close) clearTimeout(timers.current.close); + timers.current.close = setTimeout(() => setOpen(null), 280); + }; + const hold = () => { + if (timers.current.close) clearTimeout(timers.current.close); + }; + + const valFromEvent = (el: HTMLDivElement, clientY: number) => { + const r = el.getBoundingClientRect(); + return Math.max(0, Math.min(1, 1 - (clientY - r.top) / r.height)); + }; + const down = (e: ReactPointerEvent, i: number) => { + e.currentTarget.setPointerCapture?.(e.pointerId); + drag.current = { + i, + moved: false, + startY: e.clientY, + el: e.currentTarget, + alt: e.altKey || e.metaKey, + }; + }; + const move = (e: ReactPointerEvent, i: number) => { + const d = drag.current; + if (d.i !== i) return; + if (Math.abs(e.clientY - d.startY) > 3) d.moved = true; + if (d.moved && !d.alt && d.el) onChange(i, { val: valFromEvent(d.el, e.clientY) }); + }; + const up = (e: ReactPointerEvent, i: number) => { + const d = drag.current; + if (d.i !== i) return; + if (d.alt && !d.moved) onChange(i, { status: RS_NEXT[params[i].status] || 'live' }); + else if (!d.moved && d.el) onChange(i, { val: valFromEvent(d.el, e.clientY) }); + drag.current = { i: -1, moved: false, startY: 0, el: null, alt: false }; + }; + + return ( +
+ + {params.map((p, i) => { + const eff = values[i] ?? 0; + const gc = `var(${RS_GROUP_COLOR[p.group] || '--accent'})`; + const dim = p.status === 'off'; + const placeRight = i > params.length - 5; + return ( +
+
scheduleOpen(i)} + style={{ + textAlign: 'center', + fontSize: 8, + fontFamily: 'var(--font-mono)', + lineHeight: '11px', + cursor: 'help', + color: p.status === 'live' ? 'var(--fg-dim)' : gc, + overflow: 'hidden', + whiteSpace: 'nowrap', + textOverflow: 'ellipsis', + }} + > + {p.name} +
+
down(e, i)} + onPointerMove={(e) => move(e, i)} + onPointerUp={(e) => up(e, i)} + onPointerCancel={(e) => up(e, i)} + style={{ + position: 'relative', + flex: 1, + background: 'var(--bg)', + borderRadius: 2, + overflow: 'hidden', + cursor: 'ns-resize', + opacity: dim ? 0.5 : 1, + touchAction: 'none', + }} + > +
+ {p.status === 'live' && ( +
+ )} + {p.status !== 'live' && ( +
+ {p.status === 'fixed' ? '⊟' : '∅'} +
+ )} +
+ {open === i && ( + onChange(i, patch)} + onHold={hold} + onLeave={scheduleClose} + place={{ top: 'calc(100% + 6px)', [placeRight ? 'right' : 'left']: 0 }} + /> + )} +
+ ); + })} +
+ ); +} diff --git a/manifold/src/console/SplitStage.tsx b/manifold/src/console/SplitStage.tsx new file mode 100644 index 0000000..924ad46 --- /dev/null +++ b/manifold/src/console/SplitStage.tsx @@ -0,0 +1,58 @@ +/** + * SplitStage — input and output given EQUAL prominence, side by side. Left = + * Manifold (input), right = OutputStage (output field). Ported from `SplitStage.jsx`. + */ +import { Manifold } from './Manifold'; +import { OutputStage } from './OutputStage'; +import type { MFParam } from './model'; +import type { FeedbackMarker, Pin } from './types'; + +export interface SplitStageProps { + pos: [number, number]; + onMove: (x: number, y: number) => void; + noiseCap: number; + pins: Pin[]; + markers?: FeedbackMarker[]; + variant?: 'rectangular' | 'circular'; + follow: boolean; + onLongPress: (p: [number, number]) => void; + params: MFParam[]; + values: number[]; + onChange: (i: number, patch: Partial) => void; +} + +export function SplitStage({ + pos, + onMove, + noiseCap, + pins, + markers = [], + variant = 'rectangular', + follow, + onLongPress, + params, + values, + onChange, +}: SplitStageProps) { + return ( +
+
+ +
+
+ +
+
+ ); +} diff --git a/manifold/src/console/VerdictCluster.tsx b/manifold/src/console/VerdictCluster.tsx new file mode 100644 index 0000000..2dddf54 --- /dev/null +++ b/manifold/src/console/VerdictCluster.tsx @@ -0,0 +1,200 @@ +/** + * VerdictCluster — floating bottom-centre control, the app's main verdict. + * ▽ perturb (thumbs-down) · ↺ undo · △ commit (thumbs-up), + A/B toggle. + * Long-press perturb = full re-roll. Ported from `VerdictCluster.jsx`. + * + * The cluster reflects the ACTIVE feedback mode (workstream B; rl-feedback §0): + * + * Explore & place (Mode 2, default): + * thumbs-DOWN = enter explore / cancel explore (NEVER a dislike); + * thumbs-UP = place (when exploring) / commit a like (when not). + * Geometric dislike (Mode 1): + * thumbs-DOWN = dislike (push away); + * thumbs-UP = like + train. + * + * Wiring (in ConsoleApp): onCommit / onPerturb dispatch on the mode; onReroll = + * re-roll the scratchpad (Mode 2) or the real net. + */ +import { useRef, useState } from 'react'; +import type { CSSProperties } from 'react'; +import type { FeedbackModeUI } from './types'; + +function ThumbIcon({ size = 24, down = false }: { size?: number; down?: boolean }) { + return ( + + ); +} + +export interface VerdictClusterProps { + onPerturb: () => void; + onUndo: () => void; + onCommit: () => void; + onReroll: () => void; + canUndo: boolean; + ab: 'A' | 'B'; + onToggleAB: () => void; + onHoldA: (holding: boolean) => void; + firstSession: boolean; + /** Active feedback mode — drives the cluster's labels/tones (rl-feedback §0). */ + feedbackMode: FeedbackModeUI; + /** True while a Mode-2 scratchpad session is active. */ + exploring: boolean; + /** True while awaiting a manifold location pick after "place". */ + picking: boolean; +} + +export function VerdictCluster({ + onPerturb, + onUndo, + onCommit, + onReroll, + canUndo, + ab, + onToggleAB, + onHoldA, + firstSession, + feedbackMode, + exploring, + picking, +}: VerdictClusterProps) { + const explore = feedbackMode === 'explore-and-place'; + // Labels per mode + session state. + const downTitle = explore + ? exploring + ? 'Cancel explore — restore the real net' + : 'Explore — re-roll into a scratchpad (hold to re-roll again)' + : 'Dislike — push the sound away (hold to re-roll)'; + const upTitle = explore + ? exploring + ? picking + ? 'Tap the manifold to place this sound' + : 'Place — pick a manifold location for this sound' + : 'Commit — keep the current sound' + : 'Like — reinforce + train'; + const [hover, setHover] = useState(false); + const lp = useRef | null>(null); + const firedReroll = useRef(false); + + const perturbDown = () => { + firedReroll.current = false; + lp.current = setTimeout(() => { + firedReroll.current = true; + onReroll(); + }, 600); + }; + const perturbUp = () => { + if (lp.current) clearTimeout(lp.current); + if (!firedReroll.current) onPerturb(); + }; + + const big = (extra: CSSProperties): CSSProperties => ({ + width: 64, + height: 64, + borderRadius: '50%', + fontSize: 26, + cursor: 'pointer', + display: 'flex', + alignItems: 'center', + justifyContent: 'center', + fontFamily: 'var(--font-mono)', + border: '1px solid var(--glass-line)', + transition: 'transform var(--dur-fast) var(--ease-console), background var(--dur-fast)', + ...extra, + }); + + return ( +
setHover(true)} + onPointerLeave={() => setHover(false)} + style={{ + position: 'absolute', + bottom: 28, + left: '50%', + transform: 'translateX(-50%)', + display: 'flex', + alignItems: 'center', + gap: 'var(--sp-3)', + padding: 'var(--sp-2) var(--sp-3)', + background: 'var(--glass)', + backdropFilter: 'blur(14px)', + WebkitBackdropFilter: 'blur(14px)', + border: '1px solid var(--glass-line)', + borderRadius: 'var(--r-pill)', + boxShadow: 'var(--shadow-2)', + opacity: hover || firstSession ? 1 : 0.55, + transition: 'opacity var(--dur-med) var(--ease-console)', + zIndex: 40, + }} + > + + + + + +
+ ); +} diff --git a/manifold/src/console/icons.tsx b/manifold/src/console/icons.tsx new file mode 100644 index 0000000..6ae3b3e --- /dev/null +++ b/manifold/src/console/icons.tsx @@ -0,0 +1,221 @@ +/** + * icons.tsx — monochrome inline-SVG icon set for the dock / verdict / console + * chrome. Every icon strokes/fills with `currentColor` (1.5px stroke, ~18px), + * so colour is driven entirely by the consumer's CSS `color`: + * + * active / focused → var(--accent) (orange) + * unfocused → the Settings unfocused colour (off-white / white / orange) + * + * No multicolour emoji here. When the Settings `monochromeIcons` flag is OFF the + * dock may fall back to the prior glyph strings (see GLYPH_FALLBACK). + * + * British spelling in copy; these are presentational only. + */ +import type { CSSProperties } from 'react'; + +export interface IconProps { + size?: number; + style?: CSSProperties; +} + +function svg(size: number, style: CSSProperties | undefined, children: React.ReactNode) { + return ( + + ); +} + +/** Mode — output target/backend selector (stacked layers / target). */ +export function ModeIcon({ size = 18, style }: IconProps) { + return svg( + size, + style, + <> + + + + , + ); +} + +/** Learning — a brain-ish node graph. */ +export function LearningIcon({ size = 18, style }: IconProps) { + return svg( + size, + style, + <> + + + + + + , + ); +} + +/** Inputs — a 2D pad with a control dot. */ +export function InputsIcon({ size = 18, style }: IconProps) { + return svg( + size, + style, + <> + + + + , + ); +} + +/** Outputs — fader bank. */ +export function OutputsIcon({ size = 18, style }: IconProps) { + return svg( + size, + style, + <> + + + + + , + ); +} + +/** Settings — gear. */ +export function SettingsIcon({ size = 18, style }: IconProps) { + return svg( + size, + style, + <> + + + , + ); +} + +/** Help — question mark in a circle. */ +export function HelpIcon({ size = 18, style }: IconProps) { + return svg( + size, + style, + <> + + + + , + ); +} + +/** Close (✕). */ +export function CloseIcon({ size = 14, style }: IconProps) { + return svg(size, style, ); +} + +/** Expand / depth toggle (diagonal arrows). */ +export function ExpandIcon({ size = 14, style }: IconProps) { + return svg( + size, + style, + <> + + + , + ); +} + +/** Particle / visual mode — orbiting dots. */ +export function ParticleIcon({ size = 18, style }: IconProps) { + return svg( + size, + style, + <> + + + + + + , + ); +} + +/** MIDI — 5-pin DIN. */ +export function MidiIcon({ size = 18, style }: IconProps) { + return svg( + size, + style, + <> + + + + + + + , + ); +} + +/** OSC — concentric signal rings. */ +export function OscIcon({ size = 18, style }: IconProps) { + return svg( + size, + style, + <> + + + + , + ); +} + +/** Built-in synth — a waveform. */ +export function SynthIcon({ size = 18, style }: IconProps) { + return svg( + size, + style, + <> + + + , + ); +} + +/** MEMLNaut Editor — USB / hardware-link plug. */ +export function EditorIcon({ size = 18, style }: IconProps) { + return svg( + size, + style, + <> + + + + + + , + ); +} + +/** Prior colour-emoji glyphs, for the monochrome-OFF fallback. */ +export const GLYPH_FALLBACK = { + mode: '⊞', + learn: '🧠', + inputs: '🎚', + route: '🔀', + settings: '⚙', + help: '?', + particles: '✦', + midi: '🎹', + osc: '◉', + synth: '🔊', + editor: '🔌', +} as const; diff --git a/manifold/src/console/index.ts b/manifold/src/console/index.ts new file mode 100644 index 0000000..dc54fce --- /dev/null +++ b/manifold/src/console/index.ts @@ -0,0 +1,18 @@ +/** + * Console barrel — the convertible Console shell, wired to the real engine. + */ +export { ConsoleApp } from './ConsoleApp'; +export type { ConsoleAppProps } from './ConsoleApp'; +export { CompositeStage } from './CompositeStage'; +export { SplitStage } from './SplitStage'; +export { OutputStage } from './OutputStage'; +export { ReadoutStrip } from './ReadoutStrip'; +export { Manifold } from './Manifold'; +export { InputMini } from './InputMini'; +export { VerdictCluster } from './VerdictCluster'; +export { Dock } from './Dock'; +export { DRAWERS } from './Drawers'; +export { AltitudeNav, MiniMeters, CompactAxis } from './shared-ui'; +export { MF_MODES, shapeValues, applyCurve, seededGradient, modeEngineId } from './model'; +export type { MFMode, MFParam, ParamStatus } from './model'; +export type { Focus, ConsoleCtx } from './types'; diff --git a/manifold/src/console/model.ts b/manifold/src/console/model.ts new file mode 100644 index 0000000..63b2010 --- /dev/null +++ b/manifold/src/console/model.ts @@ -0,0 +1,246 @@ +/** + * Console — shared instrument model: the static modes catalogue + per-param + * shaping helpers. Ported from the window-global `model.jsx`. + * + * KEY CHANGE vs the JSX reference: the pseudo-inference `MF_infer` (sin/cos + * placeholder) and the `useInstrument` hook are GONE. The `values` every + * consumer reads now come from the REAL engine (`engine.getOutputs()`), mapped + * onto a mode's params here via {@link shapeValues}. This file keeps only the + * mode/param DATA + the pure shaping maths. + * + * The `c15` mode and its synth label are relabelled to "Powerful Synth Engine" + * — the string "C15" must never appear in the UI (it survives only as an + * internal mode id). + */ + +export type ParamStatus = 'off' | 'fixed' | 'live'; +export type ParamGroup = 'formant' | 'pitch' | 'amp' | 'filter' | 'fx' | 'mod'; +export type ModeClass = 'Synth' | 'Sequencer' | 'Controller' | 'Visual'; +export type ModeInput = 'xy' | 'joystick' | 'audio_in'; + +/** + * Per-output control row — the unified store used by both the stage + * (OutputStage / ReadoutStrip) and the Outputs/Routing dock. `status` is the + * model-control tri-state; `muted` and `armed` are ORTHOGONAL modifiers + * (dock-spec §3.2 — the deliberate split of the deployed conflated + * frozen↔muted field). Backend-specific specs are populated by the active + * backend adapter (dock-spec §4); their shapes live in dock/output-state.ts and + * are re-declared here loosely to avoid a console→dock import cycle. + */ +export interface MFParam { + name: string; + group: string; + status: ParamStatus; + val: number; + min: number; + max: number; + curve: number; + /** Downstream silence — still computed + visible (distinct from `off`). */ + muted?: boolean; + /** Solo / arm — focus training on this output (dock-spec §1.2). */ + armed?: boolean; + /** MIDI CC backend spec ({ cc, channel, name, value }). */ + midi?: { cc: number; channel: number; name: string; value: number }; + /** OSC backend spec ({ path, rangeMin, rangeMax }). */ + osc?: { path: string; rangeMin: number; rangeMax: number }; + /** VCV backend spec ({ bipolar }). */ + vcv?: { bipolar: boolean }; +} + +export interface MFMode { + id: string; + label: string; + cls: ModeClass; + glyph: string; + input: ModeInput; + params: MFParam[]; + placeholder?: boolean; + badge?: string; +} + +type Spec = ReadonlyArray]>; + +function mkParams(spec: Spec): MFParam[] { + const out: MFParam[] = []; + for (const [group, names] of spec) { + names.forEach((name) => + out.push({ name, group, status: 'live', val: 0.5, min: 0, max: 1, curve: 0.5 }), + ); + } + return out; +} + +export const MF_MODES: MFMode[] = [ + { + id: 'paf_synth', + label: 'PAF Synth', + cls: 'Synth', + glyph: '∿', + input: 'xy', + params: mkParams([ + ['formant', ['F1', 'F2', 'F3', 'tilt', 'spread', 'skirt']], + ['pitch', ['root', 'glide', 'detune']], + ['amp', ['gain', 'attack', 'decay']], + ['filter', ['cutoff', 'res', 'env']], + ['fx', ['drive', 'air', 'width']], + ]), + }, + { + id: 'channel_strip', + label: 'Channel Strip', + cls: 'Synth', + glyph: '▤', + input: 'joystick', + params: mkParams([ + ['filter', ['lo', 'loMid', 'hiMid', 'hi']], + ['amp', ['comp', 'gate', 'makeup']], + ['fx', ['sat', 'width', 'glue', 'tilt', 'air']], + ]), + }, + { + id: 'verb_fx', + label: 'Verb FX', + cls: 'Synth', + glyph: '◞', + input: 'joystick', + params: mkParams([ + ['fx', ['size', 'decay', 'damp', 'diff']], + ['mod', ['rate', 'depth']], + ['filter', ['lo', 'hi']], + ]), + }, + { + id: 'elysiamorf', + label: 'Elysiamorf', + cls: 'Synth', + glyph: '❋', + input: 'xy', + params: mkParams([ + ['formant', ['grain', 'size', 'pos', 'spray']], + ['mod', ['rate', 'depth', 'jitter']], + ['amp', ['gain', 'env']], + ['filter', ['cutoff', 'res']], + ['fx', ['blur', 'shimmer', 'freeze', 'width']], + ]), + }, + { + id: 'memlcelium', + label: 'MEML Celium', + cls: 'Sequencer', + glyph: '☷', + input: 'xy', + params: mkParams([ + ['mod', ['cvA', 'cvB', 'gate', 'div']], + ['pitch', ['root', 'scale', 'oct']], + ['amp', ['vca', 'slew']], + ]), + }, + { + id: 'breakor', + label: 'Breakor', + cls: 'Sequencer', + glyph: '⊟', + input: 'joystick', + params: mkParams([ + ['mod', ['density', 'swing', 'fill', 'stutter']], + ['amp', ['punch', 'decay']], + ['filter', ['tone', 'crush']], + ['fx', ['glitch', 'rev']], + ]), + }, + { + id: 'sound_analysis_midi', + label: 'Sound Analysis → MIDI', + cls: 'Controller', + glyph: '⇉', + input: 'audio_in', + badge: '1-input', + params: mkParams([ + ['mod', ['cc1', 'cc2', 'cc3', 'cc4']], + ['pitch', ['note', 'bend']], + ['amp', ['vel', 'press']], + ]), + }, + { + id: 'visualizer', + label: 'Visualizer', + cls: 'Visual', + glyph: '◑', + input: 'xy', + params: mkParams([ + ['mod', ['hue', 'sat', 'flow', 'warp']], + ['amp', ['bloom', 'fade']], + ['fx', ['grain', 'trail']], + ]), + }, + { + // Internal id stays `c15`; the UI label is "Powerful Synth Engine". + id: 'c15', + label: 'Powerful Synth Engine', + cls: 'Synth', + glyph: '◆', + input: 'xy', + placeholder: true, + badge: 'soon', + params: mkParams([['amp', ['a', 'b']]]), + }, +]; + +/** Mirrors the engine's `applyCurve` (≈0.43 ≈ linear). */ +export function applyCurve(v: number, c: number): number { + const e = 0.25 + c * 1.75; + return Math.pow(Math.max(0, Math.min(1, v)), e); +} + +/** + * Map the engine's raw output vector onto a mode's params, applying each + * param's status / min / max / curve. Replaces `MF_infer`: + * off → 0 (muted) + * fixed → p.val (held static) + * live → engine output[i], shaped by min/max/curve + * + * The engine output is 126-dim; a mode with N params uses the first N. + */ +export function shapeValues(params: MFParam[], engineOut: Float32Array | null): number[] { + return params.map((p, i) => { + if (p.status === 'off') return 0; + if (p.status === 'fixed') return p.val ?? 0.5; + const raw = engineOut && i < engineOut.length ? engineOut[i] : 0.5; + const v = p.min + applyCurve(raw, p.curve) * (p.max - p.min); + return Math.max(0, Math.min(1, v)); + }); +} + +/** Deterministic per-revision gradient-flow stub (visual only; ported as-is). */ +export function seededGradient(rev: number): { + norms: number[]; + status: string[]; +} { + const n = 4; + const norms: number[] = []; + const status: string[] = []; + for (let i = 0; i < n; i++) { + const r = Math.abs((Math.sin((rev + 1) * (i + 1) * 12.9898) * 43758.5453) % 1); + norms.push(0.2 + r * 0.8); + status.push(r > 0.85 ? 'exploding' : r < 0.18 ? 'vanishing' : r < 0.3 ? 'converged' : 'healthy'); + } + return { norms, status }; +} + +/** Map a mode's `input` kind → the engine backend id to drive audio. */ +export function modeEngineId(modeId: string): string { + // Mode ids align with engine ids except the relabelled `c15`. + switch (modeId) { + case 'paf_synth': + case 'channel_strip': + case 'verb_fx': + case 'elysiamorf': + case 'memlcelium': + case 'breakor': + return modeId; + case 'sound_analysis_midi': + return 'analysis'; + default: + return 'thru'; + } +} diff --git a/manifold/src/console/output-mode.ts b/manifold/src/console/output-mode.ts new file mode 100644 index 0000000..d33c1eb --- /dev/null +++ b/manifold/src/console/output-mode.ts @@ -0,0 +1,89 @@ +/** + * output-mode.ts — the TOP dock selector catalogue (operator dock restructure). + * + * "Mode" here = the active OUTPUT BACKEND/target. Five options, in order, the + * first the default: + * • Particle System (visual) — DEFAULT + * • MIDI + * • OSC + * • Built-in Synth — the synth backend; NEVER the string "C15" + * • MEMLNaut Editor — hardware-connection mode (Web Serial) + * + * Selecting a Mode sets the active backend: where audio applies it maps to the + * dock's BackendId (output-state.ts) and, for the synth, engine.audio.setBackend; + * Particle + Editor are non-audio. + * + * British spelling in copy. + */ +import type { OutputMode } from './types'; +import type { BackendId } from '../dock/output-state'; +import type { + ParticleIcon, + MidiIcon, + OscIcon, + SynthIcon, + EditorIcon, +} from './icons'; + +export interface OutputModeDescriptor { + id: OutputMode; + label: string; + description: string; + /** True when this mode drives the audio engine (synth). */ + audio: boolean; + /** The dock BackendId this mode selects (drives the Outputs per-output rows). */ + backend: BackendId; +} + +/** The five Modes, in operator order; index 0 is the default. */ +export const OUTPUT_MODES: readonly OutputModeDescriptor[] = [ + { + id: 'particles', + label: 'Particle System', + description: 'Flow-field visualiser driven by the model outputs (no audio).', + audio: false, + backend: 'particles', + }, + { + id: 'midi', + label: 'MIDI', + description: 'Web MIDI CC out — per-output CC#/channel.', + audio: false, + backend: 'midi', + }, + { + id: 'osc', + label: 'OSC', + description: 'OSC bridge — named paths + physical ranges.', + audio: false, + backend: 'osc', + }, + { + id: 'synth', + label: 'Built-in Synth', + description: 'Firmware-parity built-in audio engine.', + audio: true, + backend: 'synth', + }, + { + id: 'editor', + label: 'MEMLNaut Editor', + description: 'Connect to the MEMLNaut hardware over USB serial (configure / save / restore).', + audio: false, + backend: 'synth', + }, +] as const; + +export const DEFAULT_OUTPUT_MODE: OutputMode = OUTPUT_MODES[0].id; + +export function outputModeDescriptor(id: OutputMode): OutputModeDescriptor { + return OUTPUT_MODES.find((m) => m.id === id) ?? OUTPUT_MODES[0]; +} + +/** The monochrome icon component for a Mode (resolved by the dock). */ +export type ModeIconComponent = + | typeof ParticleIcon + | typeof MidiIcon + | typeof OscIcon + | typeof SynthIcon + | typeof EditorIcon; diff --git a/manifold/src/console/shared-ui.tsx b/manifold/src/console/shared-ui.tsx new file mode 100644 index 0000000..3d9c4a0 --- /dev/null +++ b/manifold/src/console/shared-ui.tsx @@ -0,0 +1,181 @@ +/** + * Console — shared chrome for the simpler altitudes. Ported from `shared-ui.jsx`. + * + * AltitudeNav no longer navigates to separate HTML files (the JSX's href model); + * the focus switch is driven by React state via `onFocus`. The altitude pills + * (Console / Perform / Zen) are inert here — Manifold ships a single altitude. + */ +import type { CSSProperties } from 'react'; +import type { Focus } from './types'; +import type { MFParam } from './model'; + +const FOCI: [Focus, string, string][] = [ + ['in', 'IN', 'Input-first'], + ['split', 'DUAL', 'Input + output equal'], + ['out', 'OUT', 'Output-first'], + ['composite', 'FLEX', 'Composite — drag to rebalance'], +]; + +export interface AltitudeNavProps { + current?: string; + focus?: Focus; + onFocus?: (f: Focus) => void; + style?: CSSProperties; +} + +export function AltitudeNav({ current = 'console', focus = 'in', onFocus, style }: AltitudeNavProps) { + const items = [ + { id: 'console', dots: '◆◆◆', label: 'Console' }, + { id: 'perform', dots: '◆◆', label: 'Perform' }, + { id: 'zen', dots: '◆', label: 'Zen' }, + ]; + const pill = (on: boolean): CSSProperties => ({ + textDecoration: 'none', + fontSize: 11, + padding: '2px 8px', + borderRadius: 'var(--r-pill)', + color: on ? 'var(--accent)' : 'var(--fg-dim)', + background: on ? 'rgba(255,106,0,0.14)' : 'transparent', + border: 'none', + cursor: 'pointer', + fontFamily: 'var(--font-mono)', + }); + return ( +
+ {items.map((it) => ( + + {it.dots} + + ))} + + {FOCI.map(([f, label, title]) => ( + + ))} +
+ ); +} + +const MM_GROUP_COLOR: Record = { + formant: '--accent', + pitch: '--accent-2', + amp: '--good', + filter: '--warn', + fx: '--info', + mod: '--accent-3', +}; + +/** MiniMeters — glanceable read-only output bars (no interaction). */ +export function MiniMeters({ params, values }: { params: MFParam[]; values: number[] }) { + return ( +
+ {values.map((v, i) => ( +
+
+
+ ))} +
+ ); +} + +/** CompactAxis — slim labelled feel slider (Perform bar; kept for parity). */ +export function CompactAxis({ + label, + value, + onChange, + accent = 'var(--accent)', +}: { + label: string; + value: number; + onChange: (v: number) => void; + accent?: string; +}) { + return ( + + ); +} diff --git a/manifold/src/console/types.ts b/manifold/src/console/types.ts new file mode 100644 index 0000000..4d45dbd --- /dev/null +++ b/manifold/src/console/types.ts @@ -0,0 +1,176 @@ +/** + * Console — shared prop/context types used across the stage + dock components. + */ +import type { MFMode, MFParam } from './model'; +import type { BackendId } from '../dock/output-state'; +import type { FeedbackMode } from '../engine/types'; +import type { BackendStatus } from '../backends/backend'; + +/** The two product feedback modes (dock-spec §1.1; rl-feedback-design §0). */ +export type FeedbackModeUI = 'explore-and-place' | 'geometric-dislike'; + +/** Solo / arm gradient-mask variant (rl-feedback-design §0, §3). */ +export type SoloMode = 'mask-gradients' | 'zero-loss' | 'dont-care'; + +export interface Pin { + x: number; + y: number; + color?: string; +} + +/** + * The active OUTPUT MODE (target/backend). This is the TOP dock selector + * (operator dock restructure). "Built-in Synth" is the synth backend — the + * string "C15" must NEVER appear. Particle + Editor are non-audio. + */ +export type OutputMode = 'particles' | 'midi' | 'osc' | 'synth' | 'editor'; + +/** Feedback marker plotted on the 2D map at the input location it was given. */ +export interface FeedbackMarker { + /** Input-map location in [0,1]². */ + x: number; + y: number; + /** Polarity — positive (like / placed anchor) vs negative (dislike). */ + polarity: 'positive' | 'negative'; +} + +export interface Snapshot { + id: number; + tag: string; + noise: number; + seed: number; +} + +export type DrawerKey = 'learn' | 'inputs' | 'route' | 'settings' | 'help'; +export type DrawerDepth = 'peek' | 'expand' | 'full'; +export type Focus = 'in' | 'split' | 'out' | 'composite'; + +export interface Axes { + boldness: number; + memory: number; + precision: number; +} + +/** The flat context the Dock + drawers read. */ +export interface ConsoleCtx { + modes: MFMode[]; + modeId: string; + setModeId: (id: string) => void; + mode: MFMode; + + axes: Axes; + setAxis: (k: keyof Axes, v: number) => void; + + preset: string; + setPreset: (p: string) => void; + offsetActive: boolean; + + datasetCount: number; + loss: number[]; + busy: boolean; + addingExample: boolean; + onAddExample: () => void; + onTrain: () => void; + onClear: () => void; + + snapshots: Snapshot[]; + onJump: (id: number) => void; + + params: MFParam[]; + cycleStatus: (i: number) => void; + /** Patch one output row in the shared store (drives stage + dock in sync). */ + setParam: (i: number, patch: Partial) => void; + outputBackend: BackendId; + setOutputBackend: (v: BackendId) => void; + + // ---- Output backend transport (backends-spec §1–§5) ---- + /** Live status of the active output backend (MIDI/OSC connect state, etc.). */ + backendStatus: BackendStatus; + /** Available Web MIDI output ports (for the MIDI config picker). */ + midiPorts: { id: string; name: string }[]; + refreshMidiPorts: () => void; + /** MIDI backend settings (selected port + number of CCs mapped). */ + midiOutputId: string | null; + setMidiOutputId: (id: string | null) => void; + midiCcCount: number; + setMidiCcCount: (n: number) => void; + /** OSC backend settings (bridge URL + send-raw toggle). */ + oscUrl: string; + setOscUrl: (u: string) => void; + oscSendRaw: boolean; + setOscSendRaw: (v: boolean) => void; + /** Replace the whole params array (used when restoring a named preset). */ + setParams: (next: MFParam[]) => void; + + // ---- Active output MODE / target (TOP dock selector) ---- + outputMode: OutputMode; + setOutputMode: (m: OutputMode) => void; + + // ---- Feedback markers on the 2D map (both polarities) ---- + /** Markers plotted at the input location where each feedback was given. */ + markers: FeedbackMarker[]; + + health: number; + gradient: number[]; + gradientStatus: string[]; + weightsRevision: number; + + spread: boolean; + setSpread: (v: boolean) => void; + tame: number; + setTame: (v: number) => void; + noiseCap: number; + setNoiseCap: (v: number) => void; + + // ---- Learning-behaviour (dock-spec §1; rl-feedback-design) ---- + feedbackMode: FeedbackModeUI; + setFeedbackMode: (m: FeedbackModeUI) => void; + soloMode: SoloMode; + setSoloMode: (m: SoloMode) => void; + /** True while the feedback controller is exploring (engine.feedback.exploring). */ + exploring: boolean; + /** True while learning is paused (engine.feedback.learningPaused). */ + learningPaused: boolean; + /** Count of currently-armed (soloed) outputs. */ + armedCount: number; + /** Clear all arm flags ("Arm all"). */ + clearArmed: () => void; + + // ---- Live training params (dock-spec §1.3) ---- + learningRate: number; + setLearningRate: (v: number) => void; + decay: number; + setDecay: (v: number) => void; + spreadLevel: number; + setSpreadLevel: (v: number) => void; + + // ---- Synth engine (dock-spec §5) ---- + audioStarted: boolean; + onToggleAudio: () => void; + volume: number; + setVolume: (v: number) => void; + bpm: number; + setBpm: (v: number) => void; + + // ---- Explore-and-place scratchpad session (workstream B; rl-feedback §2.2) ---- + /** True while awaiting a manifold location pick after pressing "place". */ + picking: boolean; + /** Anchors placed in the current (not-yet-finalised) explore session. */ + anchorCount: number; + /** Scratchpad undo-stack depth (rerolls + nudges that can be undone). */ + undoDepth: number; + /** Enter the scratchpad / re-roll the whole net (Mode-2 explore). */ + onExplore: () => void; + /** Re-roll the scratchpad net ("meh, randomise…"). */ + onScratchReroll: () => void; + /** Small bounded weight nudge on the scratchpad (undoable). */ + onScratchNudge: () => void; + /** Begin placing the current candidate → pick a manifold location next. */ + onPlace: () => void; + /** Undo the last scratchpad op (reroll / nudge). */ + onScratchUndo: () => void; + /** Finalise: restore the real net + warm-start to interpolate all anchors. */ + onFinalise: () => void; + /** Cancel the whole explore session (discard scratchpad + anchors). */ + onCancelExplore: () => void; +} diff --git a/manifold/src/debug/probe.ts b/manifold/src/debug/probe.ts new file mode 100644 index 0000000..97d0236 --- /dev/null +++ b/manifold/src/debug/probe.ts @@ -0,0 +1,213 @@ +/** + * Debug probe: window.__nisps + * + * Synchronous-or-immediate Promise API for Playwright tests and dev console + * use. Ported from `playground/src/debug/probe.ts` to read the framework-neutral + * `EngineApi` instead of SolidJS stores. Gated behind `?debug=1` (see + * `installDebugProbe`). + * + * Test contract (unchanged): every method returns a value, returns null/empty, + * or returns an immediately-resolved Promise. None throw — bad input is + * silently ignored. + * + * Scope note: the playground probe also covered features that live in Solid + * feature-stores (snapshots, A/B, region pins, heatmap, session presets, + * compound axes). Those stores don't exist in Manifold's engine layer yet + * (they belong to later BUILD-PLAN steps). Their probe methods are present but + * inert (no-op / empty) so the probe surface stays stable and never throws; + * they'll be wired when the corresponding Manifold features land. + */ + +import type { EngineApi } from '../engine/engine-api'; +import type { FeedbackMode, LayerStats } from '../engine/types'; + +export interface DebugProbe { + // ---- Core engine surface (live) ---- + getOutputs(): Float32Array; + routedOutputs(): Float32Array; + getLoss(): number | null; + getLossHistory(): ReadonlyArray; + getWeights(): Float32Array; + getExampleCount(): number; + setInputs(x: number, y: number): void; + thumbsUp(): number; + thumbsDown(): number; + setFeedbackMode(mode: FeedbackMode): void; + getFeedbackMode(): FeedbackMode | null; + setFocus(mask: ReadonlyArray | null): void; + exploring(): boolean; + train(): number; + trainAsync(): Promise; + randomise(): void; + clearExamples(): void; + saveState(): void; + evalLoss(): number | null; + inferBatch(points: ReadonlyArray): Float32Array; + getLayerStats(): Float32Array; + addExample(features: ReadonlyArray, labels: ReadonlyArray): boolean; + + // ---- Audio ---- + audioStart(): Promise; + audioStop(): Promise; + setMuted(muted: boolean): void; + setBackend(id: string): void; + + // ---- Bus ---- + on(event: string, handler: (payload?: unknown) => void): () => void; + + readonly __ready: boolean; +} + +declare global { + interface Window { + __nisps?: DebugProbe; + } +} + +const EMPTY_F32 = new Float32Array(0); + +function makeProbe(engine: EngineApi): DebugProbe { + return { + get __ready(): boolean { + return engine.getState().ready; + }, + + getOutputs(): Float32Array { + return engine.getOutputs(); + }, + + routedOutputs(): Float32Array { + return engine.routedOutput() ?? EMPTY_F32; + }, + + getLoss(): number | null { + return engine.getState().lastLoss; + }, + + getLossHistory(): ReadonlyArray { + return engine.getState().lossHistory; + }, + + getWeights(): Float32Array { + return engine.getWeights(); + }, + + getExampleCount(): number { + return engine.getState().exampleCount; + }, + + setInputs(x: number, y: number): void { + engine.setInput(x, y); + }, + + thumbsUp(): number { + const a = engine.feedback.thumbsUp(); + engine.process(); + return a; + }, + + thumbsDown(): number { + const a = engine.feedback.thumbsDown(); + engine.process(); + return a; + }, + + setFeedbackMode(mode: FeedbackMode): void { + engine.feedback.setMode(mode); + }, + + getFeedbackMode(): FeedbackMode | null { + try { + return engine.feedback.getMode(); + } catch { + return null; + } + }, + + setFocus(mask: ReadonlyArray | null): void { + engine.feedback.setFocus(mask ? Uint8Array.from(mask) : null); + }, + + exploring(): boolean { + return engine.feedback.exploring(); + }, + + train(): number { + const loss = engine.train(); + engine.process(); + return loss; + }, + + async trainAsync(): Promise { + const loss = await engine.trainAsync(); + engine.process(); + return loss; + }, + + randomise(): void { + engine.randomise(); + }, + + clearExamples(): void { + engine.clearExamples(); + }, + + saveState(): void { + engine.saveState(); + }, + + evalLoss(): number | null { + try { + return engine.evalLoss(); + } catch { + return null; + } + }, + + inferBatch(points): Float32Array { + return engine.inferBatch(points); + }, + + getLayerStats(): Float32Array { + return engine.getLayerStatsFlat(); + }, + + addExample(features, labels): boolean { + return engine.addExample(features, labels); + }, + + audioStart(): Promise { + return engine.audio.start(); + }, + + audioStop(): Promise { + return engine.audio.stop(); + }, + + setMuted(muted: boolean): void { + engine.audio.setMuted(muted); + }, + + setBackend(id: string): void { + // Lossy cast — the probe is intentionally weakly typed. + engine.audio.setBackend(id as Parameters[0]); + }, + + on(event: string, handler): () => void { + return engine.on(event, handler); + }, + }; +} + +/** Type-only re-export so consumers can reference the stat shape. */ +export type { LayerStats }; + +/** + * Install the probe on window iff `?debug=1` is present. Idempotent. + */ +export function installDebugProbe(engine: EngineApi): void { + if (typeof window === 'undefined') return; + const params = new URLSearchParams(window.location.search); + if (params.get('debug') !== '1') return; + window.__nisps = makeProbe(engine); +} diff --git a/manifold/src/dock/BackendAdvanced.tsx b/manifold/src/dock/BackendAdvanced.tsx new file mode 100644 index 0000000..c1832b0 --- /dev/null +++ b/manifold/src/dock/BackendAdvanced.tsx @@ -0,0 +1,282 @@ +/** + * BackendAdvanced — the FULL-depth advanced backend modal bodies (dock-spec §4). + * One editor per backend. All backends share the §3.1 baseline (rendered as + * OutputControlRow elsewhere); these add the backend-specific fields. + * + * The backend transport (backends-spec workstream E) is now LIVE: editing these + * fields writes the shared MFParam store, which the BackendManager reads to send + * real Web MIDI CC / OSC-over-WS. This modal is the full-depth duplicate of the + * inline config in OutputsBackendConfig; both write the same store. + */ +import type { MFParam } from '../console/model'; +import type { BackendId } from './output-state'; +import { defaultMidiSpec, defaultOscSpec } from './output-state'; + +function num(s: string, fallback: number): number { + const v = parseFloat(s); + return Number.isFinite(v) ? v : fallback; +} + +const cellInput: React.CSSProperties = { + width: '100%', + background: 'var(--bg-1)', + border: '1px solid var(--line)', + borderRadius: 'var(--r-1)', + color: 'var(--fg)', + fontFamily: 'var(--font-mono)', + fontSize: 'var(--fs-xs)', + padding: '3px 6px', +}; + +function Th({ children }: { children: React.ReactNode }) { + return ( + + {children} + + ); +} + +export interface BackendAdvancedProps { + backend: BackendId; + params: MFParam[]; + setParam: (i: number, patch: Partial) => void; +} + +export function BackendAdvanced({ backend, params, setParam }: BackendAdvancedProps) { + switch (backend) { + case 'midi': + return ; + case 'osc': + return ; + case 'vcv': + case 'cvgate': + return ; + default: + return ; + } +} + +// ---- MIDI (dock-spec §4.1) ------------------------------------------------- + +function MidiCcEditor({ + params, + setParam, +}: { + params: MFParam[]; + setParam: (i: number, patch: Partial) => void; +}) { + return ( +
+

+ {params.length} CCs · live Web MIDI out (backends-spec §2.3). Editing the CC map here sends in + real time once a MIDI port is selected in the Outputs panel. +

+
+ + + + + + + + + + + {params.map((p, i) => { + const m = p.midi ?? defaultMidiSpec(i); + return ( + + + + + + + ); + })} + +
NameCC#ChState
+ setParam(i, { midi: { ...m, name: e.target.value } })} + /> + + + setParam(i, { + midi: { ...m, cc: Math.max(0, Math.min(127, num(e.target.value, m.cc))) }, + }) + } + /> + + + setParam(i, { + midi: { + ...m, + channel: Math.max(1, Math.min(16, num(e.target.value, m.channel))), + }, + }) + } + /> + + {p.status} + {p.muted ? ' · muted' : ''} +
+
+
+ ); +} + +// ---- OSC (dock-spec §4.2) -------------------------------------------------- + +function OscPathEditor({ + params, + setParam, +}: { + params: MFParam[]; + setParam: (i: number, patch: Partial) => void; +}) { + return ( +
+

+ Live OSC over the WebSocket bridge (backends-spec §2.4). Set the bridge URL + per-output paths in + the Outputs panel; emits only while the bridge process is connected. +

+
+ + + + + + + + + + {params.map((p, i) => { + const o = p.osc ?? defaultOscSpec(p.name); + return ( + + + + + + ); + })} + +
OSC pathrange minrange max
+ setParam(i, { osc: { ...o, path: e.target.value } })} + /> + + setParam(i, { osc: { ...o, rangeMin: num(e.target.value, o.rangeMin) } })} + /> + + setParam(i, { osc: { ...o, rangeMax: num(e.target.value, o.rangeMax) } })} + /> +
+
+
+ ); +} + +// ---- VCV / CV (dock-spec §4.3) --------------------------------------------- + +function VcvChannelEditor({ + params, + setParam, +}: { + params: MFParam[]; + setParam: (i: number, patch: Partial) => void; +}) { + return ( +
+

+ VCV adds nothing beyond the baseline (min/max = range, fixed = freeze) plus per-channel polarity. + {/* TODO(backends-spec §2.6): the VCV browser↔module bridge transport is not yet wired here. */} +

+
+ {params.map((p, i) => { + const bipolar = p.vcv?.bipolar ?? false; + return ( +
+ {p.name} + + {p.min.toFixed(2)}–{p.max.toFixed(2)} · {p.status === 'fixed' ? 'frozen' : 'live'} + + +
+ ); + })} +
+
+ ); +} + +function SynthGroupNote({ params }: { params: MFParam[] }) { + const groups = Array.from(new Set(params.map((p) => p.group))); + return ( +
+

+ The synth backend's advanced surface is the group-override matrix — see the Powerful Synth Engine + drawer's full depth (dock-spec §4.4 / §5). Groups: {groups.join(' · ')}. +

+
+ ); +} diff --git a/manifold/src/dock/OutputControlRow.tsx b/manifold/src/dock/OutputControlRow.tsx new file mode 100644 index 0000000..7d8bbe4 --- /dev/null +++ b/manifold/src/dock/OutputControlRow.tsx @@ -0,0 +1,303 @@ +/** + * OutputControlRow — the shared per-output baseline control row (dock-spec §3.2). + * Reused across the Routing, Synth and Visual drawers. Renders the FULL baseline: + * + * name · M (mute) · S (solo/arm) · [off|fixed|live] · dual-range · curve · value + * + * Writes eagerly through `onChange` into the single shared MFParam store + * (ConsoleApp owns it) — never a second data path (dock-spec §3.2, §8). + */ +import { useRef } from 'react'; +import type { PointerEvent as ReactPointerEvent } from 'react'; +import type { MFParam, ParamStatus } from '../console/model'; +import { CurvePad } from '../console/CurvePad'; + +const STATE_META: { v: ParamStatus; label: string; color: string }[] = [ + { v: 'off', label: 'off', color: 'var(--fg-dim)' }, + { v: 'fixed', label: 'fixed', color: 'var(--accent-2)' }, + { v: 'live', label: 'live', color: 'var(--accent)' }, +]; + +const GROUP_COLOR: Record = { + formant: '--accent', + pitch: '--accent-2', + amp: '--good', + filter: '--warn', + fx: '--info', + mod: '--accent-3', +}; + +/** A compact dual-thumb min/max range (min blue, max orange — dock-spec §3.1). */ +function DualRange({ + min, + max, + onMin, + onMax, +}: { + min: number; + max: number; + onMin: (v: number) => void; + onMax: (v: number) => void; +}) { + const track = useRef(null); + const drag = useRef<{ which: 'min' | 'max' | null }>({ which: null }); + const valAt = (clientX: number) => { + const el = track.current; + if (!el) return 0; + const r = el.getBoundingClientRect(); + return Math.max(0, Math.min(1, (clientX - r.left) / r.width)); + }; + const down = (e: ReactPointerEvent) => { + e.currentTarget.setPointerCapture?.(e.pointerId); + const v = valAt(e.clientX); + drag.current.which = Math.abs(v - min) <= Math.abs(v - max) ? 'min' : 'max'; + apply(v); + }; + const apply = (v: number) => { + if (drag.current.which === 'min') onMin(Math.min(v, max)); + else if (drag.current.which === 'max') onMax(Math.max(v, min)); + }; + const move = (e: ReactPointerEvent) => { + if (drag.current.which) apply(valAt(e.clientX)); + }; + const up = () => { + drag.current.which = null; + }; + return ( +
+
+ + +
+ ); +} + +function Thumb({ pct, color }: { pct: number; color: string }) { + return ( +
+ ); +} + +function GlyphToggle({ + on, + glyph, + title, + color, + onClick, +}: { + on: boolean; + glyph: string; + title: string; + color: string; + onClick: () => void; +}) { + return ( + + ); +} + +export interface OutputControlRowProps { + param: MFParam; + /** Live (computed) value for the value bar. */ + value: number; + onChange: (patch: Partial) => void; + /** Show the curve pad inline (expand depth); hidden in compact rows. */ + showCurve?: boolean; +} + +export function OutputControlRow({ param, value, onChange, showCurve = false }: OutputControlRowProps) { + const gc = `var(${GROUP_COLOR[param.group] || '--accent'})`; + const muted = param.muted ?? false; + const armed = param.armed ?? false; + const off = param.status === 'off'; + const barVal = param.status === 'fixed' ? param.val : value; + return ( +
+
+ + {param.name} + + {param.group} + onChange({ muted: !muted })} + /> + onChange({ armed: !armed })} + /> +
+ +
+ {/* tri-state segmented */} +
+ {STATE_META.map((s) => { + const on = param.status === s.v; + return ( + + ); + })} +
+ onChange({ min: v })} + onMax={(v) => onChange({ max: v })} + /> +
+ + {/* value bar (live model value, or held fixed value) */} +
+
+
+ + {param.status === 'fixed' && ( + + )} + + {showCurve && ( +
+ onChange({ curve: c })} size={88} /> +
+ )} +
+ ); +} diff --git a/manifold/src/dock/OutputsBackendConfig.tsx b/manifold/src/dock/OutputsBackendConfig.tsx new file mode 100644 index 0000000..116d9da --- /dev/null +++ b/manifold/src/dock/OutputsBackendConfig.tsx @@ -0,0 +1,432 @@ +/** + * OutputsBackendConfig — the editable, per-backend specialisation of the Outputs + * panel (backends-spec §4) plus the named-preset bar (§5). + * + * Layout: ONE preset bar (save-as / restore / rename / delete, per-backend + * namespace) on top, then a per-backend config section: + * - MIDI → output-port picker, number-of-CCs, per-output CC#/channel/name. + * - OSC → bridge URL + connect status + send-raw toggle, per-output path/range. + * - VCV/CV→ per-output polarity (delegates to the existing BackendAdvanced body). + * - Synth/Particle/Editor → handled by ModeConfig in Drawers (no extra config here). + * + * Everything is editable inline; writes go through the shared MFParam store + * (ctx.setParam) — never a second data path. The full-depth modal reuses the + * same sections via BackendAdvanced. + */ +import { useEffect, useState } from 'react'; +import type { ConsoleCtx } from '../console/types'; +import type { BackendId } from './output-state'; +import { defaultMidiSpec, defaultOscSpec } from './output-state'; +import { + applyPreset, + deletePreset, + getPreset, + listPresets, + renamePreset, + savePreset, + type OutputPreset, +} from '../backends/presets'; + +function num(s: string, fallback: number): number { + const v = parseFloat(s); + return Number.isFinite(v) ? v : fallback; +} + +const cellInput: React.CSSProperties = { + width: '100%', + background: 'var(--bg-1)', + border: '1px solid var(--line)', + borderRadius: 'var(--r-1)', + color: 'var(--fg)', + fontFamily: 'var(--font-mono)', + fontSize: 'var(--fs-xs)', + padding: '3px 6px', + boxSizing: 'border-box', +}; + +const btn = (color: string): React.CSSProperties => ({ + fontSize: 'var(--fs-xs)', + fontFamily: 'var(--font-mono)', + padding: '3px 9px', + cursor: 'pointer', + borderRadius: 'var(--r-pill)', + border: `1px solid ${color}`, + background: 'transparent', + color, +}); + +function SectionLabel({ children }: { children: React.ReactNode }) { + return ( +
+ {children} +
+ ); +} + +function Th({ children }: { children: React.ReactNode }) { + return ( + + {children} + + ); +} + +// ---- Named-preset bar (backends-spec §5) ----------------------------------- + +function PresetBar({ ctx, backend }: { ctx: ConsoleCtx; backend: BackendId }) { + const [presets, setPresets] = useState([]); + const [name, setName] = useState(''); + const [selected, setSelected] = useState(''); + + const refresh = () => setPresets(listPresets(backend)); + useEffect(() => { + refresh(); + setSelected(''); + setName(''); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [backend]); + + const backendSettings = (): Record => { + if (backend === 'midi') return { outputId: ctx.midiOutputId, ccCount: ctx.midiCcCount }; + if (backend === 'osc') return { url: ctx.oscUrl, sendRaw: ctx.oscSendRaw }; + return {}; + }; + + const applySettings = (s?: Record) => { + if (!s) return; + if (backend === 'midi') { + if ('outputId' in s) ctx.setMidiOutputId((s.outputId as string | null) ?? null); + if ('ccCount' in s) ctx.setMidiCcCount(Number(s.ccCount) || ctx.midiCcCount); + } else if (backend === 'osc') { + if ('url' in s) ctx.setOscUrl(String(s.url)); + if ('sendRaw' in s) ctx.setOscSendRaw(Boolean(s.sendRaw)); + } + }; + + const doSave = () => { + const n = name.trim(); + if (!n) return; + savePreset(backend, n, ctx.params, backendSettings()); + refresh(); + setSelected(n); + }; + const doRestore = (n: string) => { + const p = getPreset(backend, n); + if (!p) return; + ctx.setParams(applyPreset(ctx.params, p)); + applySettings(p.settings); + }; + const doDelete = () => { + if (!selected) return; + deletePreset(backend, selected); + refresh(); + setSelected(''); + }; + const doRename = () => { + const to = name.trim(); + if (!selected || !to) return; + if (renamePreset(backend, selected, to)) { + refresh(); + setSelected(to); + } + }; + + return ( +
+ + Presets · {backend} + + setName(e.target.value)} + /> + + + + +
+ ); +} + +// ---- MIDI config (backends-spec §2.3 / §4.1) ------------------------------- + +function MidiConfig({ ctx }: { ctx: ConsoleCtx }) { + const s = ctx.backendStatus; + const statusColor = + s.state === 'ready' ? 'var(--good)' : s.state === 'error' || s.state === 'unavailable' ? 'var(--danger)' : 'var(--warn)'; + return ( + <> + MIDI output +
+ + + {s.message} +
+ + Per-output CC · name · channel +
+ + + + + + + + + + + {ctx.params.slice(0, ctx.midiCcCount).map((p, i) => { + const m = p.midi ?? defaultMidiSpec(i); + return ( + + + + + + + ); + })} + +
NameCC#ChState
+ ctx.setParam(i, { midi: { ...m, name: e.target.value } })} + /> + + + ctx.setParam(i, { midi: { ...m, cc: Math.max(0, Math.min(127, num(e.target.value, m.cc))) } }) + } + /> + + + ctx.setParam(i, { + midi: { ...m, channel: Math.max(1, Math.min(16, num(e.target.value, m.channel))) }, + }) + } + /> + + {p.status} + {p.muted ? ' · muted' : ''} +
+
+ + ); +} + +// ---- OSC config (backends-spec §2.4 / §4.2) -------------------------------- + +function OscConfig({ ctx }: { ctx: ConsoleCtx }) { + const s = ctx.backendStatus; + const statusColor = s.state === 'ready' ? 'var(--good)' : s.state === 'connecting' ? 'var(--warn)' : 'var(--danger)'; + const [draftUrl, setDraftUrl] = useState(ctx.oscUrl); + useEffect(() => setDraftUrl(ctx.oscUrl), [ctx.oscUrl]); + return ( + <> + OSC bridge +
+ setDraftUrl(e.target.value)} + onBlur={() => ctx.setOscUrl(draftUrl)} + placeholder="ws://localhost:8765" + /> + + + {s.message} +
+

+ The Deno OSC bridge process must be running locally (see manifold/osc-bridge). The browser sends over + WebSocket; the bridge encodes OSC and forwards over UDP. +

+ + Per-output address · physical range +
+ + + + + + + + + + {ctx.params.map((p, i) => { + const o = p.osc ?? defaultOscSpec(p.name); + return ( + + + + + + ); + })} + +
OSC pathrange minrange max
+ ctx.setParam(i, { osc: { ...o, path: e.target.value } })} + /> + + ctx.setParam(i, { osc: { ...o, rangeMin: num(e.target.value, o.rangeMin) } })} + /> + + ctx.setParam(i, { osc: { ...o, rangeMax: num(e.target.value, o.rangeMax) } })} + /> +
+
+ + ); +} + +// ---- Public entry ---------------------------------------------------------- + +export interface OutputsBackendConfigProps { + ctx: ConsoleCtx; + backend: BackendId; +} + +/** The specialised, editable per-backend config + preset bar for the Outputs panel. */ +export function OutputsBackendConfig({ ctx, backend }: OutputsBackendConfigProps) { + // Only MIDI / OSC carry a config + preset surface here; synth/particle/editor + // config is rendered by ModeConfig in Drawers. VCV/CV polarity stays in the + // full-depth BackendAdvanced modal. + if (backend !== 'midi' && backend !== 'osc') return null; + return ( +
+ + {backend === 'midi' ? : } +
+ ); +} + +export { PresetBar as OutputPresetBar }; + +/** Tiny status pill for the Outputs drawer header. */ +export function BackendStatusChip({ ctx }: { ctx: ConsoleCtx }) { + const s = ctx.backendStatus; + if (s.state === 'idle') return null; + const color = + s.state === 'ready' + ? 'var(--good)' + : s.state === 'error' || s.state === 'unavailable' + ? 'var(--danger)' + : 'var(--warn)'; + return ( + + {s.message} + + ); +} diff --git a/manifold/src/dock/output-state.ts b/manifold/src/dock/output-state.ts new file mode 100644 index 0000000..52c912d --- /dev/null +++ b/manifold/src/dock/output-state.ts @@ -0,0 +1,132 @@ +/** + * The per-output control model for the Outputs / Routing dock (workstream D, + * docs/redesign/dock-spec.md §3.2). + * + * DELIBERATE DIVERGENCE from the deployed a-immersive app (dock-spec §3.3 note, + * open choice 3): the deployed override system conflates "frozen" (heatmap + * popup) and "muted" (group drawer) onto ONE underlying field. This model splits + * the three orthogonal concepts into distinct fields: + * + * - `state` : 'off' | 'fixed' | 'live' — the model-control tri-state. + * - `muted` : boolean — downstream silence (still computed + visible). + * - `armed` : boolean — solo / focus-training (=arm). + * + * They compose freely (e.g. an output can be `off` AND `muted` AND `armed`). + * Recorded in ALIGNMENT.md. + * + * To keep the dock tri-state and the existing OutputStage / ReadoutStrip + * tri-state in sync WITHOUT a second data path, this model is folded onto the + * existing `MFParam` (model.ts) — `MFParam.status` carries `state`, and the new + * `muted` / `armed` / backend fields live alongside it. ConsoleApp owns the + * single `MFParam[]` store; the dock and the stage both read/write it. + */ + +import type { MFParam, ParamStatus } from '../console/model'; + +/** The model-control tri-state (alias of the console ParamStatus). */ +export type OutputState = ParamStatus; // 'off' | 'fixed' | 'live' + +/** The selectable output backend (dock-spec §3.4; backends-spec §1). */ +export type BackendId = 'synth' | 'particles' | 'midi' | 'osc' | 'cvgate' | 'vcv'; + +export interface BackendDescriptor { + id: BackendId; + /** Dock label — NEVER "C15" (backends-spec naming guard). */ + label: string; + description: string; +} + +/** The backend roster surfaced in the dock's backend selector. */ +export const BACKENDS: readonly BackendDescriptor[] = [ + { id: 'synth', label: 'Powerful Synth Engine', description: 'Firmware-parity built-in audio engine.' }, + { id: 'midi', label: 'MIDI', description: 'Web MIDI CC out — per-output CC#/channel.' }, + { id: 'osc', label: 'OSC', description: 'OSC bridge — named paths + physical ranges.' }, + { id: 'cvgate', label: 'CV', description: 'CV / gate (via VCV bridge or DC-coupled audio).' }, + { id: 'vcv', label: 'VCV', description: 'VCV Rack module — 16 CV outs with LED rings.' }, + { id: 'particles', label: 'Particle', description: 'Flow-field visualiser (no audio).' }, +] as const; + +// ---- Backend-specific per-output specs (dock-spec §4) ---------------------- + +/** MIDI CC backend per-output extras (dock-spec §4.1). */ +export interface MidiCcSpec { + cc: number; // 0..127 + channel: number; // 1..16 + name: string; + value: number; // last sent, round(v*127) +} + +/** OSC backend per-output extras (dock-spec §4.2). */ +export interface OscSpec { + path: string; // e.g. "/synth/cutoff" + rangeMin: number; // physical (engineering) units, NOT [0,1] + rangeMax: number; +} + +/** VCV backend per-output extras (dock-spec §4.3) — baseline min/max IS the range. */ +export interface VcvSpec { + bipolar: boolean; // unipolar 0..10V vs bipolar ±5V +} + +/** + * The full per-output control. This is the spec's `OutputControl` (dock-spec + * §3.2). It is represented on `MFParam` for the shared store; this interface + * documents the complete contract and is what {@link toOutputControl} yields. + */ +export interface OutputControl { + index: number; + name: string; + group: string; + state: OutputState; // off | fixed | live + muted: boolean; // downstream silence; still computed + armed: boolean; // solo / focus-training (=arm) + min: number; // [0,1] + max: number; // [0,1], min<=max + curve: number; // [0,1], 0.5 linear + fixedValue: number; // held value when state==='fixed' + // backend-specific, populated by the active backend adapter: + midi?: MidiCcSpec; + osc?: OscSpec; + vcv?: VcvSpec; +} + +/** Project an MFParam (the shared store row) into the full OutputControl view. */ +export function toOutputControl(p: MFParam, index: number): OutputControl { + return { + index, + name: p.name, + group: p.group, + state: p.status, + muted: p.muted ?? false, + armed: p.armed ?? false, + min: p.min, + max: p.max, + curve: p.curve, + fixedValue: p.val, + midi: p.midi, + osc: p.osc, + vcv: p.vcv, + }; +} + +/** + * Build the focus / solo mask from the per-row armed flags (dock-spec §1.2). + * Returns null when nothing is armed (⇒ all outputs active / no focus). + */ +export function buildArmMask(params: MFParam[]): Uint8Array | null { + const anyArmed = params.some((p) => p.armed); + if (!anyArmed) return null; + const mask = new Uint8Array(params.length); + for (let i = 0; i < params.length; i++) mask[i] = params[i].armed ? 1 : 0; + return mask; +} + +/** Default MIDI CC spec for a freshly-added output, auto-named by index. */ +export function defaultMidiSpec(index: number): MidiCcSpec { + return { cc: index % 128, channel: 1, name: `CC ${index % 128}`, value: 0 }; +} + +/** Default OSC spec for an output. */ +export function defaultOscSpec(name: string): OscSpec { + return { path: `/nisps/${name.toLowerCase()}`, rangeMin: 0, rangeMax: 1 }; +} diff --git a/manifold/src/engine/EngineProvider.tsx b/manifold/src/engine/EngineProvider.tsx new file mode 100644 index 0000000..fe5b3f0 --- /dev/null +++ b/manifold/src/engine/EngineProvider.tsx @@ -0,0 +1,55 @@ +/** + * EngineProvider — the React binding layer for the headless EngineApi. + * + * This is the ONLY place (with useEngine.ts) where `engine/` touches React. + * The lint rule "skins may not import engine internals; engine may not import + * React" is satisfied: the engine is React-free, and this provider only + * consumes the public `EngineApi` façade. + * + * The engine is created asynchronously (the WASM must load). Until it's ready, + * `useEngine()` returns null; consumers should guard on it. + */ + +import { + createContext, + useEffect, + useState, + type ReactNode, +} from 'react'; +import { createEngine, EngineApi, type EngineApiOptions } from './engine-api'; + +export const EngineContext = createContext(null); + +export interface EngineProviderProps { + children: ReactNode; + options?: EngineApiOptions; + /** Optional fallback rendered until the engine has loaded. */ + fallback?: ReactNode; +} + +export function EngineProvider(props: EngineProviderProps): JSX.Element { + const [engine, setEngine] = useState(null); + + useEffect(() => { + let disposed = false; + let created: EngineApi | null = null; + void createEngine(props.options ?? {}).then((eng) => { + if (disposed) { + eng.dispose(); + return; + } + created = eng; + setEngine(eng); + }); + return () => { + disposed = true; + created?.dispose(); + }; + // Recreate only if the options object identity changes. + }, [props.options]); + + if (!engine) { + return <>{props.fallback ?? null}; + } + return {props.children}; +} diff --git a/manifold/src/engine/curves.ts b/manifold/src/engine/curves.ts new file mode 100644 index 0000000..a58bbcb --- /dev/null +++ b/manifold/src/engine/curves.ts @@ -0,0 +1,128 @@ +/** + * Curve catalog — TypeScript mirror of the named curves from + * nisps/core/math.hpp (forthcoming, stream 1). All inputs and outputs are in + * [0, 1] unless noted otherwise. + * + * IMPORTANT: This file MUST stay in lockstep with the C++ side. The + * authoritative reference is `nisps/core/math.hpp`. Golden-vector tests + * (stream 11) compare WASM-computed vs TS-computed outputs and fail on + * any drift. + * + * Architecture §5.3: + * linear, exp, log, square, sqrt, sigmoid, cubic, centered_power + * + * The "centered_power" variant comes from the legacy input/output pipelines + * and shapes around 0.5 instead of 0.0. Kept as a named curve because both + * input and output pipelines use it. + */ + +export type CurveName = + | 'linear' + | 'exp' + | 'log' + | 'square' + | 'sqrt' + | 'sigmoid' + | 'cubic' + | 'centered_power'; + +/** Hard clamp to [0, 1]. */ +export function clamp01(v: number): number { + if (v < 0) return 0; + if (v > 1) return 1; + return v; +} + +/** Generic clamp. */ +export function clamp(v: number, lo: number, hi: number): number { + if (v < lo) return lo; + if (v > hi) return hi; + return v; +} + +/** Linear: identity. */ +export function curveLinear(x: number): number { + return clamp01(x); +} + +/** Exponential: e^(k*x) - 1, normalized to [0,1] over [0,1] input. */ +export function curveExp(x: number, k: number = 4.0): number { + if (x <= 0) return 0; + if (x >= 1) return 1; + const denom = Math.exp(k) - 1.0; + if (denom === 0) return x; + return (Math.exp(k * x) - 1.0) / denom; +} + +/** Inverse of curveExp. */ +export function curveLog(x: number, k: number = 4.0): number { + if (x <= 0) return 0; + if (x >= 1) return 1; + const denom = Math.exp(k) - 1.0; + if (denom === 0) return x; + return Math.log(1 + x * denom) / k; +} + +/** Square: x^2. */ +export function curveSquare(x: number): number { + const v = clamp01(x); + return v * v; +} + +/** Square-root. */ +export function curveSqrt(x: number): number { + return Math.sqrt(clamp01(x)); +} + +/** Logistic sigmoid mapped onto [0,1] domain (centered at x=0.5). */ +export function curveSigmoid(x: number, slope: number = 8.0): number { + // Sigmoid centered at 0.5 with given slope. Output is in (0, 1). + // Normalize so endpoints map exactly to 0 and 1. + const t = (x - 0.5) * slope; + const s = 1 / (1 + Math.exp(-t)); + // Anchor: when x=0, t=-slope/2; when x=1, t=+slope/2 + const sLo = 1 / (1 + Math.exp(slope / 2)); + const sHi = 1 / (1 + Math.exp(-slope / 2)); + return (s - sLo) / (sHi - sLo); +} + +/** Cubic ease-in-out. */ +export function curveCubic(x: number): number { + const v = clamp01(x); + // Smoothstep cubic: 3v^2 - 2v^3 + return v * v * (3 - 2 * v); +} + +/** + * Centered power curve. Pivots around 0.5. + * + * exponent < 1 → push toward extremes + * exponent = 1 → identity + * exponent > 1 → pull toward center + */ +export function curveCenteredPower(x: number, exponent: number): number { + if (exponent === 1) return clamp01(x); + const offset = x - 0.5; + const sign = offset < 0 ? -1 : 1; + // Range [-0.5, 0.5] -> [-1, 1] for the power op, then halve back. + const shaped = (sign * Math.pow(Math.abs(offset) * 2, exponent)) / 2; + return clamp01(shaped + 0.5); +} + +/** Apply by name. `param` interpretation depends on the curve. */ +export function applyCurve(name: CurveName, x: number, param?: number): number { + switch (name) { + case 'linear': return curveLinear(x); + case 'exp': return curveExp(x, param ?? 4.0); + case 'log': return curveLog(x, param ?? 4.0); + case 'square': return curveSquare(x); + case 'sqrt': return curveSqrt(x); + case 'sigmoid': return curveSigmoid(x, param ?? 8.0); + case 'cubic': return curveCubic(x); + case 'centered_power': return curveCenteredPower(x, param ?? 1.0); + } +} + +export const CURVE_NAMES: ReadonlyArray = [ + 'linear', 'exp', 'log', 'square', 'sqrt', 'sigmoid', 'cubic', 'centered_power', +]; diff --git a/manifold/src/engine/dataset.ts b/manifold/src/engine/dataset.ts new file mode 100644 index 0000000..c8e6323 --- /dev/null +++ b/manifold/src/engine/dataset.ts @@ -0,0 +1,193 @@ +/** + * Dataset — JS-side training-example store. + * + * Why duplicate the C++ ring buffer? Two reasons: + * 1. Sample-weight computation (recency / spatial / combined) lives in JS so + * that adjusting weighting modes doesn't burn a WASM round-trip. + * 2. The dataset is part of session state we serialize to localStorage — + * the WASM heap is wiped on reload. + * + * On train() we ship features + labels into WASM via `addExample` calls. The + * order of insertion is preserved; FIFO eviction matches the C++ MLP's + * `dataset_head_` pointer so weighting stays consistent. + * + * The implementation is a faithful TypeScript port of the legacy + * `playground/_archive/js/nisps/dataset.js` with: + * - Float32Array backing instead of `Array>` + * - Stricter types + * - No `withBias` flag (the WASM bindings don't take a bias term) + */ + +export type WeightMode = 'global' | 'local' | 'combined' | 'uniform'; + +export interface ComputeWeightsParams { + /** [0,1] — how strongly to bias toward newest examples (global/combined). */ + recencyBias?: number; + /** Current input position, used for local/combined spatial weighting. */ + queryInput?: ReadonlyArray; + /** Spatial radius in input space (local/combined). */ + radius?: number; +} + +export class Dataset { + /** Maximum number of examples retained. FIFO eviction beyond this. */ + readonly maxSize: number; + /** Length of feature vectors. Set on first add(); locked thereafter. */ + private inputSize_ = 0; + /** Length of label vectors. Set on first add(); locked thereafter. */ + private outputSize_ = 0; + + /** Flat arrays — entries `[i*inputSize, (i+1)*inputSize)` belong to example i. */ + private features_: Float32Array = new Float32Array(0); + private labels_: Float32Array = new Float32Array(0); + private size_ = 0; + + constructor(maxSize = 100) { + if (maxSize <= 0) throw new Error('Dataset.maxSize must be > 0'); + this.maxSize = maxSize; + } + + /** Number of examples currently stored. */ + get size(): number { + return this.size_; + } + + isEmpty(): boolean { + return this.size_ === 0; + } + + /** + * Add a feature/label pair. Returns true on success, false if the + * dimensions don't match a previously-added example. + * + * Eviction: when at capacity, the oldest example is removed (shift), + * then the new one is appended. This matches the legacy JS behaviour + * (and is conceptually equivalent to the C++ side's ring buffer with + * `head_` advancement). + */ + add(features: ReadonlyArray, labels: ReadonlyArray): boolean { + if (this.size_ === 0) { + this.inputSize_ = features.length; + this.outputSize_ = labels.length; + // Allocate full-capacity buffers up front to avoid growth thrash. + this.features_ = new Float32Array(this.maxSize * this.inputSize_); + this.labels_ = new Float32Array(this.maxSize * this.outputSize_); + } + + if (features.length !== this.inputSize_ || labels.length !== this.outputSize_) { + return false; + } + + if (this.size_ >= this.maxSize) { + // FIFO: shift left in place. This is O(n*dim) and could be replaced + // with a head pointer; for maxSize ≤ a few hundred it's fine. + this.features_.copyWithin(0, this.inputSize_); + this.labels_.copyWithin(0, this.outputSize_); + this.size_ = this.maxSize - 1; + } + + const fOff = this.size_ * this.inputSize_; + const lOff = this.size_ * this.outputSize_; + for (let i = 0; i < this.inputSize_; ++i) this.features_[fOff + i] = features[i]; + for (let i = 0; i < this.outputSize_; ++i) this.labels_[lOff + i] = labels[i]; + this.size_++; + return true; + } + + clear(): void { + this.size_ = 0; + } + + /** Read-only view of the i-th feature vector. */ + feature(i: number): Float32Array { + if (i < 0 || i >= this.size_) throw new RangeError(`feature index ${i} out of bounds`); + return this.features_.subarray(i * this.inputSize_, (i + 1) * this.inputSize_); + } + + /** Read-only view of the i-th label vector. */ + label(i: number): Float32Array { + if (i < 0 || i >= this.size_) throw new RangeError(`label index ${i} out of bounds`); + return this.labels_.subarray(i * this.outputSize_, (i + 1) * this.outputSize_); + } + + /** Flat view of all features (size * inputSize). */ + featuresFlat(): Float32Array { + return this.features_.subarray(0, this.size_ * this.inputSize_); + } + + /** Flat view of all labels (size * outputSize). */ + labelsFlat(): Float32Array { + return this.labels_.subarray(0, this.size_ * this.outputSize_); + } + + get inputSize(): number { + return this.inputSize_; + } + get outputSize(): number { + return this.outputSize_; + } + + /** + * Compute per-sample training weights. Returns Float32Array (size=this.size) + * normalized to sum to 1. For an empty dataset returns a 0-length array; + * for a singleton, [1.0]. + * + * Modes: + * - `uniform` — every weight = 1/n. + * - `global` — exponential recency decay over insertion order. + * - `local` — within `radius` of `queryInput`, suppress older neighbours. + * - `combined` — global × local. + */ + computeWeights(mode: WeightMode = 'uniform', params: ComputeWeightsParams = {}): Float32Array { + const n = this.size_; + if (n === 0) return new Float32Array(0); + if (n === 1) return new Float32Array([1.0]); + + const weights = new Float32Array(n).fill(1.0); + + if (mode === 'global' || mode === 'combined') { + const bias = params.recencyBias ?? 0.6; + if (bias > 0) { + const decay = 1 - 0.3 * bias; + for (let i = n - 2; i >= 0; --i) weights[i] = weights[i + 1] * decay; + } + } + + if ((mode === 'local' || mode === 'combined') && params.queryInput) { + const query = params.queryInput; + const radius = params.radius ?? 0.15; + const radiusSq = radius * radius; + const dim = this.inputSize_; + + for (let i = 0; i < n; ++i) { + const fOffI = i * dim; + let distSq = 0; + for (let d = 0; d < dim; ++d) { + const diff = this.features_[fOffI + d] - (query[d] ?? 0); + distSq += diff * diff; + } + if (distSq < radiusSq) { + const proximity = 1 - Math.sqrt(distSq) / radius; + let newerNearby = 0; + for (let j = i + 1; j < n; ++j) { + const fOffJ = j * dim; + let djSq = 0; + for (let d = 0; d < dim; ++d) { + const diff = this.features_[fOffI + d] - this.features_[fOffJ + d]; + djSq += diff * diff; + } + if (djSq < radiusSq) ++newerNearby; + } + if (newerNearby > 0) { + weights[i] *= Math.pow(1 - proximity, newerNearby); + } + } + } + } + + let sum = 0; + for (let i = 0; i < n; ++i) sum += weights[i]; + if (sum > 0) for (let i = 0; i < n; ++i) weights[i] /= sum; + return weights; + } +} diff --git a/manifold/src/engine/engine-api.ts b/manifold/src/engine/engine-api.ts new file mode 100644 index 0000000..9e5fada --- /dev/null +++ b/manifold/src/engine/engine-api.ts @@ -0,0 +1,244 @@ +/** + * EngineApi — the headless façade every consumer uses. + * + * This is the boundary the design docs (engine-architecture.md, findings §4) + * call for: a framework-neutral object that owns the WasmIML (ML), the + * EngineHost (audio), and the reactive Spine, and exposes ONE coherent API. + * React talks to it through Context; the debug probe talks to it directly; a + * headless test can `await createEngine()` and drive it with no DOM framework. + * + * The engine imports NO React. The only React in `engine/` is the + * EngineProvider/useEngine binding layer (separate files). + * + * `subscribe(cb)` + `version()` are the `useSyncExternalStore` contract: React + * re-reads on a version bump but consumers read the live Float32Array + * imperatively via `getOutputs()` / `routedOutput()`. + */ + +import { EngineHost } from './engine-host'; +import { Spine, type BackendSend } from './spine'; +import type { EngineId, FeedbackMode, LayerStats } from './types'; +import { WasmIML } from './wasm-iml'; + +export interface EngineFeedbackApi { + /** Positive feedback (thumbs-up). Returns the FeedbackAction int. */ + thumbsUp(): number; + /** Negative feedback (thumbs-down). Returns the FeedbackAction int. */ + thumbsDown(speed?: number, spread?: number, pinMask?: Uint8Array): number; + /** Drag (continuous perturbation) tick. */ + drag(): number; + setMode(mode: FeedbackMode): void; + getMode(): FeedbackMode; + /** Restrict feedback to a subset of outputs (solo / column-freeze). */ + setFocus(mask: Uint8Array | null): void; + /** True while the controller is exploring (perturbed). */ + exploring(): boolean; + /** True while the controller has paused learning. */ + learningPaused(): boolean; +} + +export interface EngineAudioApi { + start(engineId?: EngineId): Promise; + stop(): Promise; + setMuted(muted: boolean): void; + setBackend(id: EngineId): void; + getBackend(): EngineId; + readonly isStarted: boolean; +} + +export interface EngineApiOptions { + seed?: number; + storageKey?: string; + maxExamples?: number; + /** Default learning rate for thumbsUp/train. */ + learningRate?: number; + /** Default RL move speed / spread for thumbsDown. */ + noiseCap?: number; + spread?: number; +} + +export class EngineApi { + readonly spine: Spine; + private iml: WasmIML; + private host: EngineHost; + + private learningRate: number; + private noiseCap: number; + private spread_: number; + + readonly feedback: EngineFeedbackApi; + readonly audio: EngineAudioApi; + + private constructor(iml: WasmIML, spine: Spine, host: EngineHost, opts: EngineApiOptions) { + this.iml = iml; + this.spine = spine; + this.host = host; + this.learningRate = opts.learningRate ?? 1.0; + this.noiseCap = opts.noiseCap ?? 0.3; + this.spread_ = opts.spread ?? 0.6; + + // Wire the spine's backend.send to push routed params into the worklet. + const send: BackendSend = (routed) => { + if (this.host.isStarted) this.host.setParams(new Float32Array(routed)); + }; + this.spine.attach(iml, send); + + this.feedback = { + thumbsUp: () => this.iml.feedbackUp(), + thumbsDown: (speed = this.noiseCap, spread = this.spread_, pinMask?: Uint8Array) => + this.iml.feedbackDown(speed, spread, this.spine.outputs(), pinMask), + drag: () => this.iml.feedbackDrag(), + setMode: (mode) => this.iml.feedbackSetMode(mode), + getMode: () => this.iml.feedbackGetMode(), + setFocus: (mask) => this.iml.feedbackSetFocus(mask), + exploring: () => this.iml.feedbackExploring(), + learningPaused: () => this.iml.feedbackLearningPaused(), + }; + + this.audio = { + start: (engineId?: EngineId) => this.host.start(engineId), + stop: () => this.host.stop(), + setMuted: (muted) => this.host.setMuted(muted), + setBackend: (id) => this.host.setEngine(id), + getBackend: () => this.host.engine, + get isStarted() { + return host.isStarted; + }, + }; + } + + static async create(opts: EngineApiOptions = {}): Promise { + const spine = new Spine(); + const iml = await WasmIML.create({ + seed: opts.seed, + storageKey: opts.storageKey, + maxExamples: opts.maxExamples, + sink: spine, + }); + const host = new EngineHost(); + return new EngineApi(iml, spine, host, opts); + } + + // ---- Input → spine ------------------------------------------------- + + /** Drive a raw XY input ∈ [0,1] through the full spine (off React render). */ + setInput(x: number, y: number): void { + this.spine.setInput(x, y); + } + + /** Set an arbitrary input vector (first two used as XY for the fixed 2→N MLP). */ + setInputs(arr: ReadonlyArray): void { + this.spine.setInput(arr[0] ?? 0.5, arr[1] ?? 0.5); + } + + /** Live post-ML output vector (reused buffer — read, don't retain). */ + getOutputs(): Float32Array { + return this.spine.outputs(); + } + + /** Live routed (post output-pipeline) vector. */ + routedOutput(): Float32Array | null { + return this.spine.routedOutput(); + } + + /** + * Re-run the LAST raw input through the spine — used after a weight change + * (train / randomise / feedback) so outputs + audio reflect the new MLP + * state without the user having to move the controller. + */ + process(): void { + this.spine.setInput(this.spine.lastRawX, this.spine.lastRawY); + } + + // ---- Training ------------------------------------------------------ + + addExample(features: ReadonlyArray, labels: ReadonlyArray): boolean { + return this.iml.addExample(features, labels); + } + + train(): number { + return this.iml.train(this.learningRate); + } + + trainAsync(): Promise { + return this.iml.trainAsync(this.learningRate); + } + + randomise(spread = this.spread_): void { + this.iml.randomiseWeights(spread); + this.process(); + } + + clearExamples(): void { + this.iml.clearExamples(); + } + + evalLoss(): number { + return this.iml.evalLoss(); + } + + inferBatch(points: ReadonlyArray): Float32Array { + return this.iml.inferBatch(points); + } + + // ---- Weights / stats ---------------------------------------------- + + getWeights(): Float32Array { + return this.iml.getWeights(); + } + + setWeights(w: Float32Array): void { + this.iml.setWeights(w); + } + + getLayerStats(): LayerStats[] { + return this.iml.getLayerStats(); + } + + getLayerStatsFlat(): Float32Array { + return this.iml.getLayerStatsFlat(); + } + + // ---- Reactive contract -------------------------------------------- + + /** Subscribe to state changes (useSyncExternalStore). Returns an unsubscribe. */ + subscribe(cb: () => void): () => void { + return this.spine.subscribe(cb); + } + + /** Monotonically-increasing counter, bumped on every state change. */ + version(): number { + return this.spine.version(); + } + + /** Subscribe to a named engine event (`ml.*`, `feedback.*`, …). */ + on(event: string, fn: (payload?: unknown) => void): () => void { + return this.spine.on(event, fn); + } + + getState() { + return this.spine.getState(); + } + + saveState(): void { + this.iml.saveNow(); + } + + get architecture() { + return this.iml.architecture; + } + + // ---- Direct handle access (advanced consumers; spine pipelines, etc.) ---- + get ml(): WasmIML { + return this.iml; + } + + dispose(): void { + this.host.dispose(); + this.iml.dispose(); + } +} + +export async function createEngine(opts: EngineApiOptions = {}): Promise { + return EngineApi.create(opts); +} diff --git a/manifold/src/engine/engine-host.ts b/manifold/src/engine/engine-host.ts new file mode 100644 index 0000000..2171f39 --- /dev/null +++ b/manifold/src/engine/engine-host.ts @@ -0,0 +1,203 @@ +/** + * EngineHost — main-thread side of the WASM AudioWorklet pipeline. + * + * Lifted from `playground/src/audio/engine-host.ts`. Changes vs the playground: + * - imports `./types` (the lifted ABI types) + * - the worklet entry is `./worklet/nisps-processor.ts?worker&url` + * - `nisps.wasm` is fetched via `import.meta.env.BASE_URL`, not a hardcoded + * `/nisps.wasm`, so the bundle works under any mount path (`/`, `/next`, …). + * + * Responsibilities: lazy-create AudioContext (user-gesture gated), register the + * worklet, hand it the `nisps.wasm` bytes (the worklet has no fetch), send + * engine selection + parameter updates over `port`, and tear down on dispose(). + * + * The DSP runs in `./worklet/nisps-processor.ts`, which holds a SECOND WASM + * instance owned by the worklet thread. + */ + +import type { EngineId } from './types'; + +// `?worker&url` makes Vite COMPILE the worklet TS→JS, bundle its imports, and +// hand back a hashed .js URL. Plain `new URL('./x.ts', import.meta.url)` does +// NOT work for audioWorklet.addModule (Vite only treats that as a worker entry +// for `new Worker(...)`) — it copies raw .ts, which the browser rejects. +import workletUrl from './worklet/nisps-processor.ts?worker&url'; + +const PROCESSOR_NAME = 'nisps-processor'; + +/** Base-aware absolute URL for an asset served from `public/`. Resolves against + * `document.baseURI` so a `base: './'` build works under any mount path + * (`/`, `/next/`, …); `location.origin` would drop the sub-path. */ +function assetUrl(file: string): string { + const base = import.meta.env.BASE_URL ?? '/'; + return new URL(base + file, document.baseURI).toString(); +} + +/** Message protocol: main → worklet. */ +export type HostToWorkletMessage = + | { + kind: 'init'; + wasmBinary: ArrayBuffer; + sampleRate: number; + } + | { + kind: 'engine'; + engineId: EngineId; + } + | { + kind: 'params'; + params: Float32Array; + } + | { + kind: 'mute'; + muted: boolean; + }; + +/** Message protocol: worklet → main. */ +export type WorkletToHostMessage = + | { kind: 'ready' } + | { kind: 'error'; message: string }; + +export interface EngineHostOptions { + /** Override sample rate (default: AudioContext.sampleRate). */ + sampleRate?: number; + /** Override worklet processor URL (testing). */ + processorUrl?: string; +} + +export class EngineHost { + private ctx: AudioContext | null = null; + private node: AudioWorkletNode | null = null; + private workletReady = false; + private currentEngine: EngineId = 'thru'; + private disposed = false; + private options: EngineHostOptions; + + private wasmBytes: ArrayBuffer | null = null; + + constructor(options: EngineHostOptions = {}) { + this.options = options; + } + + get isStarted(): boolean { + return !!this.ctx && this.workletReady; + } + + get engine(): EngineId { + return this.currentEngine; + } + + get sampleRate(): number { + return this.ctx?.sampleRate ?? this.options.sampleRate ?? 48000; + } + + /** + * Start audio. Must be called from a user gesture for AudioContext to + * resume. After this resolves, `setEngine()` and `setParams()` can be called. + */ + async start(engineId: EngineId = 'thru'): Promise { + if (this.ctx) { + this.setEngine(engineId); + await this.ctx.resume(); + return; + } + this.ctx = new AudioContext({ + sampleRate: this.options.sampleRate, + latencyHint: 'interactive', + }); + + if (!this.wasmBytes) { + this.wasmBytes = await this.fetchWasm_(); + } + + const procUrl = this.options.processorUrl ?? workletUrl; + await this.ctx.audioWorklet.addModule(procUrl); + + this.node = new AudioWorkletNode(this.ctx, PROCESSOR_NAME, { + numberOfInputs: 1, + numberOfOutputs: 1, + outputChannelCount: [2], + }); + this.node.connect(this.ctx.destination); + + this.workletReady = false; + const ready = new Promise((resolve, reject) => { + const onMsg = (ev: MessageEvent) => { + if (ev.data.kind === 'ready') { + this.workletReady = true; + this.node?.port.removeEventListener('message', onMsg); + resolve(); + } else if (ev.data.kind === 'error') { + this.node?.port.removeEventListener('message', onMsg); + reject(new Error(ev.data.message)); + } + }; + this.node!.port.addEventListener('message', onMsg); + this.node!.port.start(); + }); + + const copy = this.wasmBytes.slice(0); + this.node.port.postMessage( + { kind: 'init', wasmBinary: copy, sampleRate: this.ctx.sampleRate } satisfies HostToWorkletMessage, + [copy], + ); + + await ready; + this.currentEngine = engineId; + if (engineId !== 'thru') { + this.setEngine(engineId); + } + } + + setEngine(engineId: EngineId): void { + if (!this.node || !this.workletReady) return; + this.currentEngine = engineId; + this.node.port.postMessage({ kind: 'engine', engineId } satisfies HostToWorkletMessage); + } + + /** + * Push a fresh parameter vector. Caller should NOT reuse the buffer after + * this call — we transfer it. To keep yours, pass a copy. + */ + setParams(params: Float32Array): void { + if (!this.node || !this.workletReady) return; + this.node.port.postMessage( + { kind: 'params', params } satisfies HostToWorkletMessage, + [params.buffer], + ); + } + + setMuted(muted: boolean): void { + if (!this.node || !this.workletReady) return; + this.node.port.postMessage({ kind: 'mute', muted } satisfies HostToWorkletMessage); + } + + async stop(): Promise { + if (!this.ctx) return; + if (this.node) { + try { + this.node.disconnect(); + } catch { /* ignore */ } + this.node = null; + } + try { + await this.ctx.close(); + } catch { /* ignore */ } + this.ctx = null; + this.workletReady = false; + } + + dispose(): void { + if (this.disposed) return; + this.disposed = true; + void this.stop(); + this.wasmBytes = null; + } + + private async fetchWasm_(): Promise { + const url = assetUrl('nisps.wasm'); + const resp = await fetch(url); + if (!resp.ok) throw new Error(`fetch nisps.wasm: ${resp.status} ${resp.statusText}`); + return await resp.arrayBuffer(); + } +} diff --git a/manifold/src/engine/index.ts b/manifold/src/engine/index.ts new file mode 100644 index 0000000..dfe95f5 --- /dev/null +++ b/manifold/src/engine/index.ts @@ -0,0 +1,51 @@ +/** + * Engine layer barrel — the framework-neutral NISPS engine + its React binding. + * + * Skins import from here (or the specific hook files). The engine itself + * (everything except EngineProvider/useEngine) imports NO React. + */ + +export { EngineApi, createEngine } from './engine-api'; +export type { + EngineApiOptions, + EngineAudioApi, + EngineFeedbackApi, +} from './engine-api'; + +export { Spine } from './spine'; +export type { SpineState, BackendSend } from './spine'; + +export { WasmIML } from './wasm-iml'; +export type { WasmIMLOptions } from './wasm-iml'; + +export { EngineHost } from './engine-host'; +export { Dataset } from './dataset'; + +export { noopSink } from './sink'; +export type { EngineSink, EngineStatePatch } from './sink'; + +export type { + EngineId, + FeedbackMode, + LayerStats, + MLArchitecture, +} from './types'; + +export { EngineProvider, EngineContext } from './EngineProvider'; +export type { EngineProviderProps } from './EngineProvider'; +export { useEngine, useEngineOrThrow, useEngineVersion } from './useEngine'; + +// Pure pipelines (re-exported for consumers that need to configure them). +export { + processInput, + defaultInputConfig, + defaultInputState, +} from './input-pipeline'; +export type { InputConfig, InputState } from './input-pipeline'; +export { + processOutput, + defaultOutputConfig, + defaultOutputState, +} from './output-pipeline'; +export type { OutputConfig, OutputState } from './output-pipeline'; +export * as curves from './curves'; diff --git a/manifold/src/engine/input-pipeline.ts b/manifold/src/engine/input-pipeline.ts new file mode 100644 index 0000000..d9f791e --- /dev/null +++ b/manifold/src/engine/input-pipeline.ts @@ -0,0 +1,307 @@ +/** + * Input pipeline — pure TS port of legacy `js/ui/input-pipeline.js`. + * + * Stages (in order), each input/output in [0,1]: + * 0. Invert (per-axis flip) + * 1. Deadzone (suppress jitter near center, remap live zone to [0,1]) + * 2. Circular clamp (constrain to unit disk centered at 0.5,0.5) + * 3. Zoom (narrow window around anchor, modulated by momentum) + * 4. Centered power curve (per-axis exponent) + * 5. EMA smoothing (frame-rate-independent) + * 6. Momentum-as-zoom update (consumed next frame) + * + * `processInput` is a pure function over (raw, cfg, prev): returns the new + * processed coordinate plus the next-frame state. Consumer (input-store) + * holds the state and calls this each frame. + * + * Math is intentionally bit-for-bit equivalent to the legacy implementation. + */ + +import { clamp, curveCenteredPower } from './curves'; + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +export const ZOOM_MIN = 0.01; +export const ZOOM_MAX = 1.0; +export const FREEZE_THRESHOLD = ZOOM_MIN; + +export const DEADZONE_MAX = 0.4; +export const INPUT_CURVE_MIN = 0.2; +export const INPUT_CURVE_MAX = 5.0; +export const SMOOTHING_MAX = 0.95; +export const VELOCITY_WINDOW_DEFAULT = 150; // ms + +const REFERENCE_DT = 1 / 60; + +export type MomentumZoomMode = 'off' | 'gentle' | 'strong'; +export type AnchorMode = 'auto' | 'sticky' | 'center'; + +interface MomentumPreset { + factor: number; + minZoomMul: number; + maxZoomMul: number; +} + +const MOMENTUM_PRESETS: Record = { + off: null, + gentle: { factor: 0.6, minZoomMul: 0.3, maxZoomMul: 1.0 }, + strong: { factor: 1.5, minZoomMul: 0.15, maxZoomMul: 1.0 }, +}; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export interface InputConfig { + /** Global zoom level [0.01, 1.0] */ + zoom: number; + /** Optional per-axis zoom; overrides global when not null */ + zoomX: number | null; + zoomY: number | null; + /** Anchor point [0,1]^2 (used in sticky/auto modes) */ + anchorX: number; + anchorY: number; + anchorMode: AnchorMode; + /** Deadzone fraction of half-travel [0, 0.4] */ + deadzone: number; + /** Centered power curve exponent [0.2, 5.0] (1.0 = linear) */ + inputCurve: number; + inputCurveX: number | null; + inputCurveY: number | null; + /** EMA smoothing factor [0, 0.95] */ + smoothing: number; + /** Momentum-as-zoom preset */ + momentumZoom: MomentumZoomMode; + velocityWindow: number; + /** Per-axis inversion */ + invertX: boolean; + invertY: boolean; +} + +export interface InputState { + /** Last smoothed output x; seed at 0.5 */ + smoothedX: number; + smoothedY: number; + /** Velocity history ring used for momentum-zoom */ + velocityHistory: ReadonlyArray<{ x: number; y: number; t: number }>; + /** Most recent momentum-zoom multiplier (1 = no scale) */ + momentumZoomMultiplier: number; + /** Whether last process call returned frozen=true */ + frozen: boolean; +} + +export interface ProcessResult { + x: number; + y: number; + frozen: boolean; + /** Next frame's state — consumer should keep this and pass it back. */ + state: InputState; +} + +// --------------------------------------------------------------------------- +// Defaults +// --------------------------------------------------------------------------- + +export function defaultInputConfig(): InputConfig { + return { + zoom: 1.0, + zoomX: null, + zoomY: null, + anchorX: 0.5, + anchorY: 0.5, + anchorMode: 'center', + deadzone: 0, + inputCurve: 1.0, + inputCurveX: null, + inputCurveY: null, + smoothing: 0, + momentumZoom: 'off', + velocityWindow: VELOCITY_WINDOW_DEFAULT, + invertX: false, + invertY: false, + }; +} + +export function defaultInputState(): InputState { + return { + smoothedX: 0.5, + smoothedY: 0.5, + velocityHistory: [], + momentumZoomMultiplier: 1, + frozen: false, + }; +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function applyDeadzone(input: number, deadzone: number): number { + if (deadzone <= 0) return input; + const offset = input - 0.5; + const absOff = Math.abs(offset); + const halfDz = deadzone * 0.5; + if (absOff <= halfDz) return 0.5; + const sign = offset < 0 ? -1 : 1; + const remapped = ((absOff - halfDz) / (0.5 - halfDz)) * 0.5; + return 0.5 + sign * remapped; +} + +function applyZoom(input: number, anchor: number, zoomLevel: number): number { + return clamp(anchor + (input - 0.5) * zoomLevel, 0, 1); +} + +function emaSmooth(prev: number, raw: number, smoothing: number, dt: number): number { + if (smoothing <= 0) return raw; + const effectiveDt = dt > 0 ? dt : REFERENCE_DT; + const alpha = 1 - smoothing; + const alphaEff = 1 - Math.pow(1 - alpha, effectiveDt / REFERENCE_DT); + return prev + alphaEff * (raw - prev); +} + +function updateMomentumZoomMultiplier( + cfg: InputConfig, + state: InputState, + rawX: number, + rawY: number, + dt: number, +): { multiplier: number; history: InputState['velocityHistory'] } { + const preset = MOMENTUM_PRESETS[cfg.momentumZoom]; + if (!preset) { + return { multiplier: 1, history: [] }; + } + const now = performance.now(); + const window = cfg.velocityWindow; + // Append, drop entries older than `window` ms + const trimmed = state.velocityHistory.filter((p) => now - p.t <= window); + const newHist = [...trimmed, { x: rawX, y: rawY, t: now }]; + if (newHist.length < 2) { + return { multiplier: 1, history: newHist }; + } + const a = newHist[0]!; + const b = newHist[newHist.length - 1]!; + const dtMs = b.t - a.t; + if (dtMs <= 0) return { multiplier: state.momentumZoomMultiplier, history: newHist }; + const dx = b.x - a.x; + const dy = b.y - a.y; + const dist = Math.sqrt(dx * dx + dy * dy); + const speed = dist / (dtMs / 1000); // [0,1]-space units per second + const normSpeed = clamp(speed * preset.factor, 0, 1); + // Higher speed → smaller multiplier (zoom out faster movements) + const target = preset.maxZoomMul - (preset.maxZoomMul - preset.minZoomMul) * normSpeed; + // Smooth toward target so the zoom doesn't jitter + const smoothCoeff = clamp(dt * 6, 0, 1); + const next = state.momentumZoomMultiplier + smoothCoeff * (target - state.momentumZoomMultiplier); + return { multiplier: next, history: newHist }; +} + +function resolveAnchorX(cfg: InputConfig, state: InputState): number { + if (cfg.anchorMode === 'center') return 0.5; + if (cfg.anchorMode === 'sticky') return cfg.anchorX; + // auto: use stored anchor (input-store updates it on zoom changes) + return cfg.anchorX; +} + +function resolveAnchorY(cfg: InputConfig, state: InputState): number { + if (cfg.anchorMode === 'center') return 0.5; + if (cfg.anchorMode === 'sticky') return cfg.anchorY; + return cfg.anchorY; +} + +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + +/** + * Process raw 2D input through the pipeline. + * + * @param raw raw input [x, y] in [0,1] + * @param cfg pipeline configuration + * @param state prior state (use {@link defaultInputState} on first call) + * @param dt seconds since last call (default 1/60) + */ +export function processInput( + raw: readonly [number, number], + cfg: InputConfig, + state: InputState, + dt: number = REFERENCE_DT, +): ProcessResult { + const safeDt = Math.max(0, dt); + const baseZoomX = cfg.zoomX ?? cfg.zoom; + const baseZoomY = cfg.zoomY ?? cfg.zoom; + + const frozenX = baseZoomX <= FREEZE_THRESHOLD; + const frozenY = baseZoomY <= FREEZE_THRESHOLD; + const fullyFrozen = frozenX && frozenY; + + if (fullyFrozen) { + return { + x: state.smoothedX, + y: state.smoothedY, + frozen: true, + state: { ...state, frozen: true }, + }; + } + + let [rawX, rawY] = raw; + + // 0. Invert + let x = cfg.invertX ? 1 - rawX : rawX; + let y = cfg.invertY ? 1 - rawY : rawY; + + // 1. Deadzone + x = applyDeadzone(x, cfg.deadzone); + y = applyDeadzone(y, cfg.deadzone); + + // 2. Circular clamp to unit disk centered at (0.5, 0.5) + { + const cx = x - 0.5; + const cy = y - 0.5; + const dist = Math.sqrt(cx * cx + cy * cy); + if (dist > 0.5 && dist > 1e-12) { + const scale = 0.5 / dist; + x = 0.5 + cx * scale; + y = 0.5 + cy * scale; + } + } + + // 3. Zoom around anchor (with momentum modulation) + const anchorX = resolveAnchorX(cfg, state); + const anchorY = resolveAnchorY(cfg, state); + const effZoomX = frozenX + ? FREEZE_THRESHOLD + : clamp(baseZoomX * state.momentumZoomMultiplier, ZOOM_MIN, ZOOM_MAX); + const effZoomY = frozenY + ? FREEZE_THRESHOLD + : clamp(baseZoomY * state.momentumZoomMultiplier, ZOOM_MIN, ZOOM_MAX); + x = frozenX ? state.smoothedX : applyZoom(x, anchorX, effZoomX); + y = frozenY ? state.smoothedY : applyZoom(y, anchorY, effZoomY); + + // 4. Centered power curve + const curveX = cfg.inputCurveX ?? cfg.inputCurve; + const curveY = cfg.inputCurveY ?? cfg.inputCurve; + if (!frozenX) x = curveCenteredPower(x, curveX); + if (!frozenY) y = curveCenteredPower(y, curveY); + + // 5. EMA smoothing + const smoothedX = frozenX ? state.smoothedX : emaSmooth(state.smoothedX, x, cfg.smoothing, safeDt); + const smoothedY = frozenY ? state.smoothedY : emaSmooth(state.smoothedY, y, cfg.smoothing, safeDt); + + // 6. Update momentum-zoom for next frame + const { multiplier, history } = updateMomentumZoomMultiplier(cfg, state, rawX, rawY, safeDt); + + return { + x: smoothedX, + y: smoothedY, + frozen: false, + state: { + smoothedX, + smoothedY, + velocityHistory: history, + momentumZoomMultiplier: multiplier, + frozen: false, + }, + }; +} diff --git a/manifold/src/engine/output-pipeline.ts b/manifold/src/engine/output-pipeline.ts new file mode 100644 index 0000000..bc0e2b6 --- /dev/null +++ b/manifold/src/engine/output-pipeline.ts @@ -0,0 +1,153 @@ +/** + * Output pipeline — pure TS port of legacy `js/ui/output-pipeline.js`. + * + * Stages (in order) for each output: + * 1. Global power curve (raw^exponent, exponent in [0.2, 5.0]) + * 2. Per-output EMA smoothing (frame-rate-independent) + * 3. Slew-rate limiting (max change per second per output) + * 4. Freeze gate (global) and per-output freeze mask + * + * `processOutput` is a pure function: takes the raw output vector, the prior + * processed vector (or null on first call), and config; returns a new + * Float32Array. Consumer (output-store) holds prev between frames. + * + * NOTE: Reuses an internal scratch buffer ONLY when a prev buffer of the + * exact same length is supplied AND `cfg.reuseBuffer === true`. Otherwise it + * allocates a fresh Float32Array (safer for cross-component sharing). + */ + +import { clamp, clamp01 } from './curves'; + +export const GLOBAL_CURVE_MIN = 0.2; +export const GLOBAL_CURVE_MAX = 5.0; +export const SMOOTHING_MAX = 0.95; +export const SLEW_RATE_MIN = 0.005; + +const REFERENCE_DT = 1 / 60; + +export interface OutputConfig { + /** Power curve exponent applied to ALL outputs. 1 = linear. */ + globalCurve: number; + /** EMA smoothing factor [0, 0.95]. */ + smoothing: number; + /** Max change per second per output. Infinity = unlimited. */ + slewRate: number; + /** Global freeze gate. */ + freezeOutput: boolean; + /** Per-output freeze mask (1 = frozen). Length must match output vector. */ + freezeMask: Uint8Array | null; + /** If true and prev buffer matches length, reuse it for processed output. */ + reuseBuffer: boolean; +} + +export interface OutputState { + /** Last processed output (kept here for slew/freeze logic). */ + prev: Float32Array | null; + /** Last EMA-smoothed values per output. */ + smoothed: Float32Array | null; +} + +export function defaultOutputConfig(): OutputConfig { + return { + globalCurve: 1.0, + smoothing: 0, + slewRate: Infinity, + freezeOutput: false, + freezeMask: null, + reuseBuffer: false, + }; +} + +export function defaultOutputState(): OutputState { + return { prev: null, smoothed: null }; +} + +function emaSmooth(prev: number, raw: number, smoothing: number, dt: number): number { + if (smoothing <= 0) return raw; + const effectiveDt = dt > 0 ? dt : REFERENCE_DT; + const alpha = 1 - smoothing; + const alphaEff = 1 - Math.pow(1 - alpha, effectiveDt / REFERENCE_DT); + return prev + alphaEff * (raw - prev); +} + +/** + * Process raw outputs through global curve → smoothing → slew → freeze gate. + * + * @param raw raw output vector (Float32Array of size N) + * @param cfg pipeline config + * @param state prior state (use {@link defaultOutputState} first call) + * @param dtMs time since last call in milliseconds + * @returns { processed, state } with the new outputs and updated state + */ +export function processOutput( + raw: Float32Array, + cfg: OutputConfig, + state: OutputState, + dtMs: number, +): { processed: Float32Array; state: OutputState } { + const n = raw.length; + const dt = Math.max(0, dtMs / 1000); + + let prev = state.prev; + let smoothed = state.smoothed; + if (!prev || prev.length !== n) { + prev = new Float32Array(n); + // Seed from raw on first call + for (let i = 0; i < n; i++) prev[i] = clamp01(raw[i] ?? 0); + } + if (!smoothed || smoothed.length !== n) { + smoothed = new Float32Array(n); + for (let i = 0; i < n; i++) smoothed[i] = clamp01(raw[i] ?? 0); + } + + let processed: Float32Array; + if (cfg.reuseBuffer && prev.length === n) { + processed = prev; + } else { + processed = new Float32Array(n); + } + + // Stage 1: global curve (mutates a working scratch via direct compute) + const exp = cfg.globalCurve; + + if (cfg.freezeOutput) { + // Output frozen: hold prior values. + if (processed !== prev) { + processed.set(prev); + } + return { processed, state: { prev: processed, smoothed } }; + } + + for (let i = 0; i < n; i++) { + const r = clamp01(raw[i] ?? 0); + const curved = exp === 1.0 ? r : Math.pow(r, exp); + + // Per-output freeze + if (cfg.freezeMask && cfg.freezeMask[i]) { + processed[i] = prev[i] ?? curved; + continue; + } + + // Stage 2: EMA smoothing + let value = emaSmooth(smoothed[i] ?? curved, curved, cfg.smoothing, dt); + smoothed[i] = value; + + // Stage 3: slew-rate limit + if (isFinite(cfg.slewRate) && cfg.slewRate > 0) { + const maxDelta = cfg.slewRate * dt; + const delta = value - (prev[i] ?? value); + if (Math.abs(delta) > maxDelta) { + value = (prev[i] ?? value) + Math.sign(delta) * maxDelta; + } + } + + processed[i] = clamp01(value); + } + + // Update prev for next call + if (processed !== prev) { + prev = new Float32Array(processed); // copy so caller can hold processed buffer freely + } + + return { processed, state: { prev, smoothed } }; +} diff --git a/manifold/src/engine/sink.ts b/manifold/src/engine/sink.ts new file mode 100644 index 0000000..b3acf85 --- /dev/null +++ b/manifold/src/engine/sink.ts @@ -0,0 +1,43 @@ +/** + * EngineSink — the framework-neutral side-effect boundary. + * + * The lifted `WasmIML` (and any other engine component) used to call directly + * into SolidJS stores (`mlStore.__setState(produce(...))`, `coreBus.emit(...)`). + * That coupled the engine to Solid. In Manifold the engine is framework-neutral: + * every mutation that should be visible to a consumer is routed through an + * injected `EngineSink` instead. + * + * The reactive spine (spine.ts) provides the concrete sink that bumps a version + * counter and notifies `useSyncExternalStore` subscribers; tests/headless use + * can pass `noopSink`. + */ + +/** Partial state patch — plain object, NOT a Solid `produce` mutator. */ +export interface EngineStatePatch { + inputSize?: number; + outputSize?: number; + exampleCount?: number; + lastLoss?: number | null; + lossHistory?: ReadonlyArray; + training?: boolean; + ready?: boolean; +} + +export interface EngineSink { + /** Merge a shallow patch into engine-visible ML state. */ + setState(patch: EngineStatePatch): void; + /** Publish a fresh output vector (already copied; caller may keep it). */ + setOutputs(out: Float32Array): void; + /** Publish a fresh flat weight array. */ + setWeights(w: Float32Array): void; + /** Emit a named engine event (`ml.*`, `mode.*`, …) with an optional payload. */ + emit(event: string, payload?: unknown): void; +} + +/** No-op default sink. Lets `WasmIML` run fully headless (tests, smoke use). */ +export const noopSink: EngineSink = { + setState() {}, + setOutputs() {}, + setWeights() {}, + emit() {}, +}; diff --git a/manifold/src/engine/spine.ts b/manifold/src/engine/spine.ts new file mode 100644 index 0000000..70186a0 --- /dev/null +++ b/manifold/src/engine/spine.ts @@ -0,0 +1,251 @@ +/** + * Reactive spine — the external store that lives BELOW React. + * + * Per findings-design-and-manifold.md §4: the SolidJS spine was + * inputRaw → memo(processed) → memo(ml) → memo(routed) → effect(backend.send) + * which recomputes on Solid's reactive graph. In React we must NOT couple the + * per-frame audio inference to the render scheduler. So the spine is a tiny + * hand-rolled observable: the `setInput` ACTION derives processed → ml → routed + * EAGERLY + SYNCHRONOUSLY (input pipeline → WasmIML.processInto → output + * pipeline) and fires the single `backend.send` at the action TAIL, off React's + * render cycle. + * + * React subscribes via `useSyncExternalStore(subscribe, version)` — the version + * counter, NOT the array — and reads the live `Float32Array` imperatively (so + * canvases never re-render per frame). + * + * Buffers are reused (no per-frame allocation): `routedBuf` is a single + * Float32Array threaded through the output pipeline and handed to the backend. + */ + +import { + defaultInputConfig, + defaultInputState, + processInput, + type InputConfig, + type InputState, +} from './input-pipeline'; +import { + defaultOutputConfig, + defaultOutputState, + processOutput, + type OutputConfig, + type OutputState, +} from './output-pipeline'; +import type { EngineSink, EngineStatePatch } from './sink'; +import type { WasmIML } from './wasm-iml'; + +/** + * Float32Array that may be backed by either a plain ArrayBuffer or a + * SharedArrayBuffer (TS 5.7+ made `Float32Array` generic over its buffer). + * The output pipeline returns the loosely-typed form; we keep our reused + * buffers loosely typed too so assignment doesn't fight the lib types. + */ +type F32 = Float32Array; + +/** The single side-effect the spine fires at the tail of each `setInput`. */ +export type BackendSend = (routed: Float32Array) => void; + +export interface SpineState { + /** Monotonically increasing; bumped on every state change. */ + version: number; + ready: boolean; + training: boolean; + exampleCount: number; + lastLoss: number | null; + lossHistory: ReadonlyArray; + inputSize: number; + outputSize: number; +} + +/** + * The spine doubles as the `EngineSink` consumed by `WasmIML`. WasmIML calls + * `setState/setOutputs/setWeights/emit`; the spine merges into its state, + * stashes the live output/weight buffers, and bumps the version counter so + * `useSyncExternalStore` consumers re-read. + */ +export class Spine implements EngineSink { + private state_: SpineState = { + version: 0, + ready: false, + training: false, + exampleCount: 0, + lastLoss: null, + lossHistory: [], + inputSize: 2, + outputSize: 126, + }; + + private listeners = new Set<() => void>(); + private eventListeners = new Map void>>(); + + // Engine handles wired in via `attach`. + private iml: WasmIML | null = null; + private backendSend: BackendSend | null = null; + + // Pipeline config + per-frame state. + inputConfig: InputConfig = defaultInputConfig(); + outputConfig: OutputConfig = { ...defaultOutputConfig(), reuseBuffer: true }; + private inputState: InputState = defaultInputState(); + private outputState: OutputState = defaultOutputState(); + + // Reused per-frame buffers — NO per-frame allocation in the hot path. + private rawInput: [number, number] = [0.5, 0.5]; + // Last raw input, so `EngineApi.process()` can re-tick after a weight change. + lastRawX = 0.5; + lastRawY = 0.5; + private mlBuf: F32 = new Float32Array(126); + private routedBuf: F32 | null = null; + + // Last live output (post-ML, pre-routing) and weights, read imperatively. + private liveOutputs: F32 = new Float32Array(126); + private liveWeights: F32 = new Float32Array(0); + + private lastTickMs = 0; + + // ---- EngineSink ---------------------------------------------------- + + setState(patch: EngineStatePatch): void { + let changed = false; + const s = this.state_; + if (patch.inputSize !== undefined && patch.inputSize !== s.inputSize) { s.inputSize = patch.inputSize; changed = true; } + if (patch.outputSize !== undefined && patch.outputSize !== s.outputSize) { + s.outputSize = patch.outputSize; + // Resize hot buffers to the resolved output size. + this.mlBuf = new Float32Array(patch.outputSize); + this.routedBuf = new Float32Array(patch.outputSize); + this.liveOutputs = new Float32Array(patch.outputSize); + changed = true; + } + if (patch.exampleCount !== undefined && patch.exampleCount !== s.exampleCount) { s.exampleCount = patch.exampleCount; changed = true; } + if (patch.lastLoss !== undefined && patch.lastLoss !== s.lastLoss) { s.lastLoss = patch.lastLoss; changed = true; } + if (patch.lossHistory !== undefined) { s.lossHistory = patch.lossHistory; changed = true; } + if (patch.training !== undefined && patch.training !== s.training) { s.training = patch.training; changed = true; } + if (patch.ready !== undefined && patch.ready !== s.ready) { s.ready = patch.ready; changed = true; } + if (changed) this.bump_(); + } + + setOutputs(out: Float32Array): void { + if (this.liveOutputs.length === out.length) this.liveOutputs.set(out); + else this.liveOutputs = new Float32Array(out); + this.bump_(); + } + + setWeights(w: Float32Array): void { + this.liveWeights = w; + this.bump_(); + } + + emit(event: string, payload?: unknown): void { + const set = this.eventListeners.get(event); + if (set) for (const fn of set) fn(payload); + // Prefix listeners ("ml." matches "ml.trained"). + for (const [prefix, fns] of this.eventListeners) { + if (prefix.endsWith('.') && event.startsWith(prefix)) { + for (const fn of fns) fn(payload); + } + } + } + + // ---- Wiring -------------------------------------------------------- + + attach(iml: WasmIML, backendSend: BackendSend | null): void { + this.iml = iml; + this.backendSend = backendSend; + if (this.routedBuf === null || this.routedBuf.length !== iml.architecture.outputSize) { + this.routedBuf = new Float32Array(iml.architecture.outputSize); + } + } + + setBackendSend(backendSend: BackendSend | null): void { + this.backendSend = backendSend; + } + + // ---- The hot action ------------------------------------------------ + + /** + * Drive a raw [0,1] XY input through processed → ml → routed eagerly and + * synchronously, then fire the single backend.send at the tail. Off render. + * Returns the routed buffer (live, reused — do not retain across calls). + */ + setInput(x: number, y: number): Float32Array | null { + const iml = this.iml; + if (!iml) return null; + + const now = (typeof performance !== 'undefined' ? performance.now() : Date.now()); + const dt = this.lastTickMs > 0 ? (now - this.lastTickMs) / 1000 : 1 / 60; + this.lastTickMs = now; + + // 1. processed (pure input pipeline) + this.rawInput[0] = x; + this.rawInput[1] = y; + this.lastRawX = x; + this.lastRawY = y; + const proc = processInput(this.rawInput, this.inputConfig, this.inputState, dt); + this.inputState = proc.state; + + // 2. ml (inference into the reused buffer; no alloc) + iml.setInput(0, proc.x); + iml.setInput(1, proc.y); + iml.processInto(this.mlBuf); + // Mirror to liveOutputs for imperative reads + bump. + this.liveOutputs.set(this.mlBuf.subarray(0, this.liveOutputs.length)); + + // 3. routed (output pipeline → reused routedBuf) + const routedRes = processOutput(this.mlBuf, this.outputConfig, this.outputState, dt * 1000); + this.outputState = routedRes.state; + const routed = routedRes.processed; + if (this.routedBuf && this.routedBuf.length === routed.length) { + this.routedBuf.set(routed); + } else { + this.routedBuf = routed; + } + + // 4. single backend.send at the tail (off React render) + if (this.backendSend && this.routedBuf) this.backendSend(this.routedBuf); + + this.bump_(); + return this.routedBuf; + } + + // ---- Imperative reads (canvas consumers bypass React) -------------- + + /** Live post-ML output vector. Reused — read, don't retain. */ + outputs(): Float32Array { + return this.liveOutputs; + } + + /** Live routed (post output-pipeline) vector. Reused — read, don't retain. */ + routedOutput(): Float32Array | null { + return this.routedBuf; + } + + weights(): Float32Array { + return this.liveWeights; + } + + // ---- useSyncExternalStore plumbing --------------------------------- + + subscribe = (cb: () => void): (() => void) => { + this.listeners.add(cb); + return () => { this.listeners.delete(cb); }; + }; + + version = (): number => this.state_.version; + + getState(): Readonly { + return this.state_; + } + + on(event: string, fn: (payload?: unknown) => void): () => void { + let set = this.eventListeners.get(event); + if (!set) { set = new Set(); this.eventListeners.set(event, set); } + set.add(fn); + return () => { set!.delete(fn); }; + } + + private bump_(): void { + this.state_ = { ...this.state_, version: this.state_.version + 1 }; + for (const fn of this.listeners) fn(); + } +} diff --git a/manifold/src/engine/types.ts b/manifold/src/engine/types.ts new file mode 100644 index 0000000..21c5a37 --- /dev/null +++ b/manifold/src/engine/types.ts @@ -0,0 +1,191 @@ +/** + * TypeScript types matching the C API surface in `nisps/wasm/bindings.cpp`. + * + * Lifted from `playground/src/ml/types.ts` and EXTENDED with the + * `nisps_ml_feedback_*` exports (already present in the WASM build per + * `scripts/build-wasm.sh` EXPORTED_FUNCTIONS, but not previously bound in the + * playground's `NispsModule` interface). These power the `feedback.*` surface + * of `EngineApi`. + */ + +/** + * Shape of the loaded WASM module — the subset we use. Emscripten generates + * more; we type only what we need. `_*` methods are the raw exported C + * functions (numbers in, numbers out — pointers + primitives). + */ +export interface NispsModule { + // Memory views (re-bound after grow). + HEAP8: Int8Array; + HEAP16: Int16Array; + HEAP32: Int32Array; + HEAPU8: Uint8Array; + HEAPU16: Uint16Array; + HEAPU32: Uint32Array; + HEAPF32: Float32Array; + HEAPF64: Float64Array; + + _malloc(bytes: number): number; + _free(ptr: number): void; + + // ML lifecycle. Seed is uint32_t (not 64-bit) — see bindings.cpp file comment. + _nisps_ml_create(input_size: number, output_size: number, hidden_ptr: number, n_hidden: number, seed: number): number; + _nisps_ml_destroy(ml: number): void; + _nisps_ml_reset(ml: number): void; + + // ML inference. + _nisps_ml_set_input(ml: number, idx: number, v: number): void; + _nisps_ml_process(ml: number): void; + _nisps_ml_outputs(ml: number): number; // returns float* into HEAPF32 + _nisps_ml_infer_batch(ml: number, points_ptr: number, n_points: number, out_ptr: number): void; + + // ML training. + _nisps_ml_add_example(ml: number, features_ptr: number, labels_ptr: number): void; + _nisps_ml_train(ml: number, lr: number, max_iter: number, min_err: number, sample_weights_ptr: number): number; + _nisps_ml_eval_loss(ml: number): number; + + // ML examples. + _nisps_ml_clear_examples(ml: number): void; + _nisps_ml_example_count(ml: number): number; + + // ML weights. + _nisps_ml_weight_count(ml: number): number; + _nisps_ml_get_weights(ml: number, out_ptr: number): void; + _nisps_ml_set_weights(ml: number, in_ptr: number): void; + _nisps_ml_draw_weights(ml: number, spread: number): void; + _nisps_ml_move_weights(ml: number, speed: number, spread: number, mask_ptr: number): void; + _nisps_ml_get_layer_stats(ml: number, out_ptr: number): void; + _nisps_ml_describe(out_ptr: number): void; + + // ML feedback — the "Down Action" state machine (Avoid / RandomiseOutputs / + // RandomiseMlp). Mode ints: 0=Avoid 1=RandomiseOutputs 2=RandomiseMlp. + // Action return ints come from FeedbackController::on_*; see feedback.hpp. + _nisps_ml_feedback_set_mode(ml: number, mode: number): void; + _nisps_ml_feedback_get_mode(ml: number): number; + _nisps_ml_feedback_exploring(ml: number): number; // 1 = exploring + _nisps_ml_feedback_learning_paused(ml: number): number; // 1 = paused + _nisps_ml_feedback_set_focus(ml: number, mask_ptr: number, n: number): void; + _nisps_ml_feedback_down( + ml: number, + current_out_ptr: number, + speed: number, + spread: number, + pin_mask_ptr: number, + ): number; + _nisps_ml_feedback_up(ml: number): number; + _nisps_ml_feedback_drag(ml: number): number; + // Returns 1 if `out` holds a static-bypass vector (skip process()); else 0. + _nisps_ml_feedback_static_output(ml: number, out_ptr: number): number; + + // Engines. + _nisps_engine_create(id_ptr: number, sample_rate: number): number; + _nisps_engine_destroy(engine: number): void; + _nisps_engine_set_params(engine: number, params_ptr: number, n_params: number): void; + _nisps_engine_process_block( + engine: number, + in_l_ptr: number, in_r_ptr: number, + out_l_ptr: number, out_r_ptr: number, + n_samples: number, + ): void; +} + +/** Factory function exposed by the Emscripten glue. */ +export type NispsModuleFactory = (opts?: { + locateFile?: (path: string, prefix: string) => string; + wasmBinary?: ArrayBuffer | Uint8Array; + print?: (msg: string) => void; + printErr?: (msg: string) => void; +}) => Promise; + +/** Architecture descriptor returned from `nisps_ml_describe`. */ +export interface MLArchitecture { + inputSize: number; + hidden: [number, number, number]; + outputSize: number; + numLayers: number; +} + +/** Per-layer weight health record (one per layer). */ +export interface LayerStats { + meanAbs: number; + maxAbs: number; + deadFrac: number; + saturatingFrac: number; +} + +/** The `engine_id` strings the C++ side recognises. Anything else → "thru". */ +export type EngineId = + | 'thru' + | 'paf_synth' + | 'channel_strip' + | 'xiasri' + | 'verb_fx' + | 'memlcelium' + | 'breakor' + | 'elysiamorf' + | 'analysis'; + +/** Feedback "Down Action" mode. Mirrors `nisps::ml::FeedbackMode`. */ +export type FeedbackMode = 'avoid' | 'randomise_outputs' | 'randomise_mlp'; + +export const FEEDBACK_MODE_TO_INT: Record = { + avoid: 0, + randomise_outputs: 1, + randomise_mlp: 2, +}; + +export const FEEDBACK_MODE_FROM_INT: ReadonlyArray = [ + 'avoid', + 'randomise_outputs', + 'randomise_mlp', +]; + +/** Message protocol between main thread and `wasm-worker.ts`. */ +export type WorkerRequest = + | { + kind: 'init'; + seed: number; + // Absolute deploy base (e.g. "https://host/next/") computed on the main + // thread from document.baseURI — the worker has no document to resolve + // `./nisps.js` against, and resolving against its own bundle URL points at + // /assets/, not the public root. + assetBase: string; + } + | { + kind: 'train'; + requestId: number; + // Flat features: nExamples * inputSize floats. + features: Float32Array; + // Flat labels: nExamples * outputSize floats. + labels: Float32Array; + // Optional per-example weights, sums to 1. Empty = uniform. + sampleWeights: Float32Array; + // Current weights to seed worker MLP. + weights: Float32Array; + lr: number; + maxIter: number; + minErr: number; + inputSize: number; + outputSize: number; + } + | { + kind: 'dispose'; + }; + +export type WorkerResponse = + | { + kind: 'ready'; + } + | { + kind: 'result'; + requestId: number; + loss: number; + weights: Float32Array; + // Loss curve (per-iteration). Currently always single-element — the C++ + // MLP exposes loss_history but the WASM bridge does not yet plumb it. + lossHistory: Float32Array; + } + | { + kind: 'error'; + requestId: number; + message: string; + }; diff --git a/manifold/src/engine/useEngine.ts b/manifold/src/engine/useEngine.ts new file mode 100644 index 0000000..c5113de --- /dev/null +++ b/manifold/src/engine/useEngine.ts @@ -0,0 +1,43 @@ +/** + * useEngine / useEngineVersion — React hooks over the EngineApi. + * + * `useEngine()` returns the EngineApi from context (or null before it loads). + * + * `useEngineVersion()` subscribes to engine state changes via + * `useSyncExternalStore(engine.subscribe, engine.version)`. It returns the + * VERSION COUNTER (a number), NOT the output array — so a component re-renders + * when engine state changes but reads the live `Float32Array` imperatively + * (`engine.getOutputs()` / `engine.routedOutput()`) inside a rAF loop or on + * render. This keeps per-frame audio inference off React's render cycle. + */ + +import { useContext, useSyncExternalStore } from 'react'; +import type { EngineApi } from './engine-api'; +import { EngineContext } from './EngineProvider'; + +/** The EngineApi from context, or null until the WASM has loaded. */ +export function useEngine(): EngineApi | null { + return useContext(EngineContext); +} + +/** Like {@link useEngine} but throws if used outside a ready provider. */ +export function useEngineOrThrow(): EngineApi { + const engine = useContext(EngineContext); + if (!engine) { + throw new Error('useEngineOrThrow: no EngineApi in context (still loading or no provider)'); + } + return engine; +} + +/** + * Subscribe to the engine's monotonic version counter. Re-renders the caller + * on any engine state change; the returned number is the counter (read the + * live arrays imperatively from the engine). Returns 0 when there's no engine. + */ +export function useEngineVersion(engine: EngineApi | null): number { + return useSyncExternalStore( + (cb) => (engine ? engine.subscribe(cb) : () => {}), + () => (engine ? engine.version() : 0), + () => 0, + ); +} diff --git a/manifold/src/engine/wasm-iml.ts b/manifold/src/engine/wasm-iml.ts new file mode 100644 index 0000000..2941655 --- /dev/null +++ b/manifold/src/engine/wasm-iml.ts @@ -0,0 +1,670 @@ +/** + * WasmIML — main-thread ML interface backed by `nisps.wasm`. + * + * Lifted from `playground/src/ml/wasm-iml.ts`. The ONLY changes from the + * parity-tested original are framework-decoupling and base-awareness: + * + * - The Solid coupling is gone. Where the playground called + * `mlStore.__setState(produce(...))` / `mlStore.__setOutputs(...)` / + * `mlStore.__setWeights(...)` / `coreBus.emit(...)`, this class calls the + * injected {@link EngineSink} (`sink.setState({...})` with a PLAIN patch + * object — no `produce` mutator, `sink.setOutputs/setWeights/emit`). + * - Glue + WASM URLs resolve via `import.meta.env.BASE_URL` (not `/nisps.*`). + * - The `nisps_ml_feedback_*` C ABI (already exported by the WASM build) is + * now bound and surfaced via the `feedback*` methods. The playground never + * wired these. + * + * Owns one `nisps.wasm` instance, one MLP handle, a JS-side `Dataset`, + * pre-allocated heap buffers, and a lazy `WasmTrainer` worker. + */ + +import { Dataset } from './dataset'; +import { noopSink, type EngineSink } from './sink'; +import { + FEEDBACK_MODE_FROM_INT, + FEEDBACK_MODE_TO_INT, + type FeedbackMode, + type LayerStats, + type MLArchitecture, + type NispsModule, + type NispsModuleFactory, +} from './types'; +import { createTrainer, type WasmTrainer } from './wasm-worker'; + +/** Default architecture matches `nisps/wasm/bindings.cpp` instantiation. */ +const DEFAULT_INPUT_SIZE = 2; +const DEFAULT_OUTPUT_SIZE = 126; + +/** Base-aware absolute URL for an asset served from `public/`. Resolves against + * `document.baseURI` (the page URL) so a `base: './'` build works under any + * mount path — `/`, `/next/`, etc. Resolving against `location.origin` would + * drop the sub-path and fetch from the site root (404 → text/html). */ +function assetUrl(file: string): string { + const base = import.meta.env.BASE_URL ?? '/'; + return new URL(base + file, document.baseURI).toString(); +} + +let cachedFactory: NispsModuleFactory | null = null; + +async function getFactory(): Promise { + if (cachedFactory) return cachedFactory; + // `nisps.js` is Emscripten MODULARIZE glue WITHOUT ES6 exports — it assigns a + // global `createNispsModule` (CommonJS/AMD fallbacks only). `import()` of it + // yields an empty module namespace, so fetch the source and indirect-eval it + // in global scope, which installs `globalThis.createNispsModule`. + const g = globalThis as unknown as { createNispsModule?: NispsModuleFactory }; + if (!g.createNispsModule) { + const src = await (await fetch(assetUrl('nisps.js'))).text(); + (0, eval)(src); + } + const factory = g.createNispsModule; + if (!factory) throw new Error('[wasm-iml] nisps.js did not define createNispsModule'); + cachedFactory = factory; + return factory; +} + +/** Aligned float-array allocation helper. Returns ptr + a view. */ +class HeapBuffer { + readonly ptr: number; + readonly view: Float32Array; + constructor(private mod: NispsModule, public readonly count: number) { + this.ptr = mod._malloc(count * 4); + if (!this.ptr) throw new Error(`malloc(${count * 4}) failed`); + this.view = new Float32Array(mod.HEAPF32.buffer, this.ptr, count); + } + rebind(): void { + Object.defineProperty(this, 'view', { + value: new Float32Array(this.mod.HEAPF32.buffer, this.ptr, this.count), + writable: false, + }); + } + free(): void { + this.mod._free(this.ptr); + } +} + +class HeapU8 { + readonly ptr: number; + readonly view: Uint8Array; + constructor(private mod: NispsModule, public readonly count: number) { + this.ptr = mod._malloc(count); + if (!this.ptr) throw new Error(`malloc(${count}) failed`); + this.view = new Uint8Array(mod.HEAPU8.buffer, this.ptr, count); + } + rebind(): void { + Object.defineProperty(this, 'view', { + value: new Uint8Array(this.mod.HEAPU8.buffer, this.ptr, this.count), + writable: false, + }); + } + free(): void { + this.mod._free(this.ptr); + } +} + +export interface WasmIMLOptions { + inputSize?: number; + outputSize?: number; + hiddenLayers?: ReadonlyArray; + seed?: number; + /** localStorage key the loaded weights/dataset will be persisted under. */ + storageKey?: string; + maxExamples?: number; + /** Injected side-effect boundary. Defaults to a no-op sink (headless use). */ + sink?: EngineSink; +} + +export class WasmIML { + private module!: NispsModule; + private mlHandle = 0; + private weightCount_ = 0; + + private arch_: MLArchitecture = { + inputSize: DEFAULT_INPUT_SIZE, + hidden: [10, 14, 18], + outputSize: DEFAULT_OUTPUT_SIZE, + numLayers: 4, + }; + + private featuresBuf!: HeapBuffer; + private labelsBuf!: HeapBuffer; + private weightsBuf!: HeapBuffer; + private statsBuf!: HeapBuffer; + private batchInBuf!: HeapBuffer; + private batchOutBuf!: HeapBuffer; + private pinMaskBuf!: HeapU8; + private feedbackBuf!: HeapBuffer; // kDefaultOutputs scratch for feedback static/down + private describePtr = 0; + + readonly dataset: Dataset; + private readonly sink: EngineSink; + private lastLoss_: number | null = null; + private trainer: WasmTrainer | null = null; + private storageKey: string; + private saveTimer: number | null = null; + private destroyed = false; + + static MAX_BATCH = 4096; + + private constructor(opts: WasmIMLOptions) { + this.dataset = new Dataset(opts.maxExamples ?? 100); + this.storageKey = opts.storageKey ?? 'nisps:wasm-iml'; + this.sink = opts.sink ?? noopSink; + } + + static async create(opts: WasmIMLOptions = {}): Promise { + const inst = new WasmIML(opts); + await inst.init_(opts); + return inst; + } + + private async init_(opts: WasmIMLOptions): Promise { + const factory = await getFactory(); + this.module = await factory({ + locateFile: (path: string) => (path.endsWith('.wasm') ? assetUrl('nisps.wasm') : path), + }); + + this.describePtr = this.module._malloc(6 * 4); + this.module._nisps_ml_describe(this.describePtr); + const dims = new Int32Array(this.module.HEAP32.buffer, this.describePtr, 6); + this.arch_ = { + inputSize: dims[0], + hidden: [dims[1], dims[2], dims[3]], + outputSize: dims[4], + numLayers: dims[5], + }; + + const wantedIn = opts.inputSize ?? this.arch_.inputSize; + const wantedOut = opts.outputSize ?? this.arch_.outputSize; + if (wantedIn !== this.arch_.inputSize || wantedOut !== this.arch_.outputSize) { + console.warn( + `[wasm-iml] requested ${wantedIn}->${wantedOut} but WASM build is fixed at ` + + `${this.arch_.inputSize}->${this.arch_.outputSize}; extras are ignored.`, + ); + } + + const seed = (opts.seed ?? (Date.now() >>> 0)) >>> 0; + this.mlHandle = this.module._nisps_ml_create( + this.arch_.inputSize, + this.arch_.outputSize, + 0, + 0, + seed, + ); + if (!this.mlHandle) throw new Error('[wasm-iml] nisps_ml_create returned null'); + + this.weightCount_ = this.module._nisps_ml_weight_count(this.mlHandle); + + this.featuresBuf = new HeapBuffer(this.module, this.arch_.inputSize); + this.labelsBuf = new HeapBuffer(this.module, this.arch_.outputSize); + this.weightsBuf = new HeapBuffer(this.module, this.weightCount_); + this.statsBuf = new HeapBuffer(this.module, this.arch_.numLayers * 4); + this.batchInBuf = new HeapBuffer(this.module, WasmIML.MAX_BATCH * this.arch_.inputSize); + this.batchOutBuf = new HeapBuffer(this.module, WasmIML.MAX_BATCH * this.arch_.outputSize); + this.pinMaskBuf = new HeapU8(this.module, this.arch_.outputSize); + this.feedbackBuf = new HeapBuffer(this.module, this.arch_.outputSize); + + this.sink.setState({ + inputSize: this.arch_.inputSize, + outputSize: this.arch_.outputSize, + exampleCount: 0, + lastLoss: null, + lossHistory: [], + training: false, + ready: true, + }); + this.sink.setOutputs(new Float32Array(this.arch_.outputSize)); + this.publishWeights_(); + + this.tryLoadFromStorage_(); + } + + // ------------------------------------------------------------------- + // Lifecycle + // ------------------------------------------------------------------- + + dispose(): void { + if (this.destroyed) return; + this.destroyed = true; + if (this.saveTimer !== null) { + clearTimeout(this.saveTimer); + this.saveTimer = null; + } + if (this.trainer) { + this.trainer.dispose(); + this.trainer = null; + } + if (this.module && this.mlHandle) { + this.module._nisps_ml_destroy(this.mlHandle); + this.mlHandle = 0; + } + if (this.featuresBuf) this.featuresBuf.free(); + if (this.labelsBuf) this.labelsBuf.free(); + if (this.weightsBuf) this.weightsBuf.free(); + if (this.statsBuf) this.statsBuf.free(); + if (this.batchInBuf) this.batchInBuf.free(); + if (this.batchOutBuf) this.batchOutBuf.free(); + if (this.pinMaskBuf) this.pinMaskBuf.free(); + if (this.feedbackBuf) this.feedbackBuf.free(); + if (this.describePtr) this.module._free(this.describePtr); + this.sink.setState({ ready: false }); + } + + get architecture(): MLArchitecture { + return this.arch_; + } + get weightCount(): number { + return this.weightCount_; + } + get exampleCount(): number { + return this.dataset.size; + } + get lastLoss(): number | null { + return this.lastLoss_; + } + + // ------------------------------------------------------------------- + // Inference + // ------------------------------------------------------------------- + + setInput(idx: number, value: number): void { + this.module._nisps_ml_set_input(this.mlHandle, idx, value); + } + + process(): Float32Array { + this.module._nisps_ml_process(this.mlHandle); + const ptr = this.module._nisps_ml_outputs(this.mlHandle); + const view = new Float32Array(this.module.HEAPF32.buffer, ptr, this.arch_.outputSize); + const out = new Float32Array(view); // copy + this.sink.setOutputs(out); + return out; + } + + /** + * Like {@link process} but writes into a caller-provided buffer instead of + * allocating. Used by the reactive spine to avoid per-frame allocation. + * Returns the number of values written. Does NOT call `sink.setOutputs`. + */ + processInto(dst: Float32Array): number { + this.module._nisps_ml_process(this.mlHandle); + const ptr = this.module._nisps_ml_outputs(this.mlHandle); + const n = Math.min(dst.length, this.arch_.outputSize); + const view = new Float32Array(this.module.HEAPF32.buffer, ptr, this.arch_.outputSize); + dst.set(view.subarray(0, n)); + return n; + } + + /** Convenience: setInput(0,x); setInput(1,y); process(). */ + inferXY(x: number, y: number): Float32Array { + this.setInput(0, x); + this.setInput(1, y); + return this.process(); + } + + inferBatch(points: ReadonlyArray>): Float32Array { + const n = points.length; + const inSz = this.arch_.inputSize; + const outSz = this.arch_.outputSize; + const result = new Float32Array(n * outSz); + + let written = 0; + for (let offset = 0; offset < n; offset += WasmIML.MAX_BATCH) { + const chunk = Math.min(WasmIML.MAX_BATCH, n - offset); + for (let i = 0; i < chunk; ++i) { + const src = points[offset + i]; + const base = i * inSz; + for (let j = 0; j < inSz; ++j) this.batchInBuf.view[base + j] = src[j] ?? 0; + } + this.module._nisps_ml_infer_batch( + this.mlHandle, + this.batchInBuf.ptr, + chunk, + this.batchOutBuf.ptr, + ); + const slice = this.batchOutBuf.view.subarray(0, chunk * outSz); + result.set(slice, written); + written += chunk * outSz; + } + return result; + } + + // ------------------------------------------------------------------- + // Training + // ------------------------------------------------------------------- + + addExample(features: ReadonlyArray, labels: ReadonlyArray): boolean { + const ok = this.dataset.add(features, labels); + if (!ok) return false; + this.copyExampleToWasm_(features, labels); + this.sink.setState({ exampleCount: this.dataset.size }); + this.sink.emit('ml.example_added', { count: this.dataset.size }); + this.scheduleSave_(); + return true; + } + + private copyExampleToWasm_(features: ReadonlyArray, labels: ReadonlyArray): void { + const fv = this.featuresBuf.view; + const lv = this.labelsBuf.view; + const inSz = this.arch_.inputSize; + const outSz = this.arch_.outputSize; + for (let i = 0; i < inSz; ++i) fv[i] = features[i] ?? 0; + for (let i = 0; i < outSz; ++i) lv[i] = labels[i] ?? 0; + this.module._nisps_ml_add_example(this.mlHandle, this.featuresBuf.ptr, this.labelsBuf.ptr); + } + + train(lr = 1.0, maxIter = 1000, minErr = 0.001, sampleWeights?: Float32Array): number { + if (this.dataset.isEmpty()) { + this.lastLoss_ = 0; + this.sink.setState({ lastLoss: 0 }); + return 0; + } + + let weightsPtr = 0; + let weightsHandle: HeapBuffer | null = null; + if (sampleWeights && sampleWeights.length === this.dataset.size) { + weightsHandle = new HeapBuffer(this.module, sampleWeights.length); + weightsHandle.view.set(sampleWeights); + weightsPtr = weightsHandle.ptr; + } + + this.sink.setState({ training: true }); + let loss = 0; + try { + loss = this.module._nisps_ml_train(this.mlHandle, lr, maxIter, minErr, weightsPtr); + } finally { + if (weightsHandle) weightsHandle.free(); + this.sink.setState({ training: false }); + } + + this.lastLoss_ = loss; + // The C++ MLP stores per-iter history but it isn't exposed via the WASM + // bindings yet, so this is a single-element array. + this.sink.setState({ lastLoss: loss, lossHistory: [loss] }); + this.publishWeights_(); + this.sink.emit('ml.trained', { loss }); + this.scheduleSave_(); + return loss; + } + + async trainAsync(lr = 1.0, maxIter = 1000, minErr = 0.001, sampleWeights?: Float32Array): Promise { + if (this.dataset.isEmpty()) { + this.lastLoss_ = 0; + return 0; + } + if (!this.trainer) this.trainer = await createTrainer(); + + const weights = this.getWeights(); + const features = new Float32Array(this.dataset.featuresFlat()); + const labels = new Float32Array(this.dataset.labelsFlat()); + const sw = sampleWeights ? new Float32Array(sampleWeights) : new Float32Array(0); + + this.sink.setState({ training: true }); + try { + const result = await this.trainer.train({ + weights, + features, + labels, + sampleWeights: sw, + lr, + maxIter, + minErr, + inputSize: this.arch_.inputSize, + outputSize: this.arch_.outputSize, + }); + this.setWeights(result.weights); + this.lastLoss_ = result.loss; + this.sink.setState({ lastLoss: result.loss, lossHistory: Array.from(result.lossHistory) }); + this.sink.emit('ml.trained', { loss: result.loss }); + this.scheduleSave_(); + return result.loss; + } finally { + this.sink.setState({ training: false }); + } + } + + evalLoss(): number { + return this.module._nisps_ml_eval_loss(this.mlHandle); + } + + clearExamples(): void { + this.dataset.clear(); + this.module._nisps_ml_clear_examples(this.mlHandle); + this.sink.setState({ exampleCount: 0 }); + this.sink.emit('ml.examples_cleared', undefined); + this.scheduleSave_(); + } + + // ------------------------------------------------------------------- + // RL ops + // ------------------------------------------------------------------- + + randomiseWeights(spread = 0.6): void { + this.module._nisps_ml_draw_weights(this.mlHandle, spread); + this.publishWeights_(); + this.sink.emit('ml.delta_update', { reason: 'randomise' }); + this.scheduleSave_(); + } + + moveWeights(speed: number, spread: number, pinMask?: Uint8Array): void { + const maskPtr = this.writePinMask_(pinMask); + this.module._nisps_ml_move_weights(this.mlHandle, speed, spread, maskPtr); + this.publishWeights_(); + this.sink.emit('ml.delta_update', { reason: 'thumbs_down' }); + } + + private writePinMask_(pinMask?: Uint8Array): number { + if (!pinMask) return 0; + const sz = Math.min(pinMask.length, this.arch_.outputSize); + for (let i = 0; i < sz; ++i) this.pinMaskBuf.view[i] = pinMask[i]; + for (let i = sz; i < this.arch_.outputSize; ++i) this.pinMaskBuf.view[i] = 0; + return this.pinMaskBuf.ptr; + } + + // ------------------------------------------------------------------- + // Feedback "Down Action" state machine (nisps_ml_feedback_* C ABI) + // ------------------------------------------------------------------- + + /** Set the feedback dislike mode (Avoid / RandomiseOutputs / RandomiseMlp). */ + feedbackSetMode(mode: FeedbackMode): void { + this.module._nisps_ml_feedback_set_mode(this.mlHandle, FEEDBACK_MODE_TO_INT[mode]); + this.sink.emit('feedback.mode', { mode }); + } + + feedbackGetMode(): FeedbackMode { + const i = this.module._nisps_ml_feedback_get_mode(this.mlHandle); + return FEEDBACK_MODE_FROM_INT[i] ?? 'avoid'; + } + + /** True while the controller is in an exploratory (perturbed) state. */ + feedbackExploring(): boolean { + return this.module._nisps_ml_feedback_exploring(this.mlHandle) === 1; + } + + feedbackLearningPaused(): boolean { + return this.module._nisps_ml_feedback_learning_paused(this.mlHandle) === 1; + } + + /** Restrict feedback to a subset of outputs (solo / focus). null clears it. */ + feedbackSetFocus(mask: Uint8Array | null): void { + if (!mask || mask.length === 0) { + this.module._nisps_ml_feedback_set_focus(this.mlHandle, 0, 0); + return; + } + const n = Math.min(mask.length, this.arch_.outputSize); + for (let i = 0; i < n; ++i) this.pinMaskBuf.view[i] = mask[i]; + this.module._nisps_ml_feedback_set_focus(this.mlHandle, this.pinMaskBuf.ptr, n); + } + + /** Positive feedback (thumbs-up). Returns the FeedbackAction int. */ + feedbackUp(): number { + const action = this.module._nisps_ml_feedback_up(this.mlHandle); + this.publishWeights_(); + this.sink.emit('feedback.up', { action }); + this.scheduleSave_(); + return action; + } + + /** + * Negative feedback (thumbs-down). `currentOut` is the kDefaultOutputs vector + * the user is hearing (optional). Returns the FeedbackAction int. + */ + feedbackDown(speed: number, spread: number, currentOut?: Float32Array, pinMask?: Uint8Array): number { + let outPtr = 0; + if (currentOut) { + const n = Math.min(currentOut.length, this.arch_.outputSize); + this.feedbackBuf.view.fill(0); + this.feedbackBuf.view.set(currentOut.subarray(0, n)); + outPtr = this.feedbackBuf.ptr; + } + const maskPtr = this.writePinMask_(pinMask); + const action = this.module._nisps_ml_feedback_down(this.mlHandle, outPtr, speed, spread, maskPtr); + this.publishWeights_(); + this.sink.emit('feedback.down', { action }); + this.scheduleSave_(); + return action; + } + + /** Drag (continuous perturbation) tick. Returns the FeedbackAction int. */ + feedbackDrag(): number { + const action = this.module._nisps_ml_feedback_drag(this.mlHandle); + this.publishWeights_(); + return action; + } + + /** + * If a static bypass vector is active, copies it into `out` and returns true + * (the caller should NOT call process()); otherwise returns false. + */ + feedbackStaticOutput(out: Float32Array): boolean { + const bypass = this.module._nisps_ml_feedback_static_output(this.mlHandle, this.feedbackBuf.ptr); + if (bypass === 1) { + const n = Math.min(out.length, this.arch_.outputSize); + out.set(this.feedbackBuf.view.subarray(0, n)); + return true; + } + return false; + } + + // ------------------------------------------------------------------- + // Weights I/O + // ------------------------------------------------------------------- + + getWeights(): Float32Array { + this.module._nisps_ml_get_weights(this.mlHandle, this.weightsBuf.ptr); + return new Float32Array(this.weightsBuf.view); + } + + setWeights(w: Float32Array | Uint8Array): void { + if (w.length < this.weightCount_) { + throw new Error(`setWeights: expected ${this.weightCount_} floats, got ${w.length}`); + } + this.weightsBuf.view.set(w as Float32Array, 0); + this.module._nisps_ml_set_weights(this.mlHandle, this.weightsBuf.ptr); + this.publishWeights_(); + } + + getLayerStats(): LayerStats[] { + this.module._nisps_ml_get_layer_stats(this.mlHandle, this.statsBuf.ptr); + const out: LayerStats[] = []; + for (let i = 0; i < this.arch_.numLayers; ++i) { + const base = i * 4; + out.push({ + meanAbs: this.statsBuf.view[base], + maxAbs: this.statsBuf.view[base + 1], + deadFrac: this.statsBuf.view[base + 2], + saturatingFrac: this.statsBuf.view[base + 3], + }); + } + return out; + } + + getLayerStatsFlat(): Float32Array { + this.module._nisps_ml_get_layer_stats(this.mlHandle, this.statsBuf.ptr); + return new Float32Array(this.statsBuf.view); + } + + // ------------------------------------------------------------------- + // Misc + // ------------------------------------------------------------------- + + reset(): void { + this.module._nisps_ml_reset(this.mlHandle); + this.dataset.clear(); + this.lastLoss_ = null; + this.sink.setState({ exampleCount: 0, lastLoss: null, lossHistory: [] }); + this.publishWeights_(); + this.sink.emit('ml.examples_cleared', undefined); + this.scheduleSave_(); + } + + // ------------------------------------------------------------------- + // Persistence + // ------------------------------------------------------------------- + + private scheduleSave_(): void { + if (this.saveTimer !== null) clearTimeout(this.saveTimer); + this.saveTimer = window.setTimeout(() => this.saveNow(), 500); + } + + saveNow(): void { + if (this.destroyed) return; + if (this.saveTimer !== null) { + clearTimeout(this.saveTimer); + this.saveTimer = null; + } + try { + const weights = this.getWeights(); + const payload = { + v: 1, + arch: this.arch_, + weights: Array.from(weights), + features: Array.from(this.dataset.featuresFlat()), + labels: Array.from(this.dataset.labelsFlat()), + size: this.dataset.size, + lastLoss: this.lastLoss_, + }; + localStorage.setItem(this.storageKey, JSON.stringify(payload)); + } catch (err) { + console.warn('[wasm-iml] saveNow failed:', err); + } + } + + private tryLoadFromStorage_(): void { + try { + const raw = localStorage.getItem(this.storageKey); + if (!raw) return; + const payload = JSON.parse(raw) as { + v: number; + weights: number[]; + features: number[]; + labels: number[]; + size: number; + lastLoss: number | null; + }; + if (payload.v !== 1) return; + const inSz = this.arch_.inputSize; + const outSz = this.arch_.outputSize; + if (payload.size > 0 && payload.features.length === payload.size * inSz && + payload.labels.length === payload.size * outSz) { + for (let i = 0; i < payload.size; ++i) { + const f = payload.features.slice(i * inSz, (i + 1) * inSz); + const l = payload.labels.slice(i * outSz, (i + 1) * outSz); + this.dataset.add(f, l); + this.copyExampleToWasm_(f, l); + } + } + if (payload.weights.length === this.weightCount_) { + this.setWeights(new Float32Array(payload.weights)); + } + this.lastLoss_ = payload.lastLoss; + this.sink.setState({ exampleCount: this.dataset.size, lastLoss: this.lastLoss_ }); + } catch (err) { + console.warn('[wasm-iml] tryLoadFromStorage failed:', err); + } + } + + private publishWeights_(): void { + const w = this.getWeights(); + this.sink.setWeights(w); + } +} diff --git a/manifold/src/engine/wasm-worker.ts b/manifold/src/engine/wasm-worker.ts new file mode 100644 index 0000000..f2ca35f --- /dev/null +++ b/manifold/src/engine/wasm-worker.ts @@ -0,0 +1,314 @@ +/** + * Disposable Web Worker that runs SGD off the main thread. + * + * Lifted from `playground/src/ml/wasm-worker.ts`. Changes vs the playground: + * - imports `./types` (the lifted, feedback-extended ABI types) + * - WASM glue + binary are resolved via `import.meta.env.BASE_URL` so the + * bundle works under any mount path (`/`, `/next`, …), not a hardcoded + * `/nisps.js` / `/nisps.wasm`. + * + * The worker holds its own `nisps.wasm` instance; the main thread sends + * current weights + dataset + hyperparameters and receives updated weights + + * final loss. + */ + +import type { NispsModule, NispsModuleFactory, WorkerRequest, WorkerResponse } from './types'; + +// --------------------------------------------------------------------------- +// Main-thread side +// --------------------------------------------------------------------------- + +export interface TrainArgs { + weights: Float32Array; + features: Float32Array; + labels: Float32Array; + /** Optional; pass empty for uniform weighting. */ + sampleWeights: Float32Array; + lr: number; + maxIter: number; + minErr: number; + inputSize: number; + outputSize: number; +} + +export interface TrainResult { + loss: number; + weights: Float32Array; + lossHistory: Float32Array; +} + +export class WasmTrainer { + private worker: Worker; + private nextId = 1; + private pending = new Map void; reject: (e: unknown) => void }>(); + private disposed = false; + + static async create(): Promise { + const trainer = new WasmTrainer(); + await trainer.init_(); + return trainer; + } + + private constructor() { + this.worker = new Worker(new URL('./wasm-worker.ts', import.meta.url), { type: 'module' }); + this.worker.onmessage = (ev) => this.onMessage_(ev.data as WorkerResponse); + this.worker.onerror = (ev) => { + for (const { reject } of this.pending.values()) reject(ev.message ?? 'worker error'); + this.pending.clear(); + }; + } + + private init_(): Promise { + return new Promise((resolve, reject) => { + const handler = (ev: MessageEvent) => { + const msg = ev.data as WorkerResponse; + if (msg.kind === 'ready') { + this.worker.removeEventListener('message', handler); + resolve(); + } else if (msg.kind === 'error') { + this.worker.removeEventListener('message', handler); + reject(new Error(msg.message)); + } + }; + this.worker.addEventListener('message', handler); + const seed = (Date.now() ^ Math.floor(Math.random() * 0xffffffff)) >>> 0; + // Resolve the deploy base on the main thread (the worker has no document). + const assetBase = new URL(import.meta.env.BASE_URL ?? '/', document.baseURI).href; + this.worker.postMessage({ kind: 'init', seed, assetBase } satisfies WorkerRequest); + }); + } + + train(args: TrainArgs): Promise { + if (this.disposed) return Promise.reject(new Error('WasmTrainer disposed')); + const requestId = this.nextId++; + return new Promise((resolve, reject) => { + this.pending.set(requestId, { resolve, reject }); + const msg: WorkerRequest = { + kind: 'train', + requestId, + weights: args.weights, + features: args.features, + labels: args.labels, + sampleWeights: args.sampleWeights, + lr: args.lr, + maxIter: args.maxIter, + minErr: args.minErr, + inputSize: args.inputSize, + outputSize: args.outputSize, + }; + this.worker.postMessage(msg, [ + args.weights.buffer, + args.features.buffer, + args.labels.buffer, + args.sampleWeights.buffer, + ]); + }); + } + + dispose(): void { + if (this.disposed) return; + this.disposed = true; + try { + this.worker.postMessage({ kind: 'dispose' } satisfies WorkerRequest); + } catch { + /* ignore */ + } + this.worker.terminate(); + for (const { reject } of this.pending.values()) reject(new Error('disposed')); + this.pending.clear(); + } + + private onMessage_(msg: WorkerResponse): void { + if (msg.kind === 'result') { + const p = this.pending.get(msg.requestId); + if (p) { + this.pending.delete(msg.requestId); + p.resolve({ loss: msg.loss, weights: msg.weights, lossHistory: msg.lossHistory }); + } + } else if (msg.kind === 'error') { + const p = this.pending.get(msg.requestId); + if (p) { + this.pending.delete(msg.requestId); + p.reject(new Error(msg.message)); + } + } + } +} + +export function createTrainer(): Promise { + return WasmTrainer.create(); +} + +// --------------------------------------------------------------------------- +// Worker-thread side +// --------------------------------------------------------------------------- + +declare const self: { + postMessage: (msg: unknown, transfer?: Transferable[]) => void; + addEventListener: (event: string, handler: (ev: MessageEvent) => void) => void; + location: { origin: string }; + importScripts?: unknown; +}; + +const isWorker = + typeof window === 'undefined' && + typeof self !== 'undefined' && + typeof (self as { importScripts?: unknown }).importScripts !== 'undefined'; + +/** Absolute deploy base injected by the main thread on `init` (e.g. + * "https://host/next/"). The worker cannot derive it: it has no document, and + * its own bundle lives under /assets/, not the public root. */ +let workerAssetBase = '/'; + +/** Base-aware absolute URL for an asset served from `public/`. */ +function assetUrl(file: string): string { + return new URL(file, workerAssetBase).toString(); +} + +if (isWorker) { + let mod: NispsModule | null = null; + let mlHandle = 0; + let weightCount = 0; + + let weightsPtr = 0; + let weightsViewLen = 0; + let featuresPtr = 0; + let featuresLen = 0; + let labelsPtr = 0; + let labelsLen = 0; + let sampleWeightsPtr = 0; + let sampleWeightsLen = 0; + + async function loadModule(seed: number): Promise { + // nisps.js is non-ES-module Emscripten glue; fetch + indirect-eval to + // install the global factory (a module worker cannot importScripts, and + // import() yields an empty namespace — see wasm-iml.getFactory). + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const g = self as any; + if (!g.createNispsModule) { + const src = await (await fetch(assetUrl('nisps.js'))).text(); + (0, eval)(src); + } + const factory: NispsModuleFactory = g.createNispsModule; + if (!factory) throw new Error('[wasm-worker] nisps.js did not define createNispsModule'); + mod = await factory({ + locateFile: (path: string) => (path.endsWith('.wasm') ? assetUrl('nisps.wasm') : path), + }); + mlHandle = mod._nisps_ml_create(0, 0, 0, 0, seed >>> 0); + weightCount = mod._nisps_ml_weight_count(mlHandle); + } + + function ensureBuffers(features: Float32Array, labels: Float32Array, sampleWeights: Float32Array, weights: Float32Array): void { + if (!mod) throw new Error('worker module not loaded'); + + if (weightsViewLen !== weightCount) { + if (weightsPtr) mod._free(weightsPtr); + weightsPtr = mod._malloc(weightCount * 4); + weightsViewLen = weightCount; + } + if (features.length !== featuresLen) { + if (featuresPtr) mod._free(featuresPtr); + featuresPtr = mod._malloc(features.length * 4); + featuresLen = features.length; + } + if (labels.length !== labelsLen) { + if (labelsPtr) mod._free(labelsPtr); + labelsPtr = mod._malloc(labels.length * 4); + labelsLen = labels.length; + } + if (sampleWeights.length !== sampleWeightsLen) { + if (sampleWeightsPtr) mod._free(sampleWeightsPtr); + sampleWeightsPtr = sampleWeights.length > 0 ? mod._malloc(sampleWeights.length * 4) : 0; + sampleWeightsLen = sampleWeights.length; + } + + new Float32Array(mod.HEAPF32.buffer, weightsPtr, weightCount).set(weights); + new Float32Array(mod.HEAPF32.buffer, featuresPtr, features.length).set(features); + new Float32Array(mod.HEAPF32.buffer, labelsPtr, labels.length).set(labels); + if (sampleWeightsPtr) { + new Float32Array(mod.HEAPF32.buffer, sampleWeightsPtr, sampleWeights.length).set(sampleWeights); + } + } + + function trainOnce(req: Extract): WorkerResponse { + if (!mod) { + return { kind: 'error', requestId: req.requestId, message: 'worker not initialised' }; + } + try { + ensureBuffers(req.features, req.labels, req.sampleWeights, req.weights); + mod._nisps_ml_set_weights(mlHandle, weightsPtr); + + mod._nisps_ml_clear_examples(mlHandle); + const inSz = req.inputSize; + const outSz = req.outputSize; + const n = req.features.length / inSz; + for (let i = 0; i < n; ++i) { + const fPtr = featuresPtr + i * inSz * 4; + const lPtr = labelsPtr + i * outSz * 4; + mod._nisps_ml_add_example(mlHandle, fPtr, lPtr); + } + + const swPtr = req.sampleWeights.length > 0 ? sampleWeightsPtr : 0; + const loss = mod._nisps_ml_train(mlHandle, req.lr, req.maxIter, req.minErr, swPtr); + + mod._nisps_ml_get_weights(mlHandle, weightsPtr); + const view = new Float32Array(mod.HEAPF32.buffer, weightsPtr, weightCount); + const outWeights = new Float32Array(view); // copy + + const lossHistory = new Float32Array([loss]); + + return { + kind: 'result', + requestId: req.requestId, + loss, + weights: outWeights, + lossHistory, + }; + } catch (err) { + return { + kind: 'error', + requestId: req.requestId, + message: err instanceof Error ? err.message : String(err), + }; + } + } + + function disposeModule(): void { + if (!mod) return; + if (mlHandle) { + mod._nisps_ml_destroy(mlHandle); + mlHandle = 0; + } + if (weightsPtr) { mod._free(weightsPtr); weightsPtr = 0; } + if (featuresPtr) { mod._free(featuresPtr); featuresPtr = 0; } + if (labelsPtr) { mod._free(labelsPtr); labelsPtr = 0; } + if (sampleWeightsPtr) { mod._free(sampleWeightsPtr); sampleWeightsPtr = 0; } + mod = null; + } + + self.addEventListener('message', async (ev: MessageEvent) => { + const req = ev.data; + if (req.kind === 'init') { + try { + workerAssetBase = req.assetBase ?? self.location.origin + '/'; + await loadModule(req.seed); + self.postMessage({ kind: 'ready' } satisfies WorkerResponse); + } catch (err) { + self.postMessage({ + kind: 'error', + requestId: 0, + message: err instanceof Error ? err.message : String(err), + } satisfies WorkerResponse); + } + } else if (req.kind === 'train') { + const res = trainOnce(req); + if (res.kind === 'result') { + self.postMessage(res, [res.weights.buffer, res.lossHistory.buffer]); + } else { + self.postMessage(res); + } + } else if (req.kind === 'dispose') { + disposeModule(); + } + }); +} diff --git a/manifold/src/engine/worklet/audioworklet-globals.d.ts b/manifold/src/engine/worklet/audioworklet-globals.d.ts new file mode 100644 index 0000000..b7feb1c --- /dev/null +++ b/manifold/src/engine/worklet/audioworklet-globals.d.ts @@ -0,0 +1,26 @@ +/** + * Type declarations for AudioWorkletGlobalScope. The default `lib.dom` + * and `lib.dom.iterable` files don't include these because they only + * exist inside an AudioWorklet thread. + * + * Keep this file minimal — only what `nisps-processor.ts` actually uses. + */ + +declare const sampleRate: number; +declare const currentFrame: number; +declare const currentTime: number; + +declare class AudioWorkletProcessor { + constructor(options?: { numberOfInputs?: number; numberOfOutputs?: number; processorOptions?: unknown }); + readonly port: MessagePort; + process( + inputs: Float32Array[][], + outputs: Float32Array[][], + parameters: Record, + ): boolean; +} + +declare function registerProcessor( + name: string, + processorCtor: new (options?: unknown) => AudioWorkletProcessor, +): void; diff --git a/manifold/src/engine/worklet/nisps-processor.ts b/manifold/src/engine/worklet/nisps-processor.ts new file mode 100644 index 0000000..16c692e --- /dev/null +++ b/manifold/src/engine/worklet/nisps-processor.ts @@ -0,0 +1,309 @@ +/** + * AudioWorkletProcessor that runs `nisps.wasm` engines. + * + * Why a separate WASM instance from the main thread? AudioWorklet runs in + * its own thread + global scope; reusing a single instance would require + * SharedArrayBuffer + locking on the heap. Architecture.md §6.4 specifies + * separate instances connected by `port` messages instead. + * + * Wasm load path: AudioWorklet has NO `fetch` and NO ESM `import`. The + * main thread fetches `nisps.wasm` once and posts the bytes here as an + * ArrayBuffer; we then `WebAssembly.compile` and `instantiate` directly, + * skipping the Emscripten glue entirely. This is fine because the + * exported functions don't need any of the JS-side runtime. + * + * Block size: AudioWorklet ALWAYS calls process() with 128-sample blocks. + * We allocate 128-sample input and output buffers in the WASM linear + * memory and shuttle samples in/out per call. + */ + +/// + +import type { EngineId } from '../types'; +import type { HostToWorkletMessage, WorkletToHostMessage } from '../engine-host'; + +const PROC_BLOCK = 128; +const MAX_PARAMS = 256; // upper bound across all engines + +interface WasmInstance { + exports: { + memory: WebAssembly.Memory; + malloc: (n: number) => number; + free: (p: number) => void; + _nisps_engine_create: (id_ptr: number, sample_rate: number) => number; + _nisps_engine_destroy: (engine: number) => void; + _nisps_engine_set_params: (engine: number, params_ptr: number, n: number) => void; + _nisps_engine_process_block: ( + engine: number, + in_l: number, in_r: number, + out_l: number, out_r: number, + n_samples: number, + ) => void; + }; +} + +class NispsProcessor extends AudioWorkletProcessor { + private instance: WasmInstance | null = null; + private engineHandle = 0; + private engineId: EngineId = 'thru'; + private muted = true; + + // Pointers + buffer views (allocated once instance is up). + private inLPtr = 0; + private inRPtr = 0; + private outLPtr = 0; + private outRPtr = 0; + private idPtr = 0; + private paramsPtr = 0; + private inLView: Float32Array | null = null; + private inRView: Float32Array | null = null; + private outLView: Float32Array | null = null; + private outRView: Float32Array | null = null; + private paramsView: Float32Array | null = null; + private idView: Uint8Array | null = null; + private mem: WebAssembly.Memory | null = null; + + // Pending params posted before the engine was ready. + private pendingParams: Float32Array | null = null; + + constructor() { + super(); + this.port.onmessage = (ev) => this.onMessage_(ev.data as HostToWorkletMessage); + } + + private async onMessage_(msg: HostToWorkletMessage): Promise { + if (msg.kind === 'init') { + try { + await this.init_(msg.wasmBinary, msg.sampleRate); + this.post_({ kind: 'ready' }); + } catch (err) { + this.post_({ + kind: 'error', + message: err instanceof Error ? err.message : String(err), + }); + } + } else if (msg.kind === 'engine') { + this.switchEngine_(msg.engineId); + } else if (msg.kind === 'params') { + this.applyParams_(msg.params); + } else if (msg.kind === 'mute') { + this.muted = msg.muted; + } + } + + private post_(msg: WorkletToHostMessage): void { + this.port.postMessage(msg); + } + + /** + * Compile + instantiate the wasm module. We provide minimal imports — + * the Emscripten module needs `__abort_js` and `_emscripten_resize_heap` + * (we keep memory non-resizing so the latter is a stub). + */ + private async init_(bytes: ArrayBuffer, sampleRate: number): Promise { + const memory = new WebAssembly.Memory({ initial: 128, maximum: 4096, shared: false }); + const imports: WebAssembly.Imports = { + // Emscripten import "a" group; field names match the generated JS. + a: { + a: () => { throw new Error('wasm aborted'); }, + b: () => false, // _emscripten_resize_heap returning 0 disables growth + }, + }; + + const compiled = await WebAssembly.compile(bytes); + // Discover the actual import shape from the module — names like "a", + // "b" depend on emcc's mangling; we accept whatever it produces. + const importDesc = WebAssembly.Module.imports(compiled); + const reshaped: WebAssembly.Imports = {}; + for (const desc of importDesc) { + if (!reshaped[desc.module]) reshaped[desc.module] = {} as WebAssembly.ModuleImports; + const mod = reshaped[desc.module] as WebAssembly.ModuleImports; + if (desc.kind === 'function') { + if (desc.name === 'c') { + // unused + } + mod[desc.name] = (() => { + // Generic stub: log + return 0. + return (..._args: unknown[]) => 0; + })(); + } else if (desc.kind === 'memory') { + mod[desc.name] = memory; + } else if (desc.kind === 'table') { + mod[desc.name] = new WebAssembly.Table({ element: 'anyfunc', initial: 0 }); + } else if (desc.kind === 'global') { + mod[desc.name] = new WebAssembly.Global({ value: 'i32', mutable: true }, 0); + } + } + // For known-needed Emscripten imports, supply real implementations. + for (const desc of importDesc) { + const mod = reshaped[desc.module] as WebAssembly.ModuleImports; + // __abort_js + if (desc.name === 'a' && desc.kind === 'function') { + mod[desc.name] = () => { throw new Error('wasm aborted'); }; + } + // _emscripten_resize_heap + if (desc.name === 'b' && desc.kind === 'function') { + mod[desc.name] = (_size: number) => 0; // refuse growth in worklet + } + } + + void imports; // silence unused + const wasmInst = await WebAssembly.instantiate(compiled, reshaped); + + // Many Emscripten exports use single-letter mangled names. Discover + // by reading the export descriptors. + const exDesc = WebAssembly.Module.exports(compiled); + const exMap = new Map(); // logical name → mangled + for (const e of exDesc) { + // The exports list includes both the original (with leading + // underscore for C funcs) and the mangled single-letter alias used + // in the import section. We only see the export side here, but + // Emscripten in modern versions also re-exports the C names with + // their leading-underscore form. Walk both. + exMap.set(e.name, e.name); + } + const exports = wasmInst.exports as Record; + + function pickFn(...names: string[]): (...args: number[]) => number { + for (const n of names) { + const v = exports[n]; + if (typeof v === 'function') return v as unknown as (...a: number[]) => number; + } + throw new Error(`worklet: missing wasm export, tried: ${names.join(', ')}`); + } + function pickFnVoid(...names: string[]): (...args: number[]) => void { + return pickFn(...names) as unknown as (...args: number[]) => void; + } + + // The exports we need. + const malloc = pickFn('_malloc', 'malloc'); + const free = pickFnVoid('_free', 'free'); + const ec = pickFn('_nisps_engine_create'); + const ed = pickFnVoid('_nisps_engine_destroy'); + const esp = pickFnVoid('_nisps_engine_set_params'); + const epb = pickFnVoid('_nisps_engine_process_block'); + + // The wasm-exported memory might be named `memory` or another mangled + // alias. Find it. + let wasmMemory: WebAssembly.Memory | null = null; + for (const e of exDesc) { + if (e.kind === 'memory') { + const v = exports[e.name]; + if (v instanceof WebAssembly.Memory) { wasmMemory = v; break; } + } + } + // If the module imports memory (which our build does — we passed it), + // there will be no exported memory; use the imported one. + this.mem = wasmMemory ?? memory; + + this.instance = { + exports: { + memory: this.mem, + malloc, + free, + _nisps_engine_create: (id, sr) => ec(id, sr), + _nisps_engine_destroy: (h) => ed(h), + _nisps_engine_set_params: (h, p, n) => esp(h, p, n), + _nisps_engine_process_block: (h, il, ir, ol, or_, n) => epb(h, il, ir, ol, or_, n), + }, + }; + + // Allocate buffers. + this.inLPtr = malloc(PROC_BLOCK * 4); + this.inRPtr = malloc(PROC_BLOCK * 4); + this.outLPtr = malloc(PROC_BLOCK * 4); + this.outRPtr = malloc(PROC_BLOCK * 4); + this.paramsPtr = malloc(MAX_PARAMS * 4); + // Engine ids are short ASCII; 32 bytes covers everything we have. + this.idPtr = malloc(32); + + const buf = this.mem.buffer; + this.inLView = new Float32Array(buf, this.inLPtr, PROC_BLOCK); + this.inRView = new Float32Array(buf, this.inRPtr, PROC_BLOCK); + this.outLView = new Float32Array(buf, this.outLPtr, PROC_BLOCK); + this.outRView = new Float32Array(buf, this.outRPtr, PROC_BLOCK); + this.paramsView = new Float32Array(buf, this.paramsPtr, MAX_PARAMS); + this.idView = new Uint8Array(buf, this.idPtr, 32); + + // Default engine: thru. + this.spawnEngine_('thru', sampleRate); + + // Apply pending params if any arrived before init completed. + if (this.pendingParams) { + this.applyParams_(this.pendingParams); + this.pendingParams = null; + } + + this.muted = false; + } + + private spawnEngine_(id: EngineId, sampleRate: number): void { + if (!this.instance || !this.idView) return; + if (this.engineHandle) { + this.instance.exports._nisps_engine_destroy(this.engineHandle); + this.engineHandle = 0; + } + // Write engine_id as ASCII into idView, NUL-terminated. + const enc = new TextEncoder(); + const bytes = enc.encode(id); + this.idView.fill(0); + this.idView.set(bytes.subarray(0, Math.min(bytes.length, 31))); + this.engineHandle = this.instance.exports._nisps_engine_create(this.idPtr, sampleRate); + this.engineId = id; + } + + private switchEngine_(id: EngineId): void { + // sampleRate global from AudioWorkletGlobalScope. + this.spawnEngine_(id, sampleRate); + } + + private applyParams_(params: Float32Array): void { + if (!this.instance || !this.paramsView) { + this.pendingParams = params; + return; + } + const n = Math.min(params.length, MAX_PARAMS); + for (let i = 0; i < n; ++i) this.paramsView[i] = params[i]; + if (this.engineHandle) { + this.instance.exports._nisps_engine_set_params(this.engineHandle, this.paramsPtr, n); + } + } + + override process(inputs: Float32Array[][], outputs: Float32Array[][]): boolean { + const out = outputs[0]; + if (!out || out.length === 0) return true; + + const outL = out[0]; + const outR = out.length > 1 ? out[1] : out[0]; + + if (this.muted || !this.instance || !this.engineHandle || + !this.inLView || !this.outLView || !this.outRView || !this.inRView) { + // Silence. + outL.fill(0); + if (out.length > 1) outR.fill(0); + return true; + } + + // Copy inputs into wasm buffers (zero-fill if missing). + const inp = inputs[0]; + if (inp && inp[0]) this.inLView.set(inp[0].subarray(0, PROC_BLOCK)); + else this.inLView.fill(0); + if (inp && inp[1]) this.inRView.set(inp[1].subarray(0, PROC_BLOCK)); + else if (inp && inp[0]) this.inRView.set(inp[0].subarray(0, PROC_BLOCK)); + else this.inRView.fill(0); + + this.instance.exports._nisps_engine_process_block( + this.engineHandle, + this.inLPtr, this.inRPtr, + this.outLPtr, this.outRPtr, + PROC_BLOCK, + ); + + outL.set(this.outLView.subarray(0, outL.length)); + if (out.length > 1) outR.set(this.outRView.subarray(0, outR.length)); + + return true; + } +} + +registerProcessor('nisps-processor', NispsProcessor); diff --git a/manifold/src/feedback/controller.ts b/manifold/src/feedback/controller.ts new file mode 100644 index 0000000..a80053e --- /dev/null +++ b/manifold/src/feedback/controller.ts @@ -0,0 +1,522 @@ +/** + * FeedbackController — framework-neutral learning-engine behaviour for the two + * feedback modes plus solo/arm, prototyped in pure TS on the EXISTING engine + * primitives (NO C++/WASM change). + * + * Authoritative design: docs/redesign/rl-feedback-design.md (Mode 2 default; + * Mode 1 selectable; SOLO default MaskGradients). Engine primitives audited in + * docs/redesign/findings-feedback-behaviour.md. + * + * This class holds NO React. ConsoleApp owns one instance and exposes its + * actions + state into the console context; VerdictCluster + Manifold drive it. + * + * It talks ONLY to the small primitive surface of EngineApi: + * getWeights / setWeights — snapshot + restore (byte round-trip) + * randomise() — draw_weights, re-roll the whole net + * setInput(x,y) / getOutputs() — synchronous forward inference (the spine) + * process() — re-run last input after a weight change + * addExample([x,y], outVec) — append a training example + * train() — SGD over the dataset + * feedback.{setFocus,thumbsDown,thumbsUp} — engine's RL primitives (Mode 1) + * + * Everything the design plans to push into the C++ core (geometric push-away, + * the scratch-undo ring, the column-freeze gradient mask, the warm-start + * interpolation loop) is implemented here in TS and CLEARLY COMMENTED as the + * approximation it is, with a pointer to where the real core primitive lands. + */ + +import { SeededRng } from './rng'; + +/** The two product feedback modes (rl-feedback-design §0). */ +export type ProtoFeedbackMode = 'explore-and-place' | 'geometric-dislike'; + +/** Solo / arm gradient-mask variant (rl-feedback-design §3). */ +export type ProtoSoloMode = 'mask-gradients' | 'zero-loss' | 'dont-care'; + +/** + * A placed positive anchor: a chosen input location → the scratchpad output + * vector heard there. The real model is warm-started to interpolate all of + * these (rl-feedback-design §2.2 step 4). + */ +export interface Anchor { + /** Chosen input location in [0,1]². */ + input: readonly [number, number]; + /** The 126-dim output vector heard at that location (copied, owned). */ + output: Float32Array; + /** + * Per-output arm mask captured at placement time (don't-care approximation — + * §3.3). `null` ⇒ assert every output. Non-null ⇒ only assert masked dims. + * In TS we can only approximate column-freeze at the EXAMPLE level (the true + * gradient column-freeze is the C++ step). + */ + mask: Uint8Array | null; +} + +/** The minimal engine surface the controller needs (decoupled from EngineApi). */ +export interface ControllerEngine { + getWeights(): Float32Array; + setWeights(w: Float32Array): void; + randomise(spread?: number): void; + setInput(x: number, y: number): void; + getOutputs(): Float32Array; + process(): void; + addExample(features: ReadonlyArray, labels: ReadonlyArray): boolean; + train(): number; + readonly feedback: { + thumbsUp(): number; + thumbsDown(speed?: number, spread?: number, pinMask?: Uint8Array): number; + setFocus(mask: Uint8Array | null): void; + }; +} + +/** Snapshot of controller-observable state, mirrored into React on demand. */ +export interface FeedbackControllerState { + mode: ProtoFeedbackMode; + soloMode: ProtoSoloMode; + /** True while a Mode-2 scratchpad session is active. */ + exploring: boolean; + /** True while a "place" gesture is pending a manifold location pick. */ + picking: boolean; + /** Anchors placed in the CURRENT (not-yet-finalised) explore session. */ + anchorCount: number; + /** Scratchpad undo-stack depth (nudges/rerolls that can be undone). */ + undoDepth: number; + /** Count of currently-armed (soloed) outputs; 0 ⇒ none armed ⇒ train all. */ + armedCount: number; +} + +export interface FeedbackControllerOptions { + /** Seed for the deterministic nudge RNG (NOT Math.random — task constraint). */ + seed?: number; + /** Master spread for randomise / nudge (mirrors the engine spread knob). */ + spread?: number; + /** Nudge perturbation standard deviation (small bounded weight jitter). */ + nudgeStddev?: number; + /** + * Undo-stack depth. WASM D=4, firmware D=2 per rl-feedback-design §2.2; the + * prototype defaults to the WASM depth. + */ + undoDepth?: number; +} + +export class FeedbackController { + private engine: ControllerEngine; + private rng: SeededRng; + private spread: number; + private nudgeStddev: number; + private maxUndo: number; + + private mode: ProtoFeedbackMode = 'explore-and-place'; + private soloMode: ProtoSoloMode = 'mask-gradients'; + + // ---- Mode-2 scratchpad session state ------------------------------- + /** The set-aside REAL trained net, restored on finalise/cancel. */ + private snapshot: Float32Array | null = null; + private exploringFlag = false; + /** Undo stack of scratchpad weight snapshots (reroll + nudge are undoable). */ + private undoStack: Float32Array[] = []; + /** Anchors placed this session (positives only — NEVER a dislike). */ + private anchors: Anchor[] = []; + /** True between place() and the manifold location pick. */ + private pickingFlag = false; + /** + * The scratchpad output vector frozen at place() time, so the heard sound is + * held while the user aims at a location (rl-feedback-design §2.2 step 3, + * "place_begin freezes the current scratchpad output"). Copied/owned. + */ + private placedOutput: Float32Array | null = null; + + // ---- Solo / arm ---------------------------------------------------- + /** Current arm mask (1=armed/soloed). null ⇒ none armed ⇒ train all. */ + private armMask: Uint8Array | null = null; + + // ---- Mode-1 dislike memory (TS approximation) ---------------------- + /** + * Disliked (input → output) pairs. The TRUE firmware geometric push (upstream + * 0a541cc, replay-backed) computes a k-NN positive centroid and pushes the + * disliked action away from it, then trains toward that target. We cannot do + * that on the existing primitives without the C++ replay store + train_targets + * hook, so the TS prototype: + * (a) calls the engine's existing feedback.thumbsDown() (AVOID/move_weights) + * as the audible baseline, AND + * (b) records the disliked pair here so subsequent training can bias AWAY + * from it (a coarse example-level approximation — see applyDislikeBias). + * Documented C++ gap: the directed geometric push-away lands in the core as + * `geo_push.hpp` + `replay.hpp` + `mlp.train_targets` (rl-feedback-design §4). + */ + private dislikes: { input: readonly [number, number]; output: Float32Array }[] = []; + + constructor(engine: ControllerEngine, opts: FeedbackControllerOptions = {}) { + this.engine = engine; + this.rng = new SeededRng(opts.seed ?? 0xfeedbacc); + this.spread = opts.spread ?? 0.6; + this.nudgeStddev = opts.nudgeStddev ?? 0.05; + this.maxUndo = Math.max(1, opts.undoDepth ?? 4); + } + + // =================================================================== + // Config + // =================================================================== + + setMode(mode: ProtoFeedbackMode): void { + if (mode === this.mode) return; + // Switching mode aborts any active scratchpad session (mirrors the C++ + // `set_mode` which aborts active exploration first — findings §2). + if (this.exploringFlag) this.cancel(); + this.mode = mode; + } + + getMode(): ProtoFeedbackMode { + return this.mode; + } + + setSoloMode(mode: ProtoSoloMode): void { + this.soloMode = mode; + } + + setSpread(spread: number): void { + this.spread = spread; + } + + /** + * Set the arm/solo mask. The dock builds this from the per-output `armed` + * flags (dock/output-state.ts buildArmMask). We RESPECT it at the example + * level in both modes (§3.4 honest-limit copy). We also forward it to the + * engine's `setFocus` so Mode-1's move_weights freezes unarmed final-layer + * columns — the only directional gating the existing primitive offers. + */ + setArmMask(mask: Uint8Array | null): void { + this.armMask = mask && mask.length ? mask : null; + this.engine.feedback.setFocus(this.armMask); + } + + // =================================================================== + // Mode 2 — "Explore & place" (DEFAULT, positive-only, NEVER a dislike) + // =================================================================== + + /** + * ENTER explore (rl-feedback-design §2.2 step 1): snapshot the REAL weights, + * set them aside, then randomise() into a scratchpad net. Mark exploring. + * Idempotent re-entry while already exploring = a re-roll (step 2). + */ + enterExplore(): void { + if (this.exploringFlag) { + // Re-press while exploring re-rolls ("meh, randomise…" — §2.2 step 2). + this.reroll(); + return; + } + // Snapshot the real trained net (byte round-trip via get/set weights). This + // is the SET-ASIDE net restored on finalise/cancel — it is NOT part of the + // scratchpad undo ring (undo stays inside the scratchpad; you leave the + // session via cancel/finalise, never by undoing back into the real net). + this.snapshot = this.engine.getWeights(); + this.undoStack = []; + this.anchors = []; + this.placedOutput = null; + this.pickingFlag = false; + this.exploringFlag = true; + // Randomise into the first scratchpad candidate, then record it as the undo + // baseline (the history holds the LIVE candidate AFTER each op). + this.engine.randomise(this.spread); + this.recordCandidate(); + } + + /** + * SCRATCHPAD OP: re-roll the whole net (§2.2 step 2). Undoable. The scratchpad + * is NEVER trained — this only generates a fresh candidate sound to audition. + */ + reroll(): void { + if (!this.exploringFlag) return; + this.engine.randomise(this.spread); + this.recordCandidate(); + } + + /** + * SCRATCHPAD OP: nudge — a small bounded gaussian weight perturbation (§2.2 + * step 2). Undoable. Deterministic via the seeded RNG (NO Math.random). + * + * --- C++ GAP ----------------------------------------------------------- + * The firmware does this with `move_weights(speed, spread)` on its own + * `nisps::Rng`. Here we read the weights, add a small seeded gaussian, and + * write them back — the TS-achievable equivalent. Becomes + * `nisps_ml_feedback_nudge` driving the engine's Rng (rl-feedback-design §4). + * ---------------------------------------------------------------------- + */ + nudge(): void { + if (!this.exploringFlag) return; + const w = this.engine.getWeights(); + // Bounded gaussian perturbation. No per-call allocation beyond the weights + // buffer the engine already returns (we mutate it in place then write back). + for (let i = 0; i < w.length; i++) { + w[i] += this.rng.nextGaussian(this.nudgeStddev); + } + this.engine.setWeights(w); + this.engine.process(); + this.recordCandidate(); + } + + /** + * UNDO the last scratchpad op (reroll or nudge). Both are undoable (§2.2). The + * undo ring holds the live scratchpad candidate after each op; undo discards + * the current candidate and restores the previous one. The baseline (first + * candidate after enter) is kept so undo never leaves the scratchpad. + */ + undo(): void { + if (!this.exploringFlag) return; + if (this.undoStack.length <= 1) return; // already at the baseline candidate + this.undoStack.pop(); // discard current candidate + const prev = this.undoStack[this.undoStack.length - 1]; + this.engine.setWeights(prev); + this.engine.process(); + } + + /** Record the CURRENT live scratchpad weights as a new undo-ring entry. */ + private recordCandidate(): void { + this.undoStack.push(this.engine.getWeights()); + // Bound the ring to maxUndo+1 (the +1 is the kept baseline at index 0). + if (this.undoStack.length > this.maxUndo + 1) { + this.undoStack.splice(1, 1); + } + } + + /** + * PLACE begin (§2.2 step 3): the user likes the current candidate. Freeze the + * scratchpad output so the heard sound is held while they aim, and enter the + * PICK-LOCATION state — the next manifold pointer-down chooses the location. + */ + place(): void { + if (!this.exploringFlag) return; + this.placedOutput = new Float32Array(this.engine.getOutputs()); + this.pickingFlag = true; + } + + /** True while a place() is awaiting a manifold location pick. */ + isPicking(): boolean { + return this.pickingFlag; + } + + /** The frozen scratchpad output held during aiming (read-only; may be null). */ + getPlacedOutput(): Float32Array | null { + return this.placedOutput; + } + + /** + * PLACE commit (§2.2 step 3): the user picked a location on the manifold. We + * move the scratchpad input there, run inference, capture the output the + * scratchpad produces AT THAT LOCATION, and store it as a positive anchor. + * + * Per the spec the captured output is "the output the scratchpad produces at + * the chosen location" (getOutputs() after setting the input there) — NOT the + * frozen audition vector. The frozen vector only kept the *audio* steady while + * aiming. Returns the new anchor count. + */ + placeCommit(x: number, y: number): number { + if (!this.exploringFlag || !this.pickingFlag) return this.anchors.length; + this.engine.setInput(x, y); + this.engine.process(); + const out = new Float32Array(this.engine.getOutputs()); + // Solo/arm respected at the EXAMPLE level: capture the arm mask so warm-start + // only asserts armed outputs ("don't-care on others" — §3.3 approximation). + const mask = this.armMask ? new Uint8Array(this.armMask) : null; + this.anchors.push({ input: [x, y], output: out, mask }); + this.pickingFlag = false; + this.placedOutput = null; + return this.anchors.length; + } + + /** Cancel a pending place() without storing an anchor (back to auditioning). */ + cancelPlace(): void { + this.pickingFlag = false; + this.placedOutput = null; + } + + /** + * RESOLVE / warm-start (§2.2 step 4): restore the set-aside REAL net, then + * warm-start it to interpolate ALL placed anchors by re-adding each as an + * example and training. ADDITIVE — anchors are added to the existing dataset + * (the user's prior thumbs-up likes are NOT clobbered). Exits exploring. + * + * --- C++ GAP ----------------------------------------------------------- + * The firmware warm-start trains anchors only on soloed dims via a gradient + * column-freeze (`train_masked`). Here we approximate that at the example + * level: when an anchor carries an arm mask we still add the FULL output + * vector (the engine's addExample takes a full label row), but we forward the + * mask to the engine's setFocus so move_weights/training freezes unarmed + * final-layer columns. True per-example gradient masking (`train_masked` + * consuming `Anchor.mask`) is the C++ step (rl-feedback-design §3.3). + * ---------------------------------------------------------------------- + */ + finalise(): number { + if (!this.exploringFlag) return 0; + if (this.snapshot) { + this.engine.setWeights(this.snapshot); // restore the real net (warm start) + } + const placed = this.anchors.length; + // Re-assert the arm focus so training honours any soloed columns. + this.engine.feedback.setFocus(this.armMask); + for (const a of this.anchors) { + this.engine.addExample([a.input[0], a.input[1]], Array.from(a.output)); + } + if (placed > 0) { + this.engine.train(); + } + this.engine.process(); + this.endSession(); + return placed; + } + + /** + * CANCEL / undo whole session (§2.2 step 5): discard scratchpad + anchors, + * restore the set-aside real net. No anchor stored. + */ + cancel(): void { + if (!this.exploringFlag) return; + if (this.snapshot) { + this.engine.setWeights(this.snapshot); + this.engine.process(); + } + this.endSession(); + } + + private endSession(): void { + this.exploringFlag = false; + this.pickingFlag = false; + this.placedOutput = null; + this.snapshot = null; + this.undoStack = []; + this.anchors = []; + } + + // =================================================================== + // Mode 1 — "Geometric dislike" (selectable) + // =================================================================== + + /** + * DISLIKE (thumbs-down in Mode 1). Push the current mapping away from the + * disliked sound. + * + * PROTOTYPE: we use the engine's existing feedback.thumbsDown() (AVOID / + * move_weights — undirected Gaussian diffusion, the baseline) as the audible + * effect, AND record the disliked (input → output) so a subsequent like+train + * can bias away from it (applyDislikeBias). + * + * --- C++ GAP (the real firmware behaviour) ----------------------------- + * The true geometric push-away (upstream 0a541cc, replay-backed, + * InterfaceRL.cpp:602-738) is: + * 1. store the negative (input, action) in a ReplayStore (dedup within 0.05) + * 2. compute the k-NN(k=4) centroid of POSITIVE memories near the input + * 3. target[j] = clamp(neg[j] + dir/||dir|| * pushStep/(1+||dir||), 0, 1) + * where dir[j] = neg[j] - meanPositive[j] (away from the liked centroid) + * 4. train the net toward that computed `target` at lr*negLRRatio + * 5. cold-start fallback when there are no positives yet. + * This needs `replay.hpp`, `geo_push.hpp`, and `mlp.train_targets` (train + * toward arbitrary COMPUTED targets, which the existing train()/addExample() + * cannot do — they only train toward STORED labels). It lands in the C++ core + * in rl-feedback-design Phase 1 (§5). Until then this TS prototype keeps the + * baseline move_weights effect plus example-level bias. + * ---------------------------------------------------------------------- + * + * @param input the control input the disliked sound was heard at + * @param output the heard 126-dim output vector (a_neg) + * @param speed move_weights speed (noise cap) + * @param spread move_weights spread + */ + dislike( + input: readonly [number, number], + output: Float32Array, + speed: number, + spread: number, + ): void { + // Record the disliked pair (the firmware ReplayStore negative). Dedup within + // a coarse radius so repeated dislikes near each other don't pile up — a + // cheap stand-in for the firmware `deepen_or_store_negative(radius=0.05)`. + const RADIUS = 0.05; + const near = this.dislikes.find( + (d) => + Math.hypot(d.input[0] - input[0], d.input[1] - input[1]) <= RADIUS, + ); + if (near) { + near.output = new Float32Array(output); + } else { + this.dislikes.push({ input: [input[0], input[1]], output: new Float32Array(output) }); + } + // Audible baseline: the engine's existing AVOID move_weights, focus-gated by + // the arm mask (the only directional gating the primitive offers today). + this.engine.feedback.thumbsDown(speed, spread, this.armMask ?? undefined); + this.engine.process(); + } + + /** + * LIKE + train (thumbs-up in Mode 1). Store the current (input → output) as a + * positive example and train. In firmware this also feeds the positive + * centroid (replay.store(+1,…)); here it is a normal addExample + train, with + * an optional bias away from recorded dislikes. + */ + like(input: readonly [number, number], output: Float32Array): void { + this.engine.feedback.setFocus(this.armMask); + this.engine.addExample([input[0], input[1]], Array.from(output)); + this.applyDislikeBias(); + this.engine.train(); + this.engine.process(); + } + + /** + * Coarse example-level bias AWAY from disliked sounds (the TS approximation of + * the geometric push). For each recorded dislike we add a "repelled" example: + * an example at the disliked input whose output is nudged away from the + * disliked vector toward the dataset mean. This is a WEAK stand-in — it biases + * the trainer rather than computing a true centroid-relative push. + * + * --- C++ GAP ----------------------------------------------------------- + * Replaced by `geo_push.compute_push_targets` + `train_targets` in the C++ + * core (rl-feedback-design §4). Intentionally conservative here so it never + * destabilises the net before any positives exist (the `posMemCount==0` + * cold-start fallback the design ports faithfully). + * ---------------------------------------------------------------------- + */ + private applyDislikeBias(): void { + // No-op when there are no dislikes; conservative cold-start (do nothing + // destabilising) when there is nothing to push away from yet. + if (this.dislikes.length === 0) return; + for (const d of this.dislikes) { + const out = new Float32Array(d.output.length); + // Push each dim of the disliked output toward its complement (0.5 pivot) — + // a direction-free repulsion stand-in. Respect the arm mask: only move + // armed dims; leave others at the disliked value (don't-care). + for (let j = 0; j < out.length; j++) { + const armed = !this.armMask || this.armMask[j] === 1; + if (armed) { + const v = d.output[j]; + out[j] = Math.max(0, Math.min(1, v + (0.5 - v) * 0.6)); + } else { + out[j] = d.output[j]; + } + } + this.engine.addExample([d.input[0], d.input[1]], Array.from(out)); + } + } + + // =================================================================== + // State snapshot + // =================================================================== + + getState(): FeedbackControllerState { + let armed = 0; + if (this.armMask) for (const m of this.armMask) if (m) armed++; + return { + mode: this.mode, + soloMode: this.soloMode, + exploring: this.exploringFlag, + picking: this.pickingFlag, + anchorCount: this.anchors.length, + // -1 for the entry-state baseline kept at index 0. + undoDepth: Math.max(0, this.undoStack.length - 1), + armedCount: armed, + }; + } + + /** Read-only view of placed anchors (current session). */ + getAnchors(): readonly Anchor[] { + return this.anchors; + } +} diff --git a/manifold/src/feedback/index.ts b/manifold/src/feedback/index.ts new file mode 100644 index 0000000..d464643 --- /dev/null +++ b/manifold/src/feedback/index.ts @@ -0,0 +1,18 @@ +/** + * The learning-engine behaviour module (workstream B) — the two feedback modes + * plus solo, prototyped in TS on the existing engine primitives. + * + * See docs/redesign/rl-feedback-design.md for the authoritative design and the + * C++ integration plan. Everything here is the TS-prototype-first layer; the + * controller comments mark each place that becomes a C++ core primitive. + */ +export { + FeedbackController, + type ProtoFeedbackMode, + type ProtoSoloMode, + type Anchor, + type ControllerEngine, + type FeedbackControllerState, + type FeedbackControllerOptions, +} from './controller'; +export { SeededRng } from './rng'; diff --git a/manifold/src/feedback/rng.ts b/manifold/src/feedback/rng.ts new file mode 100644 index 0000000..f4454c6 --- /dev/null +++ b/manifold/src/feedback/rng.ts @@ -0,0 +1,64 @@ +/** + * Deterministic seeded RNG for the feedback controller's hot path. + * + * The rl-feedback-design (§6) mandates: "every new operation is deterministic + * f32 arithmetic on the per-instance `nisps::Rng` (no libc `rand()` anywhere)". + * In the C++ core the controller owns a `nisps::Rng` seeded from + * `kSeed ^ kFeedbackSalt`. This TS prototype mirrors that discipline so that the + * `nudge` perturbation is reproducible run-to-run (no `Math.random` in the + * core path — see the task CONSTRAINTS). + * + * Implementation: a small splitmix64-style integer generator reduced to f32. + * This is NOT bit-identical to the C++ `nisps::Rng` — when the geometric push / + * nudge becomes a C++ core primitive (rl-feedback-design §4), the seeded stream + * must come from `nisps::Rng` so native==WASM parity holds. Here it only needs + * to be deterministic *within* the prototype. + * + * --- C++ GAP ------------------------------------------------------------- + * The true firmware nudge perturbs weights with `move_weights(speed, spread)` + * driven by the controller's `nisps::Rng`. This TS RNG is a stand-in so the + * prototype is reproducible; it will be REPLACED by the engine's own Rng stream + * once `nisps_ml_feedback_nudge` exists (rl-feedback-design §4 "TS"). + * ------------------------------------------------------------------------ + */ + +export class SeededRng { + // 64-bit state held as two 32-bit halves (BigInt would be cleaner but we keep + // to plain number maths to avoid any per-call BigInt allocation in the hot + // nudge loop). + private state: number; + + constructor(seed: number) { + // Fold the seed into a non-zero 32-bit state. + this.state = (seed ^ 0x9e3779b9) >>> 0; + if (this.state === 0) this.state = 0x1234567; + } + + /** Next uniform float in [0, 1). xorshift32 — deterministic, allocation-free. */ + nextFloat(): number { + let x = this.state; + x ^= x << 13; + x >>>= 0; + x ^= x >>> 17; + x ^= x << 5; + x >>>= 0; + this.state = x; + // Map to [0,1) using the top 24 bits for a clean float mantissa. + return (x >>> 8) / 0x01000000; + } + + /** Next uniform float in [-1, 1). */ + nextFloatSigned(): number { + return this.nextFloat() * 2 - 1; + } + + /** + * Approximate gaussian via the sum-of-three-uniforms method the nisps core + * uses (`gen_randn` in MEMORY.md: sum of 3 uniforms). Mean 0, the given + * standard deviation. Allocation-free. + */ + nextGaussian(stddev: number): number { + const u = this.nextFloatSigned() + this.nextFloatSigned() + this.nextFloatSigned(); + return u * stddev; + } +} diff --git a/manifold/src/main.tsx b/manifold/src/main.tsx new file mode 100644 index 0000000..abe632d --- /dev/null +++ b/manifold/src/main.tsx @@ -0,0 +1,10 @@ +import { StrictMode } from 'react'; +import { createRoot } from 'react-dom/client'; +import './styles/tokens.css'; +import { App } from './App'; + +createRoot(document.getElementById('root')!).render( + + + , +); diff --git a/manifold/src/primitives/Badge.tsx b/manifold/src/primitives/Badge.tsx new file mode 100644 index 0000000..cfaa926 --- /dev/null +++ b/manifold/src/primitives/Badge.tsx @@ -0,0 +1,61 @@ +import type { CSSProperties, ReactNode } from 'react'; + +export type BadgeTone = 'neutral' | 'accent' | 'good' | 'warn' | 'bad' | 'info'; + +export interface BadgeProps { + children?: ReactNode; + tone?: BadgeTone; + /** Prepend a glowing status dot. */ + dot?: boolean; + style?: CSSProperties; +} + +const TONES: Record = { + neutral: { fg: 'var(--fg-mute)', bd: 'var(--line)', bg: 'var(--bg-2)' }, + accent: { fg: 'var(--accent)', bd: 'rgba(255,106,0,0.4)', bg: 'rgba(255,106,0,0.12)' }, + good: { fg: 'var(--good)', bd: 'rgba(107,194,107,0.4)', bg: 'rgba(107,194,107,0.14)' }, + warn: { fg: 'var(--warn)', bd: 'rgba(245,196,94,0.4)', bg: 'rgba(245,196,94,0.14)' }, + bad: { fg: 'var(--bad)', bd: 'rgba(239,91,91,0.4)', bg: 'rgba(239,91,91,0.14)' }, + info: { fg: 'var(--info)', bd: 'rgba(91,158,239,0.4)', bg: 'rgba(91,158,239,0.14)' }, +}; + +/** + * Manifold Badge — small status capsule. `dot` prepends a status dot; + * `tone` sets the colour. Use for state labels (frozen, training, healthy). + */ +export function Badge({ children, tone = 'neutral', dot = false, style }: BadgeProps) { + const t = TONES[tone] ?? TONES.neutral; + return ( + + {dot && ( + + )} + {children} + + ); +} diff --git a/manifold/src/primitives/Button.tsx b/manifold/src/primitives/Button.tsx new file mode 100644 index 0000000..a5177d1 --- /dev/null +++ b/manifold/src/primitives/Button.tsx @@ -0,0 +1,123 @@ +import type { ButtonHTMLAttributes, CSSProperties, ReactNode } from 'react'; + +export type ButtonVariant = 'primary' | 'secondary' | 'ghost'; +export type ButtonSize = 'sm' | 'md' | 'lg'; + +export interface ButtonProps + extends Omit, 'style'> { + children?: ReactNode; + variant?: ButtonVariant; + size?: ButtonSize; + disabled?: boolean; + /** Optional leading glyph rendered before the children. */ + glyph?: ReactNode; + /** Active (pressed/selected) styling for secondary/ghost variants. */ + active?: boolean; + style?: CSSProperties; +} + +interface VariantStyle { + background: string; + borderColor: string; + color: string; + fontWeight?: number; +} + +/** + * Manifold Button — terminal-styled action. + * Variants: primary (solid orange), secondary (outlined raised), ghost (text). + * Sizes: sm, md, lg. Optional leading glyph. + */ +export function Button({ + children, + variant = 'secondary', + size = 'md', + disabled = false, + glyph, + active = false, + type = 'button', + onClick, + style, + ...rest +}: ButtonProps) { + const sizes: Record = { + sm: { padding: '4px 12px', fontSize: 'var(--fs-xs)', height: 28 }, + md: { padding: '8px 12px', fontSize: 'var(--fs-sm)', height: 34 }, + lg: { padding: '10px 18px', fontSize: 'var(--fs-md)', height: 44 }, + }; + const s = sizes[size] ?? sizes.md; + + const base: CSSProperties = { + display: 'inline-flex', + alignItems: 'center', + justifyContent: 'center', + gap: 'var(--sp-2)', + fontFamily: 'var(--font-mono)', + fontSize: s.fontSize, + height: s.height, + padding: s.padding, + borderRadius: 'var(--r-1)', + border: '1px solid var(--line)', + cursor: disabled ? 'not-allowed' : 'pointer', + userSelect: 'none', + transition: + 'background var(--dur-fast) var(--ease), border-color var(--dur-fast) var(--ease), color var(--dur-fast) var(--ease)', + whiteSpace: 'nowrap', + }; + + const variants: Record = { + primary: { + background: 'var(--accent)', + borderColor: 'var(--accent)', + color: 'var(--bg)', + fontWeight: 600, + }, + secondary: { + background: active ? 'var(--bg-3)' : 'var(--bg-2)', + borderColor: active ? 'var(--accent)' : 'var(--line)', + color: active ? 'var(--accent)' : 'var(--fg)', + }, + ghost: { + background: 'transparent', + borderColor: 'transparent', + color: active ? 'var(--accent)' : 'var(--fg-mute)', + }, + }; + + const v = variants[variant] ?? variants.secondary; + const disabledStyle: CSSProperties | null = disabled + ? { opacity: 0.45, color: 'var(--fg-dim)', boxShadow: 'none' } + : null; + + return ( + + ); +} diff --git a/manifold/src/primitives/ControlAxis.tsx b/manifold/src/primitives/ControlAxis.tsx new file mode 100644 index 0000000..cdebcf5 --- /dev/null +++ b/manifold/src/primitives/ControlAxis.tsx @@ -0,0 +1,131 @@ +import type { CSSProperties, ReactNode } from 'react'; + +export interface ControlAxisProps { + label?: ReactNode; + /** Bipolar endpoint labels, e.g. ['Caution', 'Bold']. */ + endpoints?: [ReactNode, ReactNode]; + value?: number; + onChange?: (value: number) => void; + /** Live preset tag shown next to the label. */ + preset?: ReactNode; + /** Per-axis track/thumb accent colour (any CSS colour or var()). */ + accent?: string; + disabled?: boolean; + style?: CSSProperties; +} + +/** + * Manifold ControlAxis — a named macro slider with bipolar endpoint labels + * (e.g. Boldness: Caution ↔ Bold). Shows a live preset tag and value. The + * track accent can be themed per-axis via `accent`. + * + * Relies on the `.mf-axis-input` rules in `styles/primitives.css`; the accent + * is passed via the inline `--mf-axis-accent` custom property. + */ +export function ControlAxis({ + label, + endpoints = ['', ''], + value = 0.5, + onChange, + preset, + accent = 'var(--accent)', + disabled = false, + style, +}: ControlAxisProps) { + return ( +
+
+ + {label} + + {preset && ( + + {preset} + + )} + + {value.toFixed(2)} + +
+ onChange?.(parseFloat(e.target.value))} + className="mf-axis-input" + style={ + { + WebkitAppearance: 'none', + appearance: 'none', + width: '100%', + height: 24, + background: 'transparent', + margin: 0, + cursor: 'pointer', + '--mf-axis-accent': accent, + } as CSSProperties + } + /> +
+ {endpoints[0]} + {endpoints[1]} +
+
+ ); +} diff --git a/manifold/src/primitives/CurvePlot.tsx b/manifold/src/primitives/CurvePlot.tsx new file mode 100644 index 0000000..251567a --- /dev/null +++ b/manifold/src/primitives/CurvePlot.tsx @@ -0,0 +1,131 @@ +import { useEffect, useRef } from 'react'; +import type { CSSProperties } from 'react'; + +export type CurveName = + | 'linear' + | 'exp' + | 'log' + | 'square' + | 'sqrt' + | 'sigmoid' + | 'cubic' + | 'centered_power'; + +export interface CurvePlotProps { + /** One of the named response curves. Ignored when `fn` is provided. */ + curve?: CurveName; + /** Custom response function f:[0,1]→[0,1]. Overrides `curve`. */ + fn?: (x: number) => number; + width?: number; + height?: number; + /** Stroke colour (any CSS colour or var()). */ + color?: string; + showAxes?: boolean; + ariaLabel?: string; + style?: CSSProperties; +} + +const clamp01 = (v: number): number => (v < 0 ? 0 : v > 1 ? 1 : v); + +const CURVES: Record number> = { + linear: (x) => x, + exp: (x) => (Math.exp(4 * x) - 1) / (Math.exp(4) - 1), + log: (x) => Math.log(1 + x * (Math.exp(4) - 1)) / 4, + square: (x) => x * x, + sqrt: (x) => Math.sqrt(clamp01(x)), + sigmoid: (x) => { + const s = (v: number) => 1 / (1 + Math.exp(-(v - 0.5) * 8)); + const lo = s(0); + const hi = s(1); + return (s(x) - lo) / (hi - lo); + }, + cubic: (x) => { + const v = clamp01(x); + return v * v * (3 - 2 * v); + }, + centered_power: (x) => { + const o = x - 0.5; + const sg = o < 0 ? -1 : 1; + return clamp01((sg * Math.pow(Math.abs(o) * 2, 0.5)) / 2 + 0.5); + }, +}; + +/** + * Manifold CurvePlot — renders one of the named response curves (or a custom + * function f:[0,1]→[0,1]) on the dark grid. The brand's straight-line & + * parabolic/bézier motif. + */ +export function CurvePlot({ + curve = 'cubic', + fn, + width = 200, + height = 120, + color = 'var(--accent)', + showAxes = true, + ariaLabel, + style, +}: CurvePlotProps) { + const ref = useRef(null); + + useEffect(() => { + const cv = ref.current; + if (!cv) return; + const dpr = window.devicePixelRatio || 1; + const w = width * dpr; + const h = height * dpr; + cv.width = w; + cv.height = h; + const ctx = cv.getContext('2d'); + if (!ctx) return; + ctx.clearRect(0, 0, w, h); + const cs = getComputedStyle(cv); + const stroke = color.startsWith('var(') + ? cs.getPropertyValue(color.slice(4, -1).trim()).trim() || '#ff6a00' + : color; + const pad = 6 * dpr; + + if (showAxes) { + ctx.strokeStyle = 'rgba(255,255,255,0.06)'; + ctx.lineWidth = 1; + ctx.strokeRect(0.5, 0.5, w - 1, h - 1); + ctx.beginPath(); + ctx.moveTo(0, h / 2); + ctx.lineTo(w, h / 2); + ctx.stroke(); + ctx.beginPath(); + ctx.moveTo(w / 2, 0); + ctx.lineTo(w / 2, h); + ctx.stroke(); + } + const f = fn || CURVES[curve] || CURVES.linear; + ctx.strokeStyle = stroke; + ctx.lineWidth = 2 * dpr; + ctx.beginPath(); + for (let p = 0; p <= 120; p++) { + const x = p / 120; + const y = clamp01(f(x)); + const px = pad + x * (w - 2 * pad); + const py = h - pad - y * (h - 2 * pad); + if (p === 0) ctx.moveTo(px, py); + else ctx.lineTo(px, py); + } + ctx.stroke(); + }, [curve, fn, width, height, color, showAxes]); + + return ( + + ); +} diff --git a/manifold/src/primitives/Panel.tsx b/manifold/src/primitives/Panel.tsx new file mode 100644 index 0000000..0ed88b7 --- /dev/null +++ b/manifold/src/primitives/Panel.tsx @@ -0,0 +1,82 @@ +import type { CSSProperties, ReactNode } from 'react'; + +export interface PanelProps { + title?: ReactNode; + /** Small uppercase eyebrow shown before the title. */ + label?: ReactNode; + /** Right-aligned header actions. */ + actions?: ReactNode; + children?: ReactNode; + padding?: string; + style?: CSSProperties; +} + +/** + * Manifold Panel — the house surface: bg-1 fill, 1px hairline border, 8px + * radius, no shadow. Optional header row with an uppercase title + actions, + * separated by a hairline. + */ +export function Panel({ + title, + label, + actions, + children, + padding = 'var(--sp-3)', + style, +}: PanelProps) { + return ( +
+ {(title || label || actions) && ( +
+ {label && ( + + {label} + + )} + {title && ( +

+ {title} +

+ )} + {actions && ( +
+ {actions} +
+ )} +
+ )} +
{children}
+
+ ); +} diff --git a/manifold/src/primitives/PillToggle.tsx b/manifold/src/primitives/PillToggle.tsx new file mode 100644 index 0000000..3040769 --- /dev/null +++ b/manifold/src/primitives/PillToggle.tsx @@ -0,0 +1,81 @@ +import type { CSSProperties, ReactNode } from 'react'; + +export interface PillOption { + value: T; + label: ReactNode; +} + +export interface PillToggleProps { + options?: PillOption[]; + value?: T; + onChange?: (value: T) => void; + ariaLabel?: string; + disabled?: boolean; + style?: CSSProperties; +} + +/** + * Manifold PillToggle — segmented radio control in a pill capsule. + * The selected segment fills solid orange. Options: [{value,label}]. + */ +export function PillToggle({ + options = [], + value, + onChange, + ariaLabel = 'segmented control', + disabled = false, + style, +}: PillToggleProps) { + return ( +
+ {options.map((opt) => { + const selected = value === opt.value; + return ( + + ); + })} +
+ ); +} diff --git a/manifold/src/primitives/Slider.tsx b/manifold/src/primitives/Slider.tsx new file mode 100644 index 0000000..a795251 --- /dev/null +++ b/manifold/src/primitives/Slider.tsx @@ -0,0 +1,107 @@ +import type { CSSProperties } from 'react'; + +export interface SliderProps { + label?: string; + value?: number; + min?: number; + max?: number; + step?: number; + unit?: string; + onChange?: (value: number) => void; + disabled?: boolean; + /** Custom formatter for the value readout. */ + format?: (value: number) => string; + style?: CSSProperties; +} + +/** + * Manifold Slider — labeled horizontal range with a glowing orange thumb and + * a tabular value readout. Controlled via value/onChange (0..max). + * + * Relies on the `.mf-slider-input` rules in `styles/primitives.css` for the + * track gradient and glowing thumb. The fill percentage is passed via the + * inline `--mf-pct` custom property. + */ +export function Slider({ + label, + value = 0, + min = 0, + max = 1, + step = 0.01, + unit = '', + onChange, + disabled = false, + format, + style, +}: SliderProps) { + const pct = max > min ? (value - min) / (max - min) : 0; + const display = format + ? format(value) + : Number.isInteger(step) + ? String(value) + : value.toFixed(2); + + return ( +
+ {label && ( + + {label} + + )} +
+ onChange?.(parseFloat(e.target.value))} + className="mf-slider-input" + style={ + { + flex: 1, + WebkitAppearance: 'none', + appearance: 'none', + background: 'transparent', + height: 24, + margin: 0, + cursor: 'pointer', + '--mf-pct': `${pct}`, + } as CSSProperties + } + /> + + {display} + {unit && {unit}} + +
+
+ ); +} diff --git a/manifold/src/primitives/Sparkline.tsx b/manifold/src/primitives/Sparkline.tsx new file mode 100644 index 0000000..5db5b8c --- /dev/null +++ b/manifold/src/primitives/Sparkline.tsx @@ -0,0 +1,116 @@ +import { useEffect, useRef } from 'react'; +import type { CSSProperties } from 'react'; + +export interface SparklineProps { + data?: number[]; + width?: number; + height?: number; + /** Stroke colour (any CSS colour or var()). */ + color?: string; + /** Plot on a log scale (log(max(1e-10, v) + 1)). */ + log?: boolean; + /** Render the last-value readout in the top-right. */ + showLast?: boolean; + /** Custom formatter for the last-value readout. */ + format?: (value: number) => string; + ariaLabel?: string; + style?: CSSProperties; +} + +/** + * Manifold Sparkline — a compact time-series trace (training loss, a feature + * envelope). Cyan line on a faint grid, with an optional last-value readout. + */ +export function Sparkline({ + data = [], + width = 320, + height = 70, + color = 'var(--accent-2)', + log = false, + showLast = true, + format, + ariaLabel = 'time series', + style, +}: SparklineProps) { + const ref = useRef(null); + + useEffect(() => { + const cv = ref.current; + if (!cv) return; + const dpr = window.devicePixelRatio || 1; + const w = width * dpr; + const h = height * dpr; + cv.width = w; + cv.height = h; + const ctx = cv.getContext('2d'); + if (!ctx) return; + ctx.clearRect(0, 0, w, h); + if (!data.length) return; + + const cs = getComputedStyle(cv); + const stroke = color.startsWith('var(') + ? cs.getPropertyValue(color.slice(4, -1).trim()).trim() || '#00ccff' + : color; + + const ys = data.map((v) => (log ? Math.log(Math.max(1e-10, v) + 1) : v)); + let lo = Infinity; + let hi = -Infinity; + for (const y of ys) { + if (y < lo) lo = y; + if (y > hi) hi = y; + } + if (hi === lo) hi = lo + 1e-6; + + ctx.strokeStyle = 'rgba(255,255,255,0.05)'; + ctx.lineWidth = 1; + for (let i = 1; i < 4; i++) { + const y = (i / 4) * h; + ctx.beginPath(); + ctx.moveTo(0, y); + ctx.lineTo(w, y); + ctx.stroke(); + } + + ctx.strokeStyle = stroke; + ctx.lineWidth = 1.5 * dpr; + ctx.beginPath(); + for (let i = 0; i < ys.length; i++) { + const x = (i / Math.max(1, ys.length - 1)) * w; + const norm = (ys[i] - lo) / (hi - lo); + const y = h - norm * h; + if (i === 0) ctx.moveTo(x, y); + else ctx.lineTo(x, y); + } + ctx.stroke(); + + if (showLast) { + const last = data[data.length - 1]; + const txt = format + ? format(last) + : typeof last === 'number' + ? last.toExponential(2) + : String(last); + ctx.fillStyle = '#9a9a9a'; + ctx.font = `${10 * dpr}px ui-monospace, monospace`; + ctx.textAlign = 'right'; + ctx.fillText(txt, w - 4 * dpr, 12 * dpr); + } + }, [data, width, height, color, log, showLast, format]); + + return ( + + ); +} diff --git a/manifold/src/primitives/StatusLine.tsx b/manifold/src/primitives/StatusLine.tsx new file mode 100644 index 0000000..fee9f5e --- /dev/null +++ b/manifold/src/primitives/StatusLine.tsx @@ -0,0 +1,74 @@ +import { Fragment } from 'react'; +import type { CSSProperties, ReactNode } from 'react'; + +export type StatusTone = 'accent' | 'cyan' | 'good' | 'warn' | 'bad'; + +export interface StatusItemObject { + label?: ReactNode; + value: ReactNode; + tone?: StatusTone; +} + +export type StatusItem = string | StatusItemObject; + +export interface StatusLineProps { + items?: StatusItem[]; + style?: CSSProperties; +} + +const TONE_COLORS: Record = { + accent: 'var(--accent)', + cyan: 'var(--accent-2)', + good: 'var(--good)', + warn: 'var(--warn)', + bad: 'var(--bad)', +}; + +/** + * Manifold StatusLine — the dim mono readout strip at the bottom of a mode. + * Pass an array of items; strings render plain, {label,value,tone} render a + * labelled readout. Items are joined with the house middle-dot separator. + */ +export function StatusLine({ items = [], style }: StatusLineProps) { + return ( +

+ {items.map((it, i) => { + const isObj = it !== null && typeof it === 'object'; + const toneColor = + isObj && it.tone ? (TONE_COLORS[it.tone] ?? null) : null; + return ( + + {i > 0 && } + {isObj ? ( + + {it.label && {it.label} } + + {it.value} + + + ) : ( + {it} + )} + + ); + })} +

+ ); +} diff --git a/manifold/src/primitives/Switch.tsx b/manifold/src/primitives/Switch.tsx new file mode 100644 index 0000000..4000598 --- /dev/null +++ b/manifold/src/primitives/Switch.tsx @@ -0,0 +1,73 @@ +import type { CSSProperties, ReactNode } from 'react'; + +export interface SwitchProps { + checked?: boolean; + onChange?: (checked: boolean) => void; + label?: ReactNode; + disabled?: boolean; + style?: CSSProperties; +} + +/** + * Manifold Switch — compact toggle. On = orange track + glow. Optional label. + */ +export function Switch({ + checked = false, + onChange, + label, + disabled = false, + style, +}: SwitchProps) { + return ( + + ); +} diff --git a/manifold/src/primitives/VirtualJoystick.tsx b/manifold/src/primitives/VirtualJoystick.tsx new file mode 100644 index 0000000..ebeacb2 --- /dev/null +++ b/manifold/src/primitives/VirtualJoystick.tsx @@ -0,0 +1,148 @@ +import { useRef, useState } from 'react'; +import type { CSSProperties, PointerEvent as ReactPointerEvent } from 'react'; + +export interface VirtualJoystickProps { + size?: number; + /** Controlled position as [x, y] in [0,1], y-up. Omit for uncontrolled. */ + position?: [number, number]; + onMove?: (x: number, y: number) => void; + onGrab?: () => void; + onRelease?: () => void; + disabled?: boolean; + ariaLabel?: string; + style?: CSSProperties; +} + +/** + * Manifold VirtualJoystick — circular control. Drag the glowing orange knob; + * motion is constrained to the circle. Emits normalised (x, y) in [0,1], y-up. + */ +export function VirtualJoystick({ + size = 200, + position, + onMove, + onGrab, + onRelease, + disabled = false, + ariaLabel = 'virtual joystick', + style, +}: VirtualJoystickProps) { + const [internal, setInternal] = useState<[number, number]>([0.5, 0.5]); + const [dragging, setDragging] = useState(false); + const ref = useRef(null); + const pos = position ?? internal; + + const update = (e: ReactPointerEvent) => { + const el = ref.current; + if (!el) return; + const r = el.getBoundingClientRect(); + let x = Math.max(0, Math.min(1, (e.clientX - r.left) / r.width)); + let y = Math.max(0, Math.min(1, 1 - (e.clientY - r.top) / r.height)); + const dx = x - 0.5; + const dy = y - 0.5; + const dist = Math.sqrt(dx * dx + dy * dy); + if (dist > 0.5 && dist > 1e-12) { + x = 0.5 + (dx / dist) * 0.5; + y = 0.5 + (dy / dist) * 0.5; + } + if (!position) setInternal([x, y]); + onMove?.(x, y); + }; + + const down = (e: ReactPointerEvent) => { + if (disabled) return; + e.currentTarget.setPointerCapture?.(e.pointerId); + setDragging(true); + onGrab?.(); + update(e); + }; + const move = (e: ReactPointerEvent) => { + if (dragging) update(e); + }; + const up = (e: ReactPointerEvent) => { + if (!dragging) return; + e.currentTarget.releasePointerCapture?.(e.pointerId); + setDragging(false); + onRelease?.(); + }; + + const [x, y] = pos; + return ( +
+
+