diff --git a/vcv/Makefile b/vcv/Makefile index d715a9d..7d004b3 100644 --- a/vcv/Makefile +++ b/vcv/Makefile @@ -2,7 +2,9 @@ RACK_DIR ?= $(HOME)/.local/share/Rack2/Rack-SDK FLAGS += -std=c++20 FLAGS += -I$(RACK_DIR)/include -I$(RACK_DIR)/dep/include -FLAGS += -I../nisps-core/include +# The retired nisps-core header tree is gone; the runtime IML/MLP is vendored +# self-contained in src/iml.hpp (see that file's header for the rationale). +FLAGS += -Isrc SOURCES += src/plugin.cpp SOURCES += src/MEMLNaut.cpp diff --git a/vcv/SPEC.md b/vcv/SPEC.md index a33b1bd..dac069d 100644 --- a/vcv/SPEC.md +++ b/vcv/SPEC.md @@ -1,5 +1,51 @@ # MEMLNaut VCV Rack Module — Specification +--- + +## ⚠️ BUILD DELTAS (2026-06-28) — AUTHORITATIVE OVERRIDES + +These supersede any conflicting detail below. They reflect the Manifold mission + the locked decisions in +`docs/redesign/BUILD-PLAN.md` and `docs/redesign/backends-spec.md`. Build to THESE. + +1. **I/O = 8 inputs × 16 outputs** (was 2→12). `NUM_ML_INPUTS = 8`, `NUM_ML_OUTPUTS = 16`. The IML is sized + 8→16 (a runtime-shaped native MLP is fine here — the module is C++, not the fixed WASM target). The 8 CV + inputs feed the model's input dims; the 16 CV outputs are the model's inference outputs (the modular N×M + envelope). Keep the control inputs (Spread CV, Learn gate, + / − triggers). +2. **LED RING around EACH of the 16 outputs** — a custom ring widget encircling each output jack whose arc + fills in proportion to that output's value (0..1 → 0..2π). Draw on `drawLayer()` layer 1 with `nvgArc` for + the proportional fill + a dim track ring. Each ring is COLOURED from a palette that MATCHES the frontend + design tokens. +3. **Palette from the frontend tokens** — generate `vcv/src/palette.hpp` from + `docs/redesign/manifold-export/tokens/colors.css`: `--accent #ff6a00` (orange), `--accent-2 #00ccff` (cyan), + and the group colours (formant→accent, pitch→accent-2, amp→`--good #6bc26b`, filter→`--warn #f5c45e`, + fx→`--info #5b9eef`, mod→`--accent-3 #ffa860`). Assign the 16 rings across these group colours (or a clean + 16-step ramp between orange and cyan) so the module reads as the same instrument as the browser. A tiny + hand-written `palette.hpp` is acceptable (no build-time codegen needed). +4. **Browser ↔ VCV bridge = WS↔OSC** (locked transport). The module runs its OSC server (`src/osc_server.hpp` + already exists — evolve it). The browser's OSC backend (`manifold/src/backends/osc-backend.ts`) sends over a + WebSocket to the Deno bridge (`manifold/osc-bridge/`), which relays UDP-OSC to the module. **Bidirectional + training**: drive + train the module FROM the browser (the verdict loop + example-placing over the bridge) + AND from the module's own panel (+/− buttons, Learn gate, triggers). OSC verbs to support both directions: + `/nisps/input` (drive), `/nisps/output` (module→browser viz), `/nisps/feedback` (thumbs up/down + place), + `/nisps/weights`, `/nisps/examples`, `/nisps/state`. Pick a fixed default UDP port (e.g. 7001) + per-instance + offset; the Deno bridge maps `ws://localhost:8765` ↔ that UDP port. +5. **Core include path** — the Makefile's `-I../nisps-core/include` points at the RETIRED `nisps-core`. Repoint + to the current core (`../nisps/`) OR vendor a minimal runtime IML inside `vcv/src/`. Goal: get it COMPILING + with an 8→16 runtime MLP that shares the firmware/browser training semantics (spread-aware draw/move_weights, + deterministic RNG) as closely as the native runtime-shaped form allows. If full nisps/ml reuse is blocked by + the templated fixed-size API, keep a self-contained IML in the module and note the alignment as a follow-up. +6. **Derived outputs** (Mean/Std/Delta/Novelty/Confidence) → move to the optional EXPANDER or a context-menu + toggle; the headline is 16 raw outputs + their LED rings. Do not let them crowd the 16-jack panel. +7. **Build** needs the VCV Rack 2 SDK (`RACK_DIR`, default `$HOME/.local/share/Rack2/Rack-SDK`, NOT installed). + The build step must fetch the Linux Rack-SDK zip from vcvrack.com, set `RACK_DIR`, and `make` to verify the + plugin compiles. Panel SVGs in `vcv/res/` exist (MEMLNaut.svg / -wide / -expander) — widen/relayout for 16 + outputs + rings as needed. + +The rest of this document is the prior (2→12) design — useful for threading, persistence, RL workflow, and +panel/build mechanics, but the I/O counts, LED rings, palette, bridge, and core path above WIN. + +--- + ## Overview A VCV Rack module that embeds the NISPS interactive ML engine (nisps-core C++ library) as a CV-to-CV mapper. Users explore high-dimensional parameter spaces via reinforcement learning feedback, producing 12 raw CV outputs and 5 derived meta-signals from configurable CV inputs. diff --git a/vcv/plugin.json b/vcv/plugin.json index 02d98ab..ac31469 100644 --- a/vcv/plugin.json +++ b/vcv/plugin.json @@ -1,7 +1,7 @@ { "slug": "MEMLNaut", "name": "MEMLNaut", - "version": "0.1.0", + "version": "0.2.0", "license": "proprietary", "brand": "MEMLNaut", "author": "MEML", diff --git a/vcv/src/LedRing.hpp b/vcv/src/LedRing.hpp new file mode 100644 index 0000000..2a41634 --- /dev/null +++ b/vcv/src/LedRing.hpp @@ -0,0 +1,75 @@ +// LedRing.hpp — Custom LED-ring widget encircling each output jack. +// +// The arc fills clockwise from 12 o'clock in proportion to the output's value +// (0..1 → 0..2π) and is drawn on drawLayer() layer 1 (so it stays bright when +// room brightness is lowered, per the VCV custom-light convention). A dim track +// ring sits underneath. Colour comes from the frontend design tokens +// (palette.hpp) so the module matches the browser. +// +// Templated on the module type so it can read the per-output value without a +// hard include cycle; MEMLNaut.cpp instantiates LedRingWidget. +#pragma once + +#include +#include "palette.hpp" + +using namespace rack; + +template +struct LedRingWidget : Widget { + TModule* module = nullptr; + int outIdx = 0; + NVGcolor ringColor = memlnaut::palette::accent(); + float radius = 9.f; // px around a PJ301M jack + + LedRingWidget() { + // Box large enough to host the ring around a ~22px jack. + box.size = Vec(radius * 2.f + 6.f, radius * 2.f + 6.f); + } + + float valueOf() const { + if (!module) return 0.f; + return clamp(module->ringValue(outIdx), 0.f, 1.f); + } + + void drawLayer(const DrawArgs& args, int layer) override { + if (layer != 1) { + Widget::drawLayer(args, layer); + return; + } + float v = valueOf(); + Vec c = box.size.div(2.f); + const float start = -M_PI / 2.f; // 12 o'clock + const float end = start + 2.f * M_PI; // full circle + + // Dim track ring (full circle). + nvgBeginPath(args.vg); + nvgArc(args.vg, c.x, c.y, radius, start, end, NVG_CW); + nvgStrokeColor(args.vg, nvgRGBA((unsigned char)(ringColor.r * 255), + (unsigned char)(ringColor.g * 255), + (unsigned char)(ringColor.b * 255), 40)); + nvgStrokeWidth(args.vg, 1.4f); + nvgStroke(args.vg); + + // Proportional value arc. + if (v > 0.001f) { + nvgBeginPath(args.vg); + nvgArc(args.vg, c.x, c.y, radius, start, start + v * 2.f * M_PI, NVG_CW); + nvgStrokeColor(args.vg, ringColor); + nvgStrokeWidth(args.vg, 1.9f); + nvgLineCap(args.vg, NVG_ROUND); + nvgStroke(args.vg); + + // Soft glow halo — the frontend "glow not shadow" signature. + nvgGlobalCompositeOperation(args.vg, NVG_LIGHTER); + nvgBeginPath(args.vg); + nvgArc(args.vg, c.x, c.y, radius, start, start + v * 2.f * M_PI, NVG_CW); + nvgStrokeColor(args.vg, nvgRGBAf(ringColor.r, ringColor.g, ringColor.b, 0.25f * v)); + nvgStrokeWidth(args.vg, 4.0f); + nvgStroke(args.vg); + nvgGlobalCompositeOperation(args.vg, NVG_SOURCE_OVER); + } + + Widget::drawLayer(args, layer); + } +}; diff --git a/vcv/src/MEMLNaut.cpp b/vcv/src/MEMLNaut.cpp index 59cfaf5..6b4f96d 100644 --- a/vcv/src/MEMLNaut.cpp +++ b/vcv/src/MEMLNaut.cpp @@ -1,6 +1,8 @@ #include "plugin.hpp" #include "osc_server.hpp" -#include +#include "iml.hpp" +#include "palette.hpp" +#include "LedRing.hpp" #include #include #include @@ -8,11 +10,18 @@ #include #include #include +#include -static constexpr int NUM_ML_INPUTS = 2; -static constexpr int NUM_ML_OUTPUTS = 12; +// ── I/O contract (SPEC BUILD DELTAS 2026-06-28): 8 inputs × 16 outputs ── +static constexpr int NUM_ML_INPUTS = 8; +static constexpr int NUM_ML_OUTPUTS = 16; static constexpr int MAX_ML_INPUTS = 8; +// OSC: a fixed default UDP listen port + a per-instance offset so multiple +// module instances in one patch don't collide. The Deno bridge maps +// ws://localhost:8765 ↔ this UDP port. +static constexpr int OSC_DEFAULT_PORT = 7001; + // ── Background job types ────────────────────────────────────────────── enum class JobType { Train, Perturb, Randomize, Clear }; struct Job { @@ -21,6 +30,17 @@ struct Job { float spread; }; +// ── Staged remote feedback op (from the OSC bridge) ─────────────────── +enum class FeedbackOp { None, Up, Down, Rand, Clear }; +struct StagedFeedback { + FeedbackOp op = FeedbackOp::None; + float spread = 0.6f; + bool hasInput = false; + bool hasOutput = false; + float input[MAX_ML_INPUTS] = {}; + float output[NUM_ML_OUTPUTS] = {}; +}; + // ── MEMLNaut Module ─────────────────────────────────────────────────── struct MEMLNaut : Module { enum ParamId { @@ -31,14 +51,13 @@ struct MEMLNaut : Module { PARAM_THUMBS_DOWN, PARAM_LEARN, PARAM_CLEAR, - PARAM_ATTEN_1, // 12 attenuverters + PARAM_ATTEN_1, // 16 attenuverters (kept in the model for range scaling) PARAM_ATTEN_LAST = PARAM_ATTEN_1 + NUM_ML_OUTPUTS - 1, PARAMS_LEN }; enum InputId { - INPUT_X, - INPUT_Y, - // IN 3–8 reserved for configurable inputs (future) + INPUT_1, // 8 model-input CV jacks + INPUT_LAST = INPUT_1 + NUM_ML_INPUTS - 1, INPUT_SPREAD_CV, INPUT_LEARN_GATE, INPUT_TRIG_POS, @@ -46,31 +65,20 @@ struct MEMLNaut : Module { INPUTS_LEN }; enum OutputId { - OUTPUT_1, OUTPUT_2, OUTPUT_3, OUTPUT_4, - OUTPUT_5, OUTPUT_6, OUTPUT_7, OUTPUT_8, - OUTPUT_9, OUTPUT_10, OUTPUT_11, OUTPUT_12, - OUTPUT_MEAN, - OUTPUT_STD, - OUTPUT_DELTA, - OUTPUT_NOVELTY, - OUTPUT_CONFIDENCE, + OUTPUT_1, // 16 inference-output CV jacks + OUTPUT_LAST = OUTPUT_1 + NUM_ML_OUTPUTS - 1, OUTPUTS_LEN }; enum LightId { LIGHT_LEARN, LIGHT_TRAINING, - LIGHT_OUT_1, // 12 output LEDs - LIGHT_OUT_LAST = LIGHT_OUT_1 + NUM_ML_OUTPUTS - 1, LIGHTS_LEN }; // ── ML Engine (double-buffered) ───────────────────────────────── - // THREADING INVARIANT: Only the audio thread touches `iml`. - // The worker thread operates exclusively on `imlShadow`. - // Communication is through atomic-flagged staging buffers: - // Audio → Worker: exampleStaging (mutex-protected) - // Worker → Audio: pendingWeights (atomic flag) - // OSC → Audio: oscStaging (atomic flag) + // THREADING INVARIANT: only the audio thread touches `iml`; the worker + // thread operates exclusively on `imlShadow`. Hand-off is through atomic- + // flagged staging buffers (see startOsc + workerLoop). nisps::IML iml{NUM_ML_INPUTS, NUM_ML_OUTPUTS, {16, 24, 16}}; nisps::IML imlShadow{NUM_ML_INPUTS, NUM_ML_OUTPUTS, {16, 24, 16}}; @@ -82,53 +90,83 @@ struct MEMLNaut : Module { nisps::MLP::mlp_weights stagedWeightsForWorker; std::vector> stagedFeatures; std::vector> stagedLabels; - std::mutex stagingMutex; // protects stagedWeightsForWorker, stagedFeatures, stagedLabels + std::mutex stagingMutex; // ── State ───────────────────────────────────────────────────────── std::atomic noiseLevel{0.1f}; float cachedOutputs[NUM_ML_OUTPUTS] = {}; float prevOutputs[NUM_ML_OUTPUTS] = {}; float slewOutputs[NUM_ML_OUTPUTS] = {}; - float lastInferenceOutputs[NUM_ML_OUTPUTS] = {}; // for linear interpolation - float lastOutputsForDelta[NUM_ML_OUTPUTS] = {}; // per-instance (NOT static) - float crossfadeProgress = 1.f; // 1 = no crossfade active + float lastOutputsForDelta[NUM_ML_OUTPUTS] = {}; + float crossfadeProgress = 1.f; float slewMs = 10.f; int sampleCounter = 0; - bool outputRangeUnipolar[NUM_ML_OUTPUTS] = {}; // true = 0-10V, false = ±5V - bool inputRangeUnipolar[MAX_ML_INPUTS] = {}; // true = 0-10V, false = ±5V + bool outputRangeUnipolar[NUM_ML_OUTPUTS] = {}; + bool inputRangeUnipolar[MAX_ML_INPUTS] = {}; float clearHoldTime = 0.f; - std::atomic cachedNovelty{10.f}; // default: everything novel (10V) - std::atomic cachedConfidence{0.f}; // default: no confidence (0V) + std::atomic cachedNovelty{10.f}; + std::atomic cachedConfidence{0.f}; float lastInputs[MAX_ML_INPUTS] = {}; - // OSC → Audio: staged data from OSC recv thread + // Derived outputs (Mean/Std/Delta/Novelty/Confidence) are OFF the main + // panel per SPEC delta #6 — kept as a context-menu computation toggle for + // future expander use. When disabled (default) they cost nothing. + bool computeDerived = false; + float derivedMean = 0.f, derivedStd = 0.f, derivedDelta = 0.f; + + // Bridged mode: when the browser streams /nisps/input, drive the model from + // those values instead of the physical CV jacks until the bridge goes quiet. + std::atomic bridgeDriveInputs{false}; + float bridgedInputs[MAX_ML_INPUTS] = {}; + std::mutex bridgedInputMutex; + + // OSC → Audio: staged JSON (state/weights) + staged feedback op std::string oscStagedJson; std::atomic oscJsonPending{false}; + StagedFeedback stagedFeedback; + std::atomic feedbackPending{false}; + std::mutex feedbackMutex; // ── OSC bridge ──────────────────────────────────────────────────── std::unique_ptr oscServer; bool oscEnabled = false; - int oscPort = 9000; + int oscPort = OSC_DEFAULT_PORT; int oscSendCounter = 0; static constexpr int OSC_SEND_INTERVAL_SAMPLES = 4410; // ~100ms at 44.1kHz + std::atomic stateDirty{false}; // set after a weight swap → push /nisps/state void startOsc() { if (oscServer && oscServer->isRunning()) return; oscServer = std::make_unique(); - // Stage received data for audio thread to apply (no direct mutation) + // Full state / weights JSON → stage for the audio thread to apply. oscServer->onState([this](const std::string& json) { - if (!oscJsonPending.load()) { - oscStagedJson = json; - oscJsonPending.store(true); - } + if (!oscJsonPending.load()) { oscStagedJson = json; oscJsonPending.store(true); } + }); + oscServer->onWeights([this](const std::string& json) { + if (!oscJsonPending.load()) { oscStagedJson = json; oscJsonPending.store(true); } }); - oscServer->onWeights([this](const std::string& json) { - if (!oscJsonPending.load()) { - oscStagedJson = json; - oscJsonPending.store(true); + // Live input vector from the browser → drive the model inputs. + oscServer->onInput([this](const std::vector& values) { + { + std::lock_guard lock(bridgedInputMutex); + for (int i = 0; i < NUM_ML_INPUTS && i < (int)values.size(); i++) + bridgedInputs[i] = clamp(values[i], 0.f, 1.f); } + bridgeDriveInputs.store(true); + }); + + // Verdict op from the browser → stage for the audio thread, which routes + // it through the SAME enqueueJob/add_example path the panel buttons use. + oscServer->onFeedback([this](const std::string& json) { + StagedFeedback fb = parseFeedback(json); + if (fb.op == FeedbackOp::None) return; + { + std::lock_guard lock(feedbackMutex); + stagedFeedback = fb; + } + feedbackPending.store(true); }); if (!oscServer->start(oscPort)) { @@ -140,11 +178,75 @@ struct MEMLNaut : Module { } void stopOsc() { - if (oscServer) { - oscServer->stop(); - oscServer.reset(); - } + if (oscServer) { oscServer->stop(); oscServer.reset(); } oscEnabled = false; + bridgeDriveInputs.store(false); + } + + // Minimal JSON-ish parse of the feedback op (avoids pulling jansson into the + // OSC recv thread). Reads "op", "spread", and optional "input"/"output". + static StagedFeedback parseFeedback(const std::string& s) { + StagedFeedback fb; + auto findStr = [&](const char* key) -> std::string { + std::string k = std::string("\"") + key + "\""; + size_t p = s.find(k); + if (p == std::string::npos) return ""; + p = s.find(':', p); + if (p == std::string::npos) return ""; + size_t q = s.find('"', p); + if (q == std::string::npos) return ""; + size_t r = s.find('"', q + 1); + if (r == std::string::npos) return ""; + return s.substr(q + 1, r - q - 1); + }; + auto findNum = [&](const char* key, float def) -> float { + std::string k = std::string("\"") + key + "\""; + size_t p = s.find(k); + if (p == std::string::npos) return def; + p = s.find(':', p); + if (p == std::string::npos) return def; + return (float)atof(s.c_str() + p + 1); + }; + auto findArr = [&](const char* key, float* out, int maxN) -> int { + std::string k = std::string("\"") + key + "\""; + size_t p = s.find(k); + if (p == std::string::npos) return 0; + p = s.find('[', p); + if (p == std::string::npos) return 0; + size_t e = s.find(']', p); + if (e == std::string::npos) return 0; + int n = 0; size_t cur = p + 1; + while (cur < e && n < maxN) { + while (cur < e && (s[cur] == ' ' || s[cur] == ',')) cur++; + if (cur >= e) break; + out[n++] = (float)atof(s.c_str() + cur); + size_t nx = s.find(',', cur); + if (nx == std::string::npos || nx > e) break; + cur = nx + 1; + } + return n; + }; + + std::string op = findStr("op"); + if (op == "up") fb.op = FeedbackOp::Up; + else if (op == "down") fb.op = FeedbackOp::Down; + else if (op == "rand") fb.op = FeedbackOp::Rand; + else if (op == "clear") fb.op = FeedbackOp::Clear; + else fb.op = FeedbackOp::None; + fb.spread = clamp(findNum("spread", 0.6f), 0.f, 1.f); + fb.hasInput = findArr("input", fb.input, MAX_ML_INPUTS) > 0; + fb.hasOutput = findArr("output", fb.output, NUM_ML_OUTPUTS) > 0; + return fb; + } + + // Build a compact JSON state snapshot (for module → browser sync). + std::string buildStateJson() { + json_t* root = dataToJson(); + char* str = json_dumps(root, JSON_COMPACT); + json_decref(root); + std::string out = str ? str : "{}"; + free(str); + return out; } // ── Triggers ────────────────────────────────────────────────────── @@ -168,51 +270,40 @@ struct MEMLNaut : Module { MEMLNaut() { config(PARAMS_LEN, INPUTS_LEN, OUTPUTS_LEN, LIGHTS_LEN); - // Knobs configParam(PARAM_SPREAD, 0.f, 1.f, 0.6f, "Spread", "%", 0.f, 100.f); configParam(PARAM_RATE, 0.f, 1.f, 0.5f, "Inference rate"); - // Buttons - configButton(PARAM_RAND, "Randomize weights"); + configButton(PARAM_RAND, "Randomise weights"); configButton(PARAM_THUMBS_UP, "Thumbs up (+)"); configButton(PARAM_THUMBS_DOWN, "Thumbs down (−)"); configSwitch(PARAM_LEARN, 0.f, 1.f, 0.f, "Learn enable", {"Off", "On"}); configButton(PARAM_CLEAR, "Clear (long-press)"); - // Attenuverters for (int i = 0; i < NUM_ML_OUTPUTS; i++) { configParam(PARAM_ATTEN_1 + i, -1.f, 1.f, 1.f, string::f("Out %d attenuverter", i + 1), "%", 0.f, 100.f); } - // Inputs - configInput(INPUT_X, "X"); - configInput(INPUT_Y, "Y"); + for (int i = 0; i < NUM_ML_INPUTS; i++) + configInput(INPUT_1 + i, string::f("In %d", i + 1)); configInput(INPUT_SPREAD_CV, "Spread CV"); configInput(INPUT_LEARN_GATE, "Learn gate"); configInput(INPUT_TRIG_POS, "+ trigger"); configInput(INPUT_TRIG_NEG, "− trigger"); - // Outputs - for (int i = 0; i < NUM_ML_OUTPUTS; i++) { + for (int i = 0; i < NUM_ML_OUTPUTS; i++) configOutput(OUTPUT_1 + i, string::f("Out %d", i + 1)); - } - configOutput(OUTPUT_MEAN, "Mean"); - configOutput(OUTPUT_STD, "Std deviation"); - configOutput(OUTPUT_DELTA, "Delta (rate of change)"); - configOutput(OUTPUT_NOVELTY, "Novelty"); - configOutput(OUTPUT_CONFIDENCE, "Confidence"); - // Init ranges to unipolar for (int i = 0; i < NUM_ML_OUTPUTS; i++) outputRangeUnipolar[i] = true; for (int i = 0; i < MAX_ML_INPUTS; i++) inputRangeUnipolar[i] = true; - // Randomize with default spread + // Per-instance OSC port offset (avoids collisions across instances). + oscPort = OSC_DEFAULT_PORT + (int)(id % 64); + iml.set_mode(nisps::IML::Mode::Training); iml.randomise_weights(0.6f); iml.set_mode(nisps::IML::Mode::Inference); - // Start worker thread workerThread = std::thread(&MEMLNaut::workerLoop, this); } @@ -220,9 +311,13 @@ struct MEMLNaut : Module { stopOsc(); shouldStop.store(true); jobCv.notify_one(); - if (workerThread.joinable()) { - workerThread.join(); - } + if (workerThread.joinable()) workerThread.join(); + } + + // Read by the LED-ring widget (per-output 0..1 value). + float ringValue(int i) const { + if (i < 0 || i >= NUM_ML_OUTPUTS) return 0.f; + return slewOutputs[i]; } // ── Background worker ───────────────────────────────────────────── @@ -239,7 +334,6 @@ struct MEMLNaut : Module { isTraining.store(true); - // Load staged weights + examples into shadow (safe: staging is mutex-protected) { std::lock_guard lock(stagingMutex); imlShadow.set_weights(stagedWeightsForWorker); @@ -248,7 +342,7 @@ struct MEMLNaut : Module { if (job.type == JobType::Train) { imlShadow.set_mode(nisps::IML::Mode::Training); - imlShadow.set_mode(nisps::IML::Mode::Inference); + imlShadow.set_mode(nisps::IML::Mode::Inference); // triggers train_() } else if (job.type == JobType::Perturb) { imlShadow.move_weights(job.noiseLevel, job.spread); } else if (job.type == JobType::Randomize) { @@ -263,16 +357,13 @@ struct MEMLNaut : Module { noiseLevel.store(0.1f); } - // Wait for audio thread to consume previous weights before staging new ones - while (weightsPending.load() && !shouldStop.load()) { + while (weightsPending.load() && !shouldStop.load()) std::this_thread::sleep_for(std::chrono::microseconds(100)); - } if (shouldStop.load()) break; pendingWeights = imlShadow.get_weights(); weightsPending.store(true); - // Compute novelty/confidence on shadow's dataset (safe: no concurrent access) if (imlShadow.get_example_count() > 0) { float inputs[MAX_ML_INPUTS]; for (int i = 0; i < NUM_ML_INPUTS; i++) inputs[i] = lastInputs[i]; @@ -286,7 +377,6 @@ struct MEMLNaut : Module { isTraining.store(false); - // Check for pending work { std::unique_lock lock(jobMutex); if (hasPending) { @@ -301,7 +391,6 @@ struct MEMLNaut : Module { void enqueueJob(JobType type, float noise = 0.f, float spread = 0.f) { std::unique_lock lock(jobMutex); if (hasJob || isTraining.load()) { - // Queue as pending (max depth 1, latest wins) pendingJob = {type, noise, spread}; hasPending = true; } else { @@ -311,16 +400,13 @@ struct MEMLNaut : Module { } } - // ── Helper: read spread with CV modulation ──────────────────────── float getSpread() { float spread = params[PARAM_SPREAD].getValue(); - if (inputs[INPUT_SPREAD_CV].isConnected()) { + if (inputs[INPUT_SPREAD_CV].isConnected()) spread += inputs[INPUT_SPREAD_CV].getVoltage() / 10.f; - } return clamp(spread, 0.f, 1.f); } - // ── Helper: is learning enabled ─────────────────────────────────── bool isLearnEnabled() { bool toggle = params[PARAM_LEARN].getValue() > 0.5f; bool gate = inputs[INPUT_LEARN_GATE].isConnected() && @@ -328,24 +414,52 @@ struct MEMLNaut : Module { return toggle || gate; } - // ── Helper: normalize input CV ──────────────────────────────────── - float normalizeInput(int inputId, int rangeIdx) { - float v = inputs[inputId].getVoltage(); - if (inputRangeUnipolar[rangeIdx]) { - return clamp(v / 10.f, 0.f, 1.f); - } else { - return clamp((v + 5.f) / 10.f, 0.f, 1.f); + // Normalise a model-input CV jack to [0,1]. When the bridge is driving + // inputs, that value wins. + float modelInput(int idx) { + if (bridgeDriveInputs.load()) { + std::lock_guard lock(bridgedInputMutex); + return clamp(bridgedInputs[idx], 0.f, 1.f); } + float v = inputs[INPUT_1 + idx].getVoltage(); + if (inputRangeUnipolar[idx]) return clamp(v / 10.f, 0.f, 1.f); + return clamp((v + 5.f) / 10.f, 0.f, 1.f); } - // ── Helper: scale output to CV ──────────────────────────────────── float outputToVoltage(float val01, int outIdx) { float atten = params[PARAM_ATTEN_1 + outIdx].getValue(); - if (outputRangeUnipolar[outIdx]) { - return val01 * 10.f * atten; - } else { - return (val01 - 0.5f) * 10.f * atten; - } + if (outputRangeUnipolar[outIdx]) return val01 * 10.f * atten; + return (val01 - 0.5f) * 10.f * atten; + } + + // Stage the current iml state for the worker thread. + void stageForWorker() { + std::lock_guard lock(stagingMutex); + stagedWeightsForWorker = iml.get_weights(); + stagedFeatures = iml.get_example_features(); + stagedLabels = iml.get_example_labels(); + } + + // Add the current (input,output) pair as an example + enqueue training. + void doThumbsUp(float spread) { + const float* curOuts = iml.get_outputs(); + float curInputs[MAX_ML_INPUTS]; + for (int i = 0; i < NUM_ML_INPUTS; i++) curInputs[i] = modelInput(i); + iml.set_mode(nisps::IML::Mode::Training); + iml.add_example(curInputs, NUM_ML_INPUTS, curOuts, NUM_ML_OUTPUTS); + iml.set_mode(nisps::IML::Mode::Inference); + stageForWorker(); + enqueueJob(JobType::Train); + noiseLevel.store(noiseLevel.load() * 0.97f); + (void)spread; + } + + void doThumbsDown(float spread) { + float noiseCap = 0.3f * (1.f - spread) + 0.05f * spread; + float nl = std::min(noiseLevel.load() * 1.5f, noiseCap); + noiseLevel.store(nl); + stageForWorker(); + enqueueJob(JobType::Perturb, nl, spread); } // ── Process ─────────────────────────────────────────────────────── @@ -353,51 +467,65 @@ struct MEMLNaut : Module { float spread = getSpread(); bool learn = isLearnEnabled(); - // ── Learn LED ───────────────────────────────────────────────── lights[LIGHT_LEARN].setBrightness(learn ? 1.f : 0.f); lights[LIGHT_TRAINING].setBrightness(isTraining.load() ? 1.f : 0.f); - // ── Apply staged OSC data ───────────────────────────────────── + // Apply staged OSC state/weights JSON. if (oscJsonPending.load()) { json_error_t error; json_t* root = json_loads(oscStagedJson.c_str(), 0, &error); - if (root) { - dataFromJson(root); - json_decref(root); - } + if (root) { dataFromJson(root); json_decref(root); } oscJsonPending.store(false); } - // ── Apply new weights from background thread ───────────────── + // Apply staged remote feedback (browser verdict over the bridge) — routes + // through the same paths as the panel buttons. + if (feedbackPending.load()) { + StagedFeedback fb; + { std::lock_guard lock(feedbackMutex); fb = stagedFeedback; } + feedbackPending.store(false); + float fbSpread = fb.spread; + if (fb.op == FeedbackOp::Up) { + if (fb.hasInput && fb.hasOutput) { + iml.set_mode(nisps::IML::Mode::Training); + iml.add_example(fb.input, NUM_ML_INPUTS, fb.output, NUM_ML_OUTPUTS); + iml.set_mode(nisps::IML::Mode::Inference); + stageForWorker(); + enqueueJob(JobType::Train); + noiseLevel.store(noiseLevel.load() * 0.97f); + } else { + doThumbsUp(fbSpread); + } + } else if (fb.op == FeedbackOp::Down) { + doThumbsDown(fbSpread); + } else if (fb.op == FeedbackOp::Rand) { + stageForWorker(); + enqueueJob(JobType::Randomize, 0.f, fbSpread); + } else if (fb.op == FeedbackOp::Clear) { + stageForWorker(); + enqueueJob(JobType::Clear, 0.f, fbSpread); + } + } + + // Apply new weights from the background thread. if (weightsPending.load()) { iml.set_weights(pendingWeights); weightsPending.store(false); - // Also sync examples from shadow → main (for future training rounds) auto newFeats = imlShadow.get_example_features(); auto newLabels = imlShadow.get_example_labels(); iml.load_examples(newFeats, newLabels); - // Start crossfade - for (int i = 0; i < NUM_ML_OUTPUTS; i++) { - prevOutputs[i] = cachedOutputs[i]; - } + for (int i = 0; i < NUM_ML_OUTPUTS; i++) prevOutputs[i] = cachedOutputs[i]; crossfadeProgress = 0.f; + stateDirty.store(true); // push fresh /nisps/state to the browser } - // ── Helper: stage current iml state for worker thread ───────── - auto stageForWorker = [&]() { - std::lock_guard lock(stagingMutex); - stagedWeightsForWorker = iml.get_weights(); - stagedFeatures = iml.get_example_features(); - stagedLabels = iml.get_example_labels(); - }; - - // ── Handle RAND button → enqueue Randomize job ──────────────── + // RAND button → enqueue Randomize. if (randTrigger.process(params[PARAM_RAND].getValue() > 0.f)) { stageForWorker(); enqueueJob(JobType::Randomize, 0.f, spread); } - // ── Handle CLEAR button (long-press ~1s) → enqueue Clear job ─ + // CLEAR long-press (~1s) → enqueue Clear. if (params[PARAM_CLEAR].getValue() > 0.f) { clearHoldTime += args.sampleTime; if (clearHoldTime >= 1.f) { @@ -409,132 +537,81 @@ struct MEMLNaut : Module { clearHoldTime = 0.f; } - // ── Handle RL feedback (only when learning) ─────────────────── + // RL feedback from the panel (only when learning). if (learn) { - bool thumbsUp = thumbsUpTrigger.process( - params[PARAM_THUMBS_UP].getValue() > 0.f); - bool trigPos = trigPosTrigger.process( - inputs[INPUT_TRIG_POS].getVoltage()); - if (thumbsUp || trigPos) { - // Add example to iml's dataset (audio thread owns iml) - const float* curOuts = iml.get_outputs(); - float curInputs[2] = { - normalizeInput(INPUT_X, 0), - normalizeInput(INPUT_Y, 1) - }; - iml.set_mode(nisps::IML::Mode::Training); - iml.add_example(curInputs, 2, curOuts, NUM_ML_OUTPUTS); - iml.set_mode(nisps::IML::Mode::Inference); - // Stage and enqueue training - stageForWorker(); - enqueueJob(JobType::Train); - noiseLevel.store(noiseLevel.load() * 0.97f); - } + bool thumbsUp = thumbsUpTrigger.process(params[PARAM_THUMBS_UP].getValue() > 0.f); + bool trigPos = trigPosTrigger.process(inputs[INPUT_TRIG_POS].getVoltage()); + if (thumbsUp || trigPos) doThumbsUp(spread); - bool thumbsDown = thumbsDownTrigger.process( - params[PARAM_THUMBS_DOWN].getValue() > 0.f); - bool trigNeg = trigNegTrigger.process( - inputs[INPUT_TRIG_NEG].getVoltage()); - if (thumbsDown || trigNeg) { - float noiseCap = 0.3f * (1.f - spread) + 0.05f * spread; - float nl = std::min(noiseLevel.load() * 1.5f, noiseCap); - noiseLevel.store(nl); - stageForWorker(); - enqueueJob(JobType::Perturb, nl, spread); - } + bool thumbsDown = thumbsDownTrigger.process(params[PARAM_THUMBS_DOWN].getValue() > 0.f); + bool trigNeg = trigNegTrigger.process(inputs[INPUT_TRIG_NEG].getVoltage()); + if (thumbsDown || trigNeg) doThumbsDown(spread); } - // ── Inference rate decimation ───────────────────────────────── + // Inference-rate decimation: 256 samples (block rate) → 1 (audio rate). float rate = params[PARAM_RATE].getValue(); - // Map 0→1 to period: 256 samples (block rate) → 1 sample (audio rate) - // Exponential mapping for perceptual linearity int period = std::max(1, (int)(256.f * std::pow(1.f / 256.f, rate))); sampleCounter++; bool runInference = (sampleCounter >= period); if (runInference) { sampleCounter = 0; - - // Read and normalize inputs - float x = normalizeInput(INPUT_X, 0); - float y = normalizeInput(INPUT_Y, 1); - lastInputs[0] = x; - lastInputs[1] = y; - - iml.set_input(0, x); - iml.set_input(1, y); - iml.process(); - - const float* outs = iml.get_outputs(); - for (int i = 0; i < NUM_ML_OUTPUTS; i++) { - cachedOutputs[i] = outs[i]; + for (int i = 0; i < NUM_ML_INPUTS; i++) { + float v = modelInput(i); + lastInputs[i] = v; + iml.set_input(i, v); } + iml.process(); + const float* outs = iml.get_outputs(); + for (int i = 0; i < NUM_ML_OUTPUTS; i++) cachedOutputs[i] = outs[i]; } - // ── Crossfade after weight swap ─────────────────────────────── + // Crossfade after a weight swap. float effectiveOutputs[NUM_ML_OUTPUTS]; if (crossfadeProgress < 1.f) { float slewSamples = std::max(1.f, slewMs * 0.001f * args.sampleRate); crossfadeProgress += 1.f / slewSamples; if (crossfadeProgress > 1.f) crossfadeProgress = 1.f; - for (int i = 0; i < NUM_ML_OUTPUTS; i++) { + for (int i = 0; i < NUM_ML_OUTPUTS; i++) effectiveOutputs[i] = prevOutputs[i] + crossfadeProgress * (cachedOutputs[i] - prevOutputs[i]); - } } else { - for (int i = 0; i < NUM_ML_OUTPUTS; i++) { - effectiveOutputs[i] = cachedOutputs[i]; - } + for (int i = 0; i < NUM_ML_OUTPUTS; i++) effectiveOutputs[i] = cachedOutputs[i]; } - // ── Interpolate between inference steps (slew) ──────────────── + // Interpolate between inference steps (slew). if (!runInference && period > 1) { float alpha = (float)sampleCounter / (float)period; - for (int i = 0; i < NUM_ML_OUTPUTS; i++) { + for (int i = 0; i < NUM_ML_OUTPUTS; i++) slewOutputs[i] += alpha * (effectiveOutputs[i] - slewOutputs[i]); - } } else { - for (int i = 0; i < NUM_ML_OUTPUTS; i++) { - slewOutputs[i] = effectiveOutputs[i]; - } + for (int i = 0; i < NUM_ML_OUTPUTS; i++) slewOutputs[i] = effectiveOutputs[i]; } - // ── Write raw outputs with attenuverters ────────────────────── - for (int i = 0; i < NUM_ML_OUTPUTS; i++) { + // Write the 16 outputs (with attenuverters). + for (int i = 0; i < NUM_ML_OUTPUTS; i++) outputs[OUTPUT_1 + i].setVoltage(outputToVoltage(slewOutputs[i], i)); - lights[LIGHT_OUT_1 + i].setBrightness(slewOutputs[i]); + + // Derived stats — computed only when the context-menu toggle is on. + if (computeDerived) { + float mean = 0.f; + for (int i = 0; i < NUM_ML_OUTPUTS; i++) mean += slewOutputs[i]; + mean /= NUM_ML_OUTPUTS; + float variance = 0.f, delta = 0.f; + for (int i = 0; i < NUM_ML_OUTPUTS; i++) { + float d = slewOutputs[i] - mean; variance += d * d; + float dd = slewOutputs[i] - lastOutputsForDelta[i]; + delta += dd * dd; lastOutputsForDelta[i] = slewOutputs[i]; + } + derivedMean = mean; + derivedStd = std::sqrt(variance / NUM_ML_OUTPUTS); + derivedDelta = std::sqrt(delta); } - // ── Derived outputs ─────────────────────────────────────────── - // Mean - float mean = 0.f; - for (int i = 0; i < NUM_ML_OUTPUTS; i++) mean += slewOutputs[i]; - mean /= NUM_ML_OUTPUTS; - outputs[OUTPUT_MEAN].setVoltage(mean * 10.f); - - // STD - float variance = 0.f; - for (int i = 0; i < NUM_ML_OUTPUTS; i++) { - float d = slewOutputs[i] - mean; - variance += d * d; - } - float stddev = std::sqrt(variance / NUM_ML_OUTPUTS); - outputs[OUTPUT_STD].setVoltage(stddev * 10.f); - - // Delta (L2 norm of change) - float delta = 0.f; - for (int i = 0; i < NUM_ML_OUTPUTS; i++) { - float d = slewOutputs[i] - lastOutputsForDelta[i]; - delta += d * d; - lastOutputsForDelta[i] = slewOutputs[i]; - } - outputs[OUTPUT_DELTA].setVoltage(std::sqrt(delta) * 10.f); - - // Novelty + Confidence (computed on background thread, cached) - outputs[OUTPUT_NOVELTY].setVoltage(cachedNovelty.load()); - outputs[OUTPUT_CONFIDENCE].setVoltage(cachedConfidence.load()); - - // ── OSC send (throttled to ~100ms) ─────────────────────────── + // OSC send (throttled to ~100ms), plus an immediate state push when dirty. if (oscServer && oscServer->isRunning()) { + if (stateDirty.exchange(false)) { + oscServer->sendState(buildStateJson()); + } oscSendCounter++; if (oscSendCounter >= OSC_SEND_INTERVAL_SAMPLES) { oscSendCounter = 0; @@ -547,43 +624,38 @@ struct MEMLNaut : Module { // ── Serialization ───────────────────────────────────────────────── json_t* dataToJson() override { json_t* root = json_object(); - json_object_set_new(root, "version", json_integer(1)); + json_object_set_new(root, "version", json_integer(2)); + json_object_set_new(root, "inputCount", json_integer(NUM_ML_INPUTS)); + json_object_set_new(root, "outputCount", json_integer(NUM_ML_OUTPUTS)); json_object_set_new(root, "noiseLevel", json_real(noiseLevel)); json_object_set_new(root, "slewMs", json_real(slewMs)); + json_object_set_new(root, "computeDerived", json_boolean(computeDerived)); json_object_set_new(root, "oscEnabled", json_boolean(oscEnabled)); json_object_set_new(root, "oscPort", json_integer(oscPort)); - // Output ranges json_t* outRanges = json_array(); - for (int i = 0; i < NUM_ML_OUTPUTS; i++) { + for (int i = 0; i < NUM_ML_OUTPUTS; i++) json_array_append_new(outRanges, json_boolean(outputRangeUnipolar[i])); - } json_object_set_new(root, "outputRangeUnipolar", outRanges); - // Input ranges json_t* inRanges = json_array(); - for (int i = 0; i < MAX_ML_INPUTS; i++) { + for (int i = 0; i < MAX_ML_INPUTS; i++) json_array_append_new(inRanges, json_boolean(inputRangeUnipolar[i])); - } json_object_set_new(root, "inputRangeUnipolar", inRanges); - // MLP weights (3D: layer → node → weight) auto weights = iml.get_weights(); json_t* jWeights = json_array(); for (auto& layer : weights) { json_t* jLayer = json_array(); for (auto& node : layer) { json_t* jNode = json_array(); - for (float w : node) { - json_array_append_new(jNode, json_real(w)); - } + for (float w : node) json_array_append_new(jNode, json_real(w)); json_array_append_new(jLayer, jNode); } json_array_append_new(jWeights, jLayer); } json_object_set_new(root, "weights", jWeights); - // Training examples auto features = iml.get_example_features(); auto labels = iml.get_example_labels(); json_t* jExamples = json_object(); @@ -603,11 +675,9 @@ struct MEMLNaut : Module { json_object_set_new(jExamples, "labels", jLabels); json_object_set_new(root, "examples", jExamples); - // MLP config (for validation on load) json_t* jConfig = json_object(); json_t* jLayers = json_array(); - // [3, 16, 24, 16, 12] for default config - json_array_append_new(jLayers, json_integer(NUM_ML_INPUTS + 1)); // +bias + json_array_append_new(jLayers, json_integer(NUM_ML_INPUTS + 1)); // + bias for (int h : {16, 24, 16}) json_array_append_new(jLayers, json_integer(h)); json_array_append_new(jLayers, json_integer(NUM_ML_OUTPUTS)); json_object_set_new(jConfig, "layers", jLayers); @@ -618,38 +688,25 @@ struct MEMLNaut : Module { void dataFromJson(json_t* root) override { json_t* j; - if ((j = json_object_get(root, "noiseLevel"))) - noiseLevel = json_real_value(j); - if ((j = json_object_get(root, "slewMs"))) - slewMs = json_real_value(j); + if ((j = json_object_get(root, "noiseLevel"))) noiseLevel = json_real_value(j); + if ((j = json_object_get(root, "slewMs"))) slewMs = json_real_value(j); + if ((j = json_object_get(root, "computeDerived"))) computeDerived = json_boolean_value(j); - // OSC - if ((j = json_object_get(root, "oscPort"))) - oscPort = json_integer_value(j); + if ((j = json_object_get(root, "oscPort"))) oscPort = json_integer_value(j); if ((j = json_object_get(root, "oscEnabled"))) { - if (json_boolean_value(j)) - startOsc(); - else - stopOsc(); + if (json_boolean_value(j)) startOsc(); else stopOsc(); } - // Output ranges json_t* outRanges = json_object_get(root, "outputRangeUnipolar"); - if (outRanges) { - for (int i = 0; i < NUM_ML_OUTPUTS && i < (int)json_array_size(outRanges); i++) { + if (outRanges) + for (int i = 0; i < NUM_ML_OUTPUTS && i < (int)json_array_size(outRanges); i++) outputRangeUnipolar[i] = json_boolean_value(json_array_get(outRanges, i)); - } - } - // Input ranges json_t* inRanges = json_object_get(root, "inputRangeUnipolar"); - if (inRanges) { - for (int i = 0; i < MAX_ML_INPUTS && i < (int)json_array_size(inRanges); i++) { + if (inRanges) + for (int i = 0; i < MAX_ML_INPUTS && i < (int)json_array_size(inRanges); i++) inputRangeUnipolar[i] = json_boolean_value(json_array_get(inRanges, i)); - } - } - // MLP weights json_t* jWeights = json_object_get(root, "weights"); if (jWeights && json_is_array(jWeights)) { nisps::MLP::mlp_weights weights; @@ -659,9 +716,8 @@ struct MEMLNaut : Module { for (size_t ni = 0; ni < json_array_size(jLayer); ni++) { json_t* jNode = json_array_get(jLayer, ni); std::vector node; - for (size_t wi = 0; wi < json_array_size(jNode); wi++) { + for (size_t wi = 0; wi < json_array_size(jNode); wi++) node.push_back(json_real_value(json_array_get(jNode, wi))); - } layer.push_back(node); } weights.push_back(layer); @@ -669,7 +725,6 @@ struct MEMLNaut : Module { iml.set_weights(weights); } - // Training examples json_t* jExamples = json_object_get(root, "examples"); if (jExamples) { json_t* jFeatures = json_object_get(jExamples, "features"); @@ -696,53 +751,49 @@ struct MEMLNaut : Module { } }; -// ── NanoVG Bar Graph Display ────────────────────────────────────────── +// ── NanoVG Bar Graph Display (16 bars) ──────────────────────────────── struct MEMLNautDisplay : LedDisplay { MEMLNaut* module = nullptr; void drawLayer(const DrawArgs& args, int layer) override { if (layer != 1 || !module) return; - nvgSave(args.vg); float w = box.size.x; float h = box.size.y; - float barW = (w - 4.f) / NUM_ML_OUTPUTS; float margin = 2.f; + float barW = (w - 4.f) / NUM_ML_OUTPUTS; - // Background nvgBeginPath(args.vg); nvgRect(args.vg, 0, 0, w, h); - nvgFillColor(args.vg, nvgRGB(0x10, 0x10, 0x18)); + nvgFillColor(args.vg, nvgRGB(0x0d, 0x0d, 0x0d)); // --bg nvgFill(args.vg); - // Output bars for (int i = 0; i < NUM_ML_OUTPUTS; i++) { float val = module->slewOutputs[i]; - float barH = val * (h - 16.f); - - // Color: hue based on output index - float hue = (float)i / NUM_ML_OUTPUTS; - NVGcolor color = nvgHSLA(hue, 0.7f, 0.5f, 200); - + float barH = clamp(val, 0.f, 1.f) * (h - 16.f); + NVGcolor color = memlnaut::palette::ring(i); nvgBeginPath(args.vg); nvgRect(args.vg, margin + i * barW, h - 8.f - barH, barW - 1.f, barH); nvgFillColor(args.vg, color); nvgFill(args.vg); } - // Status text nvgFontSize(args.vg, 8.f); - nvgFillColor(args.vg, nvgRGB(0xa0, 0xa0, 0xa0)); + nvgFillColor(args.vg, nvgRGB(0x9a, 0x9a, 0x9a)); // --fg-mute nvgTextAlign(args.vg, NVG_ALIGN_LEFT | NVG_ALIGN_TOP); - char buf[64]; - snprintf(buf, sizeof(buf), "N:%.3f", module->noiseLevel); + snprintf(buf, sizeof(buf), "N:%.3f %d/%d", module->noiseLevel.load(), + (int)module->iml.get_example_count(), (int)module->iml.get_max_examples()); nvgText(args.vg, 2.f, 1.f, buf, nullptr); if (module->isTraining.load()) { - nvgFillColor(args.vg, nvgRGB(0xff, 0xa0, 0x00)); - nvgText(args.vg, w - 24.f, 1.f, "TRAIN", nullptr); + nvgFillColor(args.vg, memlnaut::palette::accent()); + nvgText(args.vg, w - 28.f, 1.f, "TRAIN", nullptr); + } + if (module->bridgeDriveInputs.load()) { + nvgFillColor(args.vg, memlnaut::palette::accent2()); + nvgText(args.vg, w - 60.f, 1.f, "BRIDGE", nullptr); } nvgRestore(args.vg); @@ -753,66 +804,62 @@ struct MEMLNautDisplay : LedDisplay { struct MEMLNautWidget : ModuleWidget { MEMLNautWidget(MEMLNaut* module) { setModule(module); - setPanel(createPanel(asset::plugin(pluginInstance, "res/MEMLNaut.svg"))); + setPanel(createPanel(asset::plugin(pluginInstance, "res/MEMLNaut-wide.svg"))); - float col1 = 8.f; // left column - float col2 = 20.f; // center-left - float col3 = 32.f; // center-right - // float col4 = 44.f; // right column (for wide panel) - float y = 14.f; + float y = 13.f; - // ── Display ─────────────────────────────────────────────────── - MEMLNautDisplay* display = createWidget(mm2px(Vec(2.f, y))); - display->box.size = mm2px(Vec(36.f, 18.f)); + // Display. + MEMLNautDisplay* display = createWidget(mm2px(Vec(4.f, y))); + display->box.size = mm2px(Vec(58.f, 16.f)); display->module = module; addChild(display); - y += 22.f; + y += 20.f; - // ── SPREAD + RATE knobs ─────────────────────────────────────── - addParam(createParamCentered(mm2px(Vec(col1, y)), module, MEMLNaut::PARAM_SPREAD)); - addInput(createInputCentered(mm2px(Vec(col2, y)), module, MEMLNaut::INPUT_SPREAD_CV)); - addParam(createParamCentered(mm2px(Vec(col3, y)), module, MEMLNaut::PARAM_RATE)); - y += 10.f; + // SPREAD + RATE + Spread CV. + addParam(createParamCentered(mm2px(Vec(10.f, y)), module, MEMLNaut::PARAM_SPREAD)); + addInput(createInputCentered(mm2px(Vec(22.f, y)), module, MEMLNaut::INPUT_SPREAD_CV)); + addParam(createParamCentered(mm2px(Vec(34.f, y)), module, MEMLNaut::PARAM_RATE)); + // Buttons: + − LEARN RAND CLEAR. + addParam(createParamCentered(mm2px(Vec(46.f, y)), module, MEMLNaut::PARAM_THUMBS_UP)); + addParam(createParamCentered(mm2px(Vec(52.f, y)), module, MEMLNaut::PARAM_THUMBS_DOWN)); + addParam(createParamCentered(mm2px(Vec(58.f, y)), module, MEMLNaut::PARAM_LEARN)); + addChild(createLightCentered>(mm2px(Vec(58.f, y - 5.f)), module, MEMLNaut::LIGHT_LEARN)); + y += 9.f; + addParam(createParamCentered(mm2px(Vec(46.f, y)), module, MEMLNaut::PARAM_RAND)); + addParam(createParamCentered(mm2px(Vec(52.f, y)), module, MEMLNaut::PARAM_CLEAR)); + addChild(createLightCentered>(mm2px(Vec(58.f, y)), module, MEMLNaut::LIGHT_TRAINING)); - // ── Buttons row: + − LEARN RAND CLEAR ──────────────────────── - addParam(createParamCentered(mm2px(Vec(col1 - 2.f, y)), module, MEMLNaut::PARAM_THUMBS_UP)); - addParam(createParamCentered(mm2px(Vec(col1 + 6.f, y)), module, MEMLNaut::PARAM_THUMBS_DOWN)); - addParam(createParamCentered(mm2px(Vec(col2 + 2.f, y)), module, MEMLNaut::PARAM_LEARN)); - addChild(createLightCentered>(mm2px(Vec(col2 + 2.f, y - 4.f)), module, MEMLNaut::LIGHT_LEARN)); - addParam(createParamCentered(mm2px(Vec(col3, y)), module, MEMLNaut::PARAM_RAND)); - addParam(createParamCentered(mm2px(Vec(col3 + 8.f, y)), module, MEMLNaut::PARAM_CLEAR)); - addChild(createLightCentered>(mm2px(Vec(col3 + 8.f, y - 4.f)), module, MEMLNaut::LIGHT_TRAINING)); - y += 10.f; - - // ── Trigger / gate inputs ───────────────────────────────────── - addInput(createInputCentered(mm2px(Vec(col1, y)), module, MEMLNaut::INPUT_X)); - addInput(createInputCentered(mm2px(Vec(col2, y)), module, MEMLNaut::INPUT_Y)); - addInput(createInputCentered(mm2px(Vec(col3, y)), module, MEMLNaut::INPUT_LEARN_GATE)); - y += 8.f; - addInput(createInputCentered(mm2px(Vec(col1, y)), module, MEMLNaut::INPUT_TRIG_POS)); - addInput(createInputCentered(mm2px(Vec(col2, y)), module, MEMLNaut::INPUT_TRIG_NEG)); - y += 10.f; - - // ── Outputs: 3 columns of 4, with attenuverter + LED + jack ── - for (int i = 0; i < NUM_ML_OUTPUTS; i++) { - int col = i % 3; - int row = i / 3; - float ox = 6.f + col * 13.f; - float oy = y + row * 9.f; - - addParam(createParamCentered(mm2px(Vec(ox, oy)), module, MEMLNaut::PARAM_ATTEN_1 + i)); - addChild(createLightCentered>(mm2px(Vec(ox + 4.5f, oy)), module, MEMLNaut::LIGHT_OUT_1 + i)); - addOutput(createOutputCentered(mm2px(Vec(ox + 9.f, oy)), module, MEMLNaut::OUTPUT_1 + i)); + // 8 input jacks (2 rows × 4). + float iy = 27.f; + for (int i = 0; i < NUM_ML_INPUTS; i++) { + int col = i % 4; + int row = i / 4; + float ix = 8.f + col * 10.f; + addInput(createInputCentered(mm2px(Vec(ix, iy + row * 9.f)), module, MEMLNaut::INPUT_1 + i)); } - y += 4 * 9.f + 2.f; + // Control inputs to the right of the input block. + addInput(createInputCentered(mm2px(Vec(50.f, iy)), module, MEMLNaut::INPUT_LEARN_GATE)); + addInput(createInputCentered(mm2px(Vec(58.f, iy)), module, MEMLNaut::INPUT_TRIG_POS)); + addInput(createInputCentered(mm2px(Vec(58.f, iy + 9.f)), module, MEMLNaut::INPUT_TRIG_NEG)); - // ── Derived outputs ─────────────────────────────────────────── - float dox = 4.f; - addOutput(createOutputCentered(mm2px(Vec(dox, y)), module, MEMLNaut::OUTPUT_MEAN)); - addOutput(createOutputCentered(mm2px(Vec(dox + 8.f, y)), module, MEMLNaut::OUTPUT_STD)); - addOutput(createOutputCentered(mm2px(Vec(dox + 16.f, y)), module, MEMLNaut::OUTPUT_DELTA)); - addOutput(createOutputCentered(mm2px(Vec(dox + 24.f, y)), module, MEMLNaut::OUTPUT_NOVELTY)); - addOutput(createOutputCentered(mm2px(Vec(dox + 32.f, y)), module, MEMLNaut::OUTPUT_CONFIDENCE)); + // 16 output jacks (4 rows × 4), each encircled by an LED ring. + float oyTop = 52.f; + for (int i = 0; i < NUM_ML_OUTPUTS; i++) { + int col = i % 4; + int row = i / 4; + float ox = 9.f + col * 16.f; + float oy = oyTop + row * 16.f; + + // LED ring (behind the jack). + auto* ring = new LedRingWidget(); + ring->module = module; + ring->outIdx = i; + ring->ringColor = memlnaut::palette::ring(i); + ring->box.pos = mm2px(Vec(ox, oy)).minus(ring->box.size.div(2.f)); + addChild(ring); + + addOutput(createOutputCentered(mm2px(Vec(ox, oy)), module, MEMLNaut::OUTPUT_1 + i)); + } } void appendContextMenu(Menu* menu) override { @@ -821,7 +868,6 @@ struct MEMLNautWidget : ModuleWidget { menu->addChild(new MenuSeparator); menu->addChild(createMenuLabel("Output ranges")); - for (int i = 0; i < NUM_ML_OUTPUTS; i++) { menu->addChild(createCheckMenuItem( string::f("Out %d: Bipolar (±5V)", i + 1), "", @@ -832,19 +878,23 @@ struct MEMLNautWidget : ModuleWidget { menu->addChild(new MenuSeparator); menu->addChild(createMenuLabel("Input ranges")); - - std::string inputNames[] = {"X", "Y"}; for (int i = 0; i < NUM_ML_INPUTS; i++) { menu->addChild(createCheckMenuItem( - string::f("Input %s: Bipolar (±5V)", inputNames[i].c_str()), "", + string::f("In %d: Bipolar (±5V)", i + 1), "", [=]() { return !module->inputRangeUnipolar[i]; }, [=]() { module->inputRangeUnipolar[i] = !module->inputRangeUnipolar[i]; } )); } menu->addChild(new MenuSeparator); - menu->addChild(createMenuLabel("Slew")); + menu->addChild(createCheckMenuItem( + "Compute derived stats (Mean/Std/Delta)", "", + [=]() { return module->computeDerived; }, + [=]() { module->computeDerived = !module->computeDerived; } + )); + menu->addChild(new MenuSeparator); + menu->addChild(createMenuLabel("Slew")); menu->addChild(createSubmenuItem("Output slew", string::f("%.0f ms", module->slewMs), [=](Menu* childMenu) { for (float ms : {0.f, 5.f, 10.f, 20.f, 50.f, 100.f}) { childMenu->addChild(createCheckMenuItem( @@ -855,92 +905,57 @@ struct MEMLNautWidget : ModuleWidget { } })); - // ── Preset save/load ────────────────────────────────────────── menu->addChild(new MenuSeparator); menu->addChild(createMenuLabel("Presets (.nisps)")); - menu->addChild(createMenuItem("Save .nisps preset...", "", [=]() { osdialog_filters* filters = osdialog_filters_parse("NISPS preset:nisps"); char* path = osdialog_file(OSDIALOG_SAVE, nullptr, "preset.nisps", filters); osdialog_filters_free(filters); if (!path) return; - json_t* root = module->dataToJson(); - // Also save all param values json_t* jParams = json_array(); - for (int i = 0; i < MEMLNaut::PARAMS_LEN; i++) { + for (int i = 0; i < MEMLNaut::PARAMS_LEN; i++) json_array_append_new(jParams, json_real(module->params[i].getValue())); - } json_object_set_new(root, "params", jParams); - char* jsonStr = json_dumps(root, JSON_INDENT(2)); json_decref(root); - std::ofstream file(path); - if (file.is_open()) { - file << jsonStr; - file.close(); - } + if (file.is_open()) { file << jsonStr; file.close(); } free(jsonStr); free(path); })); - menu->addChild(createMenuItem("Load .nisps preset...", "", [=]() { osdialog_filters* filters = osdialog_filters_parse("NISPS preset:nisps"); char* path = osdialog_file(OSDIALOG_OPEN, nullptr, nullptr, filters); osdialog_filters_free(filters); if (!path) return; - std::ifstream file(path); free(path); if (!file.is_open()) return; - - std::string content((std::istreambuf_iterator(file)), - std::istreambuf_iterator()); + std::string content((std::istreambuf_iterator(file)), std::istreambuf_iterator()); file.close(); - json_error_t error; json_t* root = json_loads(content.c_str(), 0, &error); if (!root) return; - - // Validate version json_t* jVersion = json_object_get(root, "version"); - if (!jVersion || json_integer_value(jVersion) < 1) { - json_decref(root); - return; - } - + if (!jVersion || json_integer_value(jVersion) < 1) { json_decref(root); return; } module->dataFromJson(root); - - // Restore param values if present json_t* jParams = json_object_get(root, "params"); - if (jParams && json_is_array(jParams)) { - for (size_t i = 0; i < json_array_size(jParams) && i < MEMLNaut::PARAMS_LEN; i++) { + if (jParams && json_is_array(jParams)) + for (size_t i = 0; i < json_array_size(jParams) && i < MEMLNaut::PARAMS_LEN; i++) module->params[i].setValue(json_real_value(json_array_get(jParams, i))); - } - } - json_decref(root); })); - // ── OSC bridge ─────────────────────────────────────────────── menu->addChild(new MenuSeparator); - menu->addChild(createMenuLabel("OSC Bridge")); - + menu->addChild(createMenuLabel("Browser bridge (WS↔OSC)")); menu->addChild(createCheckMenuItem( string::f("Enable OSC server (port %d)", module->oscPort), "", [=]() { return module->oscEnabled; }, - [=]() { - if (module->oscEnabled) { - module->stopOsc(); - } else { - module->startOsc(); - } - } + [=]() { if (module->oscEnabled) module->stopOsc(); else module->startOsc(); } )); - menu->addChild(createSubmenuItem("OSC listen port", string::f("%d", module->oscPort), [=](Menu* childMenu) { - for (int port : {9000, 9001, 9002, 8000, 7000}) { + for (int port : {7001, 7002, 7003, 9000, 9001}) { childMenu->addChild(createCheckMenuItem( string::f("%d", port), "", [=]() { return module->oscPort == port; }, diff --git a/vcv/src/iml.hpp b/vcv/src/iml.hpp new file mode 100644 index 0000000..3c631f5 --- /dev/null +++ b/vcv/src/iml.hpp @@ -0,0 +1,409 @@ +// iml.hpp — Self-contained runtime IML/MLP for the MEMLNaut VCV module. +// +// This is a VENDORED, runtime-shaped re-implementation of the nisps core +// `nisps::IML` / `nisps::MLP` surface the VCV module relies on. +// The retired `nisps-core` header tree (`-I../nisps-core/include`) is gone, and +// the templated firmware/WASM `nisps/ml` core is fixed-size — neither is a clean +// fit for a runtime 8→16 module. So we ship a small native MLP here that matches +// the firmware/browser TRAINING SEMANTICS as closely as a runtime form allows: +// +// • ReLU hidden layers, sigmoid output (sigmoid maps to [0,1]). +// • A trailing bias node (1.0) appended to the input vector. +// • spread-aware weight init: uniform [-1,1] (spread=0) → Xavier 1/√fan_in +// (spread=1), interpolated per layer. +// • spread-aware perturbation (RL "move weights"): flat noise (spread=0) → +// per-layer Xavier-scaled noise + 10%·spread weight decay (spread=1). +// • Plain SGD / MSE training over the example dataset. +// • DETERMINISTIC per-instance RNG (seeded), so behaviour is reproducible and +// the threading double-buffer stays race-free (each MLP owns its own RNG). +// +// It is NOT bit-identical to the firmware core (different optimiser internals), +// and that divergence is an accepted follow-up (see vcv/SPEC.md delta #5). The +// public method names mirror the core so `MEMLNaut.cpp` is unchanged in spirit. +// +// MPL-2.0 in spirit with the rest of nisps; wrapper code under the VCV module's +// licence. British spelling in comments where it reads naturally. +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace nisps { + +// ── Tiny deterministic RNG (xorshift128) ────────────────────────────── +// Per-instance state; seeded in the constructor. No std::random_device, no +// shared global generator — this is what keeps the audio/worker MLP pair free +// of data races and makes parity reproducible. +class DetRng { +public: + explicit DetRng(uint32_t seed = 0x1234567u) { reseed(seed); } + void reseed(uint32_t seed) { + s_[0] = seed ? seed : 0xA5A5A5A5u; + s_[1] = s_[0] ^ 0x9E3779B9u; + s_[2] = s_[0] * 0x85EBCA6Bu + 1u; + s_[3] = s_[0] * 0xC2B2AE35u + 0x27D4EB2Fu; + } + uint32_t next_u32() { + uint32_t t = s_[3]; + uint32_t const u = s_[0]; + s_[3] = s_[2]; s_[2] = s_[1]; s_[1] = u; + t ^= t << 11; + t ^= t >> 8; + s_[0] = t ^ u ^ (u >> 19); + return s_[0]; + } + // Uniform in [0,1) + float uniform01() { return (next_u32() >> 8) * (1.0f / 16777216.0f); } + // Uniform in [-1,1) + float uniform_pm1() { return uniform01() * 2.0f - 1.0f; } + // Approx standard-normal: sum of 3 uniforms (matches the core's gen_randn shape) + float gaussian() { + return (uniform_pm1() + uniform_pm1() + uniform_pm1()) * 0.5773502692f; // /√3 → unit-ish variance + } +private: + uint32_t s_[4]; +}; + +// ── MLP ─────────────────────────────────────────────────────────────── +template +class MLP { +public: + // 3D weight store: [layer][node][weight] where the final weight per node is + // the bias (the previous layer's activations get a trailing 1.0). + using mlp_weights = std::vector>>; + + // layers_nodes: full sizes including input (with bias) and output, e.g. + // {n_in + 1, h0, h1, h2, n_out} + MLP(const std::vector& layers_nodes, uint32_t seed) + : layers_nodes_(layers_nodes), rng_(seed) { + build_(); + draw_weights_spread_(static_cast(0)); + } + + size_t num_layers() const { return weights_.size(); } + + // Forward pass. `input_with_bias` has the trailing 1.0 already appended. + void forward(const std::vector& input_with_bias, std::vector& out) const { + std::vector act = input_with_bias; + for (size_t l = 0; l < weights_.size(); ++l) { + const auto& layer = weights_[l]; + const bool is_output = (l + 1 == weights_.size()); + std::vector next(layer.size()); + for (size_t n = 0; n < layer.size(); ++n) { + const auto& w = layer[n]; + T sum = 0; + // w has act.size()+1 entries? No: w spans the *current* input + // size (which already includes the bias slot of `act`). + const size_t lim = std::min(w.size(), act.size()); + for (size_t k = 0; k < lim; ++k) sum += w[k] * act[k]; + next[n] = is_output ? sigmoid_(sum) : relu_(sum); + } + // For hidden layers, append a bias term for the next layer's input. + if (!is_output) next.push_back(static_cast(1)); + act = std::move(next); + } + out = std::move(act); + } + + // ── spread-aware weight init ────────────────────────────────────── + void draw_weights_spread(T spread) { draw_weights_spread_(spread); } + + // ── spread-aware perturbation (RL move_weights) ─────────────────── + void move_weights_spread(T speed, T spread) { + const T decay = static_cast(1) - static_cast(0.1) * spread; + for (size_t l = 0; l < weights_.size(); ++l) { + const T fan_in = static_cast(input_size_of_layer_(l)); + const T xavier = (fan_in > 0) ? static_cast(1) / std::sqrt(fan_in) : static_cast(1); + // spread=0 → flat noise (scale 1); spread=1 → per-layer Xavier scale + const T noiseScale = (static_cast(1) - spread) + spread * xavier; + for (auto& node : weights_[l]) { + for (auto& w : node) { + if (spread > 0) w *= decay; // weight decay only when spread>0 + w += rng_.gaussian() * speed * noiseScale; + } + } + } + } + + // ── plain SGD / MSE training ────────────────────────────────────── + // features: each row is input WITHOUT bias; labels: target outputs in [0,1]. + void train(const std::vector>& features, + const std::vector>& labels, + int max_iterations, T learning_rate, T convergence) { + const size_t n = std::min(features.size(), labels.size()); + if (n == 0) return; + for (int iter = 0; iter < max_iterations; ++iter) { + T epoch_loss = 0; + for (size_t s = 0; s < n; ++s) { + std::vector in = features[s]; + in.push_back(static_cast(1)); // bias + epoch_loss += backprop_(in, labels[s], learning_rate); + } + epoch_loss /= static_cast(n); + if (epoch_loss < convergence) break; + } + } + + mlp_weights get_weights() const { return weights_; } + void set_weights(const mlp_weights& w) { + // Only adopt if the topology matches; otherwise ignore (keeps the audio + // path safe against malformed snapshots from the bridge / patch files). + if (w.size() != weights_.size()) return; + for (size_t l = 0; l < w.size(); ++l) { + if (w[l].size() != weights_[l].size()) return; + } + weights_ = w; + } + +private: + static T relu_(T x) { return x > 0 ? x : 0; } + static T sigmoid_(T x) { return static_cast(1) / (static_cast(1) + std::exp(-x)); } + static T dsigmoid_from_out_(T y) { return y * (static_cast(1) - y); } + + size_t input_size_of_layer_(size_t l) const { + // The number of weights per node in layer l (incl. bias slot). + return weights_[l].empty() ? 0 : weights_[l][0].size(); + } + + void build_() { + weights_.clear(); + // layers_nodes_[0] is the input layer WITH bias already counted. + for (size_t l = 1; l < layers_nodes_.size(); ++l) { + const size_t in_sz = layers_nodes_[l - 1]; // includes bias slot + const size_t out_sz = layers_nodes_[l]; + std::vector> layer(out_sz, std::vector(in_sz, 0)); + weights_.push_back(std::move(layer)); + } + } + + void draw_weights_spread_(T spread) { + for (size_t l = 0; l < weights_.size(); ++l) { + const T fan_in = static_cast(layers_nodes_[l]); // incl. bias + const T xavier = (fan_in > 0) ? static_cast(1) / std::sqrt(fan_in) : static_cast(1); + const T scale = (static_cast(1) - spread) + spread * xavier; + for (auto& node : weights_[l]) { + for (size_t k = 0; k < node.size(); ++k) { + // bias (last weight) initialised to 0, like the core + const bool is_bias = (k + 1 == node.size()); + node[k] = is_bias ? static_cast(0) : rng_.uniform_pm1() * scale; + } + } + } + } + + // One SGD step on a single example; returns the MSE for this example. + T backprop_(const std::vector& in_with_bias, const std::vector& target, + T lr) { + // Forward, caching activations per layer. + std::vector> acts; + acts.reserve(weights_.size() + 1); + acts.push_back(in_with_bias); + std::vector act = in_with_bias; + for (size_t l = 0; l < weights_.size(); ++l) { + const auto& layer = weights_[l]; + const bool is_output = (l + 1 == weights_.size()); + std::vector next(layer.size()); + for (size_t nidx = 0; nidx < layer.size(); ++nidx) { + const auto& w = layer[nidx]; + T sum = 0; + const size_t lim = std::min(w.size(), act.size()); + for (size_t k = 0; k < lim; ++k) sum += w[k] * act[k]; + next[nidx] = is_output ? sigmoid_(sum) : relu_(sum); + } + if (!is_output) next.push_back(static_cast(1)); + acts.push_back(next); + act = next; + } + + // Output error. + const size_t L = weights_.size(); + std::vector& out = acts[L]; + T loss = 0; + std::vector delta(out.size()); + for (size_t o = 0; o < out.size(); ++o) { + const T t = (o < target.size()) ? target[o] : static_cast(0); + const T e = out[o] - t; + loss += e * e; + delta[o] = e * dsigmoid_from_out_(out[o]); // MSE × sigmoid' + } + loss /= static_cast(out.size() ? out.size() : 1); + + // Backprop through layers L-1 .. 0. + std::vector nextDelta; + for (size_t li = L; li-- > 0;) { + const auto& prevAct = acts[li]; // input activations to layer li + auto& layer = weights_[li]; + const bool is_output = (li + 1 == L); + // Compute delta to propagate to the previous layer (excludes bias node). + const size_t prevSize = prevAct.size(); // includes bias slot + std::vector propagate(prevSize, 0); + for (size_t nidx = 0; nidx < layer.size(); ++nidx) { + const T d = delta[nidx]; + auto& w = layer[nidx]; + const size_t lim = std::min(w.size(), prevSize); + for (size_t k = 0; k < lim; ++k) { + propagate[k] += d * w[k]; + w[k] -= lr * d * prevAct[k]; // gradient step + } + } + // Turn `propagate` into next-layer delta via ReLU' (skip for input). + if (li > 0) { + const auto& actPrev = acts[li]; // activations of layer li-1's output + nextDelta.assign(actPrev.size(), 0); + for (size_t k = 0; k < actPrev.size(); ++k) { + const T relud = actPrev[k] > 0 ? static_cast(1) : static_cast(0); + nextDelta[k] = propagate[k] * relud; + } + // Drop the trailing bias slot's delta (it has no upstream weights). + if (!nextDelta.empty()) nextDelta.pop_back(); + delta = nextDelta; + } + (void)is_output; + } + return loss; + } + + std::vector layers_nodes_; + mlp_weights weights_; + mutable DetRng rng_; +}; + +// ── Dataset (FIFO ring, max 100 examples) ───────────────────────────── +template +class Dataset { +public: + static constexpr size_t kMax_examples = 100; + void add(const std::vector& feat, const std::vector& label) { + if (features_.size() >= kMax_examples) { + features_.erase(features_.begin()); + labels_.erase(labels_.begin()); + } + features_.push_back(feat); + labels_.push_back(label); + } + void clear() { features_.clear(); labels_.clear(); } + size_t count() const { return features_.size(); } + const std::vector>& features() const { return features_; } + const std::vector>& labels() const { return labels_; } + void load(const std::vector>& f, const std::vector>& l) { + clear(); + const size_t n = std::min(f.size(), l.size()); + for (size_t i = 0; i < n; ++i) add(f[i], l[i]); + } +private: + std::vector> features_; + std::vector> labels_; +}; + +// ── IML ─────────────────────────────────────────────────────────────── +template +class IML { +public: + enum class Mode { Inference, Training }; + + IML(size_t n_inputs, size_t n_outputs, + std::vector hidden_layers = {16, 24, 16}, + size_t max_iterations = 200, + Float learning_rate = static_cast(0.1), + Float convergence_threshold = static_cast(0.00001), + uint32_t seed = 0xC0FFEEu) + : n_inputs_(n_inputs), n_outputs_(n_outputs), + max_iterations_(max_iterations), learning_rate_(learning_rate), + convergence_threshold_(convergence_threshold) { + std::vector sizes; + sizes.push_back(n_inputs_ + 1); // + bias + for (size_t h : hidden_layers) sizes.push_back(h); + sizes.push_back(n_outputs_); + mlp_ = std::make_unique>(sizes, seed); + input_state_.assign(n_inputs_, static_cast(0.5)); + output_state_.assign(n_outputs_, static_cast(0)); + } + + size_t num_inputs() const { return n_inputs_; } + size_t num_outputs() const { return n_outputs_; } + + void set_input(size_t i, Float v) { + if (i >= n_inputs_) return; + input_state_[i] = std::clamp(v, static_cast(0), static_cast(1)); + input_updated_ = true; + } + + const Float* get_outputs() const { return output_state_.data(); } + + void process() { + if (!input_updated_) return; + std::vector in = input_state_; + in.push_back(static_cast(1)); + mlp_->forward(in, output_state_); + if (output_state_.size() < n_outputs_) output_state_.resize(n_outputs_, 0); + input_updated_ = false; + } + + void set_mode(Mode m) { + if (m == Mode::Inference && mode_ == Mode::Training) train_(); + mode_ = m; + } + Mode get_mode() const { return mode_; } + + void add_example(const Float* inputs, size_t n_in, const Float* outputs, size_t n_out) { + std::vector in(inputs, inputs + std::min(n_in, n_inputs_)); + in.resize(n_inputs_, static_cast(0)); + std::vector out(outputs, outputs + std::min(n_out, n_outputs_)); + out.resize(n_outputs_, static_cast(0)); + dataset_.add(in, out); + } + void clear_dataset() { dataset_.clear(); } + + void randomise_weights(Float spread) { mlp_->draw_weights_spread(spread); refresh_(); } + void move_weights(Float speed, Float spread) { mlp_->move_weights_spread(speed, spread); refresh_(); } + + typename MLP::mlp_weights get_weights() const { return mlp_->get_weights(); } + void set_weights(typename MLP::mlp_weights& w) { mlp_->set_weights(w); } + + size_t get_example_count() const { return dataset_.count(); } + size_t get_max_examples() const { return Dataset::kMax_examples; } + std::vector> get_example_features() const { return dataset_.features(); } + std::vector> get_example_labels() const { return dataset_.labels(); } + void load_examples(const std::vector>& f, + const std::vector>& l) { dataset_.load(f, l); } + + Float nearest_example_distance(const Float* input, size_t n_in) const { + const auto& feats = dataset_.features(); + if (feats.empty()) return static_cast(-1); + Float best = std::numeric_limits::max(); + const size_t dims = std::min(n_in, n_inputs_); + for (const auto& f : feats) { + Float d = 0; + for (size_t k = 0; k < dims && k < f.size(); ++k) { + const Float diff = f[k] - input[k]; + d += diff * diff; + } + best = std::min(best, std::sqrt(d)); + } + return best; + } + +private: + void refresh_() { input_updated_ = true; process(); } + void train_() { + if (dataset_.count() == 0) return; + mlp_->train(dataset_.features(), dataset_.labels(), + static_cast(max_iterations_), learning_rate_, + convergence_threshold_); + refresh_(); + } + + size_t n_inputs_, n_outputs_, max_iterations_; + Float learning_rate_, convergence_threshold_; + Mode mode_ = Mode::Inference; + bool input_updated_ = false; + std::vector input_state_, output_state_; + Dataset dataset_; + std::unique_ptr> mlp_; +}; + +} // namespace nisps diff --git a/vcv/src/osc_server.hpp b/vcv/src/osc_server.hpp index 6112494..2352cbf 100644 --- a/vcv/src/osc_server.hpp +++ b/vcv/src/osc_server.hpp @@ -136,6 +136,7 @@ inline std::vector messageString(const std::string& address, class OscServer { public: using StringCallback = std::function; + using FloatVecCallback = std::function&)>; OscServer() = default; ~OscServer() { stop(); } @@ -144,9 +145,15 @@ public: OscServer(const OscServer&) = delete; OscServer& operator=(const OscServer&) = delete; - // Register handlers before starting + // Register handlers before starting. + // onState — full JSON state snapshot (/nisps/state ) + // onWeights — weights-only JSON (/nisps/weights ) + // onInput — live input vector (browser drives the model) (/nisps/input ) + // onFeedback— verdict op JSON (thumbs/place/rand/clear) (/nisps/feedback ) void onState(StringCallback cb) { stateCallback_ = std::move(cb); } void onWeights(StringCallback cb) { weightsCallback_ = std::move(cb); } + void onInput(FloatVecCallback cb) { inputCallback_ = std::move(cb); } + void onFeedback(StringCallback cb) { feedbackCallback_ = std::move(cb); } // Set the target address for sending (where the webapp bridge listens). // Default: 127.0.0.1:9001 @@ -240,12 +247,24 @@ public: sendPacket(msg); } - // Send current input values (2 floats) + // Send current input values (N floats) void sendInputs(const float* values, size_t count) { auto msg = osc::messageFloats("/nisps/input", values, count); sendPacket(msg); } + // Send a full JSON state snapshot (module → browser). + void sendState(const std::string& json) { + auto msg = osc::messageString("/nisps/state", json); + sendPacket(msg); + } + + // Send weights-only JSON (module → browser). + void sendWeights(const std::string& json) { + auto msg = osc::messageString("/nisps/weights", json); + sendPacket(msg); + } + private: void recvLoop() { uint8_t buf[65536]; @@ -291,6 +310,21 @@ private: std::string payload = osc::readString(buf, len, offset); if (weightsCallback_) weightsCallback_(payload); } + } else if (address == "/nisps/feedback") { + // Verdict op as a JSON string: + // {"op":"up|down|rand|clear","spread":f,"input":[…],"output":[…]} + if (tags.size() >= 2 && tags[1] == 's') { + std::string payload = osc::readString(buf, len, offset); + if (feedbackCallback_) feedbackCallback_(payload); + } + } else if (address == "/nisps/input") { + // Live input vector from the browser → drive the model inputs. + std::vector values; + for (size_t i = 1; i < tags.size(); ++i) { + if (tags[i] == 'f') values.push_back(osc::readFloat(buf, len, offset)); + else break; + } + if (!values.empty() && inputCallback_) inputCallback_(values); } // Unknown addresses are silently ignored } @@ -337,6 +371,8 @@ private: // Callbacks StringCallback stateCallback_; StringCallback weightsCallback_; + StringCallback feedbackCallback_; + FloatVecCallback inputCallback_; // Send target std::mutex sendMutex_; diff --git a/vcv/src/palette.hpp b/vcv/src/palette.hpp new file mode 100644 index 0000000..2b08c7f --- /dev/null +++ b/vcv/src/palette.hpp @@ -0,0 +1,52 @@ +// palette.hpp — MEMLNaut ring colours, hand-synced from the frontend tokens. +// +// Source of truth: docs/redesign/manifold-export/tokens/colors.css. These are +// the exact hex values from that file, so the VCV module's LED rings read as the +// same instrument as the Manifold browser front-end. If a token changes there, +// update the matching constant here (small hand-sync; see SPEC delta #3). +// +// The 16 output rings are assigned across the design-token accents + group/pin +// colours as a clean orange→cyan-anchored ramp, so outputs in the same mode +// group glow the same colour as the browser heatmap/Console grouping. +#pragma once + +#include + +namespace memlnaut { +namespace palette { + +// ── Design tokens (colors.css) ──────────────────────────────────────── +inline NVGcolor accent() { return nvgRGB(0xff, 0x6a, 0x00); } // --accent warm primary +inline NVGcolor accent2() { return nvgRGB(0x00, 0xcc, 0xff); } // --accent-2 cool secondary +inline NVGcolor accent3() { return nvgRGB(0xff, 0xa8, 0x60); } // --accent-3 warm hover/tint +inline NVGcolor good() { return nvgRGB(0x6b, 0xc2, 0x6b); } // --good +inline NVGcolor warn() { return nvgRGB(0xf5, 0xc4, 0x5e); } // --warn +inline NVGcolor info() { return nvgRGB(0x5b, 0x9e, 0xef); } // --info +inline NVGcolor pin3() { return nvgRGB(0xb4, 0x64, 0xff); } // --pin-3 base (violet) +inline NVGcolor danger() { return nvgRGB(0xff, 0x44, 0x66); } // --danger (bipolar / perturbed) +inline NVGcolor bgTrack() { return nvgRGB(0x24, 0x24, 0x24); } // --bg-3 (ring track) + +// ── 16-ring palette ─────────────────────────────────────────────────── +// Groups cycle through the token accents/group colours. Outputs 0..15 read as a +// coherent orange→cyan family with the semantic accents woven in. +inline NVGcolor ring(int outIdx) { + static const NVGcolor kRing[16] = { + // group 0 — formant/primary (orange family) + nvgRGB(0xff, 0x6a, 0x00), nvgRGB(0xff, 0x82, 0x2a), nvgRGB(0xff, 0xa8, 0x60), nvgRGB(0xff, 0xc4, 0x90), + // group 1 — amp (green) + nvgRGB(0x6b, 0xc2, 0x6b), nvgRGB(0x86, 0xcf, 0x86), + // group 2 — filter (amber/warn) + nvgRGB(0xf5, 0xc4, 0x5e), nvgRGB(0xf8, 0xd4, 0x84), + // group 3 — mod (violet/pin-3) + nvgRGB(0xb4, 0x64, 0xff), nvgRGB(0xc6, 0x86, 0xff), + // group 4 — fx (blue/info) + nvgRGB(0x5b, 0x9e, 0xef), nvgRGB(0x82, 0xb6, 0xf3), + // group 5 — pitch/data (cyan family → accent-2) + nvgRGB(0x3a, 0xd0, 0xf0), nvgRGB(0x1d, 0xce, 0xf7), nvgRGB(0x00, 0xcc, 0xff), nvgRGB(0x55, 0xdd, 0xff), + }; + if (outIdx < 0) outIdx = 0; + return kRing[outIdx % 16]; +} + +} // namespace palette +} // namespace memlnaut