diff --git a/CLAUDE.md b/CLAUDE.md index 3e3d944..00ded4d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -58,7 +58,8 @@ firmware/MEMLNaut-NISPS/ │ ├── midi_io.hpp # MIDI in → mode handlers; drain ControlEvent ring → MIDI UART │ ├── mode_select.hpp # type aliases firmware mode name → nisps::modes::*Mode │ ├── input_router.hpp # wire_inputs() entry point -│ └── output_router.hpp # drain_outputs() entry point +│ ├── output_router.hpp # drain_outputs() entry point +│ └── settings_view.hpp # wire_settings(): TFT/rotary menu (Joystick Dual/Single for 4-in modes) └── src/{memllib,daisysp,nisps} # symlinks (Arduino-CLI sketch tree convention) ``` diff --git a/MAP.md b/MAP.md index 3c5cb40..6834e1e 100644 --- a/MAP.md +++ b/MAP.md @@ -23,6 +23,7 @@ MEMLNaut-NISPS — Neural Interactive Shaping of Parameter Spaces. One C++20 cod - `mode_select.hpp` — type aliases mapping firmware mode identifiers to `nisps::modes::*Mode` C++ types. Build script rewrites the active line. Includes the six `MEMLNautModeExtSynth*` external-synth variants (one per device template in `nisps/midi`, e.g. `MEMLNautModeExtSynthSub37`). Also defines the `MEMLNautModeSelfTest` pseudo-variant (tag type) + the `NISPS_ST_*`/`NISPS_ST_CAT` token-paste macros the `.ino` uses to compute `NISPS_SELFTEST`. Note: `src/nisps/` exposes each referenced top-level nisps subdir as a symlink — `midi` was added alongside `core/dsp/engines/ml/modes`. - `selftest.hpp` — standalone guided hardware self-test rig (`SelfTest` variant; no engine/ML). Step-driven state machine on a `SelfTestView`: TFT prompts the operator through every control, auto-advances on detection, encoder-press skips. Ends with optional L/R/BOTH sine-sweep headphone check (core 1 block callback) + MIDI loopback-cable test. Lives firmware-side (touches TFT + raw pins) so it stays out of platform-agnostic `nisps/`. - `input_router.hpp`, `output_router.hpp` — top-level `wire_inputs()` / `drain_outputs()` entry points. + - `settings_view.hpp` — `wire_settings(mode)`: adds on-device settings views to the MEMLNaut display carousel (TFT + rotary encoder). Joystick Dual/Single toggle for the 4-input ("two 2-D joystick") modes — "Single" pins ML input channels 2,3 to neutral via `ModeBase::set_input_pinned` (no net rebuild). Registered in the `.ino` after `addSystemInfoView()`. - `firmware/MEMLNaut-NISPS/src/{memllib,daisysp,nisps}` — symlinks (Arduino-CLI requires sketch-tree includes; preprocessor refuses `..` in headers). - `firmware/README.md` — structure + build instructions. diff --git a/assets/media/learn/index.html b/assets/media/learn/index.html new file mode 100644 index 0000000..0e0565f --- /dev/null +++ b/assets/media/learn/index.html @@ -0,0 +1,484 @@ + + + + + +Manifold — the parameter landscape + + + +
+ + +
drag to rotate · top-down view = the flat 2-D map
+ + +
+
input · 2-D control
+ +
+ params + + 1 + +
+
+ + +
+
outputs
+
+
+ + +
+ + + +
+ + +
+
+ + +
+
+ + + +
loading the  NISPS  engine…
+
+ + +
+

The parameter landscape

+

placeholder copy — to be reworked

+

Two numbers go in — your X / Y control position — and a whole fistful of parameters comes + out. You can't picture a dozen dimensions at once, so picture them stacked: each slab is one + parameter, drawn as a 3-D landscape whose height at any spot is that parameter's value. One probe + reads every slab at the same control position.

+

Seen straight from above the slabs overlap into a single flat map — it looks 2-D. Tilt it and the + reveal lands: it was a stack of 3-D surfaces all along. Add parameters to grow the sandwich.

+

Every surface here is the real NISPS network sampled across the input space. 👍 keep teaches + the net that this sound belongs at this spot and the landscape bends to honour it; 👎 either pushes the + mapping away from the current sound or (in explore mode) re-rolls a fresh candidate to audition. nudge + and randomise perturb the net; undo steps back.

+
Values under the probe
+
+

← back to the instrument · animated explainers →

+
+ + + + + diff --git a/firmware/MEMLNaut-NISPS/MEMLNaut-NISPS.ino b/firmware/MEMLNaut-NISPS/MEMLNaut-NISPS.ino index 0a3a2a1..9cfa242 100644 --- a/firmware/MEMLNaut-NISPS/MEMLNaut-NISPS.ino +++ b/firmware/MEMLNaut-NISPS/MEMLNaut-NISPS.ino @@ -31,6 +31,7 @@ #include "glue/midi_io.hpp" #include "glue/mode_select.hpp" #include "glue/output_router.hpp" +#include "glue/settings_view.hpp" // ---- Mode selection ---- // Build script rewrites which line is uncommented. @@ -142,6 +143,8 @@ void setup() { } MEMLNaut::Instance()->addSystemInfoView(); + // Settings menu (e.g. Joystick: Dual/Single for the 4-input modes). + nisps_firmware::wire_settings(g_mode); Serial.println("Finished initialising core 0."); } diff --git a/firmware/MEMLNaut-NISPS/glue/settings_view.hpp b/firmware/MEMLNaut-NISPS/glue/settings_view.hpp new file mode 100644 index 0000000..9c5ab29 --- /dev/null +++ b/firmware/MEMLNaut-NISPS/glue/settings_view.hpp @@ -0,0 +1,64 @@ +// glue/settings_view.hpp — on-device settings menu views (TFT + rotary encoder). +// +// Adds entries to the MEMLNaut display carousel (the same DisplayDriver the +// SystemView and SelfTest use). Navigation: rotate to move between views; +// press to focus a view; rotate-while-focused to change its value; press +// again to unfocus. See src/memllib/.../display/DisplayDriver.hpp. +// +// Currently provides: +// * Joystick: Dual / Single — for the 4-input ("two 2-D joystick") modes, +// toggles whether the SECOND joystick (input channels 2,3) feeds the ML +// engine. "Single" pins those channels to the neutral value (0.5) via the +// platform-agnostic ModeBase pin mechanism, so the network is never +// rebuilt and trained state survives toggling. Default is Dual (the 4-D +// variant naturally uses both joysticks). +// +// Only registered for modes whose input arity is exactly 4 (the two-joystick +// modes); other modes (e.g. SoundAnalysisMIDI, whose joystick channels are not +// 2,3) get no joystick toggle. + +#pragma once + +#include +#include +#include +#include + +#include "../src/memllib/hardware/memlnaut/MEMLNaut.hpp" +#include "../src/memllib/hardware/memlnaut/display/SingleSelectView.hpp" + +namespace nisps_firmware { + +// The second 2-D joystick maps to ML input channels 2 and 3 (joy_z / joy_w). +inline constexpr std::size_t kSecondJoystickCh0 = 2u; +inline constexpr std::size_t kSecondJoystickCh1 = 3u; + +// Add the settings views for `mode` to the display carousel. Must run AFTER +// MEMLNaut::Initialize() (so the display + TFT exist) and after +// addSystemInfoView(), on core 0. +template +inline void wire_settings(ModeT& mode) { + if constexpr (ModeT::input_channel_count() == 4u) { + MEMLNaut* meml = MEMLNaut::Instance(); + if (!meml || !meml->disp) return; + + // Kept alive for the program lifetime (the carousel holds a shared_ptr + // too, but this guarantees ownership regardless of view churn). + static std::shared_ptr joystick_view; + joystick_view = std::make_shared("Joystick"); + meml->disp->AddView(joystick_view); // calls Setup() → creates the selector + + // Index 0 = Dual (default, no pinning); index 1 = Single. + std::array options = { String("Dual"), String("Single") }; + joystick_view->setOptions(std::span(options.data(), options.size())); + joystick_view->setNewVoiceCallback([&mode](std::size_t idx) { + const bool single = (idx == 1u); + mode.set_input_pinned(kSecondJoystickCh0, single); + mode.set_input_pinned(kSecondJoystickCh1, single); + }); + } else { + (void)mode; + } +} + +} // namespace nisps_firmware diff --git a/manifold/ONBOARDING.md b/manifold/ONBOARDING.md new file mode 100644 index 0000000..c698181 --- /dev/null +++ b/manifold/ONBOARDING.md @@ -0,0 +1,265 @@ +# Manifold UI — Agent Onboarding + +> **Purpose of this doc:** everything an agent needs to make a tweak or fix to the Manifold +> front-end without re-grepping the tree. Read this + the relevant section's source files; you +> should not need to read the 40KB design specs for routine work (they're pointers at the end). +> +> **Voice:** ground-truth map. If something here contradicts the code, the code wins — fix this doc +> in the same commit. + +--- + +## 1. What Manifold is + +A **convertible-mode React front-end** on the real, parity-tested NISPS ML+audio engine (same +`nisps/` C++ core that the firmware and the `playground/` use, compiled to WASM). It lets you drive +a neural net's mapping from a small input space (XY pad / gamepad / MIDI) to many outputs, shape +that mapping with interactive ML feedback ("explore & place" / dislike), and route the outputs to +several backends (built-in synth, MIDI, OSC, particles, VCV, MEMLNaut hardware). + +- **Stack:** Vite + React 18 + TypeScript (strict). No CSS-in-JS — design-token CSS variables. +- **Lives at:** `manifold/` (beside `playground/`). Deploys to **https://meml.lnfinitemonkeys.org/next/**. +- **Synth is labelled "Powerful Synth Engine" / "Built-in Synth" — NEVER "C15".** British spelling. + +--- + +## 2. Run / build / deploy / test + +```bash +cd manifold +bun install +bun run dev # Vite dev server, port 5273 (COOP/COEP headers set) +bun run typecheck # tsc --noEmit ← run before every commit +bun run build # tsc --noEmit && vite build → dist/ +bun run preview # serve dist/ on :4273 (honours COOP/COEP — needed by WASM/worklet) +bun run test:e2e # Playwright smoke (needs `bun run build` first; runs against preview) +``` + +- **Typecheck is the cheap gate.** It catches most regressions; run it after any edit. +- **E2E on the VPS** needs a non-snap node runner (bun is snap-confined and hides libs from + Chromium): `PLAYWRIGHT_BROWSERS_PATH=/home/w1n5t0n/snap/bun-js/87/.cache/ms-playwright node node_modules/.bin/playwright test`. Preview via bun is fine. The smoke spec (`tests/e2e/smoke.spec.ts`) + asserts: engine WASM loads, spine invariant (setInputs → outputs change), feedback runs, console + renders, **no "C15" in the bundle**, no console errors. `shot.spec.ts` takes screenshots. +- **Deploy is automatic on push to GitHub `main`** → webhook → builds `manifold/` → rsyncs to the + live `/next/` subdir. See the `manifold-deploy-pipeline` memory for the full chain and gotchas + (the `cp index.html a-immersive.html` 403 workaround; git-ignored `bun.lock`). +- **`?debug=1`** installs `window.__nisps` (synchronous probe for Playwright/console — see + `src/debug/probe.ts`). **`base: './'`** in `vite.config.ts` keeps asset URLs relative so one + `dist/` mounts at both `/` and `/next/`. **WASM URLs must resolve via `document.baseURI`**, never + hardcoded (see the assetUrl gotcha in §6). + +--- + +## 3. The big picture — three layers + +``` + ┌─────────────────────────────────────────────────────────────────┐ + │ UI layer (React) src/console/ src/dock/ src/primitives/ + │ ConsoleApp = spine of the UI: holds state, picks a Stage, │ + │ renders the Dock. "Convertible" = swappable Stages. │ + └───────────────┬─────────────────────────────────────────────────-┘ + │ reads engine.version (useSyncExternalStore), reads buffers imperatively + │ writes via engine.setInput / setParam / feedback.* + ┌───────────────▼─────────────────────────────────────────────────┐ + │ Engine layer (NO React) src/engine/ src/inputs/ src/feedback/ src/backends/ + │ Spine = reactive store below React. Per-frame ML inference is │ + │ eager + synchronous, OFF the render cycle. React only watches │ + │ a monotonic version counter. │ + └───────────────┬─────────────────────────────────────────────────┘ + │ C ABI + ┌───────────────▼─────────────────────────────────────────────────┐ + │ WASM (nisps.wasm) — built from the C++ nisps/ core │ + │ Loaded TWICE: main thread (ML inference + training + feedback) │ + │ and AudioWorklet (audio DSP, separate instance). │ + └──────────────────────────────────────────────────────────────────┘ +``` + +**Golden rule of the reactivity model:** components subscribe to `engine.version()` via +`useEngineVersion()` and then read live `Float32Array`s **imperatively** (`engine.getOutputs()`, +`engine.routedOutput()`). They do **not** get outputs as React state — that was a deliberate perf +decision. Buffers are reused frame-to-frame; never assume a fresh array. + +--- + +## 4. UI layer (where most tweaks happen) + +### Entry & composition +- `src/App.tsx` → mounts `` (async WASM load, shows `Loading`) → ``. Installs the `?debug=1` probe once the engine is live. +- `src/console/ConsoleApp.tsx` (**~1040 lines — the UI spine**). Holds nearly all UI state: + `focus`, `modeId`, `params` (`MFParam[]`), `pos` (2D input), feedback state + (`feedbackMode`/`soloMode`/`exploring`/`picking`/`anchorCount`), output config + (`outputMode`/`midiOutputId`/`oscUrl`/`vcvUrl`), and UI flags (`sandwich`, `split`, `stripPinned`, + `snapshots`, `markers`, `health`, `rev`). Builds the flat `ConsoleCtx` passed to the Dock + drawers. + +### The "convertible" Stages (one renders at a time, chosen by `focus` + `outputMode`) +| Stage | File | Renders when | What it is | +|---|---|---|---| +| Manifold | `Manifold.tsx` | `focus==='in'` (default input view) | Full-bleed 2D input surface; canvas trail + pins + feedback markers; pointer → `onMove`. | +| OutputStage | `OutputStage.tsx` | `focus==='out'` | Full-bleed output columns; drag bars set value; `InputMini` docked in a corner. | +| SplitStage | `SplitStage.tsx` | `focus==='split'` | Manifold left, OutputStage right, equal width. | +| CompositeStage | `CompositeStage.tsx` | `focus==='composite'` (**app default / hero**) | Draggable split-ratio; magnet-snaps to 0.14/0.33/0.5/0.66/0.86; collapses a side to a corner minimap at extremes. | +| SandwichStage | `SandwichStage.tsx` | `sandwich===true` (overrides) | 3D parameter-landscape view (input → MLP heatmap grid → outputs); drag to orbit. | +| ParticleStage | `ParticleStage.tsx` | `outputMode==='particles'` | Flow-field visualiser (`flow-field.ts`, 400-particle Canvas2D port) + macro-axis bar + corner joystick. | + +- **Output modes** (the TOP dock selector, NOT the same axis as `focus`): `src/console/output-mode.ts` + defines `OUTPUT_MODES` = **particles** (default) / midi / osc / synth / editor, each mapping to a + `BackendId`. `DEFAULT_OUTPUT_MODE='particles'`. +- `src/console/output-mode.ts`, `types.ts`, `model.ts` are the shared vocabulary — read these first + when touching anything cross-cutting: + - `types.ts`: `Focus`, `OutputMode`, `DrawerKey`, `DrawerDepth`, `FeedbackModeUI`, `SoloMode`, + `Pin`, `FeedbackMarker`, `Snapshot`, and **`ConsoleCtx`** (the flat context handed to the dock). + - `model.ts`: the static instrument catalogue `MF_MODES`, `MFParam`, `ParamStatus` + (`off|fixed|live`), `ParamGroup`, plus `shapeValues()` (applies min/max/curve to raw engine + outputs) and `seededGradient()`. + +### The Dock (right-edge rail) — `src/console/Dock.tsx` + `Drawers.tsx` +- 48px right rail: **TOP** = mode selector (the 5 output modes, popover); **MIDDLE** = 5 drawer + icons, vertically centred macOS-dock style; **BOTTOM** = sandwich toggle. +- Five drawers (`DRAWERS` in `Drawers.tsx`, each has `.render(ctx, depth)` — condensed 360px panel + vs expanded 80vw×80vh modal): + - **learn** — feedback mode (explore-and-place / geometric-dislike) + solo mode + per-output arm. + - **inputs** — enable/configure input sources (XY pad / MIDI / gamepad). + - **route** (label "Outputs") — per-output control matrix + per-backend config. + - **settings** — icon style (monochrome/colour), input-map shape (xy/joystick/rect/circular), corner radius. + - **help** — keymap pills + loop explanation. +- `src/dock/` holds the output-routing internals used by the `route` drawer: + - `output-state.ts` — the per-output control model: `OutputControl` (state/muted/armed/min/max/ + curve/fixedValue + backend specs `MidiCcSpec`/`OscSpec`/`VcvSpec`), `toOutputControl()`, + `buildArmMask()` (solo focus). + - `OutputControlRow.tsx` — one output row: name · M(mute) · S(solo/arm) · off|fixed|live · dual-range + · curve pad · live value. **Writes eagerly to the shared `MFParam` store via `onChange`.** + - `OutputsBackendConfig.tsx` — preset bar (save/restore/rename/delete) + per-backend config (MIDI + CC#/channel, OSC path/range, VCV polarity). + - `BackendAdvanced.tsx` — full-depth modal version of the same editors. + +### Primitives — `src/primitives/` (barrel: `index.ts`) +`Button`, `Slider`, `PillToggle`, `Panel`, `Badge`, `Switch`, `StatusLine`, `XYPad`, +`VirtualJoystick`, `ControlAxis`, `CurvePlot`, `Sparkline`. Dumb, reusable, no engine knowledge. +Side-effect import of `styles/primitives.css` styles the range inputs. + +### Other shared UI files +- `shared-ui.tsx` — `AltitudeNav` (focus IN/DUAL/OUT/FLEX switcher) + `MiniMeters` (read-only output bars). +- `icons.tsx` — monochrome inline-SVG icons (mode icons + drawer icons + `GLYPH_FALLBACK` for when monochrome is off). +- `ReadoutStrip.tsx` — thin horizontal heatmap strip (pinned, `focus==='in'`); same per-output control as OutputStage. +- `VerdictCluster.tsx` — floating bottom-centre feedback UI (perturb ▽ / undo ↺ / commit △ + A/B); labels adapt to feedback mode. +- `CurvePad.tsx` — square canvas curve editor (vertical drag reshapes [0,1]; ~0.43 ≈ linear). Used in OutputEditor + OutputControlRow. +- `InputMini.tsx` — compact XYPad/joystick docked in a corner when input is demoted. +- `OutputEditor.tsx` — inline min/max/curve popup for a single output (hover/click on a bar). + +### Styling — `src/styles/` +CSS-variable design tokens, no CSS-in-JS. `tokens.css` `@import`s `tokens/{base,colors,fonts, +typography,spacing,effects}.css`. All vars on `:root`, consumed via `var(--name)`. Dark theme: deep +black bg, warm-orange `--accent` (#ff6a00), cyan `--accent-2` (#00ccff). Corner radius is driven by +a setting → `--r-*` tokens. + +--- + +## 5. Engine / inputs / feedback / backends (touch when behaviour, not chrome, changes) + +### Engine — `src/engine/` (no React except the two binding files) +- `spine.ts` — **the reactive store.** `setInput(x,y)`/`setInputs(arr)` drive raw input synchronously + through input-pipeline → `WasmIML.processInto()` → output-pipeline → backend, all off the render + cycle, reusing buffers. Bumps a monotonic `version`. `reprocess()` re-ticks the last input after a + weight change (stores the full N-D vector so extra axes survive). +- `engine-api.ts` — **`EngineApi`, the framework-neutral facade** everything in the UI talks to: + `setInput/setInputs`, `getOutputs/routedOutput`, training (`addExample/train/trainAsync/evalLoss`), + weights (`getWeights/setWeights/process/randomise`), `subscribe/version/on`, plus nested + `.feedback` and `.audio` facades. +- `EngineProvider.tsx` / `useEngine.ts` — the **only** React coupling. `useEngine()` returns the API + (null until WASM ready); `useEngineVersion()` = `useSyncExternalStore(subscribe, version)`. +- `engine-host.ts` — main-thread audio wiring: AudioContext (user-gesture gated), fetch `nisps.wasm`, + register + feed the worklet. +- `wasm-iml.ts` (**~750 lines**) — the ML interface to `nisps.wasm`: one MLP handle, dataset, heap + buffers, feedback C-ABI bindings (`nisps_ml_feedback_*`), lazy training worker. +- `wasm-worker.ts` — off-thread training worker. `worklet/nisps-processor.ts` — the AudioWorklet's + separate WASM instance (raw `WebAssembly.instantiate`, no Emscripten glue; 128-sample blocks). +- `input-pipeline.ts` — per-axis: invert → deadzone → circular clamp → zoom → centred power curve → + EMA smoothing → momentum. **First 2 axes get the full pad pipeline; axes 2+ feed raw to the spine.** +- `output-pipeline.ts` — global power curve → per-output EMA smoothing → slew limit → freeze gate. +- `dataset.ts` — JS-side example store + sample-weight modes (uniform/recency/spatial/combined). +- `curves.ts` — math primitives. **Must stay lockstep with C++ `nisps/core/math.hpp`** (golden tests + compare WASM vs TS). `sink.ts` — `EngineSink` framework boundary. `types.ts` — C-ABI surface types. + +### Inputs — `src/inputs/` +- `input-layer.ts` — composition hub. One rAF loop polls sources, pulls all axes into a vector, + forwards N→engine. **`MAX_AXES = 32`** (WASM net over-provisioned to 32 inputs). **Dedicated + dimensions, NO mean-blending** — each active axis drives its own engine slot 1:1; unused slots + zero-padded (inert). Changing axis count requires a **net reset** (UI confirm modal). +- `base-source.ts` + sources: `xy-pad-source.ts` (push, 2 axes), `gamepad-source.ts` (single=2 / + double=4 axes, deadzone 0.08), `midi-input-source.ts` (Web MIDI, batch CC-learn, multi-port). +- `useInputLayer.ts` — React binding; manages exclusive input mode + gamepad stick mode + MIDI + device/learn map; exposes `pushPad`, `sources`, `channelLayout`, etc. +- **Reshape status:** the WASM head is over-provisioned to 32 but the spine's `setInputs` historically + treated the head as effectively 2-D — confirm current behaviour in `spine.ts`/`input-layer.ts` + before relying on >2 active dims. See the `manifold-mixed-inputs` memory for the locked design + (reshapeable net, reset-on-reshape modal, adaptive slider viz when >2 dims). + +### Feedback — `src/feedback/` +- `controller.ts` (**~490 lines**) — `FeedbackController`, framework-neutral, owned by ConsoleApp. + Two modes: **explore-and-place** (default, Mode 2, positive-only — drives the C++ core's + snapshot/scratchpad/undo lifecycle; caller accumulates anchors and trains on finalise with + warm-start) and **geometric-dislike** (Mode 1, selectable). Solo/arm via per-output mask. +- `rng.ts` — `SeededRng` (deterministic xorshift32 + gaussian). **Stand-in** until the C++ nudge owns + the stream — not bit-identical to `nisps::Rng`. +- **`--- C++ GAP ---` markers** flag behaviour approximated in TS pending C++ port: true geometric + push (firmware k-NN), feedback nudge → `nisps_ml_feedback_nudge`, loss-history plumbing. Grep for + `C++ GAP` before changing feedback maths. + +### Backends — `src/backends/` +- `manager.ts` — `BackendManager`, the **single consumer of the engine spine** for output: subscribes, + reads `routedOutput()` each version bump, calls `active.send(routed)`. Gates audio (mutes synth when + a non-synth backend is active). +- Registered: `midi` (`midi-backend.ts`, CC out, ~20Hz throttle), `osc` (`osc-client.ts` → WebSocket + to the Deno bridge in `osc-bridge/bridge.ts`), `synth`/`cvgate` (`passthrough-backend.ts`), + `particles` (`particle-backend.ts`, no-op — the visualiser is a separate `flow-field.ts` consumer), + `vcv` (`vcv-backend.ts`, bidirectional). `mapping.ts`, `presets.ts`, `useBackendManager.ts` support + config. `backend.ts` defines the `OutputBackend` interface + `OutputMapping`. + +### Misc +- `src/serial/memlnaut-serial.ts` — **STUB** Web Serial scaffold for the MEMLNaut Editor mode (protocol TODO). `EditorPanel.tsx` is its UI. +- `src/settings/settings-store.ts` — localStorage settings (`mf-settings`): icon style, input-map shape, corner radius. +- `src/midi-devices/` — codegen'd external-synth device templates. +- `src/debug/probe.ts` — `window.__nisps` synchronous probe (engine/audio/bus). Some playground + feature-store methods are present-but-inert (not ported yet) to keep the surface stable. + +--- + +## 6. Gotchas & non-obvious rules + +1. **Outputs are read imperatively, not via React state.** Subscribe to `version`, then call + `getOutputs()`/`routedOutput()`. Buffers are reused — copy if you need to retain. +2. **Asset URLs must resolve against `document.baseURI`, not `location.origin`.** `base: './'` + + the `/next/` subpath mean a hardcoded `/nisps.wasm` 404s. The `assetUrl()` helper (in + `engine-host.ts`, `wasm-iml.ts`, `wasm-worker.ts`) handles this — use it. +3. **`nisps.js` is non-module Emscripten glue (no ES exports).** Workers/worklet fetch + indirect-eval + to install the global `createNispsModule`; the worklet uses raw `WebAssembly.instantiate`. +4. **WASM architecture is hardwired at build time** (`nisps/wasm/bindings.cpp`): 2→126 outputs, + fixed hidden layers, 32 input slots. Requesting other sizes in options is a logged warning, not + dynamic. Changing it means rebuilding WASM (`bash scripts/build-wasm.sh`) and updating the parity + test (`tests/cpp/parity_check.cpp`) or parity CI goes red. +5. **`curves.ts` ↔ `nisps/core/math.hpp` must stay lockstep** (golden-vector parity tests). +6. **COOP/COEP headers are mandatory** for the WASM/worklet path — set in `vite.config.ts` for + dev+preview, and at nginx server scope in prod (inherited by `/next/`). +7. **Never emit "C15"** in code or bundle — the smoke test fails on it. Synth is "Powerful Synth + Engine"/"Built-in Synth". +8. **Two `MFParam` writers** (stage controls AND dock rows) share one store — edits in either re-render both. + +--- + +## 7. Deeper references (only when this doc isn't enough) + +All in `docs/redesign/` (at the repo root, not under `manifold/`): +- `BUILD-PLAN.md` — locked decisions + the 12-step build sequence + spec pointers (the resume anchor). +- `engine-architecture.md` — full engine/spine/WASM design. +- `dock-spec.md` — dock + drawers spec. `inputs-spec.md` — mixed-input design. `backends-spec.md` — backends. +- `rl-feedback-design.md` + `feedback-modes-port-spec.md` + `findings-feedback-behaviour.md` — feedback modes (Mode 1/Mode 2). +- `aimmersive-clone-spec.md` / `playground-2026.md` — the a-immersive feature parity target. +- `src/backends/README.md` — backend wiring notes. + +**Memories** (auto-loaded): `manifold-build` (status + locked decisions), `manifold-mixed-inputs` +(N-D input design), `manifold-deploy-pipeline` (the auto-deploy chain), `single-double-joystick-toggle`. + +--- + +*Keep this in sync with the code — update it in the same commit as any change to UI structure, the +engine spine, the dock/drawers, or the build/deploy commands.* diff --git a/manifold/src/console/ConsoleApp.tsx b/manifold/src/console/ConsoleApp.tsx index d0bf780..ca8dce4 100644 --- a/manifold/src/console/ConsoleApp.tsx +++ b/manifold/src/console/ConsoleApp.tsx @@ -28,6 +28,7 @@ import { MF_MODES, modeEngineId, seededGradient, shapeValues } from './model'; import type { MFParam } from './model'; import { CompositeStage } from './CompositeStage'; import { ParticleStage } from './ParticleStage'; +import { SandwichStage } from './SandwichStage'; import { SplitStage } from './SplitStage'; import { OutputStage } from './OutputStage'; import { InputMini } from './InputMini'; @@ -105,8 +106,10 @@ export function ConsoleApp({ focus: initialFocus = 'composite' }: ConsoleAppProp const [tame, setTame] = useState(0.85); const [health, setHealth] = useState(0.8); const [rev, setRev] = useState(1); - const [active, setActive] = useState('learn'); - const [depth, setDepth] = useState('peek'); + const [active, setActive] = useState(null); + const [depth, setDepth] = useState('condensed'); + // Sandwich (parameter-landscape) centre-stage toggle — dock-bottom layers icon. + const [sandwich, setSandwich] = useState(false); // Learning-behaviour store (dock-spec §1; rl-feedback-design). Default // feedback mode = "Explore and place"; default solo = "Mask gradients". @@ -219,8 +222,8 @@ export function ConsoleApp({ focus: initialFocus = 'composite' }: ConsoleAppProp setFollow(false); setPins([]); setMarkers([]); - setActive('learn'); - setDepth('peek'); + setActive(null); + setDepth('condensed'); if (engine) engine.setInput(0.5, 0.5); // eslint-disable-next-line react-hooks/exhaustive-deps }, [modeId]); @@ -408,6 +411,23 @@ export function ConsoleApp({ focus: initialFocus = 'composite' }: ConsoleAppProp setRev((r) => r + 1); }; + /** + * Nudge — a small bounded weight perturbation of the current net (the left + * half of the secondary pill). Routes to the scratchpad while exploring, + * otherwise nudges the real net directly. + */ + const nudgeNet = () => { + const c = controllerRef.current; + if (feedbackMode === 'explore-and-place' && c?.getState().exploring) { + c.nudge(); + } else { + engine?.feedback.nudge(0.05); + engine?.process(); + } + syncController(); + setRev((r) => r + 1); + }; + /** * Undo. While exploring (Mode 2) this pops the scratchpad undo ring (reroll / * nudge). Otherwise it falls back to the UI snapshot stack (visual A/B seed). @@ -523,8 +543,8 @@ export function ConsoleApp({ focus: initialFocus = 'composite' }: ConsoleAppProp }; if (map[e.key]) { setActive((a) => (a === map[e.key] ? null : map[e.key])); - setDepth('peek'); - } else if (e.key === '\\') setDepth((d) => (d === 'full' ? 'peek' : 'full')); + setDepth('condensed'); + } else if (e.key === '\\') setDepth((d) => (d === 'expanded' ? 'condensed' : 'expanded')); else if (focus === 'composite' && e.key === '[') { e.preventDefault(); setSplit((s) => Math.max(0, s - 0.04)); @@ -771,7 +791,55 @@ export function ConsoleApp({ focus: initialFocus = 'composite' }: ConsoleAppProp bottom: 0, }} > - {outputMode === 'particles' ? ( + {sandwich ? ( + // Sandwich centre-stage: shrunken input (left) · landscape stack + // (centre, fills) · compact outputs (right). Replaces the Mode stage. +
+
+ +
+
+ {engine && ( + p.name)} + /> + )} +
+
+ +
+
+ ) : outputMode === 'particles' ? ( ) : focus === 'composite' ? ( 0 : snapshots.length > 0} ab={ab} onToggleAB={toggleAB} @@ -951,7 +1021,19 @@ export function ConsoleApp({ focus: initialFocus = 'composite' }: ConsoleAppProp - + { + setSandwich(v); + // reveal the full 3-zone layout: a drawer would overlay the output zone + if (v) setActive(null); + }} + /> ); } diff --git a/manifold/src/console/Dock.tsx b/manifold/src/console/Dock.tsx index 4f46ea7..9481502 100644 --- a/manifold/src/console/Dock.tsx +++ b/manifold/src/console/Dock.tsx @@ -1,6 +1,8 @@ /** * The Console dock — right-edge 48px rail + one mutually-exclusive drawer with - * three depth states (Peek 320 / Expand 520 / Full modal). + * two depth states: condensed (360px side panel) and expanded (centred modal, + * ~80% of the viewport). A vertically-centred tab on the panel's left edge + * toggles between them; drawers are closed by default. * * RESTRUCTURED (operator dock restructure): * - TOP, pinned: the **Mode** selector — the active OUTPUT BACKEND/target @@ -22,12 +24,12 @@ import type { ConsoleCtx, DrawerDepth, DrawerKey, OutputMode } from './types'; import { OUTPUT_MODES } from './output-mode'; import { useSettings, unfocusedIconCss } from '../settings/settings-store'; import { - ModeIcon, ParticleIcon, MidiIcon, OscIcon, SynthIcon, EditorIcon, + SandwichIcon, CloseIcon, ExpandIcon, GLYPH_FALLBACK, @@ -79,17 +81,20 @@ function ModeSelector({ width: 40, height: 40, borderRadius: 'var(--r-2)', - border: '1px solid var(--accent)', + border: '1px solid #ffffff', background: 'rgba(255,106,0,0.12)', color: 'var(--accent)', cursor: 'pointer', - fontSize: 18, + fontFamily: 'var(--font-mono)', + fontSize: 22, + fontWeight: 700, + lineHeight: 1, display: 'flex', alignItems: 'center', justifyContent: 'center', }} > - {mono ? : '⊞'} + M {open && (
void; depth: DrawerDepth; setDepth: (d: DrawerDepth) => void; + /** Sandwich (parameter-landscape) centre-stage toggle. */ + sandwich: boolean; + setSandwich: (v: boolean) => void; } const ORDER: DrawerKey[] = ['learn', 'inputs', 'route', 'settings', 'help']; -export function Dock({ ctx, active, setActive, depth, setDepth }: DockProps) { +export function Dock({ ctx, active, setActive, depth, setDepth, sandwich, setSandwich }: DockProps) { const { settings } = useSettings(); const mono = settings.monochromeIcons; const restColour = unfocusedIconCss(settings.unfocusedIconColour); @@ -192,7 +200,7 @@ export function Dock({ ctx, active, setActive, depth, setDepth }: DockProps) { title={s.label} onClick={() => { setActive(on ? null : key); - setDepth('peek'); + setDepth('condensed'); }} style={{ position: 'relative', @@ -230,107 +238,146 @@ export function Dock({ ctx, active, setActive, depth, setDepth }: DockProps) { ); }; - const width = depth === 'expand' ? 520 : 320; - const full = depth === 'full'; + const expanded = depth === 'expanded'; const section = active ? DRAWERS[active] : null; return ( <> + {/* Dim backdrop behind the centred modal; click to collapse back. */} + {section && expanded && ( +
setDepth('condensed')} + style={{ + position: 'fixed', + inset: 0, + zIndex: 85, + background: 'rgba(0,0,0,0.45)', + backdropFilter: 'blur(2px)', + WebkitBackdropFilter: 'blur(2px)', + }} + /> + )} {section && ( + // Shell: positions/sizes the drawer and is the anchor for the left-edge + // tab. overflow:visible so the tab (at left:-22) is NOT clipped; the + // inner wrapper handles scrolling. )} @@ -368,6 +415,27 @@ export function Dock({ ctx, active, setActive, depth, setDepth }: DockProps) { > {ORDER.map(iconBtn)}
+ {/* Sandwich toggle — pinned at the BOTTOM of the rail. */} + ); diff --git a/manifold/src/console/Drawers.tsx b/manifold/src/console/Drawers.tsx index 52f4aef..d0993ae 100644 --- a/manifold/src/console/Drawers.tsx +++ b/manifold/src/console/Drawers.tsx @@ -1,6 +1,6 @@ /** * Console — the FIVE real dock drawers (operator dock restructure). Each renderer - * takes (ctx, depth); what shows is gated by depth (peek | expand | full): + * takes (ctx, depth); what shows is gated by depth (condensed | expanded): * * learn — Learning : feedback-mode selector, solo/arm chooser, live params * inputs — Inputs : input source @@ -162,13 +162,13 @@ function LearningDrawer(ctx: ConsoleCtx, depth: DrawerDepth) { Down action · feedback mode - {depth !== 'peek' && ( + {depth === 'expanded' && (

{FEEDBACK_DESC[ctx.feedbackMode]}

)} - {depth !== 'peek' && ( + {depth === 'expanded' && ( <> Solo / arm scope
@@ -204,7 +204,7 @@ function LearningDrawer(ctx: ConsoleCtx, depth: DrawerDepth) { )} - {depth === 'full' && ( + {depth === 'expanded' && ( <> Feedback lab

@@ -339,21 +339,21 @@ function InputsDrawer(ctx: ConsoleCtx, depth: DrawerDepth) { Input source - {active && depth !== 'peek' && ( + {active && depth === 'expanded' && ( {active.status.message} )} {/* ---- Internal (XY pad / manifold) ---- */} - {inp.inputMode === 'internal' && depth !== 'peek' && ( + {inp.inputMode === 'internal' && depth === 'expanded' && (

Drag the on-screen manifold / XY pad. Two axes feed the net directly — this is the default.

)} {/* ---- Game Controller ---- */} - {inp.inputMode === 'gamepad' && depth !== 'peek' && ( + {inp.inputMode === 'gamepad' && depth === 'expanded' && ( <> Sticks Device {inp.midiInputs.length === 0 ? ( @@ -466,7 +466,7 @@ function InputsDrawer(ctx: ConsoleCtx, depth: DrawerDepth) { )} {/* ---- Reshape note (only when >2 axes feed the fixed WASM head) ---- */} - {reshaping && depth !== 'peek' && ( + {reshaping && depth === 'expanded' && (

{/* TODO(workstream F, docs/redesign/inputs-spec.md — "multiple WASM modules + warm-start"): give every axis its own genuine input dimension by (re)loading a @@ -509,7 +509,7 @@ function ModeConfig(ctx: ConsoleCtx, depth: DrawerDepth) { audio starts on the play gesture

- {depth !== 'peek' && ( + {depth === 'expanded' && ( <> Tempo @@ -530,7 +530,7 @@ function ModeConfig(ctx: ConsoleCtx, depth: DrawerDepth) { ); case 'particles': - return depth !== 'peek' ? ( + return depth === 'expanded' ? (

Flow-field visualiser driven by the first {Math.min(20, ctx.params.length)} outputs (no audio). {/* TODO(backends-spec §4): port FlowFieldVisualizer + visual preset chips. */} @@ -539,13 +539,13 @@ function ModeConfig(ctx: ConsoleCtx, depth: DrawerDepth) { case 'midi': // The full MIDI config (port picker, CC count, per-output CC/channel/name) // + preset bar render via OutputsBackendConfig in RoutingDrawer below. - return depth !== 'peek' ? ( + return depth === 'expanded' ? (

Each output sends a real Web MIDI CC. Pick a port and set CC# / channel per output.

) : null; case 'osc': - return depth !== 'peek' ? ( + return depth === 'expanded' ? (

Each output sends to an OSC path with a physical range, over the WebSocket bridge.

@@ -568,23 +568,8 @@ function RoutingDrawer(ctx: ConsoleCtx, depth: DrawerDepth) { const nameFor = (idx: number, fallback: string) => ctx.outputMode === 'particles' ? VISUAL_NAMES[idx] ?? fallback : fallback; - if (depth === 'full') { - return ( - <> -
- {modeDesc.label} - -
- {ModeConfig(ctx, depth)} - {/* Specialised, editable per-backend config (MIDI/OSC) + named-preset bar. */} - - Advanced · {modeDesc.label} - - - ); - } - - const rows = depth === 'peek' ? ctx.params.slice(0, 6) : ctx.params; + const expanded = depth === 'expanded'; + const rows = expanded ? ctx.params : ctx.params.slice(0, 6); return ( <>
@@ -596,15 +581,15 @@ function RoutingDrawer(ctx: ConsoleCtx, depth: DrawerDepth) {
{ModeConfig(ctx, depth)} - {/* Specialised per-backend config + named-preset bar (MIDI/OSC); hidden at peek. */} - {depth !== 'peek' && } + {/* Specialised per-backend config + named-preset bar (MIDI/OSC); hidden when condensed. */} + {expanded && } Outputs · M mute · S arm · off/fixed/live
@@ -617,14 +602,20 @@ function RoutingDrawer(ctx: ConsoleCtx, depth: DrawerDepth) { param={labelled} value={values[i] ?? 0} onChange={(patch) => ctx.setParam(i, patch)} - showCurve={depth !== 'peek'} + showCurve={expanded} /> ); })}
- {depth === 'peek' && ctx.params.length > 6 && ( + {!expanded && ctx.params.length > 6 && ( +{ctx.params.length - 6} more — expand to edit )} + {expanded && ( + <> + Advanced · {modeDesc.label} + + + )} ); } @@ -654,7 +645,7 @@ function SettingsDrawer({ depth }: { ctx: ConsoleCtx; depth: DrawerDepth }) { onChange={(v) => set('monochromeIcons', v)} label="Monochrome icons" /> - {depth !== 'peek' && ( + {depth === 'expanded' && ( <>
Unfocused icon colour
set('inputMap', v as InputMapMode)} options={INPUT_MAP_OPTS} /> - {depth !== 'peek' && ( + {depth === 'expanded' && (

The 2D input surface: a rectangular XY map or a circular joystick-style disc. "Follow mode" uses the active mode's declared input (joystick → circular, else rectangular). @@ -700,7 +691,7 @@ function SettingsDrawer({ depth }: { ctx: ConsoleCtx; depth: DrawerDepth }) { unit="px" onChange={(v) => set('cornerRadius', Math.round(v))} /> - {depth !== 'peek' && ( + {depth === 'expanded' && (

Roundness of buttons, control rows, dock icons and panels. Pills and the circular verdict buttons are intentionally exempt. Default 2px. @@ -716,7 +707,7 @@ function SettingsDrawer({ depth }: { ctx: ConsoleCtx; depth: DrawerDepth }) { const KEYS: [string, string][] = [ ['1–5', 'open drawers'], - ['\\', 'full depth'], + ['\\', 'expand drawer'], ['space / ↑', 'commit +'], ['↓', 'perturb / down −'], ['z', 'undo'], diff --git a/manifold/src/console/Manifold.tsx b/manifold/src/console/Manifold.tsx index 80f2c83..56e240b 100644 --- a/manifold/src/console/Manifold.tsx +++ b/manifold/src/console/Manifold.tsx @@ -285,22 +285,8 @@ export function Manifold({ ctx.stroke(); } - if (nc > 0.001) { - const breathe = 1 + Math.sin(now / 600) * 0.06; - const rCap = nc * Math.min(W, H) * 0.5 * breathe; - const rCur = rCap * 0.55; - ctx.setLineDash([4, 5]); - ctx.strokeStyle = 'rgba(255,106,0,0.4)'; - ctx.lineWidth = 1.5; - ctx.beginPath(); - ctx.arc(px, py, rCap, 0, Math.PI * 2); - ctx.stroke(); - ctx.strokeStyle = 'rgba(255,106,0,0.7)'; - ctx.beginPath(); - ctx.arc(px, py, rCur, 0, Math.PI * 2); - ctx.stroke(); - ctx.setLineDash([]); - } + // Noise-cap exploration rings hidden for now (kept for possible later use). + void nc; ctx.shadowColor = fz ? cyan : accent; ctx.shadowBlur = 18; diff --git a/manifold/src/console/OutputEditor.tsx b/manifold/src/console/OutputEditor.tsx index 4c0efa0..c9f935b 100644 --- a/manifold/src/console/OutputEditor.tsx +++ b/manifold/src/console/OutputEditor.tsx @@ -2,6 +2,7 @@ * OutputEditor — the per-output menu (state · min · max · static value · curve). * Opens on hover over an output column/cell. Ported from `OutputEditor.jsx`. */ +import { useLayoutEffect, useRef, useState } from 'react'; import type { CSSProperties, ReactNode } from 'react'; import { CurvePad } from './CurvePad'; import type { MFParam, ParamStatus } from './model'; @@ -51,7 +52,7 @@ function MiniSlider({ disabled={disabled} onChange={(e) => onChange(parseFloat(e.target.value))} className="mf-slider-input" - style={{ width: '100%' }} + style={{ width: '100%', ['--mf-pct' as string]: `${Math.max(0, Math.min(1, value))}` } as CSSProperties} /> ); @@ -67,8 +68,35 @@ export interface OutputEditorProps { export function OutputEditor({ param, onChange, onHold, onLeave, place }: OutputEditorProps) { const isLive = param.status === 'live'; + const ref = useRef(null); + // After the popover lands at its anchored spot, nudge it back on-screen so it + // never clips the viewport edges (matters when the window isn't full-screen). + const [shift, setShift] = useState<{ dx: number; dy: number }>({ dx: 0, dy: 0 }); + useLayoutEffect(() => { + const el = ref.current; + if (!el) return; + const fit = () => { + const r = el.getBoundingClientRect(); + // r already includes the current shift — back it out to get the natural + // anchored box, then derive the shift that pulls it inside the viewport. + const left = r.left - shift.dx; + const top = r.top - shift.dy; + const M = 8; // keep an 8px gutter from each edge + let dx = 0; + let dy = 0; + if (left + r.width > window.innerWidth - M) dx = window.innerWidth - M - (left + r.width); + if (left + dx < M) dx = M - left; + if (top + r.height > window.innerHeight - M) dy = window.innerHeight - M - (top + r.height); + if (top + dy < M) dy = M - top; // if too tall, prefer pinning the top + if (dx !== shift.dx || dy !== shift.dy) setShift({ dx, dy }); + }; + fit(); + window.addEventListener('resize', fit); + return () => window.removeEventListener('resize', fit); + }); return (

(null); + const canvasRef = useRef(null); + const gridRef = useRef(null); + const posRef = useRef(pos); + posRef.current = pos; + const Kref = useRef(layerCount); + Kref.current = Math.max(1, layerCount); + const namesRef = useRef(names); + namesRef.current = names; + const yaw = useRef(0); + const pitch = useRef(Math.PI / 2); // top-down (looks 2-D) until you tilt it + const view = useRef({ cw: 800, ch: 600, scale: 1, ceny: 0 }); + + const OUT = engine.architecture.outputSize; + + // ---- re-sample the landscape grid whenever the net changes ---- + useEffect(() => { + const pts: [number, number][] = []; + for (let j = 0; j <= N; j++) for (let i = 0; i <= N; i++) pts.push([i / N, j / N]); + try { + gridRef.current = engine.inferBatch(pts); + } catch { + gridRef.current = null; + } + }, [engine, version]); + + // ---- render loop ---- + useEffect(() => { + const canvas = canvasRef.current; + const wrap = wrapRef.current; + if (!canvas || !wrap) return; + const ctx = canvas.getContext('2d'); + if (!ctx) return; + const DPR = Math.max(1, Math.min(2, window.devicePixelRatio || 1)); + + const computeView = () => { + const v = view.current; + const K = Kref.current; + v.ceny = -(K - 1) * LAYER_GAP / 2 + ZAMP / 2; // centroid (centre vertically) + let R = 1; + for (let k = 0; k < K; k++) for (const z of [0, 1]) for (const gx of [-0.5, 0.5]) for (const gy of [-0.5, 0.5]) { + const X = gx * SX, Y = (-k * LAYER_GAP + z * ZAMP) - v.ceny, Z = gy * SZ; + const d = Math.sqrt(X * X + Y * Y + Z * Z); + if (d > R) R = d; + } + const fit = (Math.min(v.cw, v.ch) / 2 - MARGIN) / R; + v.scale = Math.min(REF_SCALE, fit); // grow with K, then cap to fit + }; + const resize = () => { + const r = wrap.getBoundingClientRect(); + view.current.cw = Math.max(160, Math.round(r.width)); + view.current.ch = Math.max(160, Math.round(r.height)); + canvas.width = view.current.cw * DPR; + canvas.height = view.current.ch * DPR; + ctx.setTransform(DPR, 0, 0, DPR, 0, 0); + computeView(); + }; + const ro = new ResizeObserver(resize); + ro.observe(wrap); + resize(); + + const project = (gx: number, gy: number, zv: number, k: number) => { + const v = view.current; + const X = gx * SX, Y = (-k * LAYER_GAP + zv * ZAMP) - v.ceny, Z = -gy * SZ; + const cyA = Math.cos(yaw.current), syA = Math.sin(yaw.current); + const x1 = X * cyA - Z * syA, z1 = X * syA + Z * cyA; + const cpA = Math.cos(pitch.current), spA = Math.sin(pitch.current); + const y2 = Y * cpA - z1 * spA, z2 = Y * spA + z1 * cpA; + return { x: v.cw / 2 + x1 * v.scale, y: v.ch / 2 - y2 * v.scale, depth: z2, wx: X, wy: Y, wz: Z }; + }; + const hueOf = (k: number, K: number) => (K > 1 ? 26 + (196 - 26) * k / (K - 1) : 26); + const colCss = (k: number, K: number, a: number) => `hsla(${hueOf(k, K).toFixed(0)},85%,60%,${a})`; + const LIGHT = (() => { const v = [0.35, 0.86, 0.36]; const m = Math.hypot(v[0], v[1], v[2]); return [v[0] / m, v[1] / m, v[2] / m]; })(); + const clamp = (x: number) => (x < 0 ? 0 : x > 1 ? 1 : x); + + let raf = 0; + const draw = () => { + const v = view.current; + const K = Kref.current; + const grid = gridRef.current; + computeView(); + ctx.clearRect(0, 0, v.cw, v.ch); + const H = (k: number, i: number, j: number) => (grid ? grid[(j * (N + 1) + i) * OUT + k] : 0.5); + + const quads: { a: any; b: any; d: any; e: any; hue: number; L: number; depth: number }[] = []; + const labels: { x: number; y: number; hue: number; name: string }[] = []; + for (let k = 0; k < K; k++) { + const hue = hueOf(k, K); + const P: { x: number; y: number; depth: number; wx: number; wy: number; wz: number }[][] = []; + for (let j = 0; j <= N; j++) { P[j] = []; for (let i = 0; i <= N; i++) P[j][i] = project(i / N - 0.5, j / N - 0.5, H(k, i, j), k); } + for (let j = 0; j < N; j++) for (let i = 0; i < N; i++) { + const a = P[j][i], b = P[j][i + 1], d = P[j + 1][i + 1], e = P[j + 1][i]; + const ux = d.wx - a.wx, uy = d.wy - a.wy, uz = d.wz - a.wz, vx = b.wx - e.wx, vy = b.wy - e.wy, vz = b.wz - e.wz; + const nx = uy * vz - uz * vy, ny = uz * vx - ux * vz, nz = ux * vy - uy * vx, nm = Math.hypot(nx, ny, nz) || 1; + const L = clamp(0.32 + 0.62 * Math.abs((nx * LIGHT[0] + ny * LIGHT[1] + nz * LIGHT[2]) / nm)); + quads.push({ a, b, d, e, hue, L, depth: (a.depth + b.depth + d.depth + e.depth) / 4 }); + } + const lp = project(-0.5, 0.5, H(k, 0, N), k); + labels.push({ x: lp.x, y: lp.y, hue, name: namesRef.current[k] || `p${k}` }); + } + quads.sort((p, q) => q.depth - p.depth); + for (const Q of quads) { + ctx.fillStyle = `hsla(${Q.hue},80%,${(26 + Q.L * 44).toFixed(0)}%,0.66)`; + ctx.strokeStyle = `hsla(${Q.hue},80%,82%,0.18)`; ctx.lineWidth = 0.6; + ctx.beginPath(); ctx.moveTo(Q.a.x, Q.a.y); ctx.lineTo(Q.b.x, Q.b.y); ctx.lineTo(Q.d.x, Q.d.y); ctx.lineTo(Q.e.x, Q.e.y); ctx.closePath(); + ctx.fill(); ctx.stroke(); + } + ctx.font = '11px var(--font-mono, monospace)'; ctx.textAlign = 'right'; ctx.textBaseline = 'middle'; + for (const Lb of labels) { ctx.fillStyle = `hsla(${Lb.hue},85%,74%,0.95)`; ctx.fillText(Lb.name, Lb.x - 6, Lb.y); } + + // probe column at the current input, marker on each slab from the LIVE output + const out = engine.getOutputs(); + const gx = posRef.current[0] - 0.5, gy = posRef.current[1] - 0.5; + const top = project(gx, gy, 1.18, 0), bot = project(gx, gy, -0.18, K - 1); + ctx.strokeStyle = 'rgba(255,255,255,0.3)'; ctx.lineWidth = 1; ctx.setLineDash([3, 4]); + ctx.beginPath(); ctx.moveTo(top.x, top.y); ctx.lineTo(bot.x, bot.y); ctx.stroke(); ctx.setLineDash([]); + for (let k = 0; k < K; k++) { + const val = clamp(out[k] ?? 0.5); + const p = project(gx, gy, val, k), fl = project(gx, gy, 0, k); + ctx.strokeStyle = colCss(k, K, 0.5); ctx.lineWidth = 1; + ctx.beginPath(); ctx.moveTo(p.x, p.y); ctx.lineTo(fl.x, fl.y); ctx.stroke(); + ctx.fillStyle = colCss(k, K, 1); ctx.shadowColor = colCss(k, K, 1); ctx.shadowBlur = 12; + ctx.beginPath(); ctx.arc(p.x, p.y, 6, 0, Math.PI * 2); ctx.fill(); ctx.shadowBlur = 0; + ctx.fillStyle = 'var(--bg, #0d0d0d)'; ctx.beginPath(); ctx.arc(p.x, p.y, 2.2, 0, Math.PI * 2); ctx.fill(); + } + raf = requestAnimationFrame(draw); + }; + raf = requestAnimationFrame(draw); + return () => { cancelAnimationFrame(raf); ro.disconnect(); }; + }, [engine, OUT]); + + // ---- drag to orbit (yaw + pitch; pitch up to π/2 = exact top-down) ---- + const rotating = useRef(false); + const last = useRef({ x: 0, y: 0 }); + return ( +
+ { + rotating.current = true; + (e.currentTarget as HTMLCanvasElement).setPointerCapture?.(e.pointerId); + last.current = { x: e.clientX, y: e.clientY }; + }} + onPointerMove={(e) => { + if (!rotating.current) return; + yaw.current += (e.clientX - last.current.x) * 0.01; + pitch.current = Math.max(0.06, Math.min(Math.PI / 2, pitch.current + (e.clientY - last.current.y) * 0.008)); + last.current = { x: e.clientX, y: e.clientY }; + }} + onPointerUp={() => { rotating.current = false; }} + onPointerCancel={() => { rotating.current = false; }} + /> +
+ PARAMETER SANDWICH · drag to rotate · top-down = the flat 2-D map +
+
+ ); +} diff --git a/manifold/src/console/VerdictCluster.tsx b/manifold/src/console/VerdictCluster.tsx index 2dddf54..60539ad 100644 --- a/manifold/src/console/VerdictCluster.tsx +++ b/manifold/src/console/VerdictCluster.tsx @@ -18,6 +18,7 @@ import { useRef, useState } from 'react'; import type { CSSProperties } from 'react'; import type { FeedbackModeUI } from './types'; +import { DiceIcon } from './icons'; function ThumbIcon({ size = 24, down = false }: { size?: number; down?: boolean }) { return ( @@ -44,6 +45,10 @@ export interface VerdictClusterProps { onUndo: () => void; onCommit: () => void; onReroll: () => void; + /** Small bounded weight perturbation of the current net (left half of the pill). */ + onNudge: () => void; + /** Full re-roll of the current net (right half of the pill). */ + onRandomise: () => void; canUndo: boolean; ab: 'A' | 'B'; onToggleAB: () => void; @@ -62,6 +67,8 @@ export function VerdictCluster({ onUndo, onCommit, onReroll, + onNudge, + onRandomise, canUndo, ab, onToggleAB, @@ -116,6 +123,22 @@ export function VerdictCluster({ ...extra, }); + // Two halves of the secondary pill (nudge | randomise). + const pillHalf = (extra: CSSProperties): CSSProperties => ({ + flex: 1, + height: 30, + border: 0, + background: 'transparent', + color: 'var(--fg-mute)', + cursor: 'pointer', + fontFamily: 'var(--font-mono)', + fontSize: 'var(--fs-xs)', + letterSpacing: '0.04em', + textTransform: 'uppercase', + transition: 'background var(--dur-fast), color var(--dur-fast)', + ...extra, + }); + return (
setHover(true)} @@ -126,75 +149,138 @@ export function VerdictCluster({ left: '50%', transform: 'translateX(-50%)', display: 'flex', + flexDirection: 'column', alignItems: 'center', - gap: 'var(--sp-3)', - padding: 'var(--sp-2) var(--sp-3)', - background: 'var(--glass)', - backdropFilter: 'blur(14px)', - WebkitBackdropFilter: 'blur(14px)', - border: '1px solid var(--glass-line)', - borderRadius: 'var(--r-pill)', - boxShadow: 'var(--shadow-2)', + gap: 'var(--sp-2)', opacity: hover || firstSession ? 1 : 0.55, transition: 'opacity var(--dur-med) var(--ease-console)', zIndex: 40, }} > - + - + - +
+ + {/* Secondary pill — two halves: nudge (small perturbation) | randomise (full re-roll). */} +
- {/* Explore-mode up is "place" (a pin glyph) once exploring. */} - {explore && exploring ? ⌖ : } - + + +
); } diff --git a/manifold/src/console/icons.tsx b/manifold/src/console/icons.tsx index 6ae3b3e..a5da360 100644 --- a/manifold/src/console/icons.tsx +++ b/manifold/src/console/icons.tsx @@ -50,6 +50,19 @@ export function ModeIcon({ size = 18, style }: IconProps) { ); } +/** Sandwich — stacked parameter-landscape slabs (the layer toggle). */ +export function SandwichIcon({ size = 18, style }: IconProps) { + return svg( + size, + style, + <> + + + + , + ); +} + /** Learning — a brain-ish node graph. */ export function LearningIcon({ size = 18, style }: IconProps) { return svg( @@ -117,6 +130,22 @@ export function HelpIcon({ size = 18, style }: IconProps) { ); } +/** Dice — explore / re-roll into a scratchpad (a die showing five pips). */ +export function DiceIcon({ size = 22, style }: IconProps) { + return svg( + size, + style, + <> + + + + + + + , + ); +} + /** Close (✕). */ export function CloseIcon({ size = 14, style }: IconProps) { return svg(size, style, ); diff --git a/manifold/src/console/shared-ui.tsx b/manifold/src/console/shared-ui.tsx index 3d9c4a0..26c5154 100644 --- a/manifold/src/console/shared-ui.tsx +++ b/manifold/src/console/shared-ui.tsx @@ -163,7 +163,13 @@ export function CompactAxis({ value={value} onChange={(e) => onChange(parseFloat(e.target.value))} className="mf-slider-input" - style={{ width: 120, ['--mf-axis-accent' as string]: accent } as CSSProperties} + style={ + { + width: 120, + ['--mf-axis-accent' as string]: accent, + ['--mf-pct' as string]: `${Math.max(0, Math.min(1, value))}`, + } as CSSProperties + } /> onChange({ val: parseFloat(e.target.value) })} className="mf-slider-input" - style={{ flex: 1 }} + style={{ flex: 1, ['--mf-pct' as string]: `${Math.max(0, Math.min(1, param.val))}` } as CSSProperties} /> )} diff --git a/manifold/src/styles/primitives.css b/manifold/src/styles/primitives.css index 7d0d2ef..3c3e5bb 100644 --- a/manifold/src/styles/primitives.css +++ b/manifold/src/styles/primitives.css @@ -13,13 +13,21 @@ */ /* ---- Slider (.mf-slider-input) ---- */ +/* Base reset so the custom track/thumb styling applies even when a consumer + doesn't set these inline (e.g. OutputControlRow's held slider). The colored + fill tracks `--mf-pct` (0..1); it falls back to 0 if a consumer forgets it. */ +.mf-slider-input { + -webkit-appearance: none; + appearance: none; + background: transparent; +} .mf-slider-input::-webkit-slider-runnable-track { height: 4px; border-radius: 999px; background: linear-gradient( to right, var(--accent) 0%, - var(--accent) calc(var(--mf-pct) * 100%), + var(--accent) calc(var(--mf-pct, 0) * 100%), var(--bg-3) 0% ); } @@ -28,6 +36,11 @@ border-radius: 999px; background: var(--bg-3); } +.mf-slider-input::-moz-range-progress { + height: 4px; + border-radius: 999px; + background: var(--accent); +} .mf-slider-input::-webkit-slider-thumb { -webkit-appearance: none; appearance: none; diff --git a/manifold/tests/e2e/smoke.spec.ts b/manifold/tests/e2e/smoke.spec.ts index 0a53e8e..c67c8db 100644 --- a/manifold/tests/e2e/smoke.spec.ts +++ b/manifold/tests/e2e/smoke.spec.ts @@ -50,7 +50,9 @@ test('engine loads, spine propagates, console renders', async ({ page }) => { // 4. The convertible Console rendered (assert on the dock drawer rail, which // is always present — the old "MEMLNaut" wordmark only shows outside // Particle mode now that the particle top bar is a heatmap strip). - await expect(page.getByText('Learning')).toBeVisible(); + // Drawers are closed by default, so assert on the rail button's title + // rather than the (hidden) drawer-header text. + await expect(page.getByTitle('Learning')).toBeVisible(); // 5. No "C15" anywhere in the rendered UI. const body = await page.evaluate(() => document.body.innerText); diff --git a/nisps/modes/base.hpp b/nisps/modes/base.hpp index 996e80e..81778c9 100644 --- a/nisps/modes/base.hpp +++ b/nisps/modes/base.hpp @@ -146,7 +146,7 @@ class ModeBase { // Run an inference at default inputs so engine has params on first // process() call, even if no input has been touched. for (std::size_t i = 0u; i < NInputs; ++i) { - ml_.set_input(i, input_channels_[i]); + ml_.set_input(i, effective_input(i)); } ml_.process(); if constexpr (kRouteOutputsToEngine) { @@ -165,6 +165,34 @@ class ModeBase { input_dirty_ = true; } + // ---- Input neutralization (single/double controller toggle) ---- + // + // A pinned channel feeds `pin_value_` (neutral, default 0.5) to the MLP + // instead of its live value, without rebuilding/resizing the network. + // Glue toggles which channels are pinned (e.g. single-joystick mode pins + // the second 2D controller's two channels). The stored live value is left + // untouched, so unpinning resumes from the controller's current position. + NISPS_FORCE_INLINE void set_input_pinned(std::size_t idx, bool pinned) noexcept { + if (idx >= NInputs) return; + input_pinned_[idx] = pinned; + input_dirty_ = true; + } + NISPS_FORCE_INLINE bool is_input_pinned(std::size_t idx) const noexcept { + return idx < NInputs && input_pinned_[idx]; + } + NISPS_FORCE_INLINE void set_pin_value(float v) noexcept { + if (v < 0.f) v = 0.f; + else if (v > 1.f) v = 1.f; + pin_value_ = v; + input_dirty_ = true; + } + float pin_value() const noexcept { return pin_value_; } + + // Effective value fed to the MLP for channel i (pin override applied). + NISPS_FORCE_INLINE float effective_input(std::size_t i) const noexcept { + return input_pinned_[i] ? pin_value_ : input_channels_[i]; + } + NISPS_HOT void tick_control() noexcept { if constexpr (requires(Derived& d) { d.on_pre_inference(); }) { static_cast(*this).on_pre_inference(); @@ -177,9 +205,10 @@ class ModeBase { if (jolt_.active()) apply_jolt_(); jolt_.tick_lr_ramp(); - // Forward (possibly Derived-mutated) channels into the MLP. + // Forward (possibly Derived-mutated) channels into the MLP, applying + // the per-channel pin override. for (std::size_t i = 0u; i < NInputs; ++i) { - ml_.set_input(i, input_channels_[i]); + ml_.set_input(i, effective_input(i)); } ml_.process(); if constexpr (kRouteOutputsToEngine) { @@ -294,6 +323,8 @@ class ModeBase { EngineT engine_{}; MLPType ml_; std::array input_channels_{}; + std::array input_pinned_{}; // false => live + float pin_value_ = 0.5f; // neutral bool input_dirty_ = false; std::size_t voice_space_idx_ = 0u; RingBuffer events_{}; diff --git a/tests/cpp/test_mode_paf_synth.cpp b/tests/cpp/test_mode_paf_synth.cpp index 737ba2f..9d18662 100644 --- a/tests/cpp/test_mode_paf_synth.cpp +++ b/tests/cpp/test_mode_paf_synth.cpp @@ -88,6 +88,62 @@ NISPS_TEST(paf_synth_mode_input_clamping) { m.set_input(99u, 0.5f); } +NISPS_TEST(paf_synth_mode_pinned_inputs_neutralized) { + // Single-joystick mode pins the second 2D controller's channels (2,3). + // A pinned channel must feed the neutral pin value (default 0.5) to the + // MLP regardless of its live value, and pinning must not resize the model. + PAFSynthMode m; + m.setup(48000.f); + NISPS_EXPECT(m.pin_value() == 0.5f); + + // Reference: channels 2,3 driven live to 0.5 (the neutral value). + m.set_input(0, 0.2f); + m.set_input(1, 0.8f); + m.set_input(2, 0.5f); + m.set_input(3, 0.5f); + m.tick_control(); + std::array ref{}; + { + const auto outs = m.ml().outputs(); + for (std::size_t i = 0u; i < outs.size(); ++i) ref[i] = outs[i]; + } + + // Pin channels 2,3, then drive them to arbitrary values. Output must match + // the reference (they are neutralized to 0.5). + m.set_input_pinned(2, true); + m.set_input_pinned(3, true); + NISPS_EXPECT(m.is_input_pinned(2)); + m.set_input(2, 0.0f); + m.set_input(3, 1.0f); + m.tick_control(); + for (std::size_t i = 0u; i < 33u; ++i) { + NISPS_EXPECT(std::fabs(m.ml().outputs()[i] - ref[i]) < 1e-6f); + } + + // The active controller (channels 0,1) still affects the output. + m.set_input(0, 0.9f); + m.tick_control(); + bool changed = false; + for (std::size_t i = 0u; i < 33u; ++i) { + if (std::fabs(m.ml().outputs()[i] - ref[i]) > 1e-6f) { changed = true; break; } + } + NISPS_EXPECT(changed); + + // Unpinning restores sensitivity on channels 2,3. + m.set_input_pinned(2, false); + m.set_input_pinned(3, false); + m.set_input(0, 0.2f); // restore active channels to the reference pose + m.set_input(1, 0.8f); + m.set_input(2, 0.0f); + m.set_input(3, 1.0f); + m.tick_control(); + bool differs = false; + for (std::size_t i = 0u; i < 33u; ++i) { + if (std::fabs(m.ml().outputs()[i] - ref[i]) > 1e-6f) { differs = true; break; } + } + NISPS_EXPECT(differs); +} + NISPS_TEST(paf_synth_mode_engine_and_ml_accessors) { PAFSynthMode m; m.setup(48000.f);