diff --git a/.beads/interactions.jsonl b/.beads/interactions.jsonl index e69de29..50d1f2f 100644 --- a/.beads/interactions.jsonl +++ b/.beads/interactions.jsonl @@ -0,0 +1,2 @@ +{"id":"int-691bec44","kind":"field_change","created_at":"2026-04-15T15:26:14.033393505Z","actor":"w1n5t0n","issue_id":"meml-kx6a","extra":{"field":"status","new_value":"in_progress","old_value":"open"}} +{"id":"int-55a0a37d","kind":"field_change","created_at":"2026-04-15T15:32:17.624534929Z","actor":"w1n5t0n","issue_id":"meml-kx6a","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Added firmware helper scripts and updated docs to the RP2350/C++20/O3 build flow"}} diff --git a/CLAUDE.md b/CLAUDE.md index d96ff9c..48220aa 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -235,17 +235,24 @@ Tests use the `?debug=1` probe (`window.__nisps`) for programmatic access to the ## Build System -This is an Arduino project targeting Raspberry Pi Pico. Build and upload using Arduino IDE or arduino-cli with the earlephilhower/pico board package. +This is an Arduino project targeting the MEMLNaut RP2350 hardware. Build and flash it with the repo-local helper scripts, which wrap the correct board target and compiler settings. ```bash # Initialize submodules (required for memllib and memlp) git submodule update --init --recursive -# Build (adjust port as needed) -arduino-cli compile --fqbn rp2040:rp2040:rpipico -b 115200 MEMLNaut-NISPS.ino -arduino-cli upload --fqbn rp2040:rp2040:rpipico -p /dev/ttyACM0 MEMLNaut-NISPS.ino +# Build only +scripts/build-firmware.sh + +# Flash a previously-built UF2 +scripts/flash-firmware.sh + +# Build then flash +scripts/build-and-flash-firmware.sh ``` +The scripts build for `rp2040:rp2040:solderparty_rp2350_stamp_xl:opt=Optimize3` and force C++20 via `compiler.cpp.extra_flags=-std=gnu++20`. + ## Architecture ### Dual-Core Design diff --git a/MAP.md b/MAP.md new file mode 100644 index 0000000..5e69e7f --- /dev/null +++ b/MAP.md @@ -0,0 +1,95 @@ +# MAP + +MEMLNaut-NISPS: Neural Interactive Shaping of Parameter Spaces. Two living artefacts share one ML core: (1) Arduino/RP2040 firmware for the MEMLNaut hardware, and (2) a browser playground that uses a WASM build of the same MLP to drive a C15 synth + other outputs. A header-only `nisps-core/` extraction is reused by the playground (via WASM bindings) and a VCV Rack module. See `CLAUDE.md` for the long-form architecture narrative. + +## Layout + +### Firmware (Arduino, RP2040) +- `MEMLNaut-NISPS.ino` — top-level sketch; dual-core setup, mode selected at compile-time via `#define MEMLNAUT_MODE_TYPE`. +- `IMLInterface.hpp` — interactive ML controller; wraps MLP + RL with STORE_VALUE / STORE_POSITION modes. +- `modes/MEMLNautMode.hpp` — concept/contract satisfied by each mode class. +- `modes/MEMLNautMode*.hpp` — mode implementations (PAFSynth, ChannelStrip, XIASRI, SoundAnalysisMIDI, VerbFX, BreakOr, Elysiamorfs). +- `modes/AudioApps/` — audio engines used by the newer modes (VerbFX, BreakOr, Elysiamorf, RatioSeq). +- `PAFSynthAudioApp.hpp`, `ChannelStripAudioApp.hpp`, `ThruAudioApp.hpp`, `XIASRIAudioApp.hpp` — older audio engines at the repo root. +- `XiasriAnalysis.{hpp,cpp}` — real-time audio feature extraction (pitch, aperiodicity, brightness, etc.). +- `voicespaces/` — lambdas mapping MLP output vectors to synth parameters. Subdir `ChannelStrip/` for EQ/comp presets. +- `src/memllib/`, `src/memlp/` — git submodules (hardware abstraction + MLP). **Not auto-initialized** — build breaks without `git submodule update --init --recursive`. +- `src/daisysp/` — vendored DSP library (filters, drums, effects). +- `data/` — preset/asset CSVs. + +### nisps-core (platform-agnostic C++20 header-only) +- `nisps-core/include/nisps/` — `iml.hpp`, `mlp.hpp`, `layer.hpp`, `node.hpp`, `dataset.hpp`, `loss.hpp`, `utils.hpp`. +- `nisps-core/test/`, `nisps-core/examples/`, `nisps-core/README.md`, `CHANGELOG.md`. + +### Playground (browser ML demo) +- `playground/index.html` — hub linking to the three variants. +- `playground/a-immersive.html` + `js/a-app.js` — **primary** app. WASM engine, full control surface, modular/engine-switcher, C15 + MIDI + audio-canvas outputs. +- `playground/b-workbench.html` + `js/b-app.js`, `c-journey.html` + `js/c-app.js` — older variants on the legacy JS engine. Feature-frozen; drift vs. a-app is intentional (see `CLAUDE.md` memory on `playground/RECONCILIATION.md` — note: file does **not** currently exist). +- `playground/designs.html`, `js/app.js` — oldest experimental app. Kept for reference. +- `playground/wasm/` — Emscripten build: `nisps_bindings.cpp` (C API, float32), `build.sh`, compiled `nisps.wasm`/`nisps.js`. +- `playground/js/nisps/` — `nisps-wasm.js` (WasmIML wrapper), `nisps-wasm-worker.js` (off-thread train), `dataset.js` (FIFO ring buffer, max 100), legacy pure-JS engine (`iml.js`, `mlp.js`, `layer.js`, `node.js`) used by b/c apps. +- `playground/js/synth/` — `c15-bridge.js`, `param-map.js` (126 curated C15 params), `presets.js` (4 tiers), `arpeggiator.js`. +- `playground/js/ui/` — UI modules. Categories: + - Input: `input-pipeline.js`, `joystick.js`, `joy-map-enhanced.js`, `gamepad.js`, `hand-tracker.js`, `eoc-*.js`. + - Control surface: `control-surface.js`, `control-surface-ui.js` (3 compound axes: Boldness / Memory / Precision). + - Training/exploration: `snapshot-stack.js`, `ab-compare.js`, `region-pin.js`, `param-pin.js`, `auto-explore.js`, `pressure-feedback.js`, `input-heatmap.js`. + - Output/debug: `output-pipeline.js`, `weight-health.js`, `gradient-flow.js`, `session-presets.js`, `visualizer.js`, `param-display.js`, `dev-panel.js`. + - Phase wiring: `phase2-ui.js`, `phase3-ui.js`, `phase4-ui.js`. + - Modular mode: `modular-ui.js` (~52k, large), `engine-switcher.js` — newer; not yet documented in `CLAUDE.md`. +- `playground/c15/`, `playground/faust/`, `playground/osc-bridge/` — external synth/bridge assets. +- `playground/SPEC-controls.md`, `SPEC-shapeseq.md`, `ARCHITECTURE.md`, `PLAN-solidjs-migration.md`, `TODOS.md`, `README.md`, `devlog/` — docs. + +### Other consumers +- `vcv/` — VCV Rack plugin using `nisps-core` (`src/MEMLNaut.cpp`, `SPEC.md`, `NISPS-FORMAT.md`). + +### Tests +- `tests/e2e/*.spec.js` — Playwright e2e against the immersive app via the `?debug=1` probe (`window.__nisps`). Covers ml-engine, wasm-api, ui-interactions, input-pipeline, persistence, engine-switching, modular-mode. Shared helpers in `tests/e2e/helpers.js`. +- `playwright.config.js`, `package.json` — auto-starts a static server on port 7331. + +### Top-level docs / planning +- `CLAUDE.md` — architecture narrative for both firmware and playground. +- `AGENTS.md` — beads/bd conventions. +- `NISPS_CORE_EXTRACTION_PLAN.md`, `NISPS_CORE_TASKS.md` — extraction task list; status unclear, likely stale now that `nisps-core/` exists. +- `README.md` — short quickstart. + +## Entry points +- **Firmware**: `scripts/build-firmware.sh`, `scripts/flash-firmware.sh`, or `scripts/build-and-flash-firmware.sh` (requires submodules initialised). The scripts target `rp2040:rp2040:solderparty_rp2350_stamp_xl:opt=Optimize3` and force C++20. Execution = `setup()`/`loop()` on Core 0, `setup1()`/`loop1()` on Core 1, audio ISR on Core 1. +- **Playground**: `cd playground && python3 -m http.server` (or `serve.sh` / `serve-coop.py`), open `a-immersive.html`. Append `?debug=1` to expose `window.__nisps`. +- **WASM rebuild**: `cd playground/wasm && ./build.sh` (needs `emcc`). +- **Tests**: `npx playwright test` (auto-spawns server on 7331). +- **VCV module**: built inside `vcv/` with the VCV Rack SDK. + +## Conventions +- Firmware mode selection is compile-time only; only one `MEMLNAUT_MODE_TYPE` uncommented at a time in `MEMLNaut-NISPS.ino`. +- RP2040 memory placement via `APP_SRAM`, `AUDIO_MEM`, `AUDIO_FUNC`, `__not_in_flash("app")`. Audio hot paths use `__force_inline` / `__hot` / `__flatten`. +- Cross-core sync: `MEMORY_BARRIER()`, `WRITE_VOLATILE`/`READ_VOLATILE`, RP2040 `queue_t`. +- Voice spaces are header-only structs whose mappings are lambdas capturing synth state — **implicit coupling** to synth members. +- Playground ML engines (`IML`, `WasmIML`) share a duck-typed interface (`inference`, `train`, `getWeights`/`setWeights`, …); WASM uses **float32**, JS engine uses float64. +- `Dataset` is a **FIFO ring buffer**, default max 100 examples; recency/spatial sample weighting is computed JS-side. +- Spread-aware weight init / RL noise (`drawWeightsSpread`, `moveWeightsEx`) live in `playground/wasm/nisps_bindings.cpp`, **not** in `nisps-core` proper — they are playground-specific. +- Playground UI modules dispatch `controlsurface:change` CustomEvents; `a-app.js` listens and reconfigures the input pipeline, spread, and RL params. +- URL params: `?tame`, `?spread`, `?preset`, `?debug=1`, `?shapeseq=1`. +- Persistent memory (`bd remember`) notes: + - ShapeSeq is gated behind `?shapeseq=1` until solid — arp remains default. + - ShapeSeq MLP plan is **switchable mode** (unified single-MLP first, then dual-MLP option). + - `playground/RECONCILIATION.md` is supposed to track features landed in a-app but not yet in b/c. File does not currently exist — if you add a-only features, either create it or explicitly accept the drift. + +## Gotchas +- `memllib` and `memlp` submodules are **not auto-checked-out** — a fresh clone will fail to compile the firmware silently. +- Double-scaled loss: C++ `Train()` and WASM `train_ex` both divide by `n` when no sample weights are supplied (known, backward-compat, tracked as `meml-ues`). +- `VerbFXAudioApp` / `MEMLNautModeVerbFX` have **entire analysis blocks commented out** — the mode was migrated from analysis-driven to joystick-driven and cleanup is unfinished. Don't assume XiasriAnalysis wiring is live there. +- `XiasriAnalysis` output struct is union-cast to a float array inside XIASRI mode — fragile if the struct layout changes. +- Recent fixes cluster around the modular voice: matrix rebuild, `mod_amp` positive-only floor, MLP bypass when untrained, worklet blob-url registration — modular mode is under active churn, so expect rough edges. +- `a-app.js` is the single source of truth. **Do not** reflexively mirror changes to `b-app.js` / `c-app.js` — they are frozen legacy variants. +- `window.__nisps` only exists with `?debug=1`; Playwright helpers expect this. + +## Open questions / smells +- `modular-ui.js` is ~52k and undocumented in `CLAUDE.md`; needs a `docs/modular.md` stub, especially given recent bug cluster. +- `engine-switcher.js` + `engine-switching.spec.js` + `modular-mode.spec.js` — newer engine-selection mechanism not described in `CLAUDE.md`. Verify whether there is now a supported alternative engine besides WASM-MLP. +- `playground/RECONCILIATION.md` is referenced by persistent memory but missing on disk. Either the memory is stale or the file needs creating. +- `NISPS_CORE_TASKS.md` / `NISPS_CORE_EXTRACTION_PLAN.md` at the repo root likely describe completed work — candidates for deletion or archiving under `docs/history/`. +- `PLAN-solidjs-migration.md` (34k) describes an unstarted rewrite. Either flag it "aspirational / not started" at the top or move to `docs/`. +- No `README.md` for `playground/wasm/` — a 10-line binding table (C API ↔ JS wrapper ↔ nisps-core call) would save future agents a trip through `nisps_bindings.cpp`. +- Firmware `IMLInterface`'s STORE_VALUE vs STORE_POSITION modes have no docs — decide which modes use which and document. +- Global `std::shared_ptr` in the sketch introduces refcount traffic on the 1 ms MIDI poll — likely benign, worth confirming. +- Two duplicated CLAUDE.md copies at `~/.claude/CLAUDE.md` and `~/.claude-gp/CLAUDE.md` (symlinked), and a per-repo one — not a repo smell, just noted so future agents don't try to "reconcile". diff --git a/README.md b/README.md index 343f605..50a26f9 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,24 @@ https://musicallyembodiedml.github.io/memlnaut/approaches/nisps +## Firmware + +The hardware firmware targets the MEMLNaut RP2350 build and uses repo-local helper +scripts for the known-good build configuration: + +```bash +git submodule update --init --recursive + +scripts/build-firmware.sh +scripts/flash-firmware.sh +scripts/build-and-flash-firmware.sh +``` + +Notes: +- The scripts build for `rp2040:rp2040:solderparty_rp2350_stamp_xl` with `Optimize3`. +- The build forces C++20 because the firmware uses `std::span` and concepts. +- `flash-firmware.sh` accepts an optional mountpoint argument, or auto-detects common UF2 bootloader mounts such as `/run/media/$USER/RP2350` and `/run/media/$USER/RPI-RP2`. + ## Web Playground Try NISPS in your browser — no hardware required: diff --git a/scripts/build-and-flash-firmware.sh b/scripts/build-and-flash-firmware.sh new file mode 100755 index 0000000..443e8da --- /dev/null +++ b/scripts/build-and-flash-firmware.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +usage() { + cat <<'EOF' +Usage: + scripts/build-and-flash-firmware.sh + scripts/build-and-flash-firmware.sh [mountpoint] + +If no mountpoint is provided, the flash step auto-detects a standard UF2 +bootloader location such as /run/media/$USER/RP2350 or /run/media/$USER/RPI-RP2. +EOF +} + +if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then + usage + exit 0 +fi + +case $# in + 0) + ;; + 1) + ;; + *) + usage >&2 + exit 1 + ;; +esac + +"$SCRIPT_DIR/build-firmware.sh" + +if [[ $# -eq 1 ]]; then + "$SCRIPT_DIR/flash-firmware.sh" "$1" +else + "$SCRIPT_DIR/flash-firmware.sh" +fi diff --git a/scripts/build-firmware.sh b/scripts/build-firmware.sh new file mode 100755 index 0000000..4200090 --- /dev/null +++ b/scripts/build-firmware.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=./firmware-common.sh +source "$SCRIPT_DIR/firmware-common.sh" + +if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then + cat <<'EOF' +Usage: + scripts/build-firmware.sh [extra arduino-cli compile args...] + +Builds the MEMLNaut firmware with the repo's known-good RP2350 target and C++20 flag. +EOF + exit 0 +fi + +ensure_arduino_cli +ensure_submodules_ready + +mkdir -p "$BUILD_DIR" + +echo "Building firmware:" +echo " sketch: $SKETCH_PATH" +echo " fqbn: $FQBN" +echo " out: $BUILD_DIR" + +compile_args=( + compile + --fqbn "$FQBN" + --build-property "$CXX20_BUILD_PROPERTY" + --output-dir "$BUILD_DIR" + "$SKETCH_PATH" +) +compile_args+=("$@") + +arduino-cli "${compile_args[@]}" + +echo +echo "UF2 ready at:" +echo " $UF2_PATH_DEFAULT" diff --git a/scripts/firmware-common.sh b/scripts/firmware-common.sh new file mode 100755 index 0000000..a6cdace --- /dev/null +++ b/scripts/firmware-common.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +SKETCH_PATH="${MEMLNAUT_FIRMWARE_SKETCH:-$REPO_ROOT/MEMLNaut-NISPS.ino}" +SKETCH_NAME="$(basename "$SKETCH_PATH")" +BUILD_DIR="${MEMLNAUT_FIRMWARE_BUILD_DIR:-/tmp/memlnaut-firmware-build}" +UF2_PATH_DEFAULT="${MEMLNAUT_FIRMWARE_UF2:-$BUILD_DIR/${SKETCH_NAME}.uf2}" + +FQBN="${MEMLNAUT_FIRMWARE_FQBN:-rp2040:rp2040:solderparty_rp2350_stamp_xl:opt=Optimize3}" +CXX20_BUILD_PROPERTY="${MEMLNAUT_FIRMWARE_CXX20_PROPERTY:-compiler.cpp.extra_flags=-std=gnu++20}" + +ensure_arduino_cli() { + if ! command -v arduino-cli >/dev/null 2>&1; then + echo "error: arduino-cli is not installed or not on PATH" >&2 + exit 1 + fi +} + +ensure_git() { + if ! command -v git >/dev/null 2>&1; then + echo "error: git is not installed or not on PATH" >&2 + exit 1 + fi +} + +ensure_submodules_ready() { + ensure_git + + local status + status="$(git -C "$REPO_ROOT" submodule status --recursive)" + if grep -qE '^[+-]' <<<"$status"; then + echo "error: submodules are not initialized or not at the recorded revision" >&2 + echo "run: git submodule update --init --recursive" >&2 + exit 1 + fi +} + +resolve_path() { + local path="$1" + + if [[ "$path" = /* ]]; then + printf '%s\n' "$path" + else + printf '%s\n' "$(pwd)/$path" + fi +} + +find_boot_mount() { + local candidates=( + "/run/media/${USER}/RP2350" + "/run/media/${USER}/RPI-RP2" + "/media/${USER}/RP2350" + "/media/${USER}/RPI-RP2" + "/mnt/RP2350" + "/mnt/RPI-RP2" + ) + local candidate + + for candidate in "${candidates[@]}"; do + if [[ -d "$candidate" ]]; then + printf '%s\n' "$candidate" + return 0 + fi + done + + local base + for base in "/run/media/${USER}" "/media/${USER}" "/mnt"; do + [[ -d "$base" ]] || continue + candidate="$(find "$base" -maxdepth 2 -type f -name INFO_UF2.TXT -printf '%h\n' 2>/dev/null | head -n 1 || true)" + if [[ -n "$candidate" ]]; then + printf '%s\n' "$candidate" + return 0 + fi + done + + return 1 +} + +assert_boot_mount() { + local mountpoint="$1" + + if [[ ! -d "$mountpoint" ]]; then + echo "error: bootloader mount does not exist: $mountpoint" >&2 + exit 1 + fi + + if [[ ! -f "$mountpoint/INFO_UF2.TXT" && ! -f "$mountpoint/INDEX.HTM" ]]; then + echo "error: $mountpoint does not look like an RP2040/RP2350 UF2 boot volume" >&2 + exit 1 + fi +} diff --git a/scripts/flash-firmware.sh b/scripts/flash-firmware.sh new file mode 100755 index 0000000..b614417 --- /dev/null +++ b/scripts/flash-firmware.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=./firmware-common.sh +source "$SCRIPT_DIR/firmware-common.sh" + +usage() { + cat <<'EOF' +Usage: + scripts/flash-firmware.sh + scripts/flash-firmware.sh [mountpoint] + scripts/flash-firmware.sh [uf2-path] [mountpoint] + +Defaults: + uf2-path -> /tmp/memlnaut-firmware-build/MEMLNaut-NISPS.ino.uf2 + mountpoint -> auto-detect from standard UF2 bootloader locations +EOF +} + +uf2_path="$UF2_PATH_DEFAULT" +mountpoint="" + +if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then + usage + exit 0 +fi + +case $# in + 0) + ;; + 1) + if [[ -f "$1" || "$1" == *.uf2 ]]; then + uf2_path="$1" + else + mountpoint="$1" + fi + ;; + 2) + uf2_path="$1" + mountpoint="$2" + ;; + *) + usage >&2 + exit 1 + ;; +esac + +uf2_path="$(resolve_path "$uf2_path")" +if [[ ! -f "$uf2_path" ]]; then + echo "error: UF2 file not found: $uf2_path" >&2 + echo "run scripts/build-firmware.sh first, or pass an explicit UF2 path" >&2 + exit 1 +fi + +if [[ -z "$mountpoint" ]]; then + mountpoint="$(find_boot_mount || true)" +fi + +if [[ -z "$mountpoint" ]]; then + echo "error: could not find a mounted UF2 bootloader volume" >&2 + echo "put the board in bootloader mode or pass the mountpoint explicitly" >&2 + exit 1 +fi + +mountpoint="$(resolve_path "$mountpoint")" +assert_boot_mount "$mountpoint" + +echo "Flashing firmware:" +echo " uf2: $uf2_path" +echo " mountpoint: $mountpoint" + +cp "$uf2_path" "$mountpoint/" +sync + +echo +echo "Flash copy completed."