From c986377b4cfe21b06b249e875aaed4c4aacd3eb8 Mon Sep 17 00:00:00 2001 From: monkey-w1n5t0n Date: Sun, 28 Jun 2026 23:44:57 +0200 Subject: [PATCH] =?UTF-8?q?feat(firmware):=20reposition=20gesture=20?= =?UTF-8?q?=E2=80=94=20relocate=20an=20existing=20example=20to=20a=20new?= =?UTF-8?q?=20input?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a "grab → move → drop" gesture that moves an existing positive example's output to a new input position, preserving the output. This is the new core's home for upstream InterfaceRL's drag-store/reposition-commit, distinct from Explore→Place (which places newly-auditioned scratchpad sounds). - nisps/ml/feedback.hpp: begin_reposition()/commit_reposition()/repositioning(). Reuses the Placing state (static_output holds the carried vector) but a reposition_ flag makes commit AND the mode-switch teardown SKIP the weight restore — the real net is never set aside here, so restoring snapshot_ would clobber the live trained net. Guards cancel_place + abort_explore_place. - firmware glue: state-gate Toggle B. Exploring → reroll/nudge (unchanged); Idle → MomB1 grab, MomB2 drop (commit + add_example + train). The 4D variant has no joystick button, so the gesture lives on the momentary toggle. Also fix a stale top-of-file control-map comment that contradicted the bindings. - tests: 4 reposition cases (hold without snapshot; commit stores carried output with no restore; mode-switch aborts without clobber; begin-only-Idle). - MAP.md: document the full ExploreAndPlace lifecycle + reposition wiring. Audio-hold (carrying the sound audibly during the move) remains the existing unwired static_output() TODO and affects Explore→Place identically. Firmware compile unverified (no arduino-cli); host tests + lint pass. --- MAP.md | 2 +- firmware/MEMLNaut-NISPS/glue/peripherals.hpp | 67 +++++++++++--- nisps/ml/feedback.hpp | 80 ++++++++++++++++- tests/cpp/test_mlp_feedback.cpp | 94 ++++++++++++++++++++ 4 files changed, 228 insertions(+), 15 deletions(-) diff --git a/MAP.md b/MAP.md index 5bf087a..0d181a0 100644 --- a/MAP.md +++ b/MAP.md @@ -18,7 +18,7 @@ MEMLNaut-NISPS — Neural Interactive Shaping of Parameter Spaces. One C++20 cod - `firmware/MEMLNaut-NISPS/MEMLNaut-NISPS.ino` — entry point. Selects active mode at compile time via `#define MEMLNAUT_MODE_TYPE`. Forks on `NISPS_SELFTEST`: normal modes run the engine/ML path; the `SelfTest` variant delegates all four entry points to `glue/selftest.hpp`. - `firmware/MEMLNaut-NISPS/glue/` — hardware bindings: - `audio_driver.hpp` — bridges memllib `AudioDriver` callback → `Mode::process(stereosample_t)`. - - `peripherals.hpp` — joystick / pots / buttons → `Mode::set_input` and ML primitives. Also wires the shared adaptive-learning gestures: **TogB1** = Jolt (held weight morph), **RVX1** = exploration amount (OU output walk). + - `peripherals.hpp` — joystick / pots / buttons → `Mode::set_input` and ML primitives. Wires the shared `FeedbackController` ExploreAndPlace lifecycle (MomA1 = enter/exit explore, MomA2 = freeze/place, TogB2 = commit; MomB1/MomB2 = reroll/nudge while exploring **or** grab/drop *reposition* while idle) plus the adaptive-learning gestures: **TogB1** = Jolt (held weight morph), **RVX1** = exploration amount (OU output walk). Reposition relocates an existing positive example's output to a new input position (`feedback.hpp` `begin_reposition`/`commit_reposition`) — no scratchpad, no weight restore. - `midi_io.hpp` — MIDI in → mode `note_on`/`update_bpm`/`set_playing`; drains `ControlEvent` ring → MIDI UART. - `mode_select.hpp` — type aliases mapping firmware mode identifiers to `nisps::modes::*Mode` C++ types. Build script rewrites the active line. Includes the six `MEMLNautModeExtSynth*` external-synth variants (one per device template in `nisps/midi`, e.g. `MEMLNautModeExtSynthSub37`). Also defines the `MEMLNautModeSelfTest` pseudo-variant (tag type) + the `NISPS_ST_*`/`NISPS_ST_CAT` token-paste macros the `.ino` uses to compute `NISPS_SELFTEST`. Note: `src/nisps/` exposes each referenced top-level nisps subdir as a symlink — `midi` was added alongside `core/dsp/engines/ml/modes`. - `selftest.hpp` — standalone guided hardware self-test rig (`SelfTest` variant; no engine/ML). Step-driven state machine on a `SelfTestView`: TFT prompts the operator through every control, auto-advances on detection, encoder-press skips. Ends with optional L/R/BOTH sine-sweep headphone check (core 1 block callback) + MIDI loopback-cable test. Lives firmware-side (touches TFT + raw pins) so it stays out of platform-agnostic `nisps/`. diff --git a/firmware/MEMLNaut-NISPS/glue/peripherals.hpp b/firmware/MEMLNaut-NISPS/glue/peripherals.hpp index 0d6bfb0..1dabe82 100644 --- a/firmware/MEMLNaut-NISPS/glue/peripherals.hpp +++ b/firmware/MEMLNaut-NISPS/glue/peripherals.hpp @@ -16,13 +16,14 @@ // `[0, N)` — that's the trivial mapping that matches every concrete mode's // schema today (joystick first, optional 4D extra pot for joy_w). // -// Buttons drive the InteractiveML primitives directly: -// MomA1 (TA up) : randomise / draw weights -// MomA2 (TA down) : clear examples (reset dataset) -// MomB1 (MA up) : randomise (synonym, deliberate) -// MomB2 (MA down) : nudge (small bounded perturbation) +// Buttons/toggles drive the SHARED ExploreAndPlace lifecycle. The authoritative +// per-control mapping is documented at the binding site below; in summary: +// MomA1 (TA up) : enter / exit explore (toggle) +// MomA2 (TA down) : like — freeze current scratchpad output (begin place) +// MomB1 (MA up) : Exploring → reroll scratchpad; Idle → grab (reposition) +// MomB2 (MA down) : Exploring → nudge scratchpad; Repositioning → drop+train // TogB1 : Jolt — held continuous weight morph (up=morph, down=freeze) -// TogB2 : train (rising-edge → call ml.train()) +// TogB2 : commit place (rising-edge) — store +1 example + train // RVX1 : exploration amount (Ornstein-Uhlenbeck output walk) // // The button block below wires the SHARED ExploreAndPlace lifecycle @@ -115,8 +116,12 @@ inline void bind_peripherals(Mode& mode) { // // MomA1 (TA up) : down — enter explore (Idle→Exploring) / // exit explore (Exploring→Idle) TOGGLE - // MomB1 (MA up) : randomise — reroll the scratchpad (Exploring) - // MomB2 (MA down) : nudge — small bounded perturbation (Exploring) + // MomB1 (MA up) : STATE-GATED — Exploring: reroll the scratchpad; + // Idle: GRAB (begin reposition — freeze the + // current trained-net output to carry it). + // MomB2 (MA down) : STATE-GATED — Exploring: nudge (small perturbation); + // Repositioning: DROP (commit the carried output + // at the current joystick input + train). // MomA2 (TA down) : like — begin place (Exploring→Placing, freeze output) // TogB2 (rising) : up — commit place (Placing→Idle) at the CURRENT // joystick input, then store the +1 example and @@ -140,14 +145,52 @@ inline void bind_peripherals(Mode& mode) { } }); - // MomB1: reroll the scratchpad (only in Exploring; no-op otherwise). + // MomB1: STATE-GATED. + // Exploring → reroll the scratchpad (a fresh random sound). + // Idle → GRAB: begin a reposition. Freezes the output currently heard + // from the trained net so the user can carry it to a new input + // location (the 4D variant has no joystick button, so the + // grab/drop gesture lives on this momentary toggle). The real + // net is NOT set aside — only the heard output is frozen. meml->setMomB1Callback([&mode]() { - feedback.reroll(mode.ml(), mode.param_schema().default_spread); + switch (feedback.explore_state()) { + case nisps::ml::ExploreState::Exploring: + feedback.reroll(mode.ml(), mode.param_schema().default_spread); + break; + case nisps::ml::ExploreState::Idle: + feedback.begin_reposition(mode.ml()); // process + capture + hold + break; + case nisps::ml::ExploreState::Placing: + break; // already holding — ignore + } }); - // MomB2: nudge the scratchpad (small bounded perturbation; undoable). + // MomB2: STATE-GATED. + // Exploring → nudge the scratchpad (small bounded perturbation). + // Repositioning → DROP: commit the carried output at the CURRENT joystick + // input, store the +1 example (new input → carried output) + // and warm-start train. This is the "move an existing + // positive example to a new position" gesture. meml->setMomB2Callback([&mode]() { - feedback.nudge(mode.ml(), 0.05f); + if (feedback.explore_state() == nisps::ml::ExploreState::Exploring) { + feedback.nudge(mode.ml(), 0.05f); + return; + } + if (feedback.repositioning()) { + // Capture the current joystick input — the DESTINATION position. + const auto in = mode.input_channels(); + std::array features{}; + for (std::size_t i = 0; i < Mode::ML::kInput; ++i) { + features[i] = (i < in.size()) ? in[i] : 0.f; + } + feedback.commit_reposition(); // no weight restore (net never set aside) + const auto label = feedback.committed_output(); // the carried output + if (!label.empty()) { + mode.ml().add_example( + std::span(features.data(), Mode::ML::kInput), label); + (void)mode.ml().train(); + } + } }); // MomA2: like → begin place. Freezes the current scratchpad output so the diff --git a/nisps/ml/feedback.hpp b/nisps/ml/feedback.hpp index 259d792..f665d1b 100644 --- a/nisps/ml/feedback.hpp +++ b/nisps/ml/feedback.hpp @@ -124,6 +124,12 @@ class FeedbackController { // ---- ExploreAndPlace state introspection -------------------------------- ExploreState explore_state() const noexcept { return ep_state_; } bool placing() const noexcept { return ep_state_ == ExploreState::Placing; } + // True while a REPOSITION hold is active (grab→move→drop). Distinguishes a + // reposition (real net never set aside) from an Explore→Place (scratchpad + + // snapshot). Both sit in ExploreState::Placing and both hold placed_out_ via + // static_output(); only commit/teardown differ (reposition does NOT restore + // weights — there is nothing to restore). + bool repositioning() const noexcept { return reposition_; } // Depth of the scratchpad undo ring currently available to pop (0..UndoDepth). std::size_t undo_depth() const noexcept { return undo_count_; } // The output vector frozen at like()/begin-place time. Valid only while @@ -359,12 +365,69 @@ class FeedbackController { // Placing→Exploring. Back out of placing without storing; resume auditioning // the scratchpad (which is still live — begin_place did not touch weights). + // A reposition hold has no scratchpad to return to, so it backs out to Idle. void cancel_place() noexcept { if (mode_ != FeedbackMode::ExploreAndPlace) return; if (ep_state_ != ExploreState::Placing) return; + if (reposition_) { + reposition_ = false; + learning_paused_ = false; + ep_state_ = ExploreState::Idle; + return; + } ep_state_ = ExploreState::Exploring; } + // ========================================================================= + // Reposition (grab → move → drop) — relocate an EXISTING positive example's + // output to a new input position. Distinct from Explore→Place: there is NO + // scratchpad and NO weight snapshot — the real (trained) net stays live the + // whole time. We only FREEZE the currently-heard output and hold it (via + // static_output) while the user moves to a new input, then the caller adds + // a +1 example (new input → carried output) and trains. This is the new + // core's home for the upstream "drag-store / reposition-commit" gesture. + // ========================================================================= + + // Idle→Placing(reposition). Freeze `current_out` — the output the user is + // hearing from the TRAINED net — and hold it. No-op unless Idle. + void begin_reposition(std::span current_out) noexcept { + if (mode_ != FeedbackMode::ExploreAndPlace) return; + if (ep_state_ != ExploreState::Idle) return; + const std::size_t n = (current_out.size() < kNOut) ? current_out.size() : kNOut; + for (std::size_t i = 0; i < n; ++i) placed_out_[i] = current_out[i]; + reposition_ = true; + learning_paused_ = true; + last_placed_valid_ = false; + ep_state_ = ExploreState::Placing; + } + + // Convenience: capture the trained net's output at its CURRENT input + // (process + capture). Equivalent to begin_reposition(mlp.outputs()). + void begin_reposition(MLP_T& mlp) noexcept { + if (mode_ != FeedbackMode::ExploreAndPlace) return; + if (ep_state_ != ExploreState::Idle) return; + mlp.process(); + const auto outs = mlp.outputs(); + const std::size_t n = (outs.size() < kNOut) ? outs.size() : kNOut; + for (std::size_t i = 0; i < n; ++i) placed_out_[i] = outs[i]; + reposition_ = true; + learning_paused_ = true; + last_placed_valid_ = false; + ep_state_ = ExploreState::Placing; + } + + // Placing(reposition)→Idle. NO weight restore (the net was never set aside). + // committed_output() then holds the carried vector so the caller can add the + // +1 example at the new input and train. No-op unless repositioning. + void commit_reposition() noexcept { + if (!reposition_ || ep_state_ != ExploreState::Placing) return; + reposition_ = false; + last_placed_valid_ = true; // committed_output() valid for the caller + learning_paused_ = false; + ep_state_ = ExploreState::Idle; + undo_count_ = 0u; + } + private: void enter_randomise_outputs(std::span seed_out) noexcept { explore_active_ = true; @@ -435,7 +498,19 @@ class FeedbackController { } void abort_explore_place(MLP_T& mlp) noexcept { - if (ep_state_ != ExploreState::Idle) restore_real_net(mlp); + if (ep_state_ == ExploreState::Idle) return; + if (reposition_) { + // A reposition never set the real net aside, so there is nothing to + // restore — clearing snapshot_ into the net here would CLOBBER the + // live trained weights. Just drop the hold. + reposition_ = false; + learning_paused_ = false; + ep_state_ = ExploreState::Idle; + undo_count_ = 0u; + last_placed_valid_ = false; + return; + } + restore_real_net(mlp); } FeedbackMode mode_ = FeedbackMode::Avoid; @@ -448,8 +523,9 @@ class FeedbackController { // ---- ExploreAndPlace state (all fixed-size, no heap) -------------------- ExploreState ep_state_ = ExploreState::Idle; - std::array placed_out_{}; // frozen audition vector + std::array placed_out_{}; // frozen audition/carried vector bool last_placed_valid_ = false; + bool reposition_ = false; // grab→move→drop hold; net NOT set aside std::array, kUndoDepth> undo_ring_{}; // bounded undo std::size_t undo_head_ = 0u; // next write slot std::size_t undo_count_ = 0u; // valid entries (0..kUndoDepth) diff --git a/tests/cpp/test_mlp_feedback.cpp b/tests/cpp/test_mlp_feedback.cpp index 2a64195..debc7e4 100644 --- a/tests/cpp/test_mlp_feedback.cpp +++ b/tests/cpp/test_mlp_feedback.cpp @@ -682,4 +682,98 @@ NISPS_TEST(ep_full_flow_two_anchors_caller_trains) { NISPS_EXPECT(m.example_count() == 2u); } +// =========================================================================== +// Reposition (grab → move → drop) — relocate an existing positive example's +// output to a new input position. NO scratchpad, NO weight snapshot/restore. +// =========================================================================== + +NISPS_TEST(ep_reposition_holds_output_without_setting_net_aside) { + SmallMLP m(0ull); + m.draw_weights(0.5f); + FB fb(0ull); + fb.set_mode(FeedbackMode::ExploreAndPlace, m); + + // The trained net's output at input A — the sound we want to carry. + m.set_input(0u, 0.2f); + m.set_input(1u, 0.8f); + m.process(); + std::array heard{}; + { auto o = m.outputs(); for (std::size_t i = 0; i < kNOut; ++i) heard[i] = o[i]; } + const auto real = snapshot_weights(m); + + fb.begin_reposition(m); // GRAB (process + capture at current input) + NISPS_EXPECT(fb.repositioning()); + NISPS_EXPECT(fb.placing()); + NISPS_EXPECT(fb.learning_paused()); + NISPS_EXPECT(weights_equal(m, real)); // real net NOT snapshotted/randomised + + // The carried output survives moving the input toward B. + m.set_input(0u, 0.9f); + m.set_input(1u, 0.1f); + std::array buf{}; + NISPS_EXPECT(fb.static_output(std::span(buf))); + for (std::size_t i = 0; i < kNOut; ++i) NISPS_EXPECT(buf[i] == heard[i]); +} + +NISPS_TEST(ep_reposition_commit_stores_carried_output_no_restore) { + SmallMLP m(0ull); + m.draw_weights(0.5f); + FB fb(0ull); + fb.set_mode(FeedbackMode::ExploreAndPlace, m); + m.set_input(0u, 0.2f); m.set_input(1u, 0.8f); m.process(); + std::array heard{}; + { auto o = m.outputs(); for (std::size_t i = 0; i < kNOut; ++i) heard[i] = o[i]; } + + fb.begin_reposition(m); + const auto held = snapshot_weights(m); // weights frozen during the hold + + fb.commit_reposition(); // DROP + NISPS_EXPECT(!fb.repositioning()); + NISPS_EXPECT(fb.explore_state() == ExploreState::Idle); + NISPS_EXPECT(!fb.learning_paused()); + NISPS_EXPECT(weights_equal(m, held)); // NO weight restore on commit + + // The committed vector is the carried (existing) output; the caller stores + // it at the NEW input B. + auto out = fb.committed_output(); + NISPS_EXPECT(out.size() == kNOut); + for (std::size_t i = 0; i < kNOut; ++i) NISPS_EXPECT(out[i] == heard[i]); + const std::array inputB{0.9f, 0.1f}; + m.add_example(std::span(inputB.data(), 2u), out); + NISPS_EXPECT(m.example_count() == 1u); +} + +NISPS_TEST(ep_reposition_mode_switch_aborts_without_clobbering_net) { + // The teardown path must NOT set_weights(stale snapshot_) for a reposition — + // that would destroy the live trained net. + SmallMLP m(0ull); + m.draw_weights(0.5f); + FB fb(0ull); + fb.set_mode(FeedbackMode::ExploreAndPlace, m); + m.process(); + const auto real = snapshot_weights(m); + fb.begin_reposition(m); + NISPS_EXPECT(fb.repositioning()); + + fb.set_mode(FeedbackMode::Avoid, m); // abort the hold + NISPS_EXPECT(!fb.repositioning()); + NISPS_EXPECT(fb.explore_state() == ExploreState::Idle); + NISPS_EXPECT(!fb.learning_paused()); + NISPS_EXPECT(weights_equal(m, real)); // live net preserved +} + +NISPS_TEST(ep_reposition_begin_only_from_idle) { + SmallMLP m(0ull); + m.draw_weights(0.5f); + FB fb(0ull); + fb.set_mode(FeedbackMode::ExploreAndPlace, m); + fb.enter_explore(m, 0.5f); // now Exploring, not Idle + fb.begin_reposition(m); // must be a no-op + NISPS_EXPECT(!fb.repositioning()); + NISPS_EXPECT(fb.explore_state() == ExploreState::Exploring); + // commit with no hold active is a no-op too. + fb.commit_reposition(); + NISPS_EXPECT(fb.explore_state() == ExploreState::Exploring); +} + } // namespace