speed optim, no clicking

IMLInterface.hpp

@@ -0,0 +1,276 @@
+#ifndef IMINTERFACE_HPP
+#define IMINTERFACE_HPP
+
+// #include "src/memllib/audio/AudioAppBase.hpp"
+#include "src/memllib/interface/InterfaceBase.hpp"
+
+#include "src/memlp/Dataset.hpp"
+#include "src/memlp/MLP.h"
+
+
+
+class IMLInterface : public InterfaceBase
+{
+public:
+    IMLInterface() : InterfaceBase() {}
+
+    void setup(size_t n_inputs, size_t n_outputs) override
+    {
+        InterfaceBase::setup(n_inputs, n_outputs);
+        // Additional setup code specific to IMLInterface
+        n_inputs_ = n_inputs;
+        n_outputs_ = n_outputs;
+
+        MLSetup_();
+        n_iterations_ = 1000;
+        input_state_.resize(n_inputs, 0.5f);
+        output_state_.resize(n_outputs, 0);
+        // Init/reset state machine
+        training_mode_ = INFERENCE_MODE;
+        perform_inference_ = true;
+        input_updated_ = false;
+
+        Serial.println("IMLInterface setup done");
+        Serial.print("Address of n_inputs_: ");
+        Serial.println(reinterpret_cast<uintptr_t>(&n_inputs_));
+        Serial.print("Inputs: ");
+        Serial.print(n_inputs_);
+        Serial.print(", Outputs: ");
+        Serial.println(n_outputs_);
+    }
+
+    enum training_mode_t {
+        INFERENCE_MODE,
+        TRAINING_MODE
+    };
+
+    void SetTrainingMode(training_mode_t training_mode)
+    {
+        Serial.print("Training mode: ");
+        Serial.println(training_mode == INFERENCE_MODE ? "Inference" : "Training");
+
+        if (training_mode == INFERENCE_MODE && training_mode_ == TRAINING_MODE) {
+            // Train the network!
+            MLTraining_();
+        }
+        training_mode_ = training_mode;
+    }
+
+    void ProcessInput()
+    {
+        // Check if input is updated
+        if (perform_inference_ && input_updated_) {
+            MLInference_(input_state_);
+            input_updated_ = false;
+        }
+    }
+
+    void SetInput(size_t index, float value)
+    {
+        // Serial.print("Input ");
+        // Serial.print(index);
+        // Serial.print(" set to: ");
+        // Serial.println(value);
+
+        if (index >= n_inputs_) {
+            Serial.print("Input index ");
+            Serial.print(index);
+            Serial.println(" out of bounds.");
+            return;
+        }
+
+        if (value < 0) {
+            value = 0;
+        } else if (value > 1.0) {
+            value = 1.0;
+        }
+
+        // Update state of input
+        input_state_[index] = value;
+        input_updated_ = true;
+    }
+
+    enum saving_mode_t {
+        STORE_VALUE_MODE,
+        STORE_POSITION_MODE,
+    };
+
+    void SaveInput(saving_mode_t mode)
+    {
+        if (STORE_VALUE_MODE == mode) {
+
+            Serial.println("Move input to position...");
+            perform_inference_ = false;
+
+        } else {  // STORE_POSITION_MODE
+
+            Serial.println("Creating example in this position.");
+            // Save pair in the dataset
+            dataset_->Add(input_state_, output_state_);
+            perform_inference_ = true;
+            MLInference_(input_state_);
+
+        }
+    }
+
+    void ClearData()
+    {
+        if (training_mode_ == TRAINING_MODE) {
+            Serial.println("Clearing dataset...");
+            dataset_->Clear();
+        }
+    }
+
+    void Randomise()
+    {
+        if (training_mode_ == TRAINING_MODE) {
+            Serial.println("Randomising weights...");
+            MLRandomise_();
+            MLInference_(input_state_);
+        }
+    }
+
+    void SetIterations(size_t iterations)
+    {
+        n_iterations_ = iterations;
+        Serial.print("Iterations set to: ");
+        Serial.println(n_iterations_);
+    }
+
+protected:
+    size_t n_inputs_;
+    size_t n_outputs_;
+    size_t n_iterations_;
+
+    // State machine
+    training_mode_t training_mode_;
+    bool perform_inference_;
+    bool input_updated_;
+
+    // Controls/sensors
+    std::vector<float> input_state_;
+    std::vector<float> output_state_;
+
+    // MLP core
+    std::unique_ptr<Dataset> dataset_;
+    std::unique_ptr<MLP<float>> mlp_;
+    MLP<float>::mlp_weights mlp_stored_weights_;
+    bool randomised_state_;
+
+    void MLSetup_()
+    {
+        // Constants for MLP init
+        const unsigned int kBias = 1;
+        const std::vector<ACTIVATION_FUNCTIONS> layers_activfuncs = {
+            RELU, RELU, RELU, SIGMOID
+        };
+        const bool use_constant_weight_init = false;
+        const float constant_weight_init = 0;
+        // Layer size definitions
+        const std::vector<size_t> layers_nodes = {
+            n_inputs_ + kBias,
+            10, 10, 14,
+            n_outputs_
+        };
+
+        // Create dataset
+        dataset_ = std::make_unique<Dataset>();
+        // Create MLP
+        mlp_ = std::make_unique<MLP<float>>(
+            layers_nodes,
+            layers_activfuncs,
+            loss::LOSS_MSE,
+            use_constant_weight_init,
+            constant_weight_init
+        );
+
+        // State machine
+        randomised_state_ = false;
+    }
+
+    void MLInference_(std::vector<float> input)
+    {
+        if (!dataset_ || !mlp_) {
+            Serial.println("ML not initialized!");
+            return;
+        }
+
+        if (input.size() != n_inputs_) {
+            Serial.print("Input size mismatch - ");
+            Serial.print("Expected: ");
+            Serial.print(n_inputs_);
+            Serial.print(", Got: ");
+            Serial.println(input.size());
+            return;
+        }
+
+        input.push_back(1.0f); // Add bias term
+        // Perform inference
+        std::vector<float> output(n_outputs_);
+        mlp_->GetOutput(input, &output);
+        // Process inferenced data
+        output_state_ = output;
+        SendParamsToQueue(output);
+    }
+
+    void MLRandomise_()
+    {
+        if (!mlp_) {
+            Serial.println("ML not initialized!");
+            return;
+        }
+
+        // Randomize weights
+        mlp_stored_weights_ = mlp_->GetWeights();
+        mlp_->DrawWeights();
+        randomised_state_ = true;
+    }
+
+    void MLTraining_()
+    {
+        if (!mlp_) {
+            Serial.println("ML not initialized!");
+            return;
+        }
+        // Restore old weights
+        if (randomised_state_) {
+            mlp_->SetWeights(mlp_stored_weights_);
+        }
+        randomised_state_ = false;
+
+        // Prepare for training
+        // Extract dataset to training pair
+        MLP<float>::training_pair_t dataset(dataset_->GetFeatures(), dataset_->GetLabels());
+        // Check and report on dataset size
+        Serial.print("Feature size ");
+        Serial.print(dataset.first.size());
+        Serial.print(", label size ");
+        Serial.println(dataset.second.size());
+        if (!dataset.first.size() || !dataset.second.size()) {
+            Serial.println("Empty dataset!");
+            return;
+        }
+        Serial.print("Feature dim ");
+        Serial.print(dataset.first[0].size());
+        Serial.print(", label dim ");
+        Serial.println(dataset.second[0].size());
+        if (!dataset.first[0].size() || !dataset.second[0].size()) {
+            Serial.println("Empty dataset dimensions!");
+            return;
+        }
+
+        // Training loop
+        Serial.print("Training for max ");
+        Serial.print(n_iterations_);
+        Serial.println(" iterations...");
+        float loss = mlp_->Train(dataset,
+                1.,
+                n_iterations_,
+                0.00001,
+                false);
+        Serial.print("Trained, loss = ");
+        Serial.println(loss, 10);
+    }
+};
+
+#endif // IMINTERFACE_HPP

MEMLNaut-PAF-IML.ino

@@ -1,305 +1,52 @@
-#include "src/memllib/interface/InterfaceBase.hpp"
+// #include "src/memllib/interface/InterfaceBase.hpp"
 #include "src/memllib/audio/AudioAppBase.hpp"
 #include "src/memllib/audio/AudioDriver.hpp"
 #include "src/memllib/hardware/memlnaut/MEMLNaut.hpp"
 #include <memory>
-
+#include "IMLInterface.hpp"
-// Includes for the IML interface
+#include "interfaceRL.hpp"
-#include "src/memlp/Dataset.hpp"
-#include "src/memlp/MLP.h"
-
-// Includes for FM Synth
-#include "src/memllib/synth/FMSynth.hpp"
-
-
-class IMLInterface : public InterfaceBase
-{
-public:
-    IMLInterface() : InterfaceBase() {}
-
-    void setup(size_t n_inputs, size_t n_outputs) override
-    {
-        InterfaceBase::setup(n_inputs, n_outputs);
-        // Additional setup code specific to IMLInterface
-        n_inputs_ = n_inputs;
-        n_outputs_ = n_outputs;
-
-        MLSetup_();
-        n_iterations_ = 1000;
-        input_state_.resize(n_inputs, 0.5f);
-        output_state_.resize(n_outputs, 0);
-        // Init/reset state machine
-        training_mode_ = INFERENCE_MODE;
-        perform_inference_ = true;
-        input_updated_ = false;
-
-        Serial.println("IMLInterface setup done");
-        Serial.print("Address of n_inputs_: ");
-        Serial.println(reinterpret_cast<uintptr_t>(&n_inputs_));
-        Serial.print("Inputs: ");
-        Serial.print(n_inputs_);
-        Serial.print(", Outputs: ");
-        Serial.println(n_outputs_);
-    }
-
-    enum training_mode_t {
-        INFERENCE_MODE,
-        TRAINING_MODE
-    };
-
-    void SetTrainingMode(training_mode_t training_mode)
-    {
-        Serial.print("Training mode: ");
-        Serial.println(training_mode == INFERENCE_MODE ? "Inference" : "Training");
-
-        if (training_mode == INFERENCE_MODE && training_mode_ == TRAINING_MODE) {
-            // Train the network!
-            MLTraining_();
-        }
-        training_mode_ = training_mode;
-    }
-
-    void ProcessInput()
-    {
-        // Check if input is updated
-        if (perform_inference_ && input_updated_) {
-            MLInference_(input_state_);
-            input_updated_ = false;
-        }
-    }
-
-    void SetInput(size_t index, float value)
-    {
-        // Serial.print("Input ");
-        // Serial.print(index);
-        // Serial.print(" set to: ");
-        // Serial.println(value);
-
-        if (index >= n_inputs_) {
-            Serial.print("Input index ");
-            Serial.print(index);
-            Serial.println(" out of bounds.");
-            return;
-        }
-
-        if (value < 0) {
-            value = 0;
-        } else if (value > 1.0) {
-            value = 1.0;
-        }
-
-        // Update state of input
-        input_state_[index] = value;
-        input_updated_ = true;
-    }
-
-    enum saving_mode_t {
-        STORE_VALUE_MODE,
-        STORE_POSITION_MODE,
-    };
-
-    void SaveInput(saving_mode_t mode)
-    {
-        if (STORE_VALUE_MODE == mode) {
-
-            Serial.println("Move input to position...");
-            perform_inference_ = false;
-
-        } else {  // STORE_POSITION_MODE
-
-            Serial.println("Creating example in this position.");
-            // Save pair in the dataset
-            dataset_->Add(input_state_, output_state_);
-            perform_inference_ = true;
-            MLInference_(input_state_);
-
-        }
-    }
-
-    void ClearData()
-    {
-        if (training_mode_ == TRAINING_MODE) {
-            Serial.println("Clearing dataset...");
-            dataset_->Clear();
-        }
-    }
-
-    void Randomise()
-    {
-        if (training_mode_ == TRAINING_MODE) {
-            Serial.println("Randomising weights...");
-            MLRandomise_();
-            MLInference_(input_state_);
-        }
-    }
-
-    void SetIterations(size_t iterations)
-    {
-        n_iterations_ = iterations;
-        Serial.print("Iterations set to: ");
-        Serial.println(n_iterations_);
-    }
-
-protected:
-    size_t n_inputs_;
-    size_t n_outputs_;
-    size_t n_iterations_;
-
-    // State machine
-    training_mode_t training_mode_;
-    bool perform_inference_;
-    bool input_updated_;
-
-    // Controls/sensors
-    std::vector<float> input_state_;
-    std::vector<float> output_state_;
-
-    // MLP core
-    std::unique_ptr<Dataset> dataset_;
-    std::unique_ptr<MLP<float>> mlp_;
-    MLP<float>::mlp_weights mlp_stored_weights_;
-    bool randomised_state_;
-
-    void MLSetup_()
-    {
-        // Constants for MLP init
-        const unsigned int kBias = 1;
-        const std::vector<ACTIVATION_FUNCTIONS> layers_activfuncs = {
-            RELU, RELU, RELU, SIGMOID
-        };
-        const bool use_constant_weight_init = false;
-        const float constant_weight_init = 0;
-        // Layer size definitions
-        const std::vector<size_t> layers_nodes = {
-            n_inputs_ + kBias,
-            10, 10, 14,
-            n_outputs_
-        };
-
-        // Create dataset
-        dataset_ = std::make_unique<Dataset>();
-        // Create MLP
-        mlp_ = std::make_unique<MLP<float>>(
-            layers_nodes,
-            layers_activfuncs,
-            loss::LOSS_MSE,
-            use_constant_weight_init,
-            constant_weight_init
-        );
-
-        // State machine
-        randomised_state_ = false;
-    }
-
-    void MLInference_(std::vector<float> input)
-    {
-        if (!dataset_ || !mlp_) {
-            Serial.println("ML not initialized!");
-            return;
-        }
-
-        if (input.size() != n_inputs_) {
-            Serial.print("Input size mismatch - ");
-            Serial.print("Expected: ");
-            Serial.print(n_inputs_);
-            Serial.print(", Got: ");
-            Serial.println(input.size());
-            return;
-        }
-
-        input.push_back(1.0f); // Add bias term
-        // Perform inference
-        std::vector<float> output(n_outputs_);
-        mlp_->GetOutput(input, &output);
-        // Process inferenced data
-        output_state_ = output;
-        SendParamsToQueue(output);
-    }
-
-    void MLRandomise_()
-    {
-        if (!mlp_) {
-            Serial.println("ML not initialized!");
-            return;
-        }
-
-        // Randomize weights
-        mlp_stored_weights_ = mlp_->GetWeights();
-        mlp_->DrawWeights();
-        randomised_state_ = true;
-    }
-
-    void MLTraining_()
-    {
-        if (!mlp_) {
-            Serial.println("ML not initialized!");
-            return;
-        }
-        // Restore old weights
-        if (randomised_state_) {
-            mlp_->SetWeights(mlp_stored_weights_);
-        }
-        randomised_state_ = false;
-
-        // Prepare for training
-        // Extract dataset to training pair
-        MLP<float>::training_pair_t dataset(dataset_->GetFeatures(), dataset_->GetLabels());
-        // Check and report on dataset size
-        Serial.print("Feature size ");
-        Serial.print(dataset.first.size());
-        Serial.print(", label size ");
-        Serial.println(dataset.second.size());
-        if (!dataset.first.size() || !dataset.second.size()) {
-            Serial.println("Empty dataset!");
-            return;
-        }
-        Serial.print("Feature dim ");
-        Serial.print(dataset.first[0].size());
-        Serial.print(", label dim ");
-        Serial.println(dataset.second[0].size());
-        if (!dataset.first[0].size() || !dataset.second[0].size()) {
-            Serial.println("Empty dataset dimensions!");
-            return;
-        }
-
-        // Training loop
-        Serial.print("Training for max ");
-        Serial.print(n_iterations_);
-        Serial.println(" iterations...");
-        float loss = mlp_->Train(dataset,
-                1.,
-                n_iterations_,
-                0.00001,
-                false);
-        Serial.print("Trained, loss = ");
-        Serial.println(loss, 10);
-    }
-};
-
 #include "src/memllib/synth/maxiPAF.hpp"
+#include "hardware/structs/bus_ctrl.h"
+
+#define APP_SRAM __not_in_flash("app")
+
+
+
+bool core1_disable_systick = true;
+bool core1_separate_stack = true;
+
+uint32_t get_rosc_entropy_seed(int bits) {
+    uint32_t seed = 0;
+    for (int i = 0; i < bits; ++i) {
+        // Wait for a bit of time to allow jitter to accumulate
+        busy_wait_us_32(5);
+        // Pull LSB from ROSC rand output
+        seed <<= 1;
+        seed |= (rosc_hw->randombit & 1);
+    }
+    return seed;
+}
 
 class PAFSynthApp : public AudioAppBase
 {
 public:
-    static constexpr size_t kN_Params = 10;
+    static constexpr size_t kN_Params = 17;
 
     PAFSynthApp() : AudioAppBase() {}
 
-    inline stereosample_t Process(const stereosample_t x) override
+    stereosample_t __force_inline Process(const stereosample_t x) override
     {
         float x1[1];
 
-        paf0.play(x1, 1, paf0_freq, paf0_cf, paf0_bw, paf0_vib, paf0_vfr, 1);
+        paf0.play(x1, 1, paf0_freq, paf0_cf, paf0_bw, paf0_vib, paf0_vfr, paf0_shift, 0);
         float y = x1[0];
 
-        // paf1.play(x1, 1, paf1_freq, paf1_cf, paf1_bw, 0, 0, 1);
+        paf1.play(x1, 1, paf1_freq, paf1_cf, paf1_bw, paf1_vib, paf1_vfr, paf1_shift, 1);
-        // y += x1[0];
+        y += x1[0];
 
-        // paf2.play(x1, 1, paf1_freq, paf1_cf, paf1_bw, paf1_vib, paf1_vfr, 1);
+        paf2.play(x1, 1, paf2_freq, paf2_cf, paf2_bw, paf2_vib, paf2_vfr, paf2_shift, 1);
-        // y += x1[0];
+        y += x1[0];
 
-        // paf2.play(x1, 1);
-        // y += x1[0];
 
         y = y * 0.3f;
 
@@ -313,33 +60,33 @@ public:
         AudioAppBase::Setup(sample_rate, interface);
         paf0.init();
         paf0.setsr(maxiSettings::getSampleRate(), 1);
-        paf0.freq(100, 0);
+        // paf0.freq(100, 0);
-        // paf0.amp(1,0);
+        // // paf0.amp(1,0);
-        paf0.bw(200,0);
+        // paf0.bw(200,0);
-        paf0.cf(210,0);
+        // paf0.cf(210,0);
-        paf0.vfr(5,0);
+        // paf0.vfr(5,0);
-        paf0.vib(0.1,0);
+        // paf0.vib(0.1,0);
-        paf0.shift(10,0);
+        // paf0.shift(10,0);
 
         paf1.init();
         paf1.setsr(maxiSettings::getSampleRate(), 1);
-        paf1.freq(150, 0);
+        // paf1.freq(150, 0);
-        // paf1.amp(1,0);
+        // // paf1.amp(1,0);
-        paf1.bw(200,0);
+        // paf1.bw(200,0);
-        paf1.cf(210,0);
+        // paf1.cf(210,0);
-        paf1.vfr(5,0);
+        // paf1.vfr(5,0);
-        paf1.vib(0.1,0);
+        // paf1.vib(0.1,0);
-        paf1.shift(10,0);
+        // paf1.shift(10,0);
 
         paf2.init();
         paf2.setsr(maxiSettings::getSampleRate(), 1);
-        paf2.freq(190, 0);
+        // paf2.freq(190, 0);
-        // paf2.amp(1,0);
+        // // paf2.amp(1,0);
-        paf2.bw(500,0);
+        // paf2.bw(500,0);
-        paf2.cf(210,0);
+        // paf2.cf(210,0);
-        paf2.vfr(5,0);
+        // paf2.vfr(5,0);
-        paf2.vib(0.1,0);
+        // paf2.vib(0.1,0);
-        paf2.shift(6,0);
+        // paf2.shift(6,0);
     }
 
     void ProcessParams(const std::vector<float>& params) override
@@ -350,19 +97,28 @@ public:
         // paf0_freq = 50.f + (params[0] * params[0] * 1000.f);
         // paf1_freq = 50.f + (params[1] * params[1] * 1000.f);
 
-        paf0_cf = paf0_freq + (params[2] * params[2] * paf0_freq * 16.f);
+        paf0_cf = paf0_freq + (params[2] * params[2] * paf0_freq * 4.f);
-        paf1_cf = 50.f + (params[3] * params[3] * 1000.f);
+        paf1_cf = paf0_freq + (params[3] * params[3] * paf1_freq * 16.f);
+        paf2_cf = paf0_freq + (params[4] * params[4] * paf2_freq * 16.f);
 
-        paf0_bw = 10.f + (params[4] * paf0_freq);
+        paf0_bw = 10.f + (params[5] * paf0_freq);
-        paf1_bw = 50.f + (params[5] * 5000.f);
+        paf1_bw = 10.f + (params[6] * paf1_freq);
+        paf2_bw = 10.f + (params[7] * paf2_freq);
 
-        paf0_vib = (params[6] * params[6] * 0.9f);
+        paf0_vib = (params[8] * params[8] * 0.99f);
-        paf1_vib = (params[7] * params[7] * 0.9f);
+        paf1_vib = (params[9] * params[9] * 0.99f);
+        paf2_vib = (params[10] * params[10] * 0.99f);
 
-        paf0_vfr = (params[8] * params[8]* 15.f);
+        paf0_vfr = (params[11] * params[11]* 15.f);
-        paf1_vfr = (params[9] * params[9] * 15.f);
+        paf1_vfr = (params[12] * params[12] * 15.f);
+        paf2_vfr = (params[13] * params[13] * 15.f);
+
+        paf0_shift = (params[14] * 1000.f);
+        paf1_shift = (params[15] * 1000.f);
+        paf2_shift = (params[16] * 1000.f);
+
+        // Serial.printf("%f %f %f %f %f\n", paf0_cf,  paf0_bw, paf0_vib, paf0_vfr, paf0_shift);
         
-        Serial.printf("%f %f %f %f\n", paf0_cf,  paf0_bw, paf0_vib, paf0_vfr);
     }
 
 protected:
@@ -375,36 +131,47 @@ protected:
 
     float paf0_freq = 100;
     float paf1_freq = 101;
+    float paf2_freq = 102;
+
     float paf0_cf = 200;
     float paf1_cf = 250;
+    float paf2_cf = 250;
 
     float paf0_bw = 100;
     float paf1_bw = 5000;
+    float paf2_bw = 5000;
 
     float paf0_vib = 0;
     float paf1_vib = 1;
+    float paf2_vib = 1;
 
     float paf0_vfr = 2;
     float paf1_vfr = 2;
+    float paf2_vfr = 2;
 
+    float paf0_shift = 0;
+    float paf1_shift = 0;
+    float paf2_shift = 0;
 };
 
 
 
 
 // Global objects
-std::shared_ptr<IMLInterface> interface;
+std::shared_ptr<IMLInterface> APP_SRAM interfaceIML;
-std::shared_ptr<PAFSynthApp> AUDIO_MEM audio_app;
+std::shared_ptr<interfaceRL> APP_SRAM RLInterface;
+
+std::shared_ptr<PAFSynthApp> __scratch_y("audio") audio_app;
 
 // Inter-core communication
-volatile bool core_0_ready = false;
+volatile bool APP_SRAM core_0_ready = false;
-volatile bool core_1_ready = false;
+volatile bool APP_SRAM core_1_ready = false;
-volatile bool serial_ready = false;
+volatile bool APP_SRAM serial_ready = false;
-volatile bool interface_ready = false;
+volatile bool APP_SRAM interface_ready = false;
 
 
 // We're only bound to the joystick inputs (x, y, rotate)
-const size_t kN_InputParams = 3;
+constexpr size_t kN_InputParams = 3;
 
 // Add these macros near other globals
 #define MEMORY_BARRIER() __sync_synchronize()
@@ -412,7 +179,50 @@ const size_t kN_InputParams = 3;
 #define READ_VOLATILE(var) ({ MEMORY_BARRIER(); typeof(var) __temp = (var); MEMORY_BARRIER(); __temp; })
 
 
-void bind_interface(std::shared_ptr<IMLInterface> interface)
+void bind_RL_interface(std::shared_ptr<interfaceRL> interface)
+{
+    // Set up momentary switch callbacks
+    MEMLNaut::Instance()->setMomA1Callback([interface] () {
+        interface->storeExperience(1.f);
+        Serial.println("Incredible");
+    });
+    MEMLNaut::Instance()->setMomA2Callback([interface] () {
+        interface->storeExperience(-1.f);
+        Serial.println("That sucks");
+    });
+    MEMLNaut::Instance()->setMomB1Callback([interface] () {
+        interface->randomiseTheActor();
+        Serial.println("The Actor is confused");
+    });
+    MEMLNaut::Instance()->setMomB2Callback([interface] () {
+        interface->randomiseTheCritic();
+        Serial.println("The Critic is confounded");
+    });
+    // Set up ADC callbacks
+    MEMLNaut::Instance()->setJoyXCallback([interface] (float value) {
+        interface->setState(0, value);
+    });
+    MEMLNaut::Instance()->setJoyYCallback([interface] (float value) {
+        interface->setState(1, value);
+    });
+    MEMLNaut::Instance()->setJoyZCallback([interface] (float value) {
+        interface->setState(2, value);
+    });
+
+    MEMLNaut::Instance()->setRVGain1Callback([interface] (float value) {
+        AudioDriver::setDACVolume(value);
+    });
+
+    // Set up loop callback
+    MEMLNaut::Instance()->setLoopCallback([interface] () {
+        interface->optimiseSometimes();
+        interface->generateAction();
+    });
+
+  
+}
+
+void bind_IML_interface(std::shared_ptr<IMLInterface> interface)
 {
     // Set up momentary switch callbacks
     MEMLNaut::Instance()->setMomA1Callback([interface] () {
@@ -456,9 +266,19 @@ void bind_interface(std::shared_ptr<IMLInterface> interface)
     });
 }
 
+enum MLMODES {IML, RL};  
+MLMODES APP_SRAM mlMode = RL;
 
 void setup()
 {
+
+
+    bus_ctrl_hw->priority = BUSCTRL_BUS_PRIORITY_DMA_W_BITS |
+        BUSCTRL_BUS_PRIORITY_DMA_R_BITS | BUSCTRL_BUS_PRIORITY_PROC1_BITS;
+
+    uint32_t seed = get_rosc_entropy_seed(32);
+    srand(seed);
+
     Serial.begin(115200);
     while (!Serial) {}
     Serial.println("Serial initialised.");
@@ -468,19 +288,39 @@ void setup()
     MEMLNaut::Initialize();
     pinMode(33, OUTPUT);
 
-    // Setup interface with memory barrier protection
+    switch(mlMode) {
+        case IML: {
             {
                 auto temp_interface = std::make_shared<IMLInterface>();
                 temp_interface->setup(kN_InputParams, PAFSynthApp::kN_Params);
                 MEMORY_BARRIER();
-        interface = temp_interface;
+                interfaceIML = temp_interface;
                 MEMORY_BARRIER();
             }
+            // Setup interface with memory barrier protection
             WRITE_VOLATILE(interface_ready, true);
-
             // Bind interface after ensuring it's fully initialized
-    bind_interface(interface);
+            bind_IML_interface(interfaceIML);
-    Serial.println("Bound interface to MEMLNaut.");
+            Serial.println("Bound IML interface to MEMLNaut.");
+        }
+        break;
+        case RL: {
+            {
+                auto temp_interface = std::make_shared<interfaceRL>();
+                temp_interface->setup(kN_InputParams, PAFSynthApp::kN_Params);
+                MEMORY_BARRIER();
+                RLInterface = temp_interface;
+                MEMORY_BARRIER();
+            }
+            // Setup interface with memory barrier protection
+            WRITE_VOLATILE(interface_ready, true);
+            // Bind interface after ensuring it's fully initialized
+            bind_RL_interface(RLInterface);
+            Serial.println("Bound RL interface to MEMLNaut.");
+        }
+        break;
+    }
+
 
     WRITE_VOLATILE(core_0_ready, true);
     while (!READ_VOLATILE(core_1_ready)) {
@@ -493,6 +333,8 @@ void setup()
 
 void loop()
 {
+        
+
     MEMLNaut::Instance()->loop();
     static int AUDIO_MEM blip_counter = 0;
     if (blip_counter++ > 100) {
@@ -504,7 +346,7 @@ void loop()
         // Un-blink LED
         digitalWrite(33, LOW);
     }
-    delay(20); // Add a small delay to avoid flooding the serial output
+    delay(10); // Add a small delay to avoid flooding the serial output
 }
 
 void setup1()
@@ -519,10 +361,20 @@ void setup1()
         delay(1);
     }
 
+
     // Create audio app with memory barrier protection
     {
         auto temp_audio_app = std::make_shared<PAFSynthApp>();
-        temp_audio_app->Setup(AudioDriver::GetSampleRate(), interface);
+        std::shared_ptr<InterfaceBase> selectedInterface;
+
+        if (mlMode == IML) {
+            selectedInterface = std::dynamic_pointer_cast<InterfaceBase>(interfaceIML);
+        } else {
+            selectedInterface = std::dynamic_pointer_cast<InterfaceBase>(RLInterface);
+        }
+
+        temp_audio_app->Setup(AudioDriver::GetSampleRate(), selectedInterface);
+        // temp_audio_app->Setup(AudioDriver::GetSampleRate(), dynamic_cast<std::shared_ptr<InterfaceBase>> (mlMode == IML ? interfaceIML : RLInterface));
         MEMORY_BARRIER();
         audio_app = temp_audio_app;
         MEMORY_BARRIER();
@@ -544,6 +396,7 @@ void loop1()
 {
     // Audio app parameter processing loop
     audio_app->loop();
+    delay(10);
 }
 
 extern "C" int getentropy (void * buffer, size_t how_many) {

interfaceRL.hpp

@@ -0,0 +1,273 @@
+#ifndef INTERFACERL_HPP
+#define INTERFACERL_HPP
+
+#include "src/memllib/interface/InterfaceBase.hpp"
+
+#include "src/memlp/MLP.h"
+#include "src/memlp/ReplayMemory.hpp"
+#include "src/memlp/OrnsteinUhlenbeckNoise.h"
+#include <memory>
+
+#define RL_MEM __not_in_flash("rlmem")
+
+
+struct trainRLItem {
+    std::vector<float> state ;
+    std::vector<float> action;
+    float reward;
+    std::vector<float> nextState;
+};
+
+
+class interfaceRL : public InterfaceBase
+{
+public:
+
+    void setup(size_t n_inputs, size_t n_outputs) override
+    {
+        InterfaceBase::setup(n_inputs, n_outputs);
+        stateSize = n_inputs;
+        actionSize = n_outputs;
+
+        actor_layers_nodes = {
+            stateSize + bias,
+            10, 10,
+            actionSize
+        };
+    
+        critic_layers_nodes = {
+            stateSize + actionSize + bias,
+            10, 10,
+            1
+        };
+
+        criticInput.resize(critic_layers_nodes[0]);
+        actorControlInput.resize(actor_layers_nodes[0]);
+        actorControlInput[actorControlInput.size()-1] = 1.f; // bias
+
+        //init networks
+        actor = std::make_shared<MLP<float> > (
+            actor_layers_nodes,
+            layers_activfuncs,
+            loss::LOSS_MSE,
+            use_constant_weight_init,
+            constant_weight_init
+        );
+
+        actorTarget = std::make_shared<MLP<float> > (
+            actor_layers_nodes,
+            layers_activfuncs,
+            loss::LOSS_MSE,
+            use_constant_weight_init,
+            constant_weight_init
+        );
+
+        critic = std::make_shared<MLP<float> > (
+            critic_layers_nodes,
+            layers_activfuncs,
+            loss::LOSS_MSE,
+            use_constant_weight_init,
+            constant_weight_init
+        );
+        criticTarget = std::make_shared<MLP<float> > (
+            critic_layers_nodes,
+            layers_activfuncs,
+            loss::LOSS_MSE,
+            use_constant_weight_init,
+            constant_weight_init
+        );
+    }
+
+    void optimise() {
+        constexpr size_t batchSize = 4;
+        std::vector<trainRLItem> sample = replayMem.sample(batchSize);
+        if (sample.size() == batchSize) {
+            //run sample through critic target, build training set for critic net
+            MLP<float>::training_pair_t ts;
+            for(size_t i = 0; i < sample.size(); i++) {
+                //---calculate y
+                //--calc next-state-action pair
+                //get next action from actorTarget given next state
+                auto nextStateInput =  sample[i].nextState;
+                nextStateInput.push_back(1.f); // bias
+                actorTarget->GetOutput(nextStateInput, &actorOutput);
+                
+                //use criticTarget to estimate value of next action given next state
+                for(size_t j=0; j < stateSize; j++) {
+                    criticInput[j] = sample[i].nextState[j];
+                }
+                for(size_t j=0; j < actionSize; j++) {
+                    criticInput[j+stateSize] = actorOutput[j];
+                }
+                criticInput[criticInput.size()-1] = 1.f; //bias
+
+                criticTarget->GetOutput(criticInput, &criticOutput);
+        
+                //calculate expected reward
+                const float y = sample[i].reward + (discountFactor *  criticOutput[0]);
+                // std::cout << "[" << i << "]: y: " << y << std::endl;
+        
+                //use criticTarget to estimate value of next action given next state
+                for(size_t j=0; j < stateSize; j++) {
+                criticInput[j] = sample[i].state[j];
+                }
+                for(size_t j=0; j < actionSize; j++) {
+                criticInput[j+stateSize] = sample[i].action[j];
+                }
+                criticInput[criticInput.size()-1] = 1.f; //bias
+
+                ts.first.push_back(criticInput);
+                ts.second.push_back({y});
+            }
+
+            //train the critic
+            float loss = critic->Train(ts, learningRate, 1);
+
+            //TODO: size limit to this log
+            criticLossLog.push_back(loss);
+        
+            //update the actor
+        
+            //for each memory in replay memory sample, and get grads from critic
+            std::vector<float> actorLoss(actionSize, 0.f);
+            std::vector<float> gradientLoss= {1.f};
+            
+            for(size_t i = 0; i < sample.size(); i++) {
+                //use criticTarget to estimate value of next action given next state
+                for(size_t j=0; j < stateSize; j++) {
+                criticInput[j] = sample[i].nextState[j];
+                }
+                for(size_t j=0; j < actionSize; j++) {
+                criticInput[j+stateSize] = sample[i].action[j];
+                }
+                criticInput[criticInput.size()-1] = 1.f; //bias
+        
+                critic->CalcGradients(criticInput, gradientLoss);
+                std::vector<float> l0Grads = critic->m_layers[0].GetGrads();
+        
+                for(size_t j=0; j < actionSize; j++) {
+                 actorLoss[j] = l0Grads[j+stateSize];
+                }
+                delay(1);
+            }
+
+            float totalLoss = 0.f;
+            for(size_t j=0; j < actorLoss.size(); j++) {
+                actorLoss[j] /= sample.size();
+                actorLoss[j] = -actorLoss[j];
+                totalLoss += actorLoss[j];                
+            }
+            // actorLossLog.push_back(actorLoss);
+            // actorLoss = -actorLoss;
+            // Serial.printf("Actor loss: %f\n", totalLoss);
+        
+            //back propagate the actor loss
+            for(size_t i = 0; i < sample.size(); i++) {
+                auto actorInput = sample[i].state;
+                actorInput.push_back(bias); 
+        
+                actor->ApplyLoss(actorInput, actorLoss, learningRate);
+                delay(1);
+            }
+        
+            // soft update the target networks
+            criticTarget->SmoothUpdateWeights(critic, smoothingAlpha);
+            actorTarget->SmoothUpdateWeights(actor, smoothingAlpha);
+        }
+    }
+
+    void setState(const size_t index, float value) {
+        actorControlInput[index] = value;
+        newInput = true;
+    }
+
+    void generateAction() {
+        if (newInput) {
+            newInput = false;
+            std::vector<float> actorOutput;
+            actorTarget->GetOutput(actorControlInput, &actorOutput);
+            SendParamsToQueue(actorOutput);
+            action = actorOutput;
+
+            // for(size_t i=0; i < actorOutput.size(); i++) {
+            //     const float noise = ou_noise.sample() * knobL;
+            //     actorOutput[i] += noise;
+            // }
+
+        }
+    }
+
+    void optimiseSometimes() {
+        if (optimiseCounter==optimiseDivisor) {
+            optimise();
+            optimiseCounter=0;
+        }else{
+            optimiseCounter++;
+        }
+    }
+
+    void storeExperience(float reward) {
+        std::vector<float> state = actorControlInput;
+        //remove bias
+        state.pop_back();
+        for(size_t i=0; i < state.size(); i++) {
+            Serial.printf("%f\t", state[i]);
+        }
+        Serial.println();
+        trainRLItem trainItem = {state, action, reward, state};
+        replayMem.add(trainItem, millis());
+    }
+
+    void randomiseTheActor()
+    {
+        actor->DrawWeights();
+        actorTarget->DrawWeights();
+    }
+
+    void randomiseTheCritic()
+    {
+        critic->DrawWeights();
+        criticTarget->DrawWeights();
+    }
+
+private:
+    static constexpr size_t bias=1;
+
+    size_t optimiseDivisor = 40;
+    size_t optimiseCounter = 0;
+
+    bool newInput=false;
+
+    const std::vector<ACTIVATION_FUNCTIONS> layers_activfuncs = {
+        RELU, RELU, TANH
+    };
+
+    size_t stateSize;
+    size_t actionSize;
+
+    std::vector<size_t> actor_layers_nodes;
+    std::vector<size_t> critic_layers_nodes;
+
+    const bool use_constant_weight_init = false;
+    const float constant_weight_init = 0;
+
+    std::shared_ptr<MLP<float> > actor, actorTarget, critic, criticTarget;
+
+    float discountFactor = 0.95;
+    float learningRate = 0.005;
+    float smoothingAlpha = 0.005;
+
+    std::vector<float> action;
+
+    ReplayMemory<trainRLItem> replayMem;
+    
+    std::vector<float> actorOutput, criticOutput;
+    std::vector<float> criticInput;
+    std::vector<float> actorControlInput;
+    
+    std::vector<float> criticLossLog, actorLossLog, log1;
+    
+    
+};
+
+#endif // INTERFACERL_HPP