From 56aa15ae100a56ca501ab6c63856cd25e0115a25 Mon Sep 17 00:00:00 2001 From: chriskiefer Date: Wed, 7 May 2025 23:58:06 +0100 Subject: [PATCH] speed optim, no clicking --- IMLInterface.hpp | 276 +++++++++++++++++++++++ MEMLNaut-PAF-IML.ino | 521 ++++++++++++++++--------------------------- interfaceRL.hpp | 273 +++++++++++++++++++++++ 3 files changed, 736 insertions(+), 334 deletions(-) create mode 100644 IMLInterface.hpp create mode 100644 interfaceRL.hpp diff --git a/IMLInterface.hpp b/IMLInterface.hpp new file mode 100644 index 0000000..b190e34 --- /dev/null +++ b/IMLInterface.hpp @@ -0,0 +1,276 @@ +#ifndef IMINTERFACE_HPP +#define IMINTERFACE_HPP + +// #include "src/memllib/audio/AudioAppBase.hpp" +#include "src/memllib/interface/InterfaceBase.hpp" + +#include "src/memlp/Dataset.hpp" +#include "src/memlp/MLP.h" + + + +class IMLInterface : public InterfaceBase +{ +public: + IMLInterface() : InterfaceBase() {} + + void setup(size_t n_inputs, size_t n_outputs) override + { + InterfaceBase::setup(n_inputs, n_outputs); + // Additional setup code specific to IMLInterface + n_inputs_ = n_inputs; + n_outputs_ = n_outputs; + + MLSetup_(); + n_iterations_ = 1000; + input_state_.resize(n_inputs, 0.5f); + output_state_.resize(n_outputs, 0); + // Init/reset state machine + training_mode_ = INFERENCE_MODE; + perform_inference_ = true; + input_updated_ = false; + + Serial.println("IMLInterface setup done"); + Serial.print("Address of n_inputs_: "); + Serial.println(reinterpret_cast(&n_inputs_)); + Serial.print("Inputs: "); + Serial.print(n_inputs_); + Serial.print(", Outputs: "); + Serial.println(n_outputs_); + } + + enum training_mode_t { + INFERENCE_MODE, + TRAINING_MODE + }; + + void SetTrainingMode(training_mode_t training_mode) + { + Serial.print("Training mode: "); + Serial.println(training_mode == INFERENCE_MODE ? "Inference" : "Training"); + + if (training_mode == INFERENCE_MODE && training_mode_ == TRAINING_MODE) { + // Train the network! + MLTraining_(); + } + training_mode_ = training_mode; + } + + void ProcessInput() + { + // Check if input is updated + if (perform_inference_ && input_updated_) { + MLInference_(input_state_); + input_updated_ = false; + } + } + + void SetInput(size_t index, float value) + { + // Serial.print("Input "); + // Serial.print(index); + // Serial.print(" set to: "); + // Serial.println(value); + + if (index >= n_inputs_) { + Serial.print("Input index "); + Serial.print(index); + Serial.println(" out of bounds."); + return; + } + + if (value < 0) { + value = 0; + } else if (value > 1.0) { + value = 1.0; + } + + // Update state of input + input_state_[index] = value; + input_updated_ = true; + } + + enum saving_mode_t { + STORE_VALUE_MODE, + STORE_POSITION_MODE, + }; + + void SaveInput(saving_mode_t mode) + { + if (STORE_VALUE_MODE == mode) { + + Serial.println("Move input to position..."); + perform_inference_ = false; + + } else { // STORE_POSITION_MODE + + Serial.println("Creating example in this position."); + // Save pair in the dataset + dataset_->Add(input_state_, output_state_); + perform_inference_ = true; + MLInference_(input_state_); + + } + } + + void ClearData() + { + if (training_mode_ == TRAINING_MODE) { + Serial.println("Clearing dataset..."); + dataset_->Clear(); + } + } + + void Randomise() + { + if (training_mode_ == TRAINING_MODE) { + Serial.println("Randomising weights..."); + MLRandomise_(); + MLInference_(input_state_); + } + } + + void SetIterations(size_t iterations) + { + n_iterations_ = iterations; + Serial.print("Iterations set to: "); + Serial.println(n_iterations_); + } + +protected: + size_t n_inputs_; + size_t n_outputs_; + size_t n_iterations_; + + // State machine + training_mode_t training_mode_; + bool perform_inference_; + bool input_updated_; + + // Controls/sensors + std::vector input_state_; + std::vector output_state_; + + // MLP core + std::unique_ptr dataset_; + std::unique_ptr> mlp_; + MLP::mlp_weights mlp_stored_weights_; + bool randomised_state_; + + void MLSetup_() + { + // Constants for MLP init + const unsigned int kBias = 1; + const std::vector layers_activfuncs = { + RELU, RELU, RELU, SIGMOID + }; + const bool use_constant_weight_init = false; + const float constant_weight_init = 0; + // Layer size definitions + const std::vector layers_nodes = { + n_inputs_ + kBias, + 10, 10, 14, + n_outputs_ + }; + + // Create dataset + dataset_ = std::make_unique(); + // Create MLP + mlp_ = std::make_unique>( + layers_nodes, + layers_activfuncs, + loss::LOSS_MSE, + use_constant_weight_init, + constant_weight_init + ); + + // State machine + randomised_state_ = false; + } + + void MLInference_(std::vector input) + { + if (!dataset_ || !mlp_) { + Serial.println("ML not initialized!"); + return; + } + + if (input.size() != n_inputs_) { + Serial.print("Input size mismatch - "); + Serial.print("Expected: "); + Serial.print(n_inputs_); + Serial.print(", Got: "); + Serial.println(input.size()); + return; + } + + input.push_back(1.0f); // Add bias term + // Perform inference + std::vector output(n_outputs_); + mlp_->GetOutput(input, &output); + // Process inferenced data + output_state_ = output; + SendParamsToQueue(output); + } + + void MLRandomise_() + { + if (!mlp_) { + Serial.println("ML not initialized!"); + return; + } + + // Randomize weights + mlp_stored_weights_ = mlp_->GetWeights(); + mlp_->DrawWeights(); + randomised_state_ = true; + } + + void MLTraining_() + { + if (!mlp_) { + Serial.println("ML not initialized!"); + return; + } + // Restore old weights + if (randomised_state_) { + mlp_->SetWeights(mlp_stored_weights_); + } + randomised_state_ = false; + + // Prepare for training + // Extract dataset to training pair + MLP::training_pair_t dataset(dataset_->GetFeatures(), dataset_->GetLabels()); + // Check and report on dataset size + Serial.print("Feature size "); + Serial.print(dataset.first.size()); + Serial.print(", label size "); + Serial.println(dataset.second.size()); + if (!dataset.first.size() || !dataset.second.size()) { + Serial.println("Empty dataset!"); + return; + } + Serial.print("Feature dim "); + Serial.print(dataset.first[0].size()); + Serial.print(", label dim "); + Serial.println(dataset.second[0].size()); + if (!dataset.first[0].size() || !dataset.second[0].size()) { + Serial.println("Empty dataset dimensions!"); + return; + } + + // Training loop + Serial.print("Training for max "); + Serial.print(n_iterations_); + Serial.println(" iterations..."); + float loss = mlp_->Train(dataset, + 1., + n_iterations_, + 0.00001, + false); + Serial.print("Trained, loss = "); + Serial.println(loss, 10); + } +}; + +#endif // IMINTERFACE_HPP diff --git a/MEMLNaut-PAF-IML.ino b/MEMLNaut-PAF-IML.ino index f2e212f..129941b 100644 --- a/MEMLNaut-PAF-IML.ino +++ b/MEMLNaut-PAF-IML.ino @@ -1,305 +1,52 @@ -#include "src/memllib/interface/InterfaceBase.hpp" +// #include "src/memllib/interface/InterfaceBase.hpp" #include "src/memllib/audio/AudioAppBase.hpp" #include "src/memllib/audio/AudioDriver.hpp" #include "src/memllib/hardware/memlnaut/MEMLNaut.hpp" #include - -// Includes for the IML interface -#include "src/memlp/Dataset.hpp" -#include "src/memlp/MLP.h" - -// Includes for FM Synth -#include "src/memllib/synth/FMSynth.hpp" - - -class IMLInterface : public InterfaceBase -{ -public: - IMLInterface() : InterfaceBase() {} - - void setup(size_t n_inputs, size_t n_outputs) override - { - InterfaceBase::setup(n_inputs, n_outputs); - // Additional setup code specific to IMLInterface - n_inputs_ = n_inputs; - n_outputs_ = n_outputs; - - MLSetup_(); - n_iterations_ = 1000; - input_state_.resize(n_inputs, 0.5f); - output_state_.resize(n_outputs, 0); - // Init/reset state machine - training_mode_ = INFERENCE_MODE; - perform_inference_ = true; - input_updated_ = false; - - Serial.println("IMLInterface setup done"); - Serial.print("Address of n_inputs_: "); - Serial.println(reinterpret_cast(&n_inputs_)); - Serial.print("Inputs: "); - Serial.print(n_inputs_); - Serial.print(", Outputs: "); - Serial.println(n_outputs_); - } - - enum training_mode_t { - INFERENCE_MODE, - TRAINING_MODE - }; - - void SetTrainingMode(training_mode_t training_mode) - { - Serial.print("Training mode: "); - Serial.println(training_mode == INFERENCE_MODE ? "Inference" : "Training"); - - if (training_mode == INFERENCE_MODE && training_mode_ == TRAINING_MODE) { - // Train the network! - MLTraining_(); - } - training_mode_ = training_mode; - } - - void ProcessInput() - { - // Check if input is updated - if (perform_inference_ && input_updated_) { - MLInference_(input_state_); - input_updated_ = false; - } - } - - void SetInput(size_t index, float value) - { - // Serial.print("Input "); - // Serial.print(index); - // Serial.print(" set to: "); - // Serial.println(value); - - if (index >= n_inputs_) { - Serial.print("Input index "); - Serial.print(index); - Serial.println(" out of bounds."); - return; - } - - if (value < 0) { - value = 0; - } else if (value > 1.0) { - value = 1.0; - } - - // Update state of input - input_state_[index] = value; - input_updated_ = true; - } - - enum saving_mode_t { - STORE_VALUE_MODE, - STORE_POSITION_MODE, - }; - - void SaveInput(saving_mode_t mode) - { - if (STORE_VALUE_MODE == mode) { - - Serial.println("Move input to position..."); - perform_inference_ = false; - - } else { // STORE_POSITION_MODE - - Serial.println("Creating example in this position."); - // Save pair in the dataset - dataset_->Add(input_state_, output_state_); - perform_inference_ = true; - MLInference_(input_state_); - - } - } - - void ClearData() - { - if (training_mode_ == TRAINING_MODE) { - Serial.println("Clearing dataset..."); - dataset_->Clear(); - } - } - - void Randomise() - { - if (training_mode_ == TRAINING_MODE) { - Serial.println("Randomising weights..."); - MLRandomise_(); - MLInference_(input_state_); - } - } - - void SetIterations(size_t iterations) - { - n_iterations_ = iterations; - Serial.print("Iterations set to: "); - Serial.println(n_iterations_); - } - -protected: - size_t n_inputs_; - size_t n_outputs_; - size_t n_iterations_; - - // State machine - training_mode_t training_mode_; - bool perform_inference_; - bool input_updated_; - - // Controls/sensors - std::vector input_state_; - std::vector output_state_; - - // MLP core - std::unique_ptr dataset_; - std::unique_ptr> mlp_; - MLP::mlp_weights mlp_stored_weights_; - bool randomised_state_; - - void MLSetup_() - { - // Constants for MLP init - const unsigned int kBias = 1; - const std::vector layers_activfuncs = { - RELU, RELU, RELU, SIGMOID - }; - const bool use_constant_weight_init = false; - const float constant_weight_init = 0; - // Layer size definitions - const std::vector layers_nodes = { - n_inputs_ + kBias, - 10, 10, 14, - n_outputs_ - }; - - // Create dataset - dataset_ = std::make_unique(); - // Create MLP - mlp_ = std::make_unique>( - layers_nodes, - layers_activfuncs, - loss::LOSS_MSE, - use_constant_weight_init, - constant_weight_init - ); - - // State machine - randomised_state_ = false; - } - - void MLInference_(std::vector input) - { - if (!dataset_ || !mlp_) { - Serial.println("ML not initialized!"); - return; - } - - if (input.size() != n_inputs_) { - Serial.print("Input size mismatch - "); - Serial.print("Expected: "); - Serial.print(n_inputs_); - Serial.print(", Got: "); - Serial.println(input.size()); - return; - } - - input.push_back(1.0f); // Add bias term - // Perform inference - std::vector output(n_outputs_); - mlp_->GetOutput(input, &output); - // Process inferenced data - output_state_ = output; - SendParamsToQueue(output); - } - - void MLRandomise_() - { - if (!mlp_) { - Serial.println("ML not initialized!"); - return; - } - - // Randomize weights - mlp_stored_weights_ = mlp_->GetWeights(); - mlp_->DrawWeights(); - randomised_state_ = true; - } - - void MLTraining_() - { - if (!mlp_) { - Serial.println("ML not initialized!"); - return; - } - // Restore old weights - if (randomised_state_) { - mlp_->SetWeights(mlp_stored_weights_); - } - randomised_state_ = false; - - // Prepare for training - // Extract dataset to training pair - MLP::training_pair_t dataset(dataset_->GetFeatures(), dataset_->GetLabels()); - // Check and report on dataset size - Serial.print("Feature size "); - Serial.print(dataset.first.size()); - Serial.print(", label size "); - Serial.println(dataset.second.size()); - if (!dataset.first.size() || !dataset.second.size()) { - Serial.println("Empty dataset!"); - return; - } - Serial.print("Feature dim "); - Serial.print(dataset.first[0].size()); - Serial.print(", label dim "); - Serial.println(dataset.second[0].size()); - if (!dataset.first[0].size() || !dataset.second[0].size()) { - Serial.println("Empty dataset dimensions!"); - return; - } - - // Training loop - Serial.print("Training for max "); - Serial.print(n_iterations_); - Serial.println(" iterations..."); - float loss = mlp_->Train(dataset, - 1., - n_iterations_, - 0.00001, - false); - Serial.print("Trained, loss = "); - Serial.println(loss, 10); - } -}; - +#include "IMLInterface.hpp" +#include "interfaceRL.hpp" #include "src/memllib/synth/maxiPAF.hpp" +#include "hardware/structs/bus_ctrl.h" + +#define APP_SRAM __not_in_flash("app") + + + +bool core1_disable_systick = true; +bool core1_separate_stack = true; + +uint32_t get_rosc_entropy_seed(int bits) { + uint32_t seed = 0; + for (int i = 0; i < bits; ++i) { + // Wait for a bit of time to allow jitter to accumulate + busy_wait_us_32(5); + // Pull LSB from ROSC rand output + seed <<= 1; + seed |= (rosc_hw->randombit & 1); + } + return seed; +} class PAFSynthApp : public AudioAppBase { public: - static constexpr size_t kN_Params = 10; + static constexpr size_t kN_Params = 17; PAFSynthApp() : AudioAppBase() {} - inline stereosample_t Process(const stereosample_t x) override + stereosample_t __force_inline Process(const stereosample_t x) override { float x1[1]; - paf0.play(x1, 1, paf0_freq, paf0_cf, paf0_bw, paf0_vib, paf0_vfr, 1); + paf0.play(x1, 1, paf0_freq, paf0_cf, paf0_bw, paf0_vib, paf0_vfr, paf0_shift, 0); float y = x1[0]; - // paf1.play(x1, 1, paf1_freq, paf1_cf, paf1_bw, 0, 0, 1); - // y += x1[0]; + paf1.play(x1, 1, paf1_freq, paf1_cf, paf1_bw, paf1_vib, paf1_vfr, paf1_shift, 1); + y += x1[0]; - // paf2.play(x1, 1, paf1_freq, paf1_cf, paf1_bw, paf1_vib, paf1_vfr, 1); - // y += x1[0]; + paf2.play(x1, 1, paf2_freq, paf2_cf, paf2_bw, paf2_vib, paf2_vfr, paf2_shift, 1); + y += x1[0]; - // paf2.play(x1, 1); - // y += x1[0]; y = y * 0.3f; @@ -313,33 +60,33 @@ public: AudioAppBase::Setup(sample_rate, interface); paf0.init(); paf0.setsr(maxiSettings::getSampleRate(), 1); - paf0.freq(100, 0); - // paf0.amp(1,0); - paf0.bw(200,0); - paf0.cf(210,0); - paf0.vfr(5,0); - paf0.vib(0.1,0); - paf0.shift(10,0); + // paf0.freq(100, 0); + // // paf0.amp(1,0); + // paf0.bw(200,0); + // paf0.cf(210,0); + // paf0.vfr(5,0); + // paf0.vib(0.1,0); + // paf0.shift(10,0); paf1.init(); paf1.setsr(maxiSettings::getSampleRate(), 1); - paf1.freq(150, 0); - // paf1.amp(1,0); - paf1.bw(200,0); - paf1.cf(210,0); - paf1.vfr(5,0); - paf1.vib(0.1,0); - paf1.shift(10,0); + // paf1.freq(150, 0); + // // paf1.amp(1,0); + // paf1.bw(200,0); + // paf1.cf(210,0); + // paf1.vfr(5,0); + // paf1.vib(0.1,0); + // paf1.shift(10,0); paf2.init(); paf2.setsr(maxiSettings::getSampleRate(), 1); - paf2.freq(190, 0); - // paf2.amp(1,0); - paf2.bw(500,0); - paf2.cf(210,0); - paf2.vfr(5,0); - paf2.vib(0.1,0); - paf2.shift(6,0); + // paf2.freq(190, 0); + // // paf2.amp(1,0); + // paf2.bw(500,0); + // paf2.cf(210,0); + // paf2.vfr(5,0); + // paf2.vib(0.1,0); + // paf2.shift(6,0); } void ProcessParams(const std::vector& params) override @@ -350,19 +97,28 @@ public: // paf0_freq = 50.f + (params[0] * params[0] * 1000.f); // paf1_freq = 50.f + (params[1] * params[1] * 1000.f); - paf0_cf = paf0_freq + (params[2] * params[2] * paf0_freq * 16.f); - paf1_cf = 50.f + (params[3] * params[3] * 1000.f); + paf0_cf = paf0_freq + (params[2] * params[2] * paf0_freq * 4.f); + paf1_cf = paf0_freq + (params[3] * params[3] * paf1_freq * 16.f); + paf2_cf = paf0_freq + (params[4] * params[4] * paf2_freq * 16.f); - paf0_bw = 10.f + (params[4] * paf0_freq); - paf1_bw = 50.f + (params[5] * 5000.f); + paf0_bw = 10.f + (params[5] * paf0_freq); + paf1_bw = 10.f + (params[6] * paf1_freq); + paf2_bw = 10.f + (params[7] * paf2_freq); - paf0_vib = (params[6] * params[6] * 0.9f); - paf1_vib = (params[7] * params[7] * 0.9f); + paf0_vib = (params[8] * params[8] * 0.99f); + paf1_vib = (params[9] * params[9] * 0.99f); + paf2_vib = (params[10] * params[10] * 0.99f); - paf0_vfr = (params[8] * params[8]* 15.f); - paf1_vfr = (params[9] * params[9] * 15.f); + paf0_vfr = (params[11] * params[11]* 15.f); + paf1_vfr = (params[12] * params[12] * 15.f); + paf2_vfr = (params[13] * params[13] * 15.f); - Serial.printf("%f %f %f %f\n", paf0_cf, paf0_bw, paf0_vib, paf0_vfr); + paf0_shift = (params[14] * 1000.f); + paf1_shift = (params[15] * 1000.f); + paf2_shift = (params[16] * 1000.f); + + // Serial.printf("%f %f %f %f %f\n", paf0_cf, paf0_bw, paf0_vib, paf0_vfr, paf0_shift); + } protected: @@ -375,36 +131,47 @@ protected: float paf0_freq = 100; float paf1_freq = 101; + float paf2_freq = 102; + float paf0_cf = 200; float paf1_cf = 250; + float paf2_cf = 250; float paf0_bw = 100; float paf1_bw = 5000; + float paf2_bw = 5000; float paf0_vib = 0; float paf1_vib = 1; + float paf2_vib = 1; float paf0_vfr = 2; float paf1_vfr = 2; + float paf2_vfr = 2; + float paf0_shift = 0; + float paf1_shift = 0; + float paf2_shift = 0; }; // Global objects -std::shared_ptr interface; -std::shared_ptr AUDIO_MEM audio_app; +std::shared_ptr APP_SRAM interfaceIML; +std::shared_ptr APP_SRAM RLInterface; + +std::shared_ptr __scratch_y("audio") audio_app; // Inter-core communication -volatile bool core_0_ready = false; -volatile bool core_1_ready = false; -volatile bool serial_ready = false; -volatile bool interface_ready = false; +volatile bool APP_SRAM core_0_ready = false; +volatile bool APP_SRAM core_1_ready = false; +volatile bool APP_SRAM serial_ready = false; +volatile bool APP_SRAM interface_ready = false; // We're only bound to the joystick inputs (x, y, rotate) -const size_t kN_InputParams = 3; +constexpr size_t kN_InputParams = 3; // Add these macros near other globals #define MEMORY_BARRIER() __sync_synchronize() @@ -412,7 +179,50 @@ const size_t kN_InputParams = 3; #define READ_VOLATILE(var) ({ MEMORY_BARRIER(); typeof(var) __temp = (var); MEMORY_BARRIER(); __temp; }) -void bind_interface(std::shared_ptr interface) +void bind_RL_interface(std::shared_ptr interface) +{ + // Set up momentary switch callbacks + MEMLNaut::Instance()->setMomA1Callback([interface] () { + interface->storeExperience(1.f); + Serial.println("Incredible"); + }); + MEMLNaut::Instance()->setMomA2Callback([interface] () { + interface->storeExperience(-1.f); + Serial.println("That sucks"); + }); + MEMLNaut::Instance()->setMomB1Callback([interface] () { + interface->randomiseTheActor(); + Serial.println("The Actor is confused"); + }); + MEMLNaut::Instance()->setMomB2Callback([interface] () { + interface->randomiseTheCritic(); + Serial.println("The Critic is confounded"); + }); + // Set up ADC callbacks + MEMLNaut::Instance()->setJoyXCallback([interface] (float value) { + interface->setState(0, value); + }); + MEMLNaut::Instance()->setJoyYCallback([interface] (float value) { + interface->setState(1, value); + }); + MEMLNaut::Instance()->setJoyZCallback([interface] (float value) { + interface->setState(2, value); + }); + + MEMLNaut::Instance()->setRVGain1Callback([interface] (float value) { + AudioDriver::setDACVolume(value); + }); + + // Set up loop callback + MEMLNaut::Instance()->setLoopCallback([interface] () { + interface->optimiseSometimes(); + interface->generateAction(); + }); + + +} + +void bind_IML_interface(std::shared_ptr interface) { // Set up momentary switch callbacks MEMLNaut::Instance()->setMomA1Callback([interface] () { @@ -456,9 +266,19 @@ void bind_interface(std::shared_ptr interface) }); } +enum MLMODES {IML, RL}; +MLMODES APP_SRAM mlMode = RL; void setup() { + + + bus_ctrl_hw->priority = BUSCTRL_BUS_PRIORITY_DMA_W_BITS | + BUSCTRL_BUS_PRIORITY_DMA_R_BITS | BUSCTRL_BUS_PRIORITY_PROC1_BITS; + + uint32_t seed = get_rosc_entropy_seed(32); + srand(seed); + Serial.begin(115200); while (!Serial) {} Serial.println("Serial initialised."); @@ -468,19 +288,39 @@ void setup() MEMLNaut::Initialize(); pinMode(33, OUTPUT); - // Setup interface with memory barrier protection - { - auto temp_interface = std::make_shared(); - temp_interface->setup(kN_InputParams, PAFSynthApp::kN_Params); - MEMORY_BARRIER(); - interface = temp_interface; - MEMORY_BARRIER(); + switch(mlMode) { + case IML: { + { + auto temp_interface = std::make_shared(); + temp_interface->setup(kN_InputParams, PAFSynthApp::kN_Params); + MEMORY_BARRIER(); + interfaceIML = temp_interface; + MEMORY_BARRIER(); + } + // Setup interface with memory barrier protection + WRITE_VOLATILE(interface_ready, true); + // Bind interface after ensuring it's fully initialized + bind_IML_interface(interfaceIML); + Serial.println("Bound IML interface to MEMLNaut."); + } + break; + case RL: { + { + auto temp_interface = std::make_shared(); + temp_interface->setup(kN_InputParams, PAFSynthApp::kN_Params); + MEMORY_BARRIER(); + RLInterface = temp_interface; + MEMORY_BARRIER(); + } + // Setup interface with memory barrier protection + WRITE_VOLATILE(interface_ready, true); + // Bind interface after ensuring it's fully initialized + bind_RL_interface(RLInterface); + Serial.println("Bound RL interface to MEMLNaut."); + } + break; } - WRITE_VOLATILE(interface_ready, true); - // Bind interface after ensuring it's fully initialized - bind_interface(interface); - Serial.println("Bound interface to MEMLNaut."); WRITE_VOLATILE(core_0_ready, true); while (!READ_VOLATILE(core_1_ready)) { @@ -493,6 +333,8 @@ void setup() void loop() { + + MEMLNaut::Instance()->loop(); static int AUDIO_MEM blip_counter = 0; if (blip_counter++ > 100) { @@ -504,7 +346,7 @@ void loop() // Un-blink LED digitalWrite(33, LOW); } - delay(20); // Add a small delay to avoid flooding the serial output + delay(10); // Add a small delay to avoid flooding the serial output } void setup1() @@ -519,10 +361,20 @@ void setup1() delay(1); } + // Create audio app with memory barrier protection { auto temp_audio_app = std::make_shared(); - temp_audio_app->Setup(AudioDriver::GetSampleRate(), interface); + std::shared_ptr selectedInterface; + + if (mlMode == IML) { + selectedInterface = std::dynamic_pointer_cast(interfaceIML); + } else { + selectedInterface = std::dynamic_pointer_cast(RLInterface); + } + + temp_audio_app->Setup(AudioDriver::GetSampleRate(), selectedInterface); + // temp_audio_app->Setup(AudioDriver::GetSampleRate(), dynamic_cast> (mlMode == IML ? interfaceIML : RLInterface)); MEMORY_BARRIER(); audio_app = temp_audio_app; MEMORY_BARRIER(); @@ -544,6 +396,7 @@ void loop1() { // Audio app parameter processing loop audio_app->loop(); + delay(10); } extern "C" int getentropy (void * buffer, size_t how_many) { diff --git a/interfaceRL.hpp b/interfaceRL.hpp new file mode 100644 index 0000000..5015af4 --- /dev/null +++ b/interfaceRL.hpp @@ -0,0 +1,273 @@ +#ifndef INTERFACERL_HPP +#define INTERFACERL_HPP + +#include "src/memllib/interface/InterfaceBase.hpp" + +#include "src/memlp/MLP.h" +#include "src/memlp/ReplayMemory.hpp" +#include "src/memlp/OrnsteinUhlenbeckNoise.h" +#include + +#define RL_MEM __not_in_flash("rlmem") + + +struct trainRLItem { + std::vector state ; + std::vector action; + float reward; + std::vector nextState; +}; + + +class interfaceRL : public InterfaceBase +{ +public: + + void setup(size_t n_inputs, size_t n_outputs) override + { + InterfaceBase::setup(n_inputs, n_outputs); + stateSize = n_inputs; + actionSize = n_outputs; + + actor_layers_nodes = { + stateSize + bias, + 10, 10, + actionSize + }; + + critic_layers_nodes = { + stateSize + actionSize + bias, + 10, 10, + 1 + }; + + criticInput.resize(critic_layers_nodes[0]); + actorControlInput.resize(actor_layers_nodes[0]); + actorControlInput[actorControlInput.size()-1] = 1.f; // bias + + //init networks + actor = std::make_shared > ( + actor_layers_nodes, + layers_activfuncs, + loss::LOSS_MSE, + use_constant_weight_init, + constant_weight_init + ); + + actorTarget = std::make_shared > ( + actor_layers_nodes, + layers_activfuncs, + loss::LOSS_MSE, + use_constant_weight_init, + constant_weight_init + ); + + critic = std::make_shared > ( + critic_layers_nodes, + layers_activfuncs, + loss::LOSS_MSE, + use_constant_weight_init, + constant_weight_init + ); + criticTarget = std::make_shared > ( + critic_layers_nodes, + layers_activfuncs, + loss::LOSS_MSE, + use_constant_weight_init, + constant_weight_init + ); + } + + void optimise() { + constexpr size_t batchSize = 4; + std::vector sample = replayMem.sample(batchSize); + if (sample.size() == batchSize) { + //run sample through critic target, build training set for critic net + MLP::training_pair_t ts; + for(size_t i = 0; i < sample.size(); i++) { + //---calculate y + //--calc next-state-action pair + //get next action from actorTarget given next state + auto nextStateInput = sample[i].nextState; + nextStateInput.push_back(1.f); // bias + actorTarget->GetOutput(nextStateInput, &actorOutput); + + //use criticTarget to estimate value of next action given next state + for(size_t j=0; j < stateSize; j++) { + criticInput[j] = sample[i].nextState[j]; + } + for(size_t j=0; j < actionSize; j++) { + criticInput[j+stateSize] = actorOutput[j]; + } + criticInput[criticInput.size()-1] = 1.f; //bias + + criticTarget->GetOutput(criticInput, &criticOutput); + + //calculate expected reward + const float y = sample[i].reward + (discountFactor * criticOutput[0]); + // std::cout << "[" << i << "]: y: " << y << std::endl; + + //use criticTarget to estimate value of next action given next state + for(size_t j=0; j < stateSize; j++) { + criticInput[j] = sample[i].state[j]; + } + for(size_t j=0; j < actionSize; j++) { + criticInput[j+stateSize] = sample[i].action[j]; + } + criticInput[criticInput.size()-1] = 1.f; //bias + + ts.first.push_back(criticInput); + ts.second.push_back({y}); + } + + //train the critic + float loss = critic->Train(ts, learningRate, 1); + + //TODO: size limit to this log + criticLossLog.push_back(loss); + + //update the actor + + //for each memory in replay memory sample, and get grads from critic + std::vector actorLoss(actionSize, 0.f); + std::vector gradientLoss= {1.f}; + + for(size_t i = 0; i < sample.size(); i++) { + //use criticTarget to estimate value of next action given next state + for(size_t j=0; j < stateSize; j++) { + criticInput[j] = sample[i].nextState[j]; + } + for(size_t j=0; j < actionSize; j++) { + criticInput[j+stateSize] = sample[i].action[j]; + } + criticInput[criticInput.size()-1] = 1.f; //bias + + critic->CalcGradients(criticInput, gradientLoss); + std::vector l0Grads = critic->m_layers[0].GetGrads(); + + for(size_t j=0; j < actionSize; j++) { + actorLoss[j] = l0Grads[j+stateSize]; + } + delay(1); + } + + float totalLoss = 0.f; + for(size_t j=0; j < actorLoss.size(); j++) { + actorLoss[j] /= sample.size(); + actorLoss[j] = -actorLoss[j]; + totalLoss += actorLoss[j]; + } + // actorLossLog.push_back(actorLoss); + // actorLoss = -actorLoss; + // Serial.printf("Actor loss: %f\n", totalLoss); + + //back propagate the actor loss + for(size_t i = 0; i < sample.size(); i++) { + auto actorInput = sample[i].state; + actorInput.push_back(bias); + + actor->ApplyLoss(actorInput, actorLoss, learningRate); + delay(1); + } + + // soft update the target networks + criticTarget->SmoothUpdateWeights(critic, smoothingAlpha); + actorTarget->SmoothUpdateWeights(actor, smoothingAlpha); + } + } + + void setState(const size_t index, float value) { + actorControlInput[index] = value; + newInput = true; + } + + void generateAction() { + if (newInput) { + newInput = false; + std::vector actorOutput; + actorTarget->GetOutput(actorControlInput, &actorOutput); + SendParamsToQueue(actorOutput); + action = actorOutput; + + // for(size_t i=0; i < actorOutput.size(); i++) { + // const float noise = ou_noise.sample() * knobL; + // actorOutput[i] += noise; + // } + + } + } + + void optimiseSometimes() { + if (optimiseCounter==optimiseDivisor) { + optimise(); + optimiseCounter=0; + }else{ + optimiseCounter++; + } + } + + void storeExperience(float reward) { + std::vector state = actorControlInput; + //remove bias + state.pop_back(); + for(size_t i=0; i < state.size(); i++) { + Serial.printf("%f\t", state[i]); + } + Serial.println(); + trainRLItem trainItem = {state, action, reward, state}; + replayMem.add(trainItem, millis()); + } + + void randomiseTheActor() + { + actor->DrawWeights(); + actorTarget->DrawWeights(); + } + + void randomiseTheCritic() + { + critic->DrawWeights(); + criticTarget->DrawWeights(); + } + +private: + static constexpr size_t bias=1; + + size_t optimiseDivisor = 40; + size_t optimiseCounter = 0; + + bool newInput=false; + + const std::vector layers_activfuncs = { + RELU, RELU, TANH + }; + + size_t stateSize; + size_t actionSize; + + std::vector actor_layers_nodes; + std::vector critic_layers_nodes; + + const bool use_constant_weight_init = false; + const float constant_weight_init = 0; + + std::shared_ptr > actor, actorTarget, critic, criticTarget; + + float discountFactor = 0.95; + float learningRate = 0.005; + float smoothingAlpha = 0.005; + + std::vector action; + + ReplayMemory replayMem; + + std::vector actorOutput, criticOutput; + std::vector criticInput; + std::vector actorControlInput; + + std::vector criticLossLog, actorLossLog, log1; + + +}; + +#endif // INTERFACERL_HPP \ No newline at end of file