From 9308af9b8486144cadd37037bd3a6179831a338b Mon Sep 17 00:00:00 2001 From: Andrea Martelloni Date: Tue, 17 Jun 2025 18:14:33 +0100 Subject: [PATCH] Using standard RL interface now --- MEMLNaut-PAF-IML.ino | 169 +++----------------------- interfaceRL.hpp | 278 ------------------------------------------- 2 files changed, 14 insertions(+), 433 deletions(-) delete mode 100644 interfaceRL.hpp diff --git a/MEMLNaut-PAF-IML.ino b/MEMLNaut-PAF-IML.ino index 13aded7..b56ba8a 100644 --- a/MEMLNaut-PAF-IML.ino +++ b/MEMLNaut-PAF-IML.ino @@ -4,10 +4,9 @@ #include "src/memllib/audio/AudioDriver.hpp" #include "src/memllib/hardware/memlnaut/MEMLNaut.hpp" #include -#include "IMLInterface.hpp" -#include "interfaceRL.hpp" #include "hardware/structs/bus_ctrl.h" #include "PAFSynthAudioApp.hpp" +#include "src/memllib/interface/InterfaceRL.hpp" #define APP_SRAM __not_in_flash("app") @@ -31,8 +30,7 @@ uint32_t get_rosc_entropy_seed(int bits) { // Global objects -std::shared_ptr APP_SRAM interfaceIML; -std::shared_ptr APP_SRAM RLInterface; +std::shared_ptr APP_SRAM RLInterface; std::shared_ptr __scratch_y("audio") audio_app; @@ -52,117 +50,6 @@ constexpr size_t kN_InputParams = 3; #define READ_VOLATILE(var) ({ MEMORY_BARRIER(); typeof(var) __temp = (var); MEMORY_BARRIER(); __temp; }) -void bind_RL_interface(std::shared_ptr interface) -{ - // Set up momentary switch callbacks - MEMLNaut::Instance()->setMomA1Callback([interface] () { - static APP_SRAM std::vector msgs = {"Wow, incredible", "Awesome", "That's amazing", "Unbelievable+","I love it!!","More of this","Yes!!!!","A-M-A-Z-I-N-G"}; - String msg = msgs[rand() % msgs.size()]; - interface->storeExperience(1.f); - Serial.println(msg); - - scr.post(msg); - }); - MEMLNaut::Instance()->setMomA2Callback([interface] () { - static APP_SRAM std::vector msgs = {"Awful!","wtf? that sucks","Get rid of this sound","Totally shite","I hate this","Why even bother?","New sound please!","No, please no!!!","Thumbs down"}; - String msg = msgs[rand() % msgs.size()]; - interface->storeExperience(-1.f); - Serial.println(msg); - scr.post(msg); - }); - MEMLNaut::Instance()->setMomB1Callback([interface] () { - interface->randomiseTheActor(); - interface->generateAction(true); - Serial.println("The Actor is confused"); - scr.post("Actor: i'm confused"); - }); - MEMLNaut::Instance()->setMomB2Callback([interface] () { - interface->randomiseTheCritic(); - interface->generateAction(true); - Serial.println("The Critic is confounded"); - scr.post("Critic: totally confounded"); - }); - // Set up ADC callbacks - MEMLNaut::Instance()->setJoyXCallback([interface] (float value) { - interface->setState(0, value); - }); - MEMLNaut::Instance()->setJoyYCallback([interface] (float value) { - interface->setState(1, value); - }); - MEMLNaut::Instance()->setJoyZCallback([interface] (float value) { - interface->setState(2, value); - }); - - MEMLNaut::Instance()->setRVGain1Callback([interface] (float value) { - AudioDriver::setDACVolume(value); - }); - - MEMLNaut::Instance()->setRVX1Callback([interface] (float value) { - size_t divisor = 1 + (value * 100); - String msg = "Optimise every " + String(divisor); - scr.post(msg); - interface->setOptimiseDivisor(divisor); - Serial.println(msg); - }); - - - // Set up loop callback - MEMLNaut::Instance()->setLoopCallback([interface] () { - interface->optimiseSometimes(); - interface->generateAction(); - }); - - -} - -void bind_IML_interface(std::shared_ptr interface) -{ - // Set up momentary switch callbacks - MEMLNaut::Instance()->setMomA1Callback([interface] () { - interface->Randomise(); - }); - MEMLNaut::Instance()->setMomA2Callback([interface] () { - interface->ClearData(); - }); - - // Set up toggle switch callbacks - MEMLNaut::Instance()->setTogA1Callback([interface] (bool state) { - interface->SetTrainingMode(state ? IMLInterface::TRAINING_MODE : IMLInterface::INFERENCE_MODE); - }); - MEMLNaut::Instance()->setJoySWCallback([interface] (bool state) { - interface->SaveInput(state ? IMLInterface::STORE_VALUE_MODE : IMLInterface::STORE_POSITION_MODE); - }); - - // Set up ADC callbacks - MEMLNaut::Instance()->setJoyXCallback([interface] (float value) { - interface->SetInput(0, value); - }); - MEMLNaut::Instance()->setJoyYCallback([interface] (float value) { - interface->SetInput(1, value); - }); - MEMLNaut::Instance()->setJoyZCallback([interface] (float value) { - interface->SetInput(2, value); - }); - MEMLNaut::Instance()->setRVZ1Callback([interface] (float value) { - // Scale value from 0-1 range to 1-3000 - value = 1.0f + (value * 2999.0f); - interface->SetIterations(static_cast(value)); - }); - - // Set up loop callback - MEMLNaut::Instance()->setLoopCallback([interface] () { - interface->ProcessInput(); - }); - - MEMLNaut::Instance()->setRVGain1Callback([interface] (float value) { - AudioDriver::setDACVolume(value); - }); -} - -enum MLMODES {IML, RL}; -MLMODES APP_SRAM mlMode = RL; - - struct repeating_timer APP_SRAM timerDisplay; inline bool __not_in_flash_func(displayUpdate)(__unused struct repeating_timer *t) { scr.update(); @@ -188,38 +75,17 @@ void setup() MEMLNaut::Initialize(); pinMode(33, OUTPUT); - switch(mlMode) { - case IML: { - { - auto temp_interface = std::make_shared(); - temp_interface->setup(kN_InputParams, PAFSynthAudioApp::kN_Params); - MEMORY_BARRIER(); - interfaceIML = temp_interface; - MEMORY_BARRIER(); - } - // Setup interface with memory barrier protection - WRITE_VOLATILE(interface_ready, true); - // Bind interface after ensuring it's fully initialized - bind_IML_interface(interfaceIML); - Serial.println("Bound IML interface to MEMLNaut."); - } - break; - case RL: { - { - auto temp_interface = std::make_shared(); - temp_interface->setup(kN_InputParams, PAFSynthAudioApp::kN_Params); - MEMORY_BARRIER(); - RLInterface = temp_interface; - MEMORY_BARRIER(); - } - // Setup interface with memory barrier protection - WRITE_VOLATILE(interface_ready, true); - // Bind interface after ensuring it's fully initialized - bind_RL_interface(RLInterface); - Serial.println("Bound RL interface to MEMLNaut."); - } - break; - } + auto temp_interface = std::make_shared(); + temp_interface->setup(kN_InputParams, PAFSynthAudioApp::kN_Params); + MEMORY_BARRIER(); + RLInterface = temp_interface; + MEMORY_BARRIER(); + + // Setup interface with memory barrier protection + WRITE_VOLATILE(interface_ready, true); + // Bind interface after ensuring it's fully initialized + RLInterface->bind_RL_interface(scr); + Serial.println("Bound RL interface to MEMLNaut."); WRITE_VOLATILE(core_0_ready, true); @@ -268,15 +134,8 @@ void setup1() // Create audio app with memory barrier protection { auto temp_audio_app = std::make_shared(); - std::shared_ptr selectedInterface; - if (mlMode == IML) { - selectedInterface = std::dynamic_pointer_cast(interfaceIML); - } else { - selectedInterface = std::dynamic_pointer_cast(RLInterface); - } - - temp_audio_app->Setup(AudioDriver::GetSampleRate(), selectedInterface); + temp_audio_app->Setup(AudioDriver::GetSampleRate(), RLInterface); // temp_audio_app->Setup(AudioDriver::GetSampleRate(), dynamic_cast> (mlMode == IML ? interfaceIML : RLInterface)); MEMORY_BARRIER(); audio_app = temp_audio_app; diff --git a/interfaceRL.hpp b/interfaceRL.hpp deleted file mode 100644 index 755ff8b..0000000 --- a/interfaceRL.hpp +++ /dev/null @@ -1,278 +0,0 @@ -#ifndef INTERFACERL_HPP -#define INTERFACERL_HPP - -#include "src/memllib/interface/InterfaceBase.hpp" - -#include "src/memlp/MLP.h" -#include "src/memlp/ReplayMemory.hpp" -#include "src/memlp/OrnsteinUhlenbeckNoise.h" -#include - -#define RL_MEM __not_in_flash("rlmem") - - -struct trainRLItem { - std::vector state ; - std::vector action; - float reward; - std::vector nextState; -}; - - -class interfaceRL : public InterfaceBase -{ -public: - - void setup(size_t n_inputs, size_t n_outputs) override - { - InterfaceBase::setup(n_inputs, n_outputs); - stateSize = n_inputs; - actionSize = n_outputs; - - actor_layers_nodes = { - stateSize + bias, - 10, 10, - actionSize - }; - - critic_layers_nodes = { - stateSize + actionSize + bias, - 10, 10, - 1 - }; - - criticInput.resize(critic_layers_nodes[0]); - actorControlInput.resize(actor_layers_nodes[0]); - actorControlInput[actorControlInput.size()-1] = 1.f; // bias - - //init networks - actor = std::make_shared > ( - actor_layers_nodes, - layers_activfuncs, - loss::LOSS_MSE, - use_constant_weight_init, - constant_weight_init - ); - - actorTarget = std::make_shared > ( - actor_layers_nodes, - layers_activfuncs, - loss::LOSS_MSE, - use_constant_weight_init, - constant_weight_init - ); - - critic = std::make_shared > ( - critic_layers_nodes, - layers_activfuncs, - loss::LOSS_MSE, - use_constant_weight_init, - constant_weight_init - ); - criticTarget = std::make_shared > ( - critic_layers_nodes, - layers_activfuncs, - loss::LOSS_MSE, - use_constant_weight_init, - constant_weight_init - ); - } - - void optimise() { - constexpr size_t batchSize = 4; - std::vector sample = replayMem.sample(batchSize); - if (sample.size() == batchSize) { - //run sample through critic target, build training set for critic net - MLP::training_pair_t ts; - for(size_t i = 0; i < sample.size(); i++) { - //---calculate y - //--calc next-state-action pair - //get next action from actorTarget given next state - auto nextStateInput = sample[i].nextState; - nextStateInput.push_back(1.f); // bias - actorTarget->GetOutput(nextStateInput, &actorOutput); - - //use criticTarget to estimate value of next action given next state - for(size_t j=0; j < stateSize; j++) { - criticInput[j] = sample[i].nextState[j]; - } - for(size_t j=0; j < actionSize; j++) { - criticInput[j+stateSize] = actorOutput[j]; - } - criticInput[criticInput.size()-1] = 1.f; //bias - - criticTarget->GetOutput(criticInput, &criticOutput); - - //calculate expected reward - const float y = sample[i].reward + (discountFactor * criticOutput[0]); - // std::cout << "[" << i << "]: y: " << y << std::endl; - - //use criticTarget to estimate value of next action given next state - for(size_t j=0; j < stateSize; j++) { - criticInput[j] = sample[i].state[j]; - } - for(size_t j=0; j < actionSize; j++) { - criticInput[j+stateSize] = sample[i].action[j]; - } - criticInput[criticInput.size()-1] = 1.f; //bias - - ts.first.push_back(criticInput); - ts.second.push_back({y}); - } - - //train the critic - float loss = critic->Train(ts, learningRate, 1); - - //TODO: size limit to this log - criticLossLog.push_back(loss); - - //update the actor - - //for each memory in replay memory sample, and get grads from critic - std::vector actorLoss(actionSize, 0.f); - std::vector gradientLoss= {1.f}; - - for(size_t i = 0; i < sample.size(); i++) { - //use criticTarget to estimate value of next action given next state - for(size_t j=0; j < stateSize; j++) { - criticInput[j] = sample[i].nextState[j]; - } - for(size_t j=0; j < actionSize; j++) { - criticInput[j+stateSize] = sample[i].action[j]; - } - criticInput[criticInput.size()-1] = 1.f; //bias - - critic->CalcGradients(criticInput, gradientLoss); - std::vector l0Grads = critic->m_layers[0].GetGrads(); - - for(size_t j=0; j < actionSize; j++) { - actorLoss[j] = l0Grads[j+stateSize]; - } - delay(1); - } - - float totalLoss = 0.f; - for(size_t j=0; j < actorLoss.size(); j++) { - actorLoss[j] /= sample.size(); - actorLoss[j] = -actorLoss[j]; - totalLoss += actorLoss[j]; - } - // actorLossLog.push_back(actorLoss); - // actorLoss = -actorLoss; - // Serial.printf("Actor loss: %f\n", totalLoss); - - //back propagate the actor loss - for(size_t i = 0; i < sample.size(); i++) { - auto actorInput = sample[i].state; - actorInput.push_back(bias); - - actor->ApplyLoss(actorInput, actorLoss, learningRate); - delay(1); - } - - // soft update the target networks - criticTarget->SmoothUpdateWeights(critic, smoothingAlpha); - actorTarget->SmoothUpdateWeights(actor, smoothingAlpha); - } - } - - void setState(const size_t index, float value) { - actorControlInput[index] = value; - newInput = true; - } - - void generateAction(bool donthesitate=false) { - if (newInput || donthesitate) { - newInput = false; - std::vector actorOutput; - actorTarget->GetOutput(actorControlInput, &actorOutput); - SendParamsToQueue(actorOutput); - action = actorOutput; - - // for(size_t i=0; i < actorOutput.size(); i++) { - // const float noise = ou_noise.sample() * knobL; - // actorOutput[i] += noise; - // } - - } - } - - void optimiseSometimes() { - if (optimiseCounter>=optimiseDivisor) { - optimise(); - optimiseCounter=0; - }else{ - optimiseCounter++; - } - } - - void storeExperience(float reward) { - std::vector state = actorControlInput; - //remove bias - state.pop_back(); - for(size_t i=0; i < state.size(); i++) { - Serial.printf("%f\t", state[i]); - } - Serial.println(); - trainRLItem trainItem = {state, action, reward, state}; - replayMem.add(trainItem, millis()); - } - - void randomiseTheActor() - { - actor->DrawWeights(); - actorTarget->DrawWeights(); - } - - void randomiseTheCritic() - { - critic->DrawWeights(); - criticTarget->DrawWeights(); - } - - void setOptimiseDivisor(size_t newDiv) { - optimiseDivisor = newDiv; - } - - -private: - static constexpr size_t bias=1; - - size_t optimiseDivisor = 40; - size_t optimiseCounter = 0; - - bool newInput=false; - - const std::vector layers_activfuncs = { - RELU, RELU, SIGMOID - }; - - size_t stateSize; - size_t actionSize; - - std::vector actor_layers_nodes; - std::vector critic_layers_nodes; - - const bool use_constant_weight_init = false; - const float constant_weight_init = 0; - - std::shared_ptr > actor, actorTarget, critic, criticTarget; - - float discountFactor = 0.95; - float learningRate = 0.005; - float smoothingAlpha = 0.005; - - std::vector action; - - ReplayMemory replayMem; - - std::vector actorOutput, criticOutput; - std::vector criticInput; - std::vector actorControlInput; - - std::vector criticLossLog, actorLossLog, log1; - - -}; - -#endif // INTERFACERL_HPP \ No newline at end of file