w1n5t0n/memlnaut-nisps
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Using standard RL interface now

Browse source
This commit is contained in:
Andrea Martelloni 2025-06-17 18:14:33 +01:00
parent 21856b195f
commit 9308af9b84
2 changed files with 14 additions and 433 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

153
MEMLNaut-PAF-IML.ino
View file

@ -4,10 +4,9 @@
#include "src/memllib/audio/AudioDriver.hpp" #include "src/memllib/audio/AudioDriver.hpp"
#include "src/memllib/hardware/memlnaut/MEMLNaut.hpp" #include "src/memllib/hardware/memlnaut/MEMLNaut.hpp"
#include <memory> #include <memory>
#include "IMLInterface.hpp"
#include "interfaceRL.hpp"
#include "hardware/structs/bus_ctrl.h" #include "hardware/structs/bus_ctrl.h"
#include "PAFSynthAudioApp.hpp" #include "PAFSynthAudioApp.hpp"
#include "src/memllib/interface/InterfaceRL.hpp"
#define APP_SRAM __not_in_flash("app") #define APP_SRAM __not_in_flash("app")
@ -31,8 +30,7 @@ uint32_t get_rosc_entropy_seed(int bits) {
// Global objects // Global objects
std::shared_ptr<IMLInterface> APP_SRAM interfaceIML; std::shared_ptr<InterfaceRL> APP_SRAM RLInterface;
std::shared_ptr<interfaceRL> APP_SRAM RLInterface;
std::shared_ptr<PAFSynthAudioApp> __scratch_y("audio") audio_app; std::shared_ptr<PAFSynthAudioApp> __scratch_y("audio") audio_app;
@ -52,117 +50,6 @@ constexpr size_t kN_InputParams = 3;
#define READ_VOLATILE(var) ({ MEMORY_BARRIER(); typeof(var) __temp = (var); MEMORY_BARRIER(); __temp; }) #define READ_VOLATILE(var) ({ MEMORY_BARRIER(); typeof(var) __temp = (var); MEMORY_BARRIER(); __temp; })
void bind_RL_interface(std::shared_ptr<interfaceRL> interface)
{
// Set up momentary switch callbacks
MEMLNaut::Instance()->setMomA1Callback([interface] () {
static APP_SRAM std::vector<String> msgs = {"Wow, incredible", "Awesome", "That's amazing", "Unbelievable+","I love it!!","More of this","Yes!!!!","A-M-A-Z-I-N-G"};
String msg = msgs[rand() % msgs.size()];
interface->storeExperience(1.f);
Serial.println(msg);
scr.post(msg);
});
MEMLNaut::Instance()->setMomA2Callback([interface] () {
static APP_SRAM std::vector<String> msgs = {"Awful!","wtf? that sucks","Get rid of this sound","Totally shite","I hate this","Why even bother?","New sound please!","No, please no!!!","Thumbs down"};
String msg = msgs[rand() % msgs.size()];
interface->storeExperience(-1.f);
Serial.println(msg);
scr.post(msg);
});
MEMLNaut::Instance()->setMomB1Callback([interface] () {
interface->randomiseTheActor();
interface->generateAction(true);
Serial.println("The Actor is confused");
scr.post("Actor: i'm confused");
});
MEMLNaut::Instance()->setMomB2Callback([interface] () {
interface->randomiseTheCritic();
interface->generateAction(true);
Serial.println("The Critic is confounded");
scr.post("Critic: totally confounded");
});
// Set up ADC callbacks
MEMLNaut::Instance()->setJoyXCallback([interface] (float value) {
interface->setState(0, value);
});
MEMLNaut::Instance()->setJoyYCallback([interface] (float value) {
interface->setState(1, value);
});
MEMLNaut::Instance()->setJoyZCallback([interface] (float value) {
interface->setState(2, value);
});
MEMLNaut::Instance()->setRVGain1Callback([interface] (float value) {
AudioDriver::setDACVolume(value);
});
MEMLNaut::Instance()->setRVX1Callback([interface] (float value) {
size_t divisor = 1 + (value * 100);
String msg = "Optimise every " + String(divisor);
scr.post(msg);
interface->setOptimiseDivisor(divisor);
Serial.println(msg);
});
// Set up loop callback
MEMLNaut::Instance()->setLoopCallback([interface] () {
interface->optimiseSometimes();
interface->generateAction();
});
}
void bind_IML_interface(std::shared_ptr<IMLInterface> interface)
{
// Set up momentary switch callbacks
MEMLNaut::Instance()->setMomA1Callback([interface] () {
interface->Randomise();
});
MEMLNaut::Instance()->setMomA2Callback([interface] () {
interface->ClearData();
});
// Set up toggle switch callbacks
MEMLNaut::Instance()->setTogA1Callback([interface] (bool state) {
interface->SetTrainingMode(state ? IMLInterface::TRAINING_MODE : IMLInterface::INFERENCE_MODE);
});
MEMLNaut::Instance()->setJoySWCallback([interface] (bool state) {
interface->SaveInput(state ? IMLInterface::STORE_VALUE_MODE : IMLInterface::STORE_POSITION_MODE);
});
// Set up ADC callbacks
MEMLNaut::Instance()->setJoyXCallback([interface] (float value) {
interface->SetInput(0, value);
});
MEMLNaut::Instance()->setJoyYCallback([interface] (float value) {
interface->SetInput(1, value);
});
MEMLNaut::Instance()->setJoyZCallback([interface] (float value) {
interface->SetInput(2, value);
});
MEMLNaut::Instance()->setRVZ1Callback([interface] (float value) {
// Scale value from 0-1 range to 1-3000
value = 1.0f + (value * 2999.0f);
interface->SetIterations(static_cast<size_t>(value));
});
// Set up loop callback
MEMLNaut::Instance()->setLoopCallback([interface] () {
interface->ProcessInput();
});
MEMLNaut::Instance()->setRVGain1Callback([interface] (float value) {
AudioDriver::setDACVolume(value);
});
}
enum MLMODES {IML, RL};
MLMODES APP_SRAM mlMode = RL;
struct repeating_timer APP_SRAM timerDisplay; struct repeating_timer APP_SRAM timerDisplay;
inline bool __not_in_flash_func(displayUpdate)(__unused struct repeating_timer *t) { inline bool __not_in_flash_func(displayUpdate)(__unused struct repeating_timer *t) {
scr.update(); scr.update();
@ -188,38 +75,17 @@ void setup()
MEMLNaut::Initialize(); MEMLNaut::Initialize();
pinMode(33, OUTPUT); pinMode(33, OUTPUT);
switch(mlMode) { auto temp_interface = std::make_shared<InterfaceRL>();
case IML: {
{
auto temp_interface = std::make_shared<IMLInterface>();
temp_interface->setup(kN_InputParams, PAFSynthAudioApp::kN_Params);
MEMORY_BARRIER();
interfaceIML = temp_interface;
MEMORY_BARRIER();
}
// Setup interface with memory barrier protection
WRITE_VOLATILE(interface_ready, true);
// Bind interface after ensuring it's fully initialized
bind_IML_interface(interfaceIML);
Serial.println("Bound IML interface to MEMLNaut.");
}
break;
case RL: {
{
auto temp_interface = std::make_shared<interfaceRL>();
temp_interface->setup(kN_InputParams, PAFSynthAudioApp::kN_Params); temp_interface->setup(kN_InputParams, PAFSynthAudioApp::kN_Params);
MEMORY_BARRIER(); MEMORY_BARRIER();
RLInterface = temp_interface; RLInterface = temp_interface;
MEMORY_BARRIER(); MEMORY_BARRIER();
}
// Setup interface with memory barrier protection // Setup interface with memory barrier protection
WRITE_VOLATILE(interface_ready, true); WRITE_VOLATILE(interface_ready, true);
// Bind interface after ensuring it's fully initialized // Bind interface after ensuring it's fully initialized
bind_RL_interface(RLInterface); RLInterface->bind_RL_interface(scr);
Serial.println("Bound RL interface to MEMLNaut."); Serial.println("Bound RL interface to MEMLNaut.");
}
break;
}
WRITE_VOLATILE(core_0_ready, true); WRITE_VOLATILE(core_0_ready, true);
@ -268,15 +134,8 @@ void setup1()
// Create audio app with memory barrier protection // Create audio app with memory barrier protection
{ {
auto temp_audio_app = std::make_shared<PAFSynthAudioApp>(); auto temp_audio_app = std::make_shared<PAFSynthAudioApp>();
std::shared_ptr<InterfaceBase> selectedInterface;
if (mlMode == IML) { temp_audio_app->Setup(AudioDriver::GetSampleRate(), RLInterface);
selectedInterface = std::dynamic_pointer_cast<InterfaceBase>(interfaceIML);
} else {
selectedInterface = std::dynamic_pointer_cast<InterfaceBase>(RLInterface);
}
temp_audio_app->Setup(AudioDriver::GetSampleRate(), selectedInterface);
// temp_audio_app->Setup(AudioDriver::GetSampleRate(), dynamic_cast<std::shared_ptr<InterfaceBase>> (mlMode == IML ? interfaceIML : RLInterface)); // temp_audio_app->Setup(AudioDriver::GetSampleRate(), dynamic_cast<std::shared_ptr<InterfaceBase>> (mlMode == IML ? interfaceIML : RLInterface));
MEMORY_BARRIER(); MEMORY_BARRIER();
audio_app = temp_audio_app; audio_app = temp_audio_app;

278
interfaceRL.hpp
View file

@ -1,278 +0,0 @@
#ifndef INTERFACERL_HPP
#define INTERFACERL_HPP
#include "src/memllib/interface/InterfaceBase.hpp"
#include "src/memlp/MLP.h"
#include "src/memlp/ReplayMemory.hpp"
#include "src/memlp/OrnsteinUhlenbeckNoise.h"
#include <memory>
#define RL_MEM __not_in_flash("rlmem")
struct trainRLItem {
std::vector<float> state ;
std::vector<float> action;
float reward;
std::vector<float> nextState;
};
class interfaceRL : public InterfaceBase
{
public:
void setup(size_t n_inputs, size_t n_outputs) override
{
InterfaceBase::setup(n_inputs, n_outputs);
stateSize = n_inputs;
actionSize = n_outputs;
actor_layers_nodes = {
stateSize + bias,
10, 10,
actionSize
};
critic_layers_nodes = {
stateSize + actionSize + bias,
10, 10,
1
};
criticInput.resize(critic_layers_nodes[0]);
actorControlInput.resize(actor_layers_nodes[0]);
actorControlInput[actorControlInput.size()-1] = 1.f; // bias
//init networks
actor = std::make_shared<MLP<float> > (
actor_layers_nodes,
layers_activfuncs,
loss::LOSS_MSE,
use_constant_weight_init,
constant_weight_init
);
actorTarget = std::make_shared<MLP<float> > (
actor_layers_nodes,
layers_activfuncs,
loss::LOSS_MSE,
use_constant_weight_init,
constant_weight_init
);
critic = std::make_shared<MLP<float> > (
critic_layers_nodes,
layers_activfuncs,
loss::LOSS_MSE,
use_constant_weight_init,
constant_weight_init
);
criticTarget = std::make_shared<MLP<float> > (
critic_layers_nodes,
layers_activfuncs,
loss::LOSS_MSE,
use_constant_weight_init,
constant_weight_init
);
}
void optimise() {
constexpr size_t batchSize = 4;
std::vector<trainRLItem> sample = replayMem.sample(batchSize);
if (sample.size() == batchSize) {
//run sample through critic target, build training set for critic net
MLP<float>::training_pair_t ts;
for(size_t i = 0; i < sample.size(); i++) {
//---calculate y
//--calc next-state-action pair
//get next action from actorTarget given next state
auto nextStateInput = sample[i].nextState;
nextStateInput.push_back(1.f); // bias
actorTarget->GetOutput(nextStateInput, &actorOutput);
//use criticTarget to estimate value of next action given next state
for(size_t j=0; j < stateSize; j++) {
criticInput[j] = sample[i].nextState[j];
}
for(size_t j=0; j < actionSize; j++) {
criticInput[j+stateSize] = actorOutput[j];
}
criticInput[criticInput.size()-1] = 1.f; //bias
criticTarget->GetOutput(criticInput, &criticOutput);
//calculate expected reward
const float y = sample[i].reward + (discountFactor * criticOutput[0]);
// std::cout << "[" << i << "]: y: " << y << std::endl;
//use criticTarget to estimate value of next action given next state
for(size_t j=0; j < stateSize; j++) {
criticInput[j] = sample[i].state[j];
}
for(size_t j=0; j < actionSize; j++) {
criticInput[j+stateSize] = sample[i].action[j];
}
criticInput[criticInput.size()-1] = 1.f; //bias
ts.first.push_back(criticInput);
ts.second.push_back({y});
}
//train the critic
float loss = critic->Train(ts, learningRate, 1);
//TODO: size limit to this log
criticLossLog.push_back(loss);
//update the actor
//for each memory in replay memory sample, and get grads from critic
std::vector<float> actorLoss(actionSize, 0.f);
std::vector<float> gradientLoss= {1.f};
for(size_t i = 0; i < sample.size(); i++) {
//use criticTarget to estimate value of next action given next state
for(size_t j=0; j < stateSize; j++) {
criticInput[j] = sample[i].nextState[j];
}
for(size_t j=0; j < actionSize; j++) {
criticInput[j+stateSize] = sample[i].action[j];
}
criticInput[criticInput.size()-1] = 1.f; //bias
critic->CalcGradients(criticInput, gradientLoss);
std::vector<float> l0Grads = critic->m_layers[0].GetGrads();
for(size_t j=0; j < actionSize; j++) {
actorLoss[j] = l0Grads[j+stateSize];
}
delay(1);
}
float totalLoss = 0.f;
for(size_t j=0; j < actorLoss.size(); j++) {
actorLoss[j] /= sample.size();
actorLoss[j] = -actorLoss[j];
totalLoss += actorLoss[j];
}
// actorLossLog.push_back(actorLoss);
// actorLoss = -actorLoss;
// Serial.printf("Actor loss: %f\n", totalLoss);
//back propagate the actor loss
for(size_t i = 0; i < sample.size(); i++) {
auto actorInput = sample[i].state;
actorInput.push_back(bias);
actor->ApplyLoss(actorInput, actorLoss, learningRate);
delay(1);
}
// soft update the target networks
criticTarget->SmoothUpdateWeights(critic, smoothingAlpha);
actorTarget->SmoothUpdateWeights(actor, smoothingAlpha);
}
}
void setState(const size_t index, float value) {
actorControlInput[index] = value;
newInput = true;
}
void generateAction(bool donthesitate=false) {
if (newInput || donthesitate) {
newInput = false;
std::vector<float> actorOutput;
actorTarget->GetOutput(actorControlInput, &actorOutput);
SendParamsToQueue(actorOutput);
action = actorOutput;
// for(size_t i=0; i < actorOutput.size(); i++) {
// const float noise = ou_noise.sample() * knobL;
// actorOutput[i] += noise;
// }
}
}
void optimiseSometimes() {
if (optimiseCounter>=optimiseDivisor) {
optimise();
optimiseCounter=0;
}else{
optimiseCounter++;
}
}
void storeExperience(float reward) {
std::vector<float> state = actorControlInput;
//remove bias
state.pop_back();
for(size_t i=0; i < state.size(); i++) {
Serial.printf("%f\t", state[i]);
}
Serial.println();
trainRLItem trainItem = {state, action, reward, state};
replayMem.add(trainItem, millis());
}
void randomiseTheActor()
{
actor->DrawWeights();
actorTarget->DrawWeights();
}
void randomiseTheCritic()
{
critic->DrawWeights();
criticTarget->DrawWeights();
}
void setOptimiseDivisor(size_t newDiv) {
optimiseDivisor = newDiv;
}
private:
static constexpr size_t bias=1;
size_t optimiseDivisor = 40;
size_t optimiseCounter = 0;
bool newInput=false;
const std::vector<ACTIVATION_FUNCTIONS> layers_activfuncs = {
RELU, RELU, SIGMOID
};
size_t stateSize;
size_t actionSize;
std::vector<size_t> actor_layers_nodes;
std::vector<size_t> critic_layers_nodes;
const bool use_constant_weight_init = false;
const float constant_weight_init = 0;
std::shared_ptr<MLP<float> > actor, actorTarget, critic, criticTarget;
float discountFactor = 0.95;
float learningRate = 0.005;
float smoothingAlpha = 0.005;
std::vector<float> action;
ReplayMemory<trainRLItem> replayMem;
std::vector<float> actorOutput, criticOutput;
std::vector<float> criticInput;
std::vector<float> actorControlInput;
std::vector<float> criticLossLog, actorLossLog, log1;
};
#endif // INTERFACERL_HPP