w1n5t0n/memlnaut-nisps
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

speed optim, no clicking

Browse source
This commit is contained in:
chriskiefer 2025-05-07 23:58:06 +01:00
parent 241a8f6d21
commit 56aa15ae10
3 changed files with 736 additions and 334 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

276
IMLInterface.hpp Normal file
View file

@ -0,0 +1,276 @@
#ifndef IMINTERFACE_HPP
#define IMINTERFACE_HPP
// #include "src/memllib/audio/AudioAppBase.hpp"
#include "src/memllib/interface/InterfaceBase.hpp"
#include "src/memlp/Dataset.hpp"
#include "src/memlp/MLP.h"
class IMLInterface : public InterfaceBase
{
public:
IMLInterface() : InterfaceBase() {}
void setup(size_t n_inputs, size_t n_outputs) override
{
InterfaceBase::setup(n_inputs, n_outputs);
// Additional setup code specific to IMLInterface
n_inputs_ = n_inputs;
n_outputs_ = n_outputs;
MLSetup_();
n_iterations_ = 1000;
input_state_.resize(n_inputs, 0.5f);
output_state_.resize(n_outputs, 0);
// Init/reset state machine
training_mode_ = INFERENCE_MODE;
perform_inference_ = true;
input_updated_ = false;
Serial.println("IMLInterface setup done");
Serial.print("Address of n_inputs_: ");
Serial.println(reinterpret_cast<uintptr_t>(&n_inputs_));
Serial.print("Inputs: ");
Serial.print(n_inputs_);
Serial.print(", Outputs: ");
Serial.println(n_outputs_);
}
enum training_mode_t {
INFERENCE_MODE,
TRAINING_MODE
};
void SetTrainingMode(training_mode_t training_mode)
{
Serial.print("Training mode: ");
Serial.println(training_mode == INFERENCE_MODE ? "Inference" : "Training");
if (training_mode == INFERENCE_MODE && training_mode_ == TRAINING_MODE) {
// Train the network!
MLTraining_();
}
training_mode_ = training_mode;
}
void ProcessInput()
{
// Check if input is updated
if (perform_inference_ && input_updated_) {
MLInference_(input_state_);
input_updated_ = false;
}
}
void SetInput(size_t index, float value)
{
// Serial.print("Input ");
// Serial.print(index);
// Serial.print(" set to: ");
// Serial.println(value);
if (index >= n_inputs_) {
Serial.print("Input index ");
Serial.print(index);
Serial.println(" out of bounds.");
return;
}
if (value < 0) {
value = 0;
} else if (value > 1.0) {
value = 1.0;
}
// Update state of input
input_state_[index] = value;
input_updated_ = true;
}
enum saving_mode_t {
STORE_VALUE_MODE,
STORE_POSITION_MODE,
};
void SaveInput(saving_mode_t mode)
{
if (STORE_VALUE_MODE == mode) {
Serial.println("Move input to position...");
perform_inference_ = false;
} else { // STORE_POSITION_MODE
Serial.println("Creating example in this position.");
// Save pair in the dataset
dataset_->Add(input_state_, output_state_);
perform_inference_ = true;
MLInference_(input_state_);
}
}
void ClearData()
{
if (training_mode_ == TRAINING_MODE) {
Serial.println("Clearing dataset...");
dataset_->Clear();
}
}
void Randomise()
{
if (training_mode_ == TRAINING_MODE) {
Serial.println("Randomising weights...");
MLRandomise_();
MLInference_(input_state_);
}
}
void SetIterations(size_t iterations)
{
n_iterations_ = iterations;
Serial.print("Iterations set to: ");
Serial.println(n_iterations_);
}
protected:
size_t n_inputs_;
size_t n_outputs_;
size_t n_iterations_;
// State machine
training_mode_t training_mode_;
bool perform_inference_;
bool input_updated_;
// Controls/sensors
std::vector<float> input_state_;
std::vector<float> output_state_;
// MLP core
std::unique_ptr<Dataset> dataset_;
std::unique_ptr<MLP<float>> mlp_;
MLP<float>::mlp_weights mlp_stored_weights_;
bool randomised_state_;
void MLSetup_()
{
// Constants for MLP init
const unsigned int kBias = 1;
const std::vector<ACTIVATION_FUNCTIONS> layers_activfuncs = {
RELU, RELU, RELU, SIGMOID
};
const bool use_constant_weight_init = false;
const float constant_weight_init = 0;
// Layer size definitions
const std::vector<size_t> layers_nodes = {
n_inputs_ + kBias,
10, 10, 14,
n_outputs_
};
// Create dataset
dataset_ = std::make_unique<Dataset>();
// Create MLP
mlp_ = std::make_unique<MLP<float>>(
layers_nodes,
layers_activfuncs,
loss::LOSS_MSE,
use_constant_weight_init,
constant_weight_init
);
// State machine
randomised_state_ = false;
}
void MLInference_(std::vector<float> input)
{
if (!dataset_ || !mlp_) {
Serial.println("ML not initialized!");
return;
}
if (input.size() != n_inputs_) {
Serial.print("Input size mismatch - ");
Serial.print("Expected: ");
Serial.print(n_inputs_);
Serial.print(", Got: ");
Serial.println(input.size());
return;
}
input.push_back(1.0f); // Add bias term
// Perform inference
std::vector<float> output(n_outputs_);
mlp_->GetOutput(input, &output);
// Process inferenced data
output_state_ = output;
SendParamsToQueue(output);
}
void MLRandomise_()
{
if (!mlp_) {
Serial.println("ML not initialized!");
return;
}
// Randomize weights
mlp_stored_weights_ = mlp_->GetWeights();
mlp_->DrawWeights();
randomised_state_ = true;
}
void MLTraining_()
{
if (!mlp_) {
Serial.println("ML not initialized!");
return;
}
// Restore old weights
if (randomised_state_) {
mlp_->SetWeights(mlp_stored_weights_);
}
randomised_state_ = false;
// Prepare for training
// Extract dataset to training pair
MLP<float>::training_pair_t dataset(dataset_->GetFeatures(), dataset_->GetLabels());
// Check and report on dataset size
Serial.print("Feature size ");
Serial.print(dataset.first.size());
Serial.print(", label size ");
Serial.println(dataset.second.size());
if (!dataset.first.size() || !dataset.second.size()) {
Serial.println("Empty dataset!");
return;
}
Serial.print("Feature dim ");
Serial.print(dataset.first[0].size());
Serial.print(", label dim ");
Serial.println(dataset.second[0].size());
if (!dataset.first[0].size() || !dataset.second[0].size()) {
Serial.println("Empty dataset dimensions!");
return;
}
// Training loop
Serial.print("Training for max ");
Serial.print(n_iterations_);
Serial.println(" iterations...");
float loss = mlp_->Train(dataset,
1.,
n_iterations_,
0.00001,
false);
Serial.print("Trained, loss = ");
Serial.println(loss, 10);
}
};
#endif // IMINTERFACE_HPP

521
MEMLNaut-PAF-IML.ino
View file

@ -1,305 +1,52 @@
#include "src/memllib/interface/InterfaceBase.hpp"
// #include "src/memllib/interface/InterfaceBase.hpp"
#include "src/memllib/audio/AudioAppBase.hpp"
#include "src/memllib/audio/AudioDriver.hpp"
#include "src/memllib/hardware/memlnaut/MEMLNaut.hpp"
#include <memory>
// Includes for the IML interface
#include "src/memlp/Dataset.hpp"
#include "src/memlp/MLP.h"
// Includes for FM Synth
#include "src/memllib/synth/FMSynth.hpp"
class IMLInterface : public InterfaceBase
{
public:
IMLInterface() : InterfaceBase() {}
void setup(size_t n_inputs, size_t n_outputs) override
{
InterfaceBase::setup(n_inputs, n_outputs);
// Additional setup code specific to IMLInterface
n_inputs_ = n_inputs;
n_outputs_ = n_outputs;
MLSetup_();
n_iterations_ = 1000;
input_state_.resize(n_inputs, 0.5f);
output_state_.resize(n_outputs, 0);
// Init/reset state machine
training_mode_ = INFERENCE_MODE;
perform_inference_ = true;
input_updated_ = false;
Serial.println("IMLInterface setup done");
Serial.print("Address of n_inputs_: ");
Serial.println(reinterpret_cast<uintptr_t>(&n_inputs_));
Serial.print("Inputs: ");
Serial.print(n_inputs_);
Serial.print(", Outputs: ");
Serial.println(n_outputs_);
}
enum training_mode_t {
INFERENCE_MODE,
TRAINING_MODE
};
void SetTrainingMode(training_mode_t training_mode)
{
Serial.print("Training mode: ");
Serial.println(training_mode == INFERENCE_MODE ? "Inference" : "Training");
if (training_mode == INFERENCE_MODE && training_mode_ == TRAINING_MODE) {
// Train the network!
MLTraining_();
}
training_mode_ = training_mode;
}
void ProcessInput()
{
// Check if input is updated
if (perform_inference_ && input_updated_) {
MLInference_(input_state_);
input_updated_ = false;
}
}
void SetInput(size_t index, float value)
{
// Serial.print("Input ");
// Serial.print(index);
// Serial.print(" set to: ");
// Serial.println(value);
if (index >= n_inputs_) {
Serial.print("Input index ");
Serial.print(index);
Serial.println(" out of bounds.");
return;
}
if (value < 0) {
value = 0;
} else if (value > 1.0) {
value = 1.0;
}
// Update state of input
input_state_[index] = value;
input_updated_ = true;
}
enum saving_mode_t {
STORE_VALUE_MODE,
STORE_POSITION_MODE,
};
void SaveInput(saving_mode_t mode)
{
if (STORE_VALUE_MODE == mode) {
Serial.println("Move input to position...");
perform_inference_ = false;
} else { // STORE_POSITION_MODE
Serial.println("Creating example in this position.");
// Save pair in the dataset
dataset_->Add(input_state_, output_state_);
perform_inference_ = true;
MLInference_(input_state_);
}
}
void ClearData()
{
if (training_mode_ == TRAINING_MODE) {
Serial.println("Clearing dataset...");
dataset_->Clear();
}
}
void Randomise()
{
if (training_mode_ == TRAINING_MODE) {
Serial.println("Randomising weights...");
MLRandomise_();
MLInference_(input_state_);
}
}
void SetIterations(size_t iterations)
{
n_iterations_ = iterations;
Serial.print("Iterations set to: ");
Serial.println(n_iterations_);
}
protected:
size_t n_inputs_;
size_t n_outputs_;
size_t n_iterations_;
// State machine
training_mode_t training_mode_;
bool perform_inference_;
bool input_updated_;
// Controls/sensors
std::vector<float> input_state_;
std::vector<float> output_state_;
// MLP core
std::unique_ptr<Dataset> dataset_;
std::unique_ptr<MLP<float>> mlp_;
MLP<float>::mlp_weights mlp_stored_weights_;
bool randomised_state_;
void MLSetup_()
{
// Constants for MLP init
const unsigned int kBias = 1;
const std::vector<ACTIVATION_FUNCTIONS> layers_activfuncs = {
RELU, RELU, RELU, SIGMOID
};
const bool use_constant_weight_init = false;
const float constant_weight_init = 0;
// Layer size definitions
const std::vector<size_t> layers_nodes = {
n_inputs_ + kBias,
10, 10, 14,
n_outputs_
};
// Create dataset
dataset_ = std::make_unique<Dataset>();
// Create MLP
mlp_ = std::make_unique<MLP<float>>(
layers_nodes,
layers_activfuncs,
loss::LOSS_MSE,
use_constant_weight_init,
constant_weight_init
);
// State machine
randomised_state_ = false;
}
void MLInference_(std::vector<float> input)
{
if (!dataset_ || !mlp_) {
Serial.println("ML not initialized!");
return;
}
if (input.size() != n_inputs_) {
Serial.print("Input size mismatch - ");
Serial.print("Expected: ");
Serial.print(n_inputs_);
Serial.print(", Got: ");
Serial.println(input.size());
return;
}
input.push_back(1.0f); // Add bias term
// Perform inference
std::vector<float> output(n_outputs_);
mlp_->GetOutput(input, &output);
// Process inferenced data
output_state_ = output;
SendParamsToQueue(output);
}
void MLRandomise_()
{
if (!mlp_) {
Serial.println("ML not initialized!");
return;
}
// Randomize weights
mlp_stored_weights_ = mlp_->GetWeights();
mlp_->DrawWeights();
randomised_state_ = true;
}
void MLTraining_()
{
if (!mlp_) {
Serial.println("ML not initialized!");
return;
}
// Restore old weights
if (randomised_state_) {
mlp_->SetWeights(mlp_stored_weights_);
}
randomised_state_ = false;
// Prepare for training
// Extract dataset to training pair
MLP<float>::training_pair_t dataset(dataset_->GetFeatures(), dataset_->GetLabels());
// Check and report on dataset size
Serial.print("Feature size ");
Serial.print(dataset.first.size());
Serial.print(", label size ");
Serial.println(dataset.second.size());
if (!dataset.first.size() || !dataset.second.size()) {
Serial.println("Empty dataset!");
return;
}
Serial.print("Feature dim ");
Serial.print(dataset.first[0].size());
Serial.print(", label dim ");
Serial.println(dataset.second[0].size());
if (!dataset.first[0].size() || !dataset.second[0].size()) {
Serial.println("Empty dataset dimensions!");
return;
}
// Training loop
Serial.print("Training for max ");
Serial.print(n_iterations_);
Serial.println(" iterations...");
float loss = mlp_->Train(dataset,
1.,
n_iterations_,
0.00001,
false);
Serial.print("Trained, loss = ");
Serial.println(loss, 10);
}
};
#include "IMLInterface.hpp"
#include "interfaceRL.hpp"
#include "src/memllib/synth/maxiPAF.hpp"
#include "hardware/structs/bus_ctrl.h"
#define APP_SRAM __not_in_flash("app")
bool core1_disable_systick = true;
bool core1_separate_stack = true;
uint32_t get_rosc_entropy_seed(int bits) {
uint32_t seed = 0;
for (int i = 0; i < bits; ++i) {
// Wait for a bit of time to allow jitter to accumulate
busy_wait_us_32(5);
// Pull LSB from ROSC rand output
seed <<= 1;
seed |= (rosc_hw->randombit & 1);
}
return seed;
}
class PAFSynthApp : public AudioAppBase
{
public:
static constexpr size_t kN_Params = 10;
static constexpr size_t kN_Params = 17;
PAFSynthApp() : AudioAppBase() {}
inline stereosample_t Process(const stereosample_t x) override
stereosample_t __force_inline Process(const stereosample_t x) override
{
float x1[1];
paf0.play(x1, 1, paf0_freq, paf0_cf, paf0_bw, paf0_vib, paf0_vfr, 1);
paf0.play(x1, 1, paf0_freq, paf0_cf, paf0_bw, paf0_vib, paf0_vfr, paf0_shift, 0);
float y = x1[0];
// paf1.play(x1, 1, paf1_freq, paf1_cf, paf1_bw, 0, 0, 1);
// y += x1[0];
paf1.play(x1, 1, paf1_freq, paf1_cf, paf1_bw, paf1_vib, paf1_vfr, paf1_shift, 1);
y += x1[0];
// paf2.play(x1, 1, paf1_freq, paf1_cf, paf1_bw, paf1_vib, paf1_vfr, 1);
// y += x1[0];
paf2.play(x1, 1, paf2_freq, paf2_cf, paf2_bw, paf2_vib, paf2_vfr, paf2_shift, 1);
y += x1[0];
// paf2.play(x1, 1);
// y += x1[0];
y = y * 0.3f;
@ -313,33 +60,33 @@ public:
AudioAppBase::Setup(sample_rate, interface);
paf0.init();
paf0.setsr(maxiSettings::getSampleRate(), 1);
paf0.freq(100, 0);
// paf0.amp(1,0);
paf0.bw(200,0);
paf0.cf(210,0);
paf0.vfr(5,0);
paf0.vib(0.1,0);
paf0.shift(10,0);
// paf0.freq(100, 0);
// // paf0.amp(1,0);
// paf0.bw(200,0);
// paf0.cf(210,0);
// paf0.vfr(5,0);
// paf0.vib(0.1,0);
// paf0.shift(10,0);
paf1.init();
paf1.setsr(maxiSettings::getSampleRate(), 1);
paf1.freq(150, 0);
// paf1.amp(1,0);
paf1.bw(200,0);
paf1.cf(210,0);
paf1.vfr(5,0);
paf1.vib(0.1,0);
paf1.shift(10,0);
// paf1.freq(150, 0);
// // paf1.amp(1,0);
// paf1.bw(200,0);
// paf1.cf(210,0);
// paf1.vfr(5,0);
// paf1.vib(0.1,0);
// paf1.shift(10,0);
paf2.init();
paf2.setsr(maxiSettings::getSampleRate(), 1);
paf2.freq(190, 0);
// paf2.amp(1,0);
paf2.bw(500,0);
paf2.cf(210,0);
paf2.vfr(5,0);
paf2.vib(0.1,0);
paf2.shift(6,0);
// paf2.freq(190, 0);
// // paf2.amp(1,0);
// paf2.bw(500,0);
// paf2.cf(210,0);
// paf2.vfr(5,0);
// paf2.vib(0.1,0);
// paf2.shift(6,0);
}
void ProcessParams(const std::vector<float>& params) override
@ -350,19 +97,28 @@ public:
// paf0_freq = 50.f + (params[0] * params[0] * 1000.f);
// paf1_freq = 50.f + (params[1] * params[1] * 1000.f);
paf0_cf = paf0_freq + (params[2] * params[2] * paf0_freq * 16.f);
paf1_cf = 50.f + (params[3] * params[3] * 1000.f);
paf0_cf = paf0_freq + (params[2] * params[2] * paf0_freq * 4.f);
paf1_cf = paf0_freq + (params[3] * params[3] * paf1_freq * 16.f);
paf2_cf = paf0_freq + (params[4] * params[4] * paf2_freq * 16.f);
paf0_bw = 10.f + (params[4] * paf0_freq);
paf1_bw = 50.f + (params[5] * 5000.f);
paf0_bw = 10.f + (params[5] * paf0_freq);
paf1_bw = 10.f + (params[6] * paf1_freq);
paf2_bw = 10.f + (params[7] * paf2_freq);
paf0_vib = (params[6] * params[6] * 0.9f);
paf1_vib = (params[7] * params[7] * 0.9f);
paf0_vib = (params[8] * params[8] * 0.99f);
paf1_vib = (params[9] * params[9] * 0.99f);
paf2_vib = (params[10] * params[10] * 0.99f);
paf0_vfr = (params[8] * params[8]* 15.f);
paf1_vfr = (params[9] * params[9] * 15.f);
paf0_vfr = (params[11] * params[11]* 15.f);
paf1_vfr = (params[12] * params[12] * 15.f);
paf2_vfr = (params[13] * params[13] * 15.f);
Serial.printf("%f %f %f %f\n", paf0_cf, paf0_bw, paf0_vib, paf0_vfr);
paf0_shift = (params[14] * 1000.f);
paf1_shift = (params[15] * 1000.f);
paf2_shift = (params[16] * 1000.f);
// Serial.printf("%f %f %f %f %f\n", paf0_cf, paf0_bw, paf0_vib, paf0_vfr, paf0_shift);
}
protected:
@ -375,36 +131,47 @@ protected:
float paf0_freq = 100;
float paf1_freq = 101;
float paf2_freq = 102;
float paf0_cf = 200;
float paf1_cf = 250;
float paf2_cf = 250;
float paf0_bw = 100;
float paf1_bw = 5000;
float paf2_bw = 5000;
float paf0_vib = 0;
float paf1_vib = 1;
float paf2_vib = 1;
float paf0_vfr = 2;
float paf1_vfr = 2;
float paf2_vfr = 2;
float paf0_shift = 0;
float paf1_shift = 0;
float paf2_shift = 0;
};
// Global objects
std::shared_ptr<IMLInterface> interface;
std::shared_ptr<PAFSynthApp> AUDIO_MEM audio_app;
std::shared_ptr<IMLInterface> APP_SRAM interfaceIML;
std::shared_ptr<interfaceRL> APP_SRAM RLInterface;
std::shared_ptr<PAFSynthApp> __scratch_y("audio") audio_app;
// Inter-core communication
volatile bool core_0_ready = false;
volatile bool core_1_ready = false;
volatile bool serial_ready = false;
volatile bool interface_ready = false;
volatile bool APP_SRAM core_0_ready = false;
volatile bool APP_SRAM core_1_ready = false;
volatile bool APP_SRAM serial_ready = false;
volatile bool APP_SRAM interface_ready = false;
// We're only bound to the joystick inputs (x, y, rotate)
const size_t kN_InputParams = 3;
constexpr size_t kN_InputParams = 3;
// Add these macros near other globals
#define MEMORY_BARRIER() __sync_synchronize()
@ -412,7 +179,50 @@ const size_t kN_InputParams = 3;
#define READ_VOLATILE(var) ({ MEMORY_BARRIER(); typeof(var) __temp = (var); MEMORY_BARRIER(); __temp; })
void bind_interface(std::shared_ptr<IMLInterface> interface)
void bind_RL_interface(std::shared_ptr<interfaceRL> interface)
{
// Set up momentary switch callbacks
MEMLNaut::Instance()->setMomA1Callback([interface] () {
interface->storeExperience(1.f);
Serial.println("Incredible");
});
MEMLNaut::Instance()->setMomA2Callback([interface] () {
interface->storeExperience(-1.f);
Serial.println("That sucks");
});
MEMLNaut::Instance()->setMomB1Callback([interface] () {
interface->randomiseTheActor();
Serial.println("The Actor is confused");
});
MEMLNaut::Instance()->setMomB2Callback([interface] () {
interface->randomiseTheCritic();
Serial.println("The Critic is confounded");
});
// Set up ADC callbacks
MEMLNaut::Instance()->setJoyXCallback([interface] (float value) {
interface->setState(0, value);
});
MEMLNaut::Instance()->setJoyYCallback([interface] (float value) {
interface->setState(1, value);
});
MEMLNaut::Instance()->setJoyZCallback([interface] (float value) {
interface->setState(2, value);
});
MEMLNaut::Instance()->setRVGain1Callback([interface] (float value) {
AudioDriver::setDACVolume(value);
});
// Set up loop callback
MEMLNaut::Instance()->setLoopCallback([interface] () {
interface->optimiseSometimes();
interface->generateAction();
});
}
void bind_IML_interface(std::shared_ptr<IMLInterface> interface)
{
// Set up momentary switch callbacks
MEMLNaut::Instance()->setMomA1Callback([interface] () {
@ -456,9 +266,19 @@ void bind_interface(std::shared_ptr<IMLInterface> interface)
});
}
enum MLMODES {IML, RL};
MLMODES APP_SRAM mlMode = RL;
void setup()
{
bus_ctrl_hw->priority = BUSCTRL_BUS_PRIORITY_DMA_W_BITS |
BUSCTRL_BUS_PRIORITY_DMA_R_BITS | BUSCTRL_BUS_PRIORITY_PROC1_BITS;
uint32_t seed = get_rosc_entropy_seed(32);
srand(seed);
Serial.begin(115200);
while (!Serial) {}
Serial.println("Serial initialised.");
@ -468,19 +288,39 @@ void setup()
MEMLNaut::Initialize();
pinMode(33, OUTPUT);
// Setup interface with memory barrier protection
{
auto temp_interface = std::make_shared<IMLInterface>();
temp_interface->setup(kN_InputParams, PAFSynthApp::kN_Params);
MEMORY_BARRIER();
interface = temp_interface;
MEMORY_BARRIER();
switch(mlMode) {
case IML: {
{
auto temp_interface = std::make_shared<IMLInterface>();
temp_interface->setup(kN_InputParams, PAFSynthApp::kN_Params);
MEMORY_BARRIER();
interfaceIML = temp_interface;
MEMORY_BARRIER();
}
// Setup interface with memory barrier protection
WRITE_VOLATILE(interface_ready, true);
// Bind interface after ensuring it's fully initialized
bind_IML_interface(interfaceIML);
Serial.println("Bound IML interface to MEMLNaut.");
}
break;
case RL: {
{
auto temp_interface = std::make_shared<interfaceRL>();
temp_interface->setup(kN_InputParams, PAFSynthApp::kN_Params);
MEMORY_BARRIER();
RLInterface = temp_interface;
MEMORY_BARRIER();
}
// Setup interface with memory barrier protection
WRITE_VOLATILE(interface_ready, true);
// Bind interface after ensuring it's fully initialized
bind_RL_interface(RLInterface);
Serial.println("Bound RL interface to MEMLNaut.");
}
break;
}
WRITE_VOLATILE(interface_ready, true);
// Bind interface after ensuring it's fully initialized
bind_interface(interface);
Serial.println("Bound interface to MEMLNaut.");
WRITE_VOLATILE(core_0_ready, true);
while (!READ_VOLATILE(core_1_ready)) {
@ -493,6 +333,8 @@ void setup()
void loop()
{
MEMLNaut::Instance()->loop();
static int AUDIO_MEM blip_counter = 0;
if (blip_counter++ > 100) {
@ -504,7 +346,7 @@ void loop()
// Un-blink LED
digitalWrite(33, LOW);
}
delay(20); // Add a small delay to avoid flooding the serial output
delay(10); // Add a small delay to avoid flooding the serial output
}
void setup1()
@ -519,10 +361,20 @@ void setup1()
delay(1);
}
// Create audio app with memory barrier protection
{
auto temp_audio_app = std::make_shared<PAFSynthApp>();
temp_audio_app->Setup(AudioDriver::GetSampleRate(), interface);
std::shared_ptr<InterfaceBase> selectedInterface;
if (mlMode == IML) {
selectedInterface = std::dynamic_pointer_cast<InterfaceBase>(interfaceIML);
} else {
selectedInterface = std::dynamic_pointer_cast<InterfaceBase>(RLInterface);
}
temp_audio_app->Setup(AudioDriver::GetSampleRate(), selectedInterface);
// temp_audio_app->Setup(AudioDriver::GetSampleRate(), dynamic_cast<std::shared_ptr<InterfaceBase>> (mlMode == IML ? interfaceIML : RLInterface));
MEMORY_BARRIER();
audio_app = temp_audio_app;
MEMORY_BARRIER();
@ -544,6 +396,7 @@ void loop1()
{
// Audio app parameter processing loop
audio_app->loop();
delay(10);
}
extern "C" int getentropy (void * buffer, size_t how_many) {

273
interfaceRL.hpp Normal file
View file

@ -0,0 +1,273 @@
#ifndef INTERFACERL_HPP
#define INTERFACERL_HPP
#include "src/memllib/interface/InterfaceBase.hpp"
#include "src/memlp/MLP.h"
#include "src/memlp/ReplayMemory.hpp"
#include "src/memlp/OrnsteinUhlenbeckNoise.h"
#include <memory>
#define RL_MEM __not_in_flash("rlmem")
struct trainRLItem {
std::vector<float> state ;
std::vector<float> action;
float reward;
std::vector<float> nextState;
};
class interfaceRL : public InterfaceBase
{
public:
void setup(size_t n_inputs, size_t n_outputs) override
{
InterfaceBase::setup(n_inputs, n_outputs);
stateSize = n_inputs;
actionSize = n_outputs;
actor_layers_nodes = {
stateSize + bias,
10, 10,
actionSize
};
critic_layers_nodes = {
stateSize + actionSize + bias,
10, 10,
1
};
criticInput.resize(critic_layers_nodes[0]);
actorControlInput.resize(actor_layers_nodes[0]);
actorControlInput[actorControlInput.size()-1] = 1.f; // bias
//init networks
actor = std::make_shared<MLP<float> > (
actor_layers_nodes,
layers_activfuncs,
loss::LOSS_MSE,
use_constant_weight_init,
constant_weight_init
);
actorTarget = std::make_shared<MLP<float> > (
actor_layers_nodes,
layers_activfuncs,
loss::LOSS_MSE,
use_constant_weight_init,
constant_weight_init
);
critic = std::make_shared<MLP<float> > (
critic_layers_nodes,
layers_activfuncs,
loss::LOSS_MSE,
use_constant_weight_init,
constant_weight_init
);
criticTarget = std::make_shared<MLP<float> > (
critic_layers_nodes,
layers_activfuncs,
loss::LOSS_MSE,
use_constant_weight_init,
constant_weight_init
);
}
void optimise() {
constexpr size_t batchSize = 4;
std::vector<trainRLItem> sample = replayMem.sample(batchSize);
if (sample.size() == batchSize) {
//run sample through critic target, build training set for critic net
MLP<float>::training_pair_t ts;
for(size_t i = 0; i < sample.size(); i++) {
//---calculate y
//--calc next-state-action pair
//get next action from actorTarget given next state
auto nextStateInput = sample[i].nextState;
nextStateInput.push_back(1.f); // bias
actorTarget->GetOutput(nextStateInput, &actorOutput);
//use criticTarget to estimate value of next action given next state
for(size_t j=0; j < stateSize; j++) {
criticInput[j] = sample[i].nextState[j];
}
for(size_t j=0; j < actionSize; j++) {
criticInput[j+stateSize] = actorOutput[j];
}
criticInput[criticInput.size()-1] = 1.f; //bias
criticTarget->GetOutput(criticInput, &criticOutput);
//calculate expected reward
const float y = sample[i].reward + (discountFactor * criticOutput[0]);
// std::cout << "[" << i << "]: y: " << y << std::endl;
//use criticTarget to estimate value of next action given next state
for(size_t j=0; j < stateSize; j++) {
criticInput[j] = sample[i].state[j];
}
for(size_t j=0; j < actionSize; j++) {
criticInput[j+stateSize] = sample[i].action[j];
}
criticInput[criticInput.size()-1] = 1.f; //bias
ts.first.push_back(criticInput);
ts.second.push_back({y});
}
//train the critic
float loss = critic->Train(ts, learningRate, 1);
//TODO: size limit to this log
criticLossLog.push_back(loss);
//update the actor
//for each memory in replay memory sample, and get grads from critic
std::vector<float> actorLoss(actionSize, 0.f);
std::vector<float> gradientLoss= {1.f};
for(size_t i = 0; i < sample.size(); i++) {
//use criticTarget to estimate value of next action given next state
for(size_t j=0; j < stateSize; j++) {
criticInput[j] = sample[i].nextState[j];
}
for(size_t j=0; j < actionSize; j++) {
criticInput[j+stateSize] = sample[i].action[j];
}
criticInput[criticInput.size()-1] = 1.f; //bias
critic->CalcGradients(criticInput, gradientLoss);
std::vector<float> l0Grads = critic->m_layers[0].GetGrads();
for(size_t j=0; j < actionSize; j++) {
actorLoss[j] = l0Grads[j+stateSize];
}
delay(1);
}
float totalLoss = 0.f;
for(size_t j=0; j < actorLoss.size(); j++) {
actorLoss[j] /= sample.size();
actorLoss[j] = -actorLoss[j];
totalLoss += actorLoss[j];
}
// actorLossLog.push_back(actorLoss);
// actorLoss = -actorLoss;
// Serial.printf("Actor loss: %f\n", totalLoss);
//back propagate the actor loss
for(size_t i = 0; i < sample.size(); i++) {
auto actorInput = sample[i].state;
actorInput.push_back(bias);
actor->ApplyLoss(actorInput, actorLoss, learningRate);
delay(1);
}
// soft update the target networks
criticTarget->SmoothUpdateWeights(critic, smoothingAlpha);
actorTarget->SmoothUpdateWeights(actor, smoothingAlpha);
}
}
void setState(const size_t index, float value) {
actorControlInput[index] = value;
newInput = true;
}
void generateAction() {
if (newInput) {
newInput = false;
std::vector<float> actorOutput;
actorTarget->GetOutput(actorControlInput, &actorOutput);
SendParamsToQueue(actorOutput);
action = actorOutput;
// for(size_t i=0; i < actorOutput.size(); i++) {
// const float noise = ou_noise.sample() * knobL;
// actorOutput[i] += noise;
// }
}
}
void optimiseSometimes() {
if (optimiseCounter==optimiseDivisor) {
optimise();
optimiseCounter=0;
}else{
optimiseCounter++;
}
}
void storeExperience(float reward) {
std::vector<float> state = actorControlInput;
//remove bias
state.pop_back();
for(size_t i=0; i < state.size(); i++) {
Serial.printf("%f\t", state[i]);
}
Serial.println();
trainRLItem trainItem = {state, action, reward, state};
replayMem.add(trainItem, millis());
}
void randomiseTheActor()
{
actor->DrawWeights();
actorTarget->DrawWeights();
}
void randomiseTheCritic()
{
critic->DrawWeights();
criticTarget->DrawWeights();
}
private:
static constexpr size_t bias=1;
size_t optimiseDivisor = 40;
size_t optimiseCounter = 0;
bool newInput=false;
const std::vector<ACTIVATION_FUNCTIONS> layers_activfuncs = {
RELU, RELU, TANH
};
size_t stateSize;
size_t actionSize;
std::vector<size_t> actor_layers_nodes;
std::vector<size_t> critic_layers_nodes;
const bool use_constant_weight_init = false;
const float constant_weight_init = 0;
std::shared_ptr<MLP<float> > actor, actorTarget, critic, criticTarget;
float discountFactor = 0.95;
float learningRate = 0.005;
float smoothingAlpha = 0.005;
std::vector<float> action;
ReplayMemory<trainRLItem> replayMem;
std::vector<float> actorOutput, criticOutput;
std::vector<float> criticInput;
std::vector<float> actorControlInput;
std::vector<float> criticLossLog, actorLossLog, log1;
};
#endif // INTERFACERL_HPP