Relintai/pmlpp
1
0
Fork 0
mirror of https://github.com/Relintai/pmlpp.git synced 2025-03-12 22:38:51 +01:00
Code Issues Packages Projects Releases Wiki Activity

MLPPTanhReg initial cleanup.

Browse Source
This commit is contained in:
Relintai 2023-02-11 01:18:25 +01:00
parent e8d0b13eed
commit 47155163b1
2 changed files with 259 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

256
mlpp/tanh_reg/tanh_reg.cpp
View File

@ -15,47 +15,91 @@
#include <iostream> #include <iostream>
#include <random> #include <random>
MLPPTanhReg::MLPPTanhReg(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, std::string reg, real_t lambda, real_t alpha) : /*
inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) { Ref<MLPPMatrix> MLPPTanhReg::get_input_set() {
y_hat.resize(n); return _input_set;
weights = MLPPUtilities::weightInitialization(k); }
bias = MLPPUtilities::biasInitialization(); void MLPPTanhReg::set_input_set(const Ref<MLPPMatrix> &val) {
_input_set = val;
_initialized = false;
} }
std::vector<real_t> MLPPTanhReg::modelSetTest(std::vector<std::vector<real_t>> X) { Ref<MLPPMatrix> MLPPTanhReg::get_output_set() {
return Evaluate(X); return _output_set;
}
void MLPPTanhReg::set_output_set(const Ref<MLPPMatrix> &val) {
_output_set = val;
_initialized = false;
} }
real_t MLPPTanhReg::modelTest(std::vector<real_t> x) { MLPPReg::RegularizationType MLPPTanhReg::get_reg() {
return Evaluate(x); return _reg;
}
void MLPPTanhReg::set_reg(const MLPPReg::RegularizationType val) {
_reg = val;
_initialized = false;
} }
void MLPPTanhReg::gradientDescent(real_t learning_rate, int max_epoch, bool UI) { real_t MLPPTanhReg::get_lambda() {
return _lambda;
}
void MLPPTanhReg::set_lambda(const real_t val) {
_lambda = val;
_initialized = false;
}
real_t MLPPTanhReg::get_alpha() {
return _alpha;
}
void MLPPTanhReg::set_alpha(const real_t val) {
_alpha = val;
_initialized = false;
}
*/
std::vector<real_t> MLPPTanhReg::model_set_test(std::vector<std::vector<real_t>> X) {
return evaluatem(X);
}
real_t MLPPTanhReg::model_test(std::vector<real_t> x) {
return evaluatev(x);
}
void MLPPTanhReg::gradient_descent(real_t learning_rate, int max_epoch, bool ui) {
MLPPActivation avn; MLPPActivation avn;
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
real_t cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
forwardPass();
forward_pass();
while (true) { while (true) {
cost_prev = Cost(y_hat, outputSet); cost_prev = cost(_y_hat, _output_set);
std::vector<real_t> error = alg.subtraction(y_hat, outputSet); std::vector<real_t> error = alg.subtraction(_y_hat, _output_set);
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputSet), alg.hadamard_product(error, avn.tanh(z, 1))))); _weights = alg.subtraction(_weights, alg.scalarMultiply(learning_rate / _n, alg.mat_vec_mult(alg.transpose(_input_set), alg.hadamard_product(error, avn.tanh(_z, 1)))));
weights = regularization.regWeights(weights, lambda, alpha, reg); //_reg
_weights = regularization.regWeights(_weights, _lambda, _alpha, "None");
// Calculating the bias gradients // Calculating the bias gradients
bias -= learning_rate * alg.sum_elements(alg.hadamard_product(error, avn.tanh(z, 1))) / n; _bias -= learning_rate * alg.sum_elements(alg.hadamard_product(error, avn.tanh(_z, 1))) / _n;
forwardPass(); forward_pass();
// UI PORTION // UI PORTION
if (UI) { if (ui) {
MLPPUtilities::CostInfo(epoch, cost_prev, Cost(y_hat, outputSet)); MLPPUtilities::CostInfo(epoch, cost_prev, cost(_y_hat, _output_set));
MLPPUtilities::UI(weights, bias); MLPPUtilities::UI(_weights, _bias);
} }
epoch++; epoch++;
if (epoch > max_epoch) { if (epoch > max_epoch) {
@ -64,35 +108,38 @@ void MLPPTanhReg::gradientDescent(real_t learning_rate, int max_epoch, bool UI)
} }
} }
void MLPPTanhReg::SGD(real_t learning_rate, int max_epoch, bool UI) { void MLPPTanhReg::sgd(real_t learning_rate, int max_epoch, bool ui) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
real_t cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
while (true) {
std::random_device rd; std::random_device rd;
std::default_random_engine generator(rd()); std::default_random_engine generator(rd());
std::uniform_int_distribution<int> distribution(0, int(n - 1)); std::uniform_int_distribution<int> distribution(0, int(_n - 1));
while (true) {
int outputIndex = distribution(generator); int outputIndex = distribution(generator);
real_t y_hat = Evaluate(inputSet[outputIndex]); real_t y_hat = evaluatev(_input_set[outputIndex]);
cost_prev = Cost({ y_hat }, { outputSet[outputIndex] }); cost_prev = cost({ _y_hat }, { _output_set[outputIndex] });
real_t error = y_hat - outputSet[outputIndex]; real_t error = y_hat - _output_set[outputIndex];
// Weight Updation // Weight Updation
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate * error * (1 - y_hat * y_hat), inputSet[outputIndex])); _weights = alg.subtraction(_weights, alg.scalarMultiply(learning_rate * error * (1 - y_hat * y_hat), _input_set[outputIndex]));
weights = regularization.regWeights(weights, lambda, alpha, reg); //_reg
_weights = regularization.regWeights(_weights, _lambda, _alpha, "None");
// Bias updation // Bias updation
bias -= learning_rate * error * (1 - y_hat * y_hat); _bias -= learning_rate * error * (1 - y_hat * y_hat);
y_hat = Evaluate({ inputSet[outputIndex] }); y_hat = evaluatev(_input_set[outputIndex]);
if (UI) { if (ui) {
MLPPUtilities::CostInfo(epoch, cost_prev, Cost({ y_hat }, { outputSet[outputIndex] })); MLPPUtilities::CostInfo(epoch, cost_prev, cost({ _y_hat }, { _output_set[outputIndex] }));
MLPPUtilities::UI(weights, bias); MLPPUtilities::UI(_weights, _bias);
} }
epoch++; epoch++;
@ -100,10 +147,11 @@ void MLPPTanhReg::SGD(real_t learning_rate, int max_epoch, bool UI) {
break; break;
} }
} }
forwardPass();
forward_pass();
} }
void MLPPTanhReg::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI) { void MLPPTanhReg::mbgd(real_t learning_rate, int max_epoch, int mini_batch_size, bool ui) {
MLPPActivation avn; MLPPActivation avn;
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
@ -112,85 +160,163 @@ void MLPPTanhReg::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size,
int epoch = 1; int epoch = 1;
// Creating the mini-batches // Creating the mini-batches
int n_mini_batch = n / mini_batch_size; int n_mini_batch = _n / mini_batch_size;
auto batches = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch); auto batches = MLPPUtilities::createMiniBatches(_input_set, _output_set, n_mini_batch);
auto inputMiniBatches = std::get<0>(batches); auto inputMiniBatches = std::get<0>(batches);
auto outputMiniBatches = std::get<1>(batches); auto outputMiniBatches = std::get<1>(batches);
while (true) { while (true) {
for (int i = 0; i < n_mini_batch; i++) { for (int i = 0; i < n_mini_batch; i++) {
std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]); std::vector<real_t> y_hat = evaluatem(inputMiniBatches[i]);
std::vector<real_t> z = propagate(inputMiniBatches[i]); std::vector<real_t> z = propagatem(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]); cost_prev = cost(y_hat, outputMiniBatches[i]);
std::vector<real_t> error = alg.subtraction(y_hat, outputMiniBatches[i]); std::vector<real_t> error = alg.subtraction(y_hat, outputMiniBatches[i]);
// Calculating the weight gradients // Calculating the weight gradients
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), alg.hadamard_product(error, avn.tanh(z, 1))))); _weights = alg.subtraction(_weights, alg.scalarMultiply(learning_rate / _n, alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), alg.hadamard_product(error, avn.tanh(z, 1)))));
weights = regularization.regWeights(weights, lambda, alpha, reg); //_reg
_weights = regularization.regWeights(_weights, _lambda, _alpha, "None");
// Calculating the bias gradients // Calculating the bias gradients
bias -= learning_rate * alg.sum_elements(alg.hadamard_product(error, avn.tanh(z, 1))) / n; _bias -= learning_rate * alg.sum_elements(alg.hadamard_product(error, avn.tanh(_z, true))) / _n;
forwardPass(); forward_pass();
y_hat = Evaluate(inputMiniBatches[i]); y_hat = evaluatem(inputMiniBatches[i]);
if (UI) { if (ui) {
MLPPUtilities::CostInfo(epoch, cost_prev, Cost(y_hat, outputMiniBatches[i])); MLPPUtilities::CostInfo(epoch, cost_prev, cost(y_hat, outputMiniBatches[i]));
MLPPUtilities::UI(weights, bias); MLPPUtilities::UI(_weights, _bias);
} }
} }
epoch++; epoch++;
if (epoch > max_epoch) { if (epoch > max_epoch) {
break; break;
} }
} }
forwardPass();
forward_pass();
} }
real_t MLPPTanhReg::score() { real_t MLPPTanhReg::score() {
MLPPUtilities util; MLPPUtilities util;
return util.performance(y_hat, outputSet);
return util.performance(_y_hat, _output_set);
} }
void MLPPTanhReg::save(std::string fileName) { void MLPPTanhReg::save(std::string file_name) {
MLPPUtilities util; MLPPUtilities util;
util.saveParameters(fileName, weights, bias);
util.saveParameters(file_name, _weights, _bias);
} }
real_t MLPPTanhReg::Cost(std::vector<real_t> y_hat, std::vector<real_t> y) { bool MLPPTanhReg::is_initialized() {
return _initialized;
}
void MLPPTanhReg::initialize() {
if (_initialized) {
return;
}
//ERR_FAIL_COND(!_input_set.is_valid() || !_output_set.is_valid());
_initialized = true;
}
MLPPTanhReg::MLPPTanhReg(std::vector<std::vector<real_t>> p_input_set, std::vector<real_t> p_output_set, MLPPReg::RegularizationType p_reg, real_t p_lambda, real_t p_alpha) {
_input_set = p_input_set;
_output_set = p_output_set;
_n = _input_set.size();
_k = _input_set[0].size();
_reg = p_reg;
_lambda = p_lambda;
_alpha = p_alpha;
_y_hat.resize(_n);
_weights = MLPPUtilities::weightInitialization(_k);
_bias = MLPPUtilities::biasInitialization();
}
MLPPTanhReg::MLPPTanhReg() {
}
MLPPTanhReg::~MLPPTanhReg() {
}
real_t MLPPTanhReg::cost(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPReg regularization; MLPPReg regularization;
class MLPPCost cost; class MLPPCost cost;
return cost.MSE(y_hat, y) + regularization.regTerm(weights, lambda, alpha, reg);
//_reg
return cost.MSE(y_hat, y) + regularization.regTerm(_weights, _lambda, _alpha, "None");
} }
std::vector<real_t> MLPPTanhReg::Evaluate(std::vector<std::vector<real_t>> X) { real_t MLPPTanhReg::evaluatev(std::vector<real_t> x) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPActivation avn; MLPPActivation avn;
return avn.tanh(alg.scalarAdd(bias, alg.mat_vec_mult(X, weights))); return avn.tanh(alg.dot(_weights, x) + _bias);
} }
std::vector<real_t> MLPPTanhReg::propagate(std::vector<std::vector<real_t>> X) { real_t MLPPTanhReg::propagatev(std::vector<real_t> x) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.scalarAdd(bias, alg.mat_vec_mult(X, weights)); return alg.dot(_weights, x) + _bias;
} }
real_t MLPPTanhReg::Evaluate(std::vector<real_t> x) { std::vector<real_t> MLPPTanhReg::evaluatem(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPActivation avn; MLPPActivation avn;
return avn.tanh(alg.dot(weights, x) + bias); return avn.tanh(alg.scalarAdd(_bias, alg.mat_vec_mult(X, _weights)));
} }
real_t MLPPTanhReg::propagate(std::vector<real_t> x) { std::vector<real_t> MLPPTanhReg::propagatem(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.dot(weights, x) + bias; return alg.scalarAdd(_bias, alg.mat_vec_mult(X, _weights));
} }
// Tanh ( wTx + b ) // Tanh ( wTx + b )
void MLPPTanhReg::forwardPass() { void MLPPTanhReg::forward_pass() {
MLPPActivation avn; MLPPActivation avn;
z = propagate(inputSet); _z = propagatem(_input_set);
y_hat = avn.tanh(z); _y_hat = avn.tanh(_z);
}
void MLPPTanhReg::_bind_methods() {
/*
ClassDB::bind_method(D_METHOD("get_input_set"), &MLPPTanhReg::get_input_set);
ClassDB::bind_method(D_METHOD("set_input_set", "val"), &MLPPTanhReg::set_input_set);
ADD_PROPERTY(PropertyInfo(Variant::OBJECT, "input_set", PROPERTY_HINT_RESOURCE_TYPE, "MLPPMatrix"), "set_input_set", "get_input_set");
ClassDB::bind_method(D_METHOD("get_output_set"), &MLPPTanhReg::get_output_set);
ClassDB::bind_method(D_METHOD("set_output_set", "val"), &MLPPTanhReg::set_output_set);
ADD_PROPERTY(PropertyInfo(Variant::OBJECT, "output_set", PROPERTY_HINT_RESOURCE_TYPE, "MLPPMatrix"), "set_output_set", "get_output_set");
ClassDB::bind_method(D_METHOD("get_reg"), &MLPPTanhReg::get_reg);
ClassDB::bind_method(D_METHOD("set_reg", "val"), &MLPPTanhReg::set_reg);
ADD_PROPERTY(PropertyInfo(Variant::INT, "reg"), "set_reg", "get_reg");
ClassDB::bind_method(D_METHOD("get_lambda"), &MLPPTanhReg::get_lambda);
ClassDB::bind_method(D_METHOD("set_lambda", "val"), &MLPPTanhReg::set_lambda);
ADD_PROPERTY(PropertyInfo(Variant::REAL, "lambda"), "set_lambda", "get_lambda");
ClassDB::bind_method(D_METHOD("get_alpha"), &MLPPTanhReg::get_alpha);
ClassDB::bind_method(D_METHOD("set_alpha", "val"), &MLPPTanhReg::set_alpha);
ADD_PROPERTY(PropertyInfo(Variant::REAL, "alpha"), "set_alpha", "get_alpha");
ClassDB::bind_method(D_METHOD("model_test", "x"), &MLPPTanhReg::model_test);
ClassDB::bind_method(D_METHOD("model_set_test", "X"), &MLPPTanhReg::model_set_test);
ClassDB::bind_method(D_METHOD("gradient_descent", "learning_rate", "max_epoch", "ui"), &MLPPTanhReg::gradient_descent, false);
ClassDB::bind_method(D_METHOD("sgd", "learning_rate", "max_epoch", "ui"), &MLPPTanhReg::sgd, false);
ClassDB::bind_method(D_METHOD("mbgd", "learning_rate", "max_epoch", "mini_batch_size", "ui"), &MLPPTanhReg::mbgd, false);
ClassDB::bind_method(D_METHOD("score"), &MLPPTanhReg::score);
ClassDB::bind_method(D_METHOD("save", "file_name"), &MLPPTanhReg::save);
ClassDB::bind_method(D_METHOD("is_initialized"), &MLPPTanhReg::is_initialized);
ClassDB::bind_method(D_METHOD("initialize"), &MLPPTanhReg::initialize);
*/
} }

95
mlpp/tanh_reg/tanh_reg.h
View File

@ -10,46 +10,85 @@
#include "core/math/math_defs.h" #include "core/math/math_defs.h"
#include "core/object/reference.h"
#include "../lin_alg/mlpp_matrix.h"
#include "../lin_alg/mlpp_vector.h"
#include "../regularization/reg.h"
#include <string> #include <string>
#include <vector> #include <vector>
class MLPPTanhReg { class MLPPTanhReg : public Reference {
GDCLASS(MLPPTanhReg, Reference);
public: public:
MLPPTanhReg(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5); /*
std::vector<real_t> modelSetTest(std::vector<std::vector<real_t>> X); Ref<MLPPMatrix> get_input_set();
real_t modelTest(std::vector<real_t> x); void set_input_set(const Ref<MLPPMatrix> &val);
void gradientDescent(real_t learning_rate, int max_epoch, bool UI = false);
void SGD(real_t learning_rate, int max_epoch, bool UI = false); Ref<MLPPMatrix> get_output_set();
void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = false); void set_output_set(const Ref<MLPPMatrix> &val);
MLPPReg::RegularizationType get_reg();
void set_reg(const MLPPReg::RegularizationType val);
real_t get_lambda();
void set_lambda(const real_t val);
real_t get_alpha();
void set_alpha(const real_t val);
*/
std::vector<real_t> model_set_test(std::vector<std::vector<real_t>> X);
real_t model_test(std::vector<real_t> x);
void gradient_descent(real_t learning_rate, int max_epoch, bool ui = false);
void sgd(real_t learning_rate, int max_epoch, bool ui = false);
void mbgd(real_t learning_rate, int max_epoch, int mini_batch_size, bool ui = false);
real_t score(); real_t score();
void save(std::string fileName);
private: void save(std::string file_name);
real_t Cost(std::vector<real_t> y_hat, std::vector<real_t> y);
std::vector<real_t> Evaluate(std::vector<std::vector<real_t>> X); bool is_initialized();
std::vector<real_t> propagate(std::vector<std::vector<real_t>> X); void initialize();
real_t Evaluate(std::vector<real_t> x);
real_t propagate(std::vector<real_t> x);
void forwardPass();
std::vector<std::vector<real_t>> inputSet; MLPPTanhReg(std::vector<std::vector<real_t>> p_input_set, std::vector<real_t> p_output_set, MLPPReg::RegularizationType p_reg = MLPPReg::REGULARIZATION_TYPE_NONE, real_t p_lambda = 0.5, real_t p_alpha = 0.5);
std::vector<real_t> outputSet;
std::vector<real_t> z;
std::vector<real_t> y_hat;
std::vector<real_t> weights;
real_t bias;
int n; MLPPTanhReg();
int k; ~MLPPTanhReg();
// UI Portion protected:
void UI(int epoch, real_t cost_prev); real_t cost(std::vector<real_t> y_hat, std::vector<real_t> y);
real_t evaluatev(std::vector<real_t> x);
real_t propagatev(std::vector<real_t> x);
std::vector<real_t> evaluatem(std::vector<std::vector<real_t>> X);
std::vector<real_t> propagatem(std::vector<std::vector<real_t>> X);
void forward_pass();
static void _bind_methods();
std::vector<std::vector<real_t>> _input_set;
std::vector<real_t> _output_set;
std::vector<real_t> _z;
std::vector<real_t> _y_hat;
std::vector<real_t> _weights;
real_t _bias;
int _n;
int _k;
// Regularization Params // Regularization Params
std::string reg; MLPPReg::RegularizationType _reg;
real_t lambda; real_t _lambda;
real_t alpha; /* This is the controlling param for Elastic Net*/ real_t _alpha; /* This is the controlling param for Elastic Net*/
bool _initialized;
}; };
#endif /* TanhReg_hpp */ #endif /* TanhReg_hpp */