From 47155163b1de32a2a1b0a9a03d9ec69e662c320c Mon Sep 17 00:00:00 2001 From: Relintai Date: Sat, 11 Feb 2023 01:18:25 +0100 Subject: [PATCH] MLPPTanhReg initial cleanup. --- mlpp/tanh_reg/tanh_reg.cpp | 258 +++++++++++++++++++++++++++---------- mlpp/tanh_reg/tanh_reg.h | 95 ++++++++++---- 2 files changed, 259 insertions(+), 94 deletions(-) diff --git a/mlpp/tanh_reg/tanh_reg.cpp b/mlpp/tanh_reg/tanh_reg.cpp index 87e7bb8..290b97e 100644 --- a/mlpp/tanh_reg/tanh_reg.cpp +++ b/mlpp/tanh_reg/tanh_reg.cpp @@ -15,47 +15,91 @@ #include #include -MLPPTanhReg::MLPPTanhReg(std::vector> inputSet, std::vector outputSet, std::string reg, real_t lambda, real_t alpha) : - inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) { - y_hat.resize(n); - weights = MLPPUtilities::weightInitialization(k); - bias = MLPPUtilities::biasInitialization(); +/* +Ref MLPPTanhReg::get_input_set() { + return _input_set; +} +void MLPPTanhReg::set_input_set(const Ref &val) { + _input_set = val; + + _initialized = false; } -std::vector MLPPTanhReg::modelSetTest(std::vector> X) { - return Evaluate(X); +Ref MLPPTanhReg::get_output_set() { + return _output_set; +} +void MLPPTanhReg::set_output_set(const Ref &val) { + _output_set = val; + + _initialized = false; } -real_t MLPPTanhReg::modelTest(std::vector x) { - return Evaluate(x); +MLPPReg::RegularizationType MLPPTanhReg::get_reg() { + return _reg; +} +void MLPPTanhReg::set_reg(const MLPPReg::RegularizationType val) { + _reg = val; + + _initialized = false; } -void MLPPTanhReg::gradientDescent(real_t learning_rate, int max_epoch, bool UI) { +real_t MLPPTanhReg::get_lambda() { + return _lambda; +} +void MLPPTanhReg::set_lambda(const real_t val) { + _lambda = val; + + _initialized = false; +} + +real_t MLPPTanhReg::get_alpha() { + return _alpha; +} +void MLPPTanhReg::set_alpha(const real_t val) { + _alpha = val; + + _initialized = false; +} +*/ + +std::vector MLPPTanhReg::model_set_test(std::vector> X) { + return evaluatem(X); +} + +real_t MLPPTanhReg::model_test(std::vector x) { + return evaluatev(x); +} + +void MLPPTanhReg::gradient_descent(real_t learning_rate, int max_epoch, bool ui) { MLPPActivation avn; MLPPLinAlg alg; MLPPReg regularization; + real_t cost_prev = 0; int epoch = 1; - forwardPass(); + + forward_pass(); while (true) { - cost_prev = Cost(y_hat, outputSet); + cost_prev = cost(_y_hat, _output_set); - std::vector error = alg.subtraction(y_hat, outputSet); + std::vector error = alg.subtraction(_y_hat, _output_set); - weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputSet), alg.hadamard_product(error, avn.tanh(z, 1))))); - weights = regularization.regWeights(weights, lambda, alpha, reg); + _weights = alg.subtraction(_weights, alg.scalarMultiply(learning_rate / _n, alg.mat_vec_mult(alg.transpose(_input_set), alg.hadamard_product(error, avn.tanh(_z, 1))))); + //_reg + _weights = regularization.regWeights(_weights, _lambda, _alpha, "None"); // Calculating the bias gradients - bias -= learning_rate * alg.sum_elements(alg.hadamard_product(error, avn.tanh(z, 1))) / n; + _bias -= learning_rate * alg.sum_elements(alg.hadamard_product(error, avn.tanh(_z, 1))) / _n; - forwardPass(); + forward_pass(); // UI PORTION - if (UI) { - MLPPUtilities::CostInfo(epoch, cost_prev, Cost(y_hat, outputSet)); - MLPPUtilities::UI(weights, bias); + if (ui) { + MLPPUtilities::CostInfo(epoch, cost_prev, cost(_y_hat, _output_set)); + MLPPUtilities::UI(_weights, _bias); } + epoch++; if (epoch > max_epoch) { @@ -64,35 +108,38 @@ void MLPPTanhReg::gradientDescent(real_t learning_rate, int max_epoch, bool UI) } } -void MLPPTanhReg::SGD(real_t learning_rate, int max_epoch, bool UI) { +void MLPPTanhReg::sgd(real_t learning_rate, int max_epoch, bool ui) { MLPPLinAlg alg; MLPPReg regularization; + real_t cost_prev = 0; int epoch = 1; + std::random_device rd; + std::default_random_engine generator(rd()); + std::uniform_int_distribution distribution(0, int(_n - 1)); + while (true) { - std::random_device rd; - std::default_random_engine generator(rd()); - std::uniform_int_distribution distribution(0, int(n - 1)); int outputIndex = distribution(generator); - real_t y_hat = Evaluate(inputSet[outputIndex]); - cost_prev = Cost({ y_hat }, { outputSet[outputIndex] }); + real_t y_hat = evaluatev(_input_set[outputIndex]); + cost_prev = cost({ _y_hat }, { _output_set[outputIndex] }); - real_t error = y_hat - outputSet[outputIndex]; + real_t error = y_hat - _output_set[outputIndex]; // Weight Updation - weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate * error * (1 - y_hat * y_hat), inputSet[outputIndex])); - weights = regularization.regWeights(weights, lambda, alpha, reg); + _weights = alg.subtraction(_weights, alg.scalarMultiply(learning_rate * error * (1 - y_hat * y_hat), _input_set[outputIndex])); + //_reg + _weights = regularization.regWeights(_weights, _lambda, _alpha, "None"); // Bias updation - bias -= learning_rate * error * (1 - y_hat * y_hat); + _bias -= learning_rate * error * (1 - y_hat * y_hat); - y_hat = Evaluate({ inputSet[outputIndex] }); + y_hat = evaluatev(_input_set[outputIndex]); - if (UI) { - MLPPUtilities::CostInfo(epoch, cost_prev, Cost({ y_hat }, { outputSet[outputIndex] })); - MLPPUtilities::UI(weights, bias); + if (ui) { + MLPPUtilities::CostInfo(epoch, cost_prev, cost({ _y_hat }, { _output_set[outputIndex] })); + MLPPUtilities::UI(_weights, _bias); } epoch++; @@ -100,10 +147,11 @@ void MLPPTanhReg::SGD(real_t learning_rate, int max_epoch, bool UI) { break; } } - forwardPass(); + + forward_pass(); } -void MLPPTanhReg::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI) { +void MLPPTanhReg::mbgd(real_t learning_rate, int max_epoch, int mini_batch_size, bool ui) { MLPPActivation avn; MLPPLinAlg alg; MLPPReg regularization; @@ -112,85 +160,163 @@ void MLPPTanhReg::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, int epoch = 1; // Creating the mini-batches - int n_mini_batch = n / mini_batch_size; - auto batches = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch); + int n_mini_batch = _n / mini_batch_size; + auto batches = MLPPUtilities::createMiniBatches(_input_set, _output_set, n_mini_batch); auto inputMiniBatches = std::get<0>(batches); auto outputMiniBatches = std::get<1>(batches); while (true) { for (int i = 0; i < n_mini_batch; i++) { - std::vector y_hat = Evaluate(inputMiniBatches[i]); - std::vector z = propagate(inputMiniBatches[i]); - cost_prev = Cost(y_hat, outputMiniBatches[i]); + std::vector y_hat = evaluatem(inputMiniBatches[i]); + std::vector z = propagatem(inputMiniBatches[i]); + cost_prev = cost(y_hat, outputMiniBatches[i]); std::vector error = alg.subtraction(y_hat, outputMiniBatches[i]); // Calculating the weight gradients - weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), alg.hadamard_product(error, avn.tanh(z, 1))))); - weights = regularization.regWeights(weights, lambda, alpha, reg); + _weights = alg.subtraction(_weights, alg.scalarMultiply(learning_rate / _n, alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), alg.hadamard_product(error, avn.tanh(z, 1))))); + //_reg + _weights = regularization.regWeights(_weights, _lambda, _alpha, "None"); // Calculating the bias gradients - bias -= learning_rate * alg.sum_elements(alg.hadamard_product(error, avn.tanh(z, 1))) / n; + _bias -= learning_rate * alg.sum_elements(alg.hadamard_product(error, avn.tanh(_z, true))) / _n; - forwardPass(); + forward_pass(); - y_hat = Evaluate(inputMiniBatches[i]); + y_hat = evaluatem(inputMiniBatches[i]); - if (UI) { - MLPPUtilities::CostInfo(epoch, cost_prev, Cost(y_hat, outputMiniBatches[i])); - MLPPUtilities::UI(weights, bias); + if (ui) { + MLPPUtilities::CostInfo(epoch, cost_prev, cost(y_hat, outputMiniBatches[i])); + MLPPUtilities::UI(_weights, _bias); } } + epoch++; + if (epoch > max_epoch) { break; } } - forwardPass(); + + forward_pass(); } real_t MLPPTanhReg::score() { MLPPUtilities util; - return util.performance(y_hat, outputSet); + + return util.performance(_y_hat, _output_set); } -void MLPPTanhReg::save(std::string fileName) { +void MLPPTanhReg::save(std::string file_name) { MLPPUtilities util; - util.saveParameters(fileName, weights, bias); + + util.saveParameters(file_name, _weights, _bias); } -real_t MLPPTanhReg::Cost(std::vector y_hat, std::vector y) { +bool MLPPTanhReg::is_initialized() { + return _initialized; +} +void MLPPTanhReg::initialize() { + if (_initialized) { + return; + } + + //ERR_FAIL_COND(!_input_set.is_valid() || !_output_set.is_valid()); + + _initialized = true; +} + +MLPPTanhReg::MLPPTanhReg(std::vector> p_input_set, std::vector p_output_set, MLPPReg::RegularizationType p_reg, real_t p_lambda, real_t p_alpha) { + _input_set = p_input_set; + _output_set = p_output_set; + _n = _input_set.size(); + _k = _input_set[0].size(); + _reg = p_reg; + _lambda = p_lambda; + _alpha = p_alpha; + + _y_hat.resize(_n); + _weights = MLPPUtilities::weightInitialization(_k); + _bias = MLPPUtilities::biasInitialization(); +} + +MLPPTanhReg::MLPPTanhReg() { +} +MLPPTanhReg::~MLPPTanhReg() { +} + +real_t MLPPTanhReg::cost(std::vector y_hat, std::vector y) { MLPPReg regularization; class MLPPCost cost; - return cost.MSE(y_hat, y) + regularization.regTerm(weights, lambda, alpha, reg); + + //_reg + return cost.MSE(y_hat, y) + regularization.regTerm(_weights, _lambda, _alpha, "None"); } -std::vector MLPPTanhReg::Evaluate(std::vector> X) { +real_t MLPPTanhReg::evaluatev(std::vector x) { MLPPLinAlg alg; MLPPActivation avn; - return avn.tanh(alg.scalarAdd(bias, alg.mat_vec_mult(X, weights))); + return avn.tanh(alg.dot(_weights, x) + _bias); } -std::vector MLPPTanhReg::propagate(std::vector> X) { +real_t MLPPTanhReg::propagatev(std::vector x) { MLPPLinAlg alg; - return alg.scalarAdd(bias, alg.mat_vec_mult(X, weights)); + return alg.dot(_weights, x) + _bias; } -real_t MLPPTanhReg::Evaluate(std::vector x) { +std::vector MLPPTanhReg::evaluatem(std::vector> X) { MLPPLinAlg alg; MLPPActivation avn; - return avn.tanh(alg.dot(weights, x) + bias); + return avn.tanh(alg.scalarAdd(_bias, alg.mat_vec_mult(X, _weights))); } -real_t MLPPTanhReg::propagate(std::vector x) { +std::vector MLPPTanhReg::propagatem(std::vector> X) { MLPPLinAlg alg; - return alg.dot(weights, x) + bias; + return alg.scalarAdd(_bias, alg.mat_vec_mult(X, _weights)); } // Tanh ( wTx + b ) -void MLPPTanhReg::forwardPass() { +void MLPPTanhReg::forward_pass() { MLPPActivation avn; - z = propagate(inputSet); - y_hat = avn.tanh(z); + _z = propagatem(_input_set); + _y_hat = avn.tanh(_z); +} + +void MLPPTanhReg::_bind_methods() { + /* + ClassDB::bind_method(D_METHOD("get_input_set"), &MLPPTanhReg::get_input_set); + ClassDB::bind_method(D_METHOD("set_input_set", "val"), &MLPPTanhReg::set_input_set); + ADD_PROPERTY(PropertyInfo(Variant::OBJECT, "input_set", PROPERTY_HINT_RESOURCE_TYPE, "MLPPMatrix"), "set_input_set", "get_input_set"); + + ClassDB::bind_method(D_METHOD("get_output_set"), &MLPPTanhReg::get_output_set); + ClassDB::bind_method(D_METHOD("set_output_set", "val"), &MLPPTanhReg::set_output_set); + ADD_PROPERTY(PropertyInfo(Variant::OBJECT, "output_set", PROPERTY_HINT_RESOURCE_TYPE, "MLPPMatrix"), "set_output_set", "get_output_set"); + + ClassDB::bind_method(D_METHOD("get_reg"), &MLPPTanhReg::get_reg); + ClassDB::bind_method(D_METHOD("set_reg", "val"), &MLPPTanhReg::set_reg); + ADD_PROPERTY(PropertyInfo(Variant::INT, "reg"), "set_reg", "get_reg"); + + ClassDB::bind_method(D_METHOD("get_lambda"), &MLPPTanhReg::get_lambda); + ClassDB::bind_method(D_METHOD("set_lambda", "val"), &MLPPTanhReg::set_lambda); + ADD_PROPERTY(PropertyInfo(Variant::REAL, "lambda"), "set_lambda", "get_lambda"); + + ClassDB::bind_method(D_METHOD("get_alpha"), &MLPPTanhReg::get_alpha); + ClassDB::bind_method(D_METHOD("set_alpha", "val"), &MLPPTanhReg::set_alpha); + ADD_PROPERTY(PropertyInfo(Variant::REAL, "alpha"), "set_alpha", "get_alpha"); + + ClassDB::bind_method(D_METHOD("model_test", "x"), &MLPPTanhReg::model_test); + ClassDB::bind_method(D_METHOD("model_set_test", "X"), &MLPPTanhReg::model_set_test); + + ClassDB::bind_method(D_METHOD("gradient_descent", "learning_rate", "max_epoch", "ui"), &MLPPTanhReg::gradient_descent, false); + ClassDB::bind_method(D_METHOD("sgd", "learning_rate", "max_epoch", "ui"), &MLPPTanhReg::sgd, false); + ClassDB::bind_method(D_METHOD("mbgd", "learning_rate", "max_epoch", "mini_batch_size", "ui"), &MLPPTanhReg::mbgd, false); + + ClassDB::bind_method(D_METHOD("score"), &MLPPTanhReg::score); + + ClassDB::bind_method(D_METHOD("save", "file_name"), &MLPPTanhReg::save); + + ClassDB::bind_method(D_METHOD("is_initialized"), &MLPPTanhReg::is_initialized); + ClassDB::bind_method(D_METHOD("initialize"), &MLPPTanhReg::initialize); + */ } diff --git a/mlpp/tanh_reg/tanh_reg.h b/mlpp/tanh_reg/tanh_reg.h index c23528b..11d1fce 100644 --- a/mlpp/tanh_reg/tanh_reg.h +++ b/mlpp/tanh_reg/tanh_reg.h @@ -10,46 +10,85 @@ #include "core/math/math_defs.h" +#include "core/object/reference.h" + +#include "../lin_alg/mlpp_matrix.h" +#include "../lin_alg/mlpp_vector.h" + +#include "../regularization/reg.h" + #include #include -class MLPPTanhReg { +class MLPPTanhReg : public Reference { + GDCLASS(MLPPTanhReg, Reference); + public: - MLPPTanhReg(std::vector> inputSet, std::vector outputSet, std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5); - std::vector modelSetTest(std::vector> X); - real_t modelTest(std::vector x); - void gradientDescent(real_t learning_rate, int max_epoch, bool UI = false); - void SGD(real_t learning_rate, int max_epoch, bool UI = false); - void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = false); + /* + Ref get_input_set(); + void set_input_set(const Ref &val); + + Ref get_output_set(); + void set_output_set(const Ref &val); + + MLPPReg::RegularizationType get_reg(); + void set_reg(const MLPPReg::RegularizationType val); + + real_t get_lambda(); + void set_lambda(const real_t val); + + real_t get_alpha(); + void set_alpha(const real_t val); + */ + + std::vector model_set_test(std::vector> X); + real_t model_test(std::vector x); + + void gradient_descent(real_t learning_rate, int max_epoch, bool ui = false); + void sgd(real_t learning_rate, int max_epoch, bool ui = false); + void mbgd(real_t learning_rate, int max_epoch, int mini_batch_size, bool ui = false); + real_t score(); - void save(std::string fileName); -private: - real_t Cost(std::vector y_hat, std::vector y); + void save(std::string file_name); - std::vector Evaluate(std::vector> X); - std::vector propagate(std::vector> X); - real_t Evaluate(std::vector x); - real_t propagate(std::vector x); - void forwardPass(); + bool is_initialized(); + void initialize(); - std::vector> inputSet; - std::vector outputSet; - std::vector z; - std::vector y_hat; - std::vector weights; - real_t bias; + MLPPTanhReg(std::vector> p_input_set, std::vector p_output_set, MLPPReg::RegularizationType p_reg = MLPPReg::REGULARIZATION_TYPE_NONE, real_t p_lambda = 0.5, real_t p_alpha = 0.5); - int n; - int k; + MLPPTanhReg(); + ~MLPPTanhReg(); - // UI Portion - void UI(int epoch, real_t cost_prev); +protected: + real_t cost(std::vector y_hat, std::vector y); + + real_t evaluatev(std::vector x); + real_t propagatev(std::vector x); + + std::vector evaluatem(std::vector> X); + std::vector propagatem(std::vector> X); + + void forward_pass(); + + static void _bind_methods(); + + std::vector> _input_set; + std::vector _output_set; + std::vector _z; + std::vector _y_hat; + std::vector _weights; + real_t _bias; + + int _n; + int _k; // Regularization Params - std::string reg; - real_t lambda; - real_t alpha; /* This is the controlling param for Elastic Net*/ + MLPPReg::RegularizationType _reg; + real_t _lambda; + real_t _alpha; /* This is the controlling param for Elastic Net*/ + + bool _initialized; }; #endif /* TanhReg_hpp */