From 1bb0cab99aaec09f36bb7b535615cab391c837fd Mon Sep 17 00:00:00 2001 From: Relintai Date: Sat, 11 Feb 2023 09:17:02 +0100 Subject: [PATCH] MLPPSoftmaxNet initial cleanup. --- mlpp/data/data.cpp | 14 +- mlpp/softmax_net/softmax_net.cpp | 367 ++++++++++++++++++++----------- mlpp/softmax_net/softmax_net.h | 106 ++++++--- test/mlpp_tests.cpp | 27 ++- 4 files changed, 349 insertions(+), 165 deletions(-) diff --git a/mlpp/data/data.cpp b/mlpp/data/data.cpp index 41d4c32..030c871 100644 --- a/mlpp/data/data.cpp +++ b/mlpp/data/data.cpp @@ -10,7 +10,7 @@ #include "core/os/file_access.h" #include "../lin_alg/lin_alg.h" -#include "../softmax_net/softmax_net.h" +#include "../softmax_net/softmax_net_old.h" #include "../stat/stat.h" #include @@ -1008,11 +1008,11 @@ std::tuple>, std::vector> MLPPData: outputSet.push_back(BOW[i]); } MLPPLinAlg alg; - MLPPSoftmaxNet *model; + MLPPSoftmaxNetOld *model; if (type == "Skipgram") { - model = new MLPPSoftmaxNet(outputSet, inputSet, dimension); + model = new MLPPSoftmaxNetOld(outputSet, inputSet, dimension); } else { // else = CBOW. We maintain it is a default. - model = new MLPPSoftmaxNet(inputSet, outputSet, dimension); + model = new MLPPSoftmaxNetOld(inputSet, outputSet, dimension); } model->gradientDescent(learning_rate, max_epoch, 1); @@ -1074,11 +1074,11 @@ MLPPData::WordsToVecResult MLPPData::word_to_vec(std::vector senten outputSet.push_back(BOW[i]); } MLPPLinAlg alg; - MLPPSoftmaxNet *model; + MLPPSoftmaxNetOld *model; if (type == "Skipgram") { - model = new MLPPSoftmaxNet(outputSet, inputSet, dimension); + model = new MLPPSoftmaxNetOld(outputSet, inputSet, dimension); } else { // else = CBOW. We maintain it is a default. - model = new MLPPSoftmaxNet(inputSet, outputSet, dimension); + model = new MLPPSoftmaxNetOld(inputSet, outputSet, dimension); } model->gradientDescent(learning_rate, max_epoch, false); diff --git a/mlpp/softmax_net/softmax_net.cpp b/mlpp/softmax_net/softmax_net.cpp index 0ee7f95..0993773 100644 --- a/mlpp/softmax_net/softmax_net.cpp +++ b/mlpp/softmax_net/softmax_net.cpp @@ -15,81 +15,114 @@ #include #include -MLPPSoftmaxNet::MLPPSoftmaxNet(std::vector> pinputSet, std::vector> poutputSet, int pn_hidden, std::string preg, real_t plambda, real_t palpha) { - inputSet = pinputSet; - outputSet = poutputSet; - n = pinputSet.size(); - k = pinputSet[0].size(); - n_hidden = pn_hidden; - n_class = poutputSet[0].size(); - reg = preg; - lambda = plambda; - alpha = palpha; +/* +Ref MLPPSoftmaxNet::get_input_set() { + return _input_set; +} +void MLPPSoftmaxNet::set_input_set(const Ref &val) { + _input_set = val; - y_hat.resize(n); - - weights1 = MLPPUtilities::weightInitialization(k, n_hidden); - weights2 = MLPPUtilities::weightInitialization(n_hidden, n_class); - bias1 = MLPPUtilities::biasInitialization(n_hidden); - bias2 = MLPPUtilities::biasInitialization(n_class); + _initialized = false; } -std::vector MLPPSoftmaxNet::modelTest(std::vector x) { - return Evaluate(x); +Ref MLPPSoftmaxNet::get_output_set() { + return _output_set; +} +void MLPPSoftmaxNet::set_output_set(const Ref &val) { + _output_set = val; + + _initialized = false; } -std::vector> MLPPSoftmaxNet::modelSetTest(std::vector> X) { - return Evaluate(X); +MLPPReg::RegularizationType MLPPSoftmaxNet::get_reg() { + return _reg; +} +void MLPPSoftmaxNet::set_reg(const MLPPReg::RegularizationType val) { + _reg = val; + + _initialized = false; } -void MLPPSoftmaxNet::gradientDescent(real_t learning_rate, int max_epoch, bool UI) { +real_t MLPPSoftmaxNet::get_lambda() { + return _lambda; +} +void MLPPSoftmaxNet::set_lambda(const real_t val) { + _lambda = val; + + _initialized = false; +} + +real_t MLPPSoftmaxNet::get_alpha() { + return _alpha; +} +void MLPPSoftmaxNet::set_alpha(const real_t val) { + _alpha = val; + + _initialized = false; +} +*/ + +std::vector MLPPSoftmaxNet::model_test(std::vector x) { + return evaluatev(x); +} + +std::vector> MLPPSoftmaxNet::model_set_test(std::vector> X) { + return evaluatem(X); +} + +void MLPPSoftmaxNet::gradient_descent(real_t learning_rate, int max_epoch, bool ui) { MLPPActivation avn; MLPPLinAlg alg; MLPPReg regularization; + real_t cost_prev = 0; int epoch = 1; - forwardPass(); + + forward_pass(); while (true) { - cost_prev = Cost(y_hat, outputSet); + cost_prev = cost(_y_hat, _output_set); // Calculating the errors - std::vector> error = alg.subtraction(y_hat, outputSet); + std::vector> error = alg.subtraction(_y_hat, _output_set); // Calculating the weight/bias gradients for layer 2 - std::vector> D2_1 = alg.matmult(alg.transpose(a2), error); + std::vector> D2_1 = alg.matmult(alg.transpose(_a2), error); // weights and bias updation for layer 2 - weights2 = alg.subtraction(weights2, alg.scalarMultiply(learning_rate, D2_1)); - weights2 = regularization.regWeights(weights2, lambda, alpha, reg); + _weights2 = alg.subtraction(_weights2, alg.scalarMultiply(learning_rate, D2_1)); + //_reg + _weights2 = regularization.regWeights(_weights2, _lambda, _alpha, "None"); - bias2 = alg.subtractMatrixRows(bias2, alg.scalarMultiply(learning_rate, error)); + _bias2 = alg.subtractMatrixRows(_bias2, alg.scalarMultiply(learning_rate, error)); //Calculating the weight/bias for layer 1 - std::vector> D1_1 = alg.matmult(error, alg.transpose(weights2)); + std::vector> D1_1 = alg.matmult(error, alg.transpose(_weights2)); - std::vector> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1)); + std::vector> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(_z2, true)); - std::vector> D1_3 = alg.matmult(alg.transpose(inputSet), D1_2); + std::vector> D1_3 = alg.matmult(alg.transpose(_input_set), D1_2); // weight an bias updation for layer 1 - weights1 = alg.subtraction(weights1, alg.scalarMultiply(learning_rate, D1_3)); - weights1 = regularization.regWeights(weights1, lambda, alpha, reg); + _weights1 = alg.subtraction(_weights1, alg.scalarMultiply(learning_rate, D1_3)); + //_reg + _weights1 = regularization.regWeights(_weights1, _lambda, _alpha, "None"); - bias1 = alg.subtractMatrixRows(bias1, alg.scalarMultiply(learning_rate, D1_2)); + _bias1 = alg.subtractMatrixRows(_bias1, alg.scalarMultiply(learning_rate, D1_2)); - forwardPass(); + forward_pass(); // UI PORTION - if (UI) { - MLPPUtilities::CostInfo(epoch, cost_prev, Cost(y_hat, outputSet)); + if (ui) { + MLPPUtilities::CostInfo(epoch, cost_prev, cost(_y_hat, _output_set)); std::cout << "Layer 1:" << std::endl; - MLPPUtilities::UI(weights1, bias1); + MLPPUtilities::UI(_weights1, _bias1); std::cout << "Layer 2:" << std::endl; - MLPPUtilities::UI(weights2, bias2); + MLPPUtilities::UI(_weights2, _bias2); } + epoch++; if (epoch > max_epoch) { @@ -98,65 +131,72 @@ void MLPPSoftmaxNet::gradientDescent(real_t learning_rate, int max_epoch, bool U } } -void MLPPSoftmaxNet::SGD(real_t learning_rate, int max_epoch, bool UI) { +void MLPPSoftmaxNet::sgd(real_t learning_rate, int max_epoch, bool ui) { MLPPActivation avn; MLPPLinAlg alg; MLPPReg regularization; + real_t cost_prev = 0; int epoch = 1; + std::random_device rd; + std::default_random_engine generator(rd()); + std::uniform_int_distribution distribution(0, int(_n - 1)); + while (true) { - std::random_device rd; - std::default_random_engine generator(rd()); - std::uniform_int_distribution distribution(0, int(n - 1)); int outputIndex = distribution(generator); - std::vector y_hat = Evaluate(inputSet[outputIndex]); + std::vector y_hat = evaluatev(_input_set[outputIndex]); - auto prop_res = propagate(inputSet[outputIndex]); + auto prop_res = propagatev(_input_set[outputIndex]); auto z2 = std::get<0>(prop_res); auto a2 = std::get<1>(prop_res); - cost_prev = Cost({ y_hat }, { outputSet[outputIndex] }); - std::vector error = alg.subtraction(y_hat, outputSet[outputIndex]); + cost_prev = cost({ y_hat }, { _output_set[outputIndex] }); + std::vector error = alg.subtraction(y_hat, _output_set[outputIndex]); // Weight updation for layer 2 std::vector> D2_1 = alg.outerProduct(error, a2); - weights2 = alg.subtraction(weights2, alg.scalarMultiply(learning_rate, alg.transpose(D2_1))); - weights2 = regularization.regWeights(weights2, lambda, alpha, reg); + _weights2 = alg.subtraction(_weights2, alg.scalarMultiply(learning_rate, alg.transpose(D2_1))); + //_reg + _weights2 = regularization.regWeights(_weights2, _lambda, _alpha, "None"); // Bias updation for layer 2 - bias2 = alg.subtraction(bias2, alg.scalarMultiply(learning_rate, error)); + _bias2 = alg.subtraction(_bias2, alg.scalarMultiply(learning_rate, error)); // Weight updation for layer 1 - std::vector D1_1 = alg.mat_vec_mult(weights2, error); + std::vector D1_1 = alg.mat_vec_mult(_weights2, error); std::vector D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, true)); - std::vector> D1_3 = alg.outerProduct(inputSet[outputIndex], D1_2); + std::vector> D1_3 = alg.outerProduct(_input_set[outputIndex], D1_2); - weights1 = alg.subtraction(weights1, alg.scalarMultiply(learning_rate, D1_3)); - weights1 = regularization.regWeights(weights1, lambda, alpha, reg); + _weights1 = alg.subtraction(_weights1, alg.scalarMultiply(learning_rate, D1_3)); + //_reg + _weights1 = regularization.regWeights(_weights1, _lambda, _alpha, "None"); // Bias updation for layer 1 - bias1 = alg.subtraction(bias1, alg.scalarMultiply(learning_rate, D1_2)); + _bias1 = alg.subtraction(_bias1, alg.scalarMultiply(learning_rate, D1_2)); - y_hat = Evaluate(inputSet[outputIndex]); - if (UI) { - MLPPUtilities::CostInfo(epoch, cost_prev, Cost({ y_hat }, { outputSet[outputIndex] })); + y_hat = evaluatev(_input_set[outputIndex]); + + if (ui) { + MLPPUtilities::CostInfo(epoch, cost_prev, cost({ y_hat }, { _output_set[outputIndex] })); std::cout << "Layer 1:" << std::endl; - MLPPUtilities::UI(weights1, bias1); + MLPPUtilities::UI(_weights1, _bias1); std::cout << "Layer 2:" << std::endl; - MLPPUtilities::UI(weights2, bias2); + MLPPUtilities::UI(_weights2, _bias2); } + epoch++; if (epoch > max_epoch) { break; } } - forwardPass(); + + forward_pass(); } -void MLPPSoftmaxNet::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI) { +void MLPPSoftmaxNet::mbgd(real_t learning_rate, int max_epoch, int mini_batch_size, bool ui) { MLPPActivation avn; MLPPLinAlg alg; MLPPReg regularization; @@ -164,40 +204,21 @@ void MLPPSoftmaxNet::MBGD(real_t learning_rate, int max_epoch, int mini_batch_si int epoch = 1; // Creating the mini-batches - int n_mini_batch = n / mini_batch_size; + int n_mini_batch = _n / mini_batch_size; - auto batches = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch); + auto batches = MLPPUtilities::createMiniBatches(_input_set, _output_set, n_mini_batch); auto inputMiniBatches = std::get<0>(batches); auto outputMiniBatches = std::get<1>(batches); - // Creating the mini-batches - for (int i = 0; i < n_mini_batch; i++) { - std::vector> currentInputSet; - std::vector> currentOutputSet; - for (int j = 0; j < n / n_mini_batch; j++) { - currentInputSet.push_back(inputSet[n / n_mini_batch * i + j]); - currentOutputSet.push_back(outputSet[n / n_mini_batch * i + j]); - } - inputMiniBatches.push_back(currentInputSet); - outputMiniBatches.push_back(currentOutputSet); - } - - if (real_t(n) / real_t(n_mini_batch) - int(n / n_mini_batch) != 0) { - for (int i = 0; i < n - n / n_mini_batch * n_mini_batch; i++) { - inputMiniBatches[n_mini_batch - 1].push_back(inputSet[n / n_mini_batch * n_mini_batch + i]); - outputMiniBatches[n_mini_batch - 1].push_back(outputSet[n / n_mini_batch * n_mini_batch + i]); - } - } - while (true) { for (int i = 0; i < n_mini_batch; i++) { - std::vector> y_hat = Evaluate(inputMiniBatches[i]); + std::vector> y_hat = evaluatem(inputMiniBatches[i]); - auto propagate_res = propagate(inputMiniBatches[i]); + auto propagate_res = propagatem(inputMiniBatches[i]); auto z2 = std::get<0>(propagate_res); auto a2 = std::get<1>(propagate_res); - cost_prev = Cost(y_hat, outputMiniBatches[i]); + cost_prev = cost(y_hat, outputMiniBatches[i]); // Calculating the errors std::vector> error = alg.subtraction(y_hat, outputMiniBatches[i]); @@ -207,102 +228,198 @@ void MLPPSoftmaxNet::MBGD(real_t learning_rate, int max_epoch, int mini_batch_si std::vector> D2_1 = alg.matmult(alg.transpose(a2), error); // weights and bias updation for layser 2 - weights2 = alg.subtraction(weights2, alg.scalarMultiply(learning_rate, D2_1)); - weights2 = regularization.regWeights(weights2, lambda, alpha, reg); + _weights2 = alg.subtraction(_weights2, alg.scalarMultiply(learning_rate, D2_1)); + //_reg + _weights2 = regularization.regWeights(_weights2, _lambda, _alpha, "None"); // Bias Updation for layer 2 - bias2 = alg.subtractMatrixRows(bias2, alg.scalarMultiply(learning_rate, error)); + _bias2 = alg.subtractMatrixRows(_bias2, alg.scalarMultiply(learning_rate, error)); //Calculating the weight/bias for layer 1 - std::vector> D1_1 = alg.matmult(error, alg.transpose(weights2)); + std::vector> D1_1 = alg.matmult(error, alg.transpose(_weights2)); - std::vector> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1)); + std::vector> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, true)); std::vector> D1_3 = alg.matmult(alg.transpose(inputMiniBatches[i]), D1_2); // weight an bias updation for layer 1 - weights1 = alg.subtraction(weights1, alg.scalarMultiply(learning_rate, D1_3)); - weights1 = regularization.regWeights(weights1, lambda, alpha, reg); + _weights1 = alg.subtraction(_weights1, alg.scalarMultiply(learning_rate, D1_3)); + //_reg + _weights1 = regularization.regWeights(_weights1, _lambda, _alpha, "None"); - bias1 = alg.subtractMatrixRows(bias1, alg.scalarMultiply(learning_rate, D1_2)); + _bias1 = alg.subtractMatrixRows(_bias1, alg.scalarMultiply(learning_rate, D1_2)); - y_hat = Evaluate(inputMiniBatches[i]); + y_hat = evaluatem(inputMiniBatches[i]); - if (UI) { - MLPPUtilities::CostInfo(epoch, cost_prev, Cost(y_hat, outputMiniBatches[i])); + if (ui) { + MLPPUtilities::CostInfo(epoch, cost_prev, cost(y_hat, outputMiniBatches[i])); std::cout << "Layer 1:" << std::endl; - MLPPUtilities::UI(weights1, bias1); + MLPPUtilities::UI(_weights1, _bias1); std::cout << "Layer 2:" << std::endl; - MLPPUtilities::UI(weights2, bias2); + MLPPUtilities::UI(_weights2, _bias2); } } + epoch++; + if (epoch > max_epoch) { break; } } - forwardPass(); + + forward_pass(); } real_t MLPPSoftmaxNet::score() { MLPPUtilities util; - return util.performance(y_hat, outputSet); + + return util.performance(_y_hat, _output_set); } void MLPPSoftmaxNet::save(std::string fileName) { MLPPUtilities util; - util.saveParameters(fileName, weights1, bias1, 0, 1); - util.saveParameters(fileName, weights2, bias2, 1, 2); + + util.saveParameters(fileName, _weights1, _bias1, false, 1); + util.saveParameters(fileName, _weights2, _bias2, true, 2); } -std::vector> MLPPSoftmaxNet::getEmbeddings() { - return weights1; +std::vector> MLPPSoftmaxNet::get_embeddings() { + return _weights1; } -real_t MLPPSoftmaxNet::Cost(std::vector> y_hat, std::vector> y) { +bool MLPPSoftmaxNet::is_initialized() { + return _initialized; +} +void MLPPSoftmaxNet::initialize() { + if (_initialized) { + return; + } + + //ERR_FAIL_COND(!_input_set.is_valid() || !_output_set.is_valid()); + + _initialized = true; +} + +MLPPSoftmaxNet::MLPPSoftmaxNet(std::vector> p_input_set, std::vector> p_output_set, int p_n_hidden, MLPPReg::RegularizationType p_reg, real_t p_lambda, real_t p_alpha) { + _input_set = p_input_set; + _output_set = p_output_set; + _n = p_input_set.size(); + _k = p_input_set[0].size(); + _n_hidden = p_n_hidden; + _n_class = p_output_set[0].size(); + _reg = p_reg; + _lambda = p_lambda; + _alpha = p_alpha; + + _y_hat.resize(_n); + + _weights1 = MLPPUtilities::weightInitialization(_k, _n_hidden); + _weights2 = MLPPUtilities::weightInitialization(_n_hidden, _n_class); + _bias1 = MLPPUtilities::biasInitialization(_n_hidden); + _bias2 = MLPPUtilities::biasInitialization(_n_class); + + _initialized = true; +} + +MLPPSoftmaxNet::MLPPSoftmaxNet() { + _initialized = false; +} +MLPPSoftmaxNet::~MLPPSoftmaxNet() { +} + +real_t MLPPSoftmaxNet::cost(std::vector> y_hat, std::vector> y) { MLPPReg regularization; MLPPData data; class MLPPCost cost; - return cost.CrossEntropy(y_hat, y) + regularization.regTerm(weights1, lambda, alpha, reg) + regularization.regTerm(weights2, lambda, alpha, reg); + + //_reg + return cost.CrossEntropy(y_hat, y) + regularization.regTerm(_weights1, _lambda, _alpha, "None") + regularization.regTerm(_weights2, _lambda, _alpha, "None"); } -std::vector> MLPPSoftmaxNet::Evaluate(std::vector> X) { +std::vector MLPPSoftmaxNet::evaluatev(std::vector x) { MLPPLinAlg alg; MLPPActivation avn; - std::vector> z2 = alg.mat_vec_add(alg.matmult(X, weights1), bias1); - std::vector> a2 = avn.sigmoid(z2); - return avn.adjSoftmax(alg.mat_vec_add(alg.matmult(a2, weights2), bias2)); + + std::vector z2 = alg.addition(alg.mat_vec_mult(alg.transpose(_weights1), x), _bias1); + std::vector a2 = avn.sigmoid(z2); + + return avn.adjSoftmax(alg.addition(alg.mat_vec_mult(alg.transpose(_weights2), a2), _bias2)); } -std::tuple>, std::vector>> MLPPSoftmaxNet::propagate(std::vector> X) { +std::tuple, std::vector> MLPPSoftmaxNet::propagatev(std::vector x) { MLPPLinAlg alg; MLPPActivation avn; - std::vector> z2 = alg.mat_vec_add(alg.matmult(X, weights1), bias1); - std::vector> a2 = avn.sigmoid(z2); + + std::vector z2 = alg.addition(alg.mat_vec_mult(alg.transpose(_weights1), x), _bias1); + std::vector a2 = avn.sigmoid(z2); + return { z2, a2 }; } -std::vector MLPPSoftmaxNet::Evaluate(std::vector x) { +std::vector> MLPPSoftmaxNet::evaluatem(std::vector> X) { MLPPLinAlg alg; MLPPActivation avn; - std::vector z2 = alg.addition(alg.mat_vec_mult(alg.transpose(weights1), x), bias1); - std::vector a2 = avn.sigmoid(z2); - return avn.adjSoftmax(alg.addition(alg.mat_vec_mult(alg.transpose(weights2), a2), bias2)); + + std::vector> z2 = alg.mat_vec_add(alg.matmult(X, _weights1), _bias1); + std::vector> a2 = avn.sigmoid(z2); + + return avn.adjSoftmax(alg.mat_vec_add(alg.matmult(a2, _weights2), _bias2)); } -std::tuple, std::vector> MLPPSoftmaxNet::propagate(std::vector x) { +std::tuple>, std::vector>> MLPPSoftmaxNet::propagatem(std::vector> X) { MLPPLinAlg alg; MLPPActivation avn; - std::vector z2 = alg.addition(alg.mat_vec_mult(alg.transpose(weights1), x), bias1); - std::vector a2 = avn.sigmoid(z2); + + std::vector> z2 = alg.mat_vec_add(alg.matmult(X, _weights1), _bias1); + std::vector> a2 = avn.sigmoid(z2); + return { z2, a2 }; } -void MLPPSoftmaxNet::forwardPass() { +void MLPPSoftmaxNet::forward_pass() { MLPPLinAlg alg; MLPPActivation avn; - z2 = alg.mat_vec_add(alg.matmult(inputSet, weights1), bias1); - a2 = avn.sigmoid(z2); - y_hat = avn.adjSoftmax(alg.mat_vec_add(alg.matmult(a2, weights2), bias2)); + + _z2 = alg.mat_vec_add(alg.matmult(_input_set, _weights1), _bias1); + _a2 = avn.sigmoid(_z2); + _y_hat = avn.adjSoftmax(alg.mat_vec_add(alg.matmult(_a2, _weights2), _bias2)); +} + +void MLPPSoftmaxNet::_bind_methods() { + /* + ClassDB::bind_method(D_METHOD("get_input_set"), &MLPPSoftmaxNet::get_input_set); + ClassDB::bind_method(D_METHOD("set_input_set", "val"), &MLPPSoftmaxNet::set_input_set); + ADD_PROPERTY(PropertyInfo(Variant::OBJECT, "input_set", PROPERTY_HINT_RESOURCE_TYPE, "MLPPMatrix"), "set_input_set", "get_input_set"); + + ClassDB::bind_method(D_METHOD("get_output_set"), &MLPPSoftmaxNet::get_output_set); + ClassDB::bind_method(D_METHOD("set_output_set", "val"), &MLPPSoftmaxNet::set_output_set); + ADD_PROPERTY(PropertyInfo(Variant::OBJECT, "output_set", PROPERTY_HINT_RESOURCE_TYPE, "MLPPMatrix"), "set_output_set", "get_output_set"); + + ClassDB::bind_method(D_METHOD("get_reg"), &MLPPSoftmaxNet::get_reg); + ClassDB::bind_method(D_METHOD("set_reg", "val"), &MLPPSoftmaxNet::set_reg); + ADD_PROPERTY(PropertyInfo(Variant::INT, "reg"), "set_reg", "get_reg"); + + ClassDB::bind_method(D_METHOD("get_lambda"), &MLPPSoftmaxNet::get_lambda); + ClassDB::bind_method(D_METHOD("set_lambda", "val"), &MLPPSoftmaxNet::set_lambda); + ADD_PROPERTY(PropertyInfo(Variant::REAL, "lambda"), "set_lambda", "get_lambda"); + + ClassDB::bind_method(D_METHOD("get_alpha"), &MLPPSoftmaxNet::get_alpha); + ClassDB::bind_method(D_METHOD("set_alpha", "val"), &MLPPSoftmaxNet::set_alpha); + ADD_PROPERTY(PropertyInfo(Variant::REAL, "alpha"), "set_alpha", "get_alpha"); + + ClassDB::bind_method(D_METHOD("model_test", "x"), &MLPPSoftmaxNet::model_test); + ClassDB::bind_method(D_METHOD("model_set_test", "X"), &MLPPSoftmaxNet::model_set_test); + + ClassDB::bind_method(D_METHOD("gradient_descent", "learning_rate", "max_epoch", "ui"), &MLPPSoftmaxNet::gradient_descent, false); + ClassDB::bind_method(D_METHOD("sgd", "learning_rate", "max_epoch", "ui"), &MLPPSoftmaxNet::sgd, false); + ClassDB::bind_method(D_METHOD("mbgd", "learning_rate", "max_epoch", "mini_batch_size", "ui"), &MLPPSoftmaxNet::mbgd, false); + + ClassDB::bind_method(D_METHOD("score"), &MLPPSoftmaxNet::score); + + ClassDB::bind_method(D_METHOD("save", "file_name"), &MLPPSoftmaxNet::save); + + ClassDB::bind_method(D_METHOD("is_initialized"), &MLPPSoftmaxNet::is_initialized); + ClassDB::bind_method(D_METHOD("initialize"), &MLPPSoftmaxNet::initialize); + */ } diff --git a/mlpp/softmax_net/softmax_net.h b/mlpp/softmax_net/softmax_net.h index 9108472..f713f5c 100644 --- a/mlpp/softmax_net/softmax_net.h +++ b/mlpp/softmax_net/softmax_net.h @@ -9,52 +9,96 @@ #include "core/math/math_defs.h" +#include "core/object/reference.h" + +#include "../lin_alg/mlpp_matrix.h" +#include "../lin_alg/mlpp_vector.h" + +#include "../regularization/reg.h" + #include #include -class MLPPSoftmaxNet { +class MLPPSoftmaxNet : public Reference { + GDCLASS(MLPPSoftmaxNet, Reference); + public: - MLPPSoftmaxNet(std::vector> inputSet, std::vector> outputSet, int n_hidden, std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5); - std::vector modelTest(std::vector x); - std::vector> modelSetTest(std::vector> X); - void gradientDescent(real_t learning_rate, int max_epoch, bool UI = false); - void SGD(real_t learning_rate, int max_epoch, bool UI = false); - void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = false); + /* + Ref get_input_set(); + void set_input_set(const Ref &val); + + Ref get_output_set(); + void set_output_set(const Ref &val); + + MLPPReg::RegularizationType get_reg(); + void set_reg(const MLPPReg::RegularizationType val); + + real_t get_lambda(); + void set_lambda(const real_t val); + + real_t get_alpha(); + void set_alpha(const real_t val); + */ + + std::vector model_test(std::vector x); + std::vector> model_set_test(std::vector> X); + + void gradient_descent(real_t learning_rate, int max_epoch, bool ui = false); + void sgd(real_t learning_rate, int max_epoch, bool ui = false); + void mbgd(real_t learning_rate, int max_epoch, int mini_batch_size, bool ui = false); + real_t score(); + void save(std::string fileName); - std::vector> getEmbeddings(); // This class is used (mostly) for word2Vec. This function returns our embeddings. -private: - real_t Cost(std::vector> y_hat, std::vector> y); + std::vector> get_embeddings(); // This class is used (mostly) for word2Vec. This function returns our embeddings. - std::vector> Evaluate(std::vector> X); - std::tuple>, std::vector>> propagate(std::vector> X); - std::vector Evaluate(std::vector x); - std::tuple, std::vector> propagate(std::vector x); - void forwardPass(); + bool is_initialized(); + void initialize(); - std::vector> inputSet; - std::vector> outputSet; - std::vector> y_hat; + MLPPSoftmaxNet(std::vector> p_input_set, std::vector> p_output_set, int p_n_hidden, MLPPReg::RegularizationType p_reg = MLPPReg::REGULARIZATION_TYPE_NONE, real_t p_lambda = 0.5, real_t p_alpha = 0.5); + //MLPPSoftmaxNet(const Ref &p_input_set, const Ref &p_output_set, MLPPReg::RegularizationType p_reg = MLPPReg::REGULARIZATION_TYPE_NONE, real_t p_lambda = 0.5, real_t p_alpha = 0.5); - std::vector> weights1; - std::vector> weights2; + MLPPSoftmaxNet(); + ~MLPPSoftmaxNet(); - std::vector bias1; - std::vector bias2; +protected: + real_t cost(std::vector> y_hat, std::vector> y); - std::vector> z2; - std::vector> a2; + std::vector evaluatev(std::vector x); + std::tuple, std::vector> propagatev(std::vector x); - int n; - int k; - int n_class; - int n_hidden; + std::vector> evaluatem(std::vector> X); + std::tuple>, std::vector>> propagatem(std::vector> X); + + void forward_pass(); + + static void _bind_methods(); + + std::vector> _input_set; + std::vector> _output_set; + std::vector> _y_hat; + + std::vector> _weights1; + std::vector> _weights2; + + std::vector _bias1; + std::vector _bias2; + + std::vector> _z2; + std::vector> _a2; + + int _n; + int _k; + int _n_class; + int _n_hidden; // Regularization Params - std::string reg; - real_t lambda; - real_t alpha; /* This is the controlling param for Elastic Net*/ + MLPPReg::RegularizationType _reg; + real_t _lambda; + real_t _alpha; /* This is the controlling param for Elastic Net*/ + + bool _initialized; }; #endif /* SoftmaxNet_hpp */ diff --git a/test/mlpp_tests.cpp b/test/mlpp_tests.cpp index edf2db8..4079a61 100644 --- a/test/mlpp_tests.cpp +++ b/test/mlpp_tests.cpp @@ -52,12 +52,30 @@ #include "../mlpp/outlier_finder/outlier_finder_old.h" #include "../mlpp/pca/pca_old.h" #include "../mlpp/probit_reg/probit_reg_old.h" +#include "../mlpp/softmax_net/softmax_net_old.h" #include "../mlpp/softmax_reg/softmax_reg_old.h" #include "../mlpp/svc/svc_old.h" #include "../mlpp/tanh_reg/tanh_reg_old.h" #include "../mlpp/uni_lin_reg/uni_lin_reg_old.h" #include "../mlpp/wgan/wgan_old.h" +/* +#include "../mlpp/ann/ann_old.h" +#include "../mlpp/bernoulli_nb/bernoulli_nb_old.h" +#include "../mlpp/c_log_log_reg/c_log_log_reg_old.h" +#include "../mlpp/dual_svc/dual_svc_old.h" +#include "../mlpp/exp_reg/exp_reg_old.h" +#include "../mlpp/gan/gan_old.h" +#include "../mlpp/gaussian_nb/gaussian_nb_old.h" +#include "../mlpp/hidden_layer/hidden_layer_old.h" +#include "../mlpp/lin_reg/lin_reg_old.h" +#include "../mlpp/log_reg/log_reg_old.h" +#include "../mlpp/mann/mann_old.h" +#include "../mlpp/multi_output_layer/multi_output_layer_old.h" +#include "../mlpp/multinomial_nb/multinomial_nb_old.h" +#include "../mlpp/output_layer/output_layer_old.h" +*/ + Vector dstd_vec_to_vec(const std::vector &in) { Vector r; @@ -490,9 +508,14 @@ void MLPPTests::test_soft_max_network(bool ui) { // SOFTMAX NETWORK Ref dt = data.load_wine(_wine_data_path); + MLPPSoftmaxNetOld model_old(dt->get_input()->to_std_vector(), dt->get_output()->to_std_vector(), 1); + model_old.gradientDescent(0.01, 100000, ui); + alg.printMatrix(model_old.modelSetTest(dt->get_input()->to_std_vector())); + std::cout << "ACCURACY: " << 100 * model_old.score() << "%" << std::endl; + MLPPSoftmaxNet model(dt->get_input()->to_std_vector(), dt->get_output()->to_std_vector(), 1); - model.gradientDescent(0.01, 100000, ui); - alg.printMatrix(model.modelSetTest(dt->get_input()->to_std_vector())); + model.gradient_descent(0.01, 100000, ui); + alg.printMatrix(model.model_set_test(dt->get_input()->to_std_vector())); std::cout << "ACCURACY: " << 100 * model.score() << "%" << std::endl; } void MLPPTests::test_autoencoder(bool ui) {