From 8544322ef02401b92faaa7f55db25f981c04d854 Mon Sep 17 00:00:00 2001 From: Relintai Date: Fri, 10 Feb 2023 14:25:28 +0100 Subject: [PATCH] Added SoftMaxRegOld. --- SCsub | 1 + mlpp/softmax_reg/softmax_reg_old.cpp | 192 +++++++++++++++++++++++++++ mlpp/softmax_reg/softmax_reg_old.h | 50 +++++++ test/mlpp_tests.cpp | 9 +- 4 files changed, 248 insertions(+), 4 deletions(-) create mode 100644 mlpp/softmax_reg/softmax_reg_old.cpp create mode 100644 mlpp/softmax_reg/softmax_reg_old.h diff --git a/SCsub b/SCsub index 89273ec..97209a3 100644 --- a/SCsub +++ b/SCsub @@ -60,6 +60,7 @@ sources = [ "mlpp/outlier_finder/outlier_finder_old.cpp", "mlpp/probit_reg/probit_reg_old.cpp", "mlpp/svc/svc_old.cpp", + "mlpp/softmax_reg/softmax_reg_old.cpp", "test/mlpp_tests.cpp", ] diff --git a/mlpp/softmax_reg/softmax_reg_old.cpp b/mlpp/softmax_reg/softmax_reg_old.cpp new file mode 100644 index 0000000..3be9f6c --- /dev/null +++ b/mlpp/softmax_reg/softmax_reg_old.cpp @@ -0,0 +1,192 @@ +// +// SoftmaxReg.cpp +// +// Created by Marc Melikyan on 10/2/20. +// + +#include "softmax_reg_old.h" +#include "../activation/activation.h" +#include "../cost/cost.h" +#include "../lin_alg/lin_alg.h" +#include "../regularization/reg.h" +#include "../utilities/utilities.h" + +#include +#include + +MLPPSoftmaxRegOld::MLPPSoftmaxRegOld(std::vector> inputSet, std::vector> outputSet, std::string reg, real_t lambda, real_t alpha) : + inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), n_class(outputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) { + y_hat.resize(n); + weights = MLPPUtilities::weightInitialization(k, n_class); + bias = MLPPUtilities::biasInitialization(n_class); +} + +std::vector MLPPSoftmaxRegOld::modelTest(std::vector x) { + return Evaluate(x); +} + +std::vector> MLPPSoftmaxRegOld::modelSetTest(std::vector> X) { + return Evaluate(X); +} + +void MLPPSoftmaxRegOld::gradientDescent(real_t learning_rate, int max_epoch, bool UI) { + MLPPLinAlg alg; + MLPPReg regularization; + real_t cost_prev = 0; + int epoch = 1; + forwardPass(); + + while (true) { + cost_prev = Cost(y_hat, outputSet); + std::vector> error = alg.subtraction(y_hat, outputSet); + + //Calculating the weight gradients + std::vector> w_gradient = alg.matmult(alg.transpose(inputSet), error); + + //Weight updation + weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate, w_gradient)); + weights = regularization.regWeights(weights, lambda, alpha, reg); + + // Calculating the bias gradients + //real_t b_gradient = alg.sum_elements(error); + + // Bias Updation + bias = alg.subtractMatrixRows(bias, alg.scalarMultiply(learning_rate, error)); + + forwardPass(); + + // UI PORTION + if (UI) { + MLPPUtilities::CostInfo(epoch, cost_prev, Cost(y_hat, outputSet)); + MLPPUtilities::UI(weights, bias); + } + epoch++; + + if (epoch > max_epoch) { + break; + } + } +} + +void MLPPSoftmaxRegOld::SGD(real_t learning_rate, int max_epoch, bool UI) { + MLPPLinAlg alg; + MLPPReg regularization; + real_t cost_prev = 0; + int epoch = 1; + + while (true) { + std::random_device rd; + std::default_random_engine generator(rd()); + std::uniform_int_distribution distribution(0, int(n - 1)); + real_t outputIndex = distribution(generator); + + std::vector y_hat = Evaluate(inputSet[outputIndex]); + cost_prev = Cost({ y_hat }, { outputSet[outputIndex] }); + + // Calculating the weight gradients + std::vector> w_gradient = alg.outerProduct(inputSet[outputIndex], alg.subtraction(y_hat, outputSet[outputIndex])); + + // Weight Updation + weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate, w_gradient)); + weights = regularization.regWeights(weights, lambda, alpha, reg); + + // Calculating the bias gradients + std::vector b_gradient = alg.subtraction(y_hat, outputSet[outputIndex]); + + // Bias updation + bias = alg.subtraction(bias, alg.scalarMultiply(learning_rate, b_gradient)); + + y_hat = Evaluate({ inputSet[outputIndex] }); + + if (UI) { + MLPPUtilities::CostInfo(epoch, cost_prev, Cost({ y_hat }, { outputSet[outputIndex] })); + MLPPUtilities::UI(weights, bias); + } + epoch++; + + if (epoch > max_epoch) { + break; + } + } + forwardPass(); +} + +void MLPPSoftmaxRegOld::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI) { + MLPPLinAlg alg; + MLPPReg regularization; + real_t cost_prev = 0; + int epoch = 1; + + // Creating the mini-batches + int n_mini_batch = n / mini_batch_size; + auto batches = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch); + auto inputMiniBatches = std::get<0>(batches); + auto outputMiniBatches = std::get<1>(batches); + + while (true) { + for (int i = 0; i < n_mini_batch; i++) { + std::vector> y_hat = Evaluate(inputMiniBatches[i]); + cost_prev = Cost(y_hat, outputMiniBatches[i]); + + std::vector> error = alg.subtraction(y_hat, outputMiniBatches[i]); + + // Calculating the weight gradients + std::vector> w_gradient = alg.matmult(alg.transpose(inputMiniBatches[i]), error); + + //Weight updation + weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate, w_gradient)); + weights = regularization.regWeights(weights, lambda, alpha, reg); + + // Calculating the bias gradients + bias = alg.subtractMatrixRows(bias, alg.scalarMultiply(learning_rate, error)); + y_hat = Evaluate(inputMiniBatches[i]); + + if (UI) { + MLPPUtilities::CostInfo(epoch, cost_prev, Cost(y_hat, outputMiniBatches[i])); + MLPPUtilities::UI(weights, bias); + } + } + epoch++; + if (epoch > max_epoch) { + break; + } + } + forwardPass(); +} + +real_t MLPPSoftmaxRegOld::score() { + MLPPUtilities util; + return util.performance(y_hat, outputSet); +} + +void MLPPSoftmaxRegOld::save(std::string fileName) { + MLPPUtilities util; + util.saveParameters(fileName, weights, bias); +} + +real_t MLPPSoftmaxRegOld::Cost(std::vector> y_hat, std::vector> y) { + MLPPReg regularization; + class MLPPCost cost; + return cost.CrossEntropy(y_hat, y) + regularization.regTerm(weights, lambda, alpha, reg); +} + +std::vector MLPPSoftmaxRegOld::Evaluate(std::vector x) { + MLPPLinAlg alg; + MLPPActivation avn; + return avn.softmax(alg.addition(bias, alg.mat_vec_mult(alg.transpose(weights), x))); +} + +std::vector> MLPPSoftmaxRegOld::Evaluate(std::vector> X) { + MLPPLinAlg alg; + MLPPActivation avn; + + return avn.softmax(alg.mat_vec_add(alg.matmult(X, weights), bias)); +} + +// softmax ( wTx + b ) +void MLPPSoftmaxRegOld::forwardPass() { + MLPPLinAlg alg; + MLPPActivation avn; + + y_hat = avn.softmax(alg.mat_vec_add(alg.matmult(inputSet, weights), bias)); +} diff --git a/mlpp/softmax_reg/softmax_reg_old.h b/mlpp/softmax_reg/softmax_reg_old.h new file mode 100644 index 0000000..864e0ee --- /dev/null +++ b/mlpp/softmax_reg/softmax_reg_old.h @@ -0,0 +1,50 @@ + +#ifndef MLPP_SOFTMAX_REG_OLD_H +#define MLPP_SOFTMAX_REG_OLD_H + +// +// SoftmaxReg.hpp +// +// Created by Marc Melikyan on 10/2/20. +// + +#include "core/math/math_defs.h" + +#include +#include + +class MLPPSoftmaxRegOld { +public: + MLPPSoftmaxRegOld(std::vector> inputSet, std::vector> outputSet, std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5); + std::vector modelTest(std::vector x); + std::vector> modelSetTest(std::vector> X); + void gradientDescent(real_t learning_rate, int max_epoch, bool UI = false); + void SGD(real_t learning_rate, int max_epoch, bool UI = false); + void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = false); + real_t score(); + void save(std::string fileName); + +private: + real_t Cost(std::vector> y_hat, std::vector> y); + + std::vector> Evaluate(std::vector> X); + std::vector Evaluate(std::vector x); + void forwardPass(); + + std::vector> inputSet; + std::vector> outputSet; + std::vector> y_hat; + std::vector> weights; + std::vector bias; + + int n; + int k; + int n_class; + + // Regularization Params + std::string reg; + real_t lambda; + real_t alpha; /* This is the controlling param for Elastic Net*/ +}; + +#endif /* SoftmaxReg_hpp */ diff --git a/test/mlpp_tests.cpp b/test/mlpp_tests.cpp index 00407a0..3092e87 100644 --- a/test/mlpp_tests.cpp +++ b/test/mlpp_tests.cpp @@ -51,6 +51,7 @@ #include "../mlpp/outlier_finder/outlier_finder_old.h" #include "../mlpp/pca/pca_old.h" #include "../mlpp/probit_reg/probit_reg_old.h" +#include "../mlpp/softmax_reg/softmax_reg_old.h" #include "../mlpp/svc/svc_old.h" #include "../mlpp/uni_lin_reg/uni_lin_reg_old.h" #include "../mlpp/wgan/wgan_old.h" @@ -399,10 +400,10 @@ void MLPPTests::test_softmax_regression(bool ui) { // SOFTMAX REGRESSION Ref dt = data.load_iris(_iris_data_path); - MLPPSoftmaxReg model(dt->get_input()->to_std_vector(), dt->get_output()->to_std_vector()); - model.SGD(0.1, 10000, ui); - alg.printMatrix(model.modelSetTest(dt->get_input()->to_std_vector())); - std::cout << "ACCURACY: " << 100 * model.score() << "%" << std::endl; + MLPPSoftmaxRegOld model_old(dt->get_input()->to_std_vector(), dt->get_output()->to_std_vector()); + model_old.SGD(0.1, 10000, ui); + alg.printMatrix(model_old.modelSetTest(dt->get_input()->to_std_vector())); + std::cout << "ACCURACY: " << 100 * model_old.score() << "%" << std::endl; } void MLPPTests::test_support_vector_classification(bool ui) { //MLPPStat stat;