From 5f63aebc99918a4fce066140216a9f055b0e73b7 Mon Sep 17 00:00:00 2001 From: Relintai Date: Sat, 4 Feb 2023 16:48:31 +0100 Subject: [PATCH] Initial cleanups to MLP. --- mlpp/mlp/mlp.cpp | 90 ++++++++++++++++++++++++++---------------------- mlpp/mlp/mlp.h | 41 +++++++++++++++------- 2 files changed, 77 insertions(+), 54 deletions(-) diff --git a/mlpp/mlp/mlp.cpp b/mlpp/mlp/mlp.cpp index 6eac1f3..c85388e 100644 --- a/mlpp/mlp/mlp.cpp +++ b/mlpp/mlp/mlp.cpp @@ -15,36 +15,25 @@ #include #include - -MLPPMLP::MLPPMLP(std::vector> inputSet, std::vector outputSet, int n_hidden, std::string reg, real_t lambda, real_t alpha) : - inputSet(inputSet), outputSet(outputSet), n_hidden(n_hidden), n(inputSet.size()), k(inputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) { - MLPPActivation avn; - y_hat.resize(n); - - weights1 = MLPPUtilities::weightInitialization(k, n_hidden); - weights2 = MLPPUtilities::weightInitialization(n_hidden); - bias1 = MLPPUtilities::biasInitialization(n_hidden); - bias2 = MLPPUtilities::biasInitialization(); +std::vector MLPPMLP::model_set_test(std::vector> X) { + return evaluate(X); } -std::vector MLPPMLP::modelSetTest(std::vector> X) { - return Evaluate(X); +real_t MLPPMLP::model_test(std::vector x) { + return evaluate(x); } -real_t MLPPMLP::modelTest(std::vector x) { - return Evaluate(x); -} - -void MLPPMLP::gradientDescent(real_t learning_rate, int max_epoch, bool UI) { +void MLPPMLP::gradient_descent(real_t learning_rate, int max_epoch, bool UI) { MLPPActivation avn; MLPPLinAlg alg; MLPPReg regularization; real_t cost_prev = 0; int epoch = 1; - forwardPass(); + + forward_pass(); while (true) { - cost_prev = Cost(y_hat, outputSet); + cost_prev = cost(y_hat, outputSet); // Calculating the errors std::vector error = alg.subtraction(y_hat, outputSet); @@ -76,11 +65,11 @@ void MLPPMLP::gradientDescent(real_t learning_rate, int max_epoch, bool UI) { bias1 = alg.subtractMatrixRows(bias1, alg.scalarMultiply(learning_rate / n, D1_2)); - forwardPass(); + forward_pass(); // UI PORTION if (UI) { - MLPPUtilities::CostInfo(epoch, cost_prev, Cost(y_hat, outputSet)); + MLPPUtilities::CostInfo(epoch, cost_prev, cost(y_hat, outputSet)); std::cout << "Layer 1:" << std::endl; MLPPUtilities::UI(weights1, bias1); std::cout << "Layer 2:" << std::endl; @@ -94,7 +83,7 @@ void MLPPMLP::gradientDescent(real_t learning_rate, int max_epoch, bool UI) { } } -void MLPPMLP::SGD(real_t learning_rate, int max_epoch, bool UI) { +void MLPPMLP::sgd(real_t learning_rate, int max_epoch, bool UI) { MLPPActivation avn; MLPPLinAlg alg; MLPPReg regularization; @@ -107,9 +96,9 @@ void MLPPMLP::SGD(real_t learning_rate, int max_epoch, bool UI) { std::uniform_int_distribution distribution(0, int(n - 1)); int outputIndex = distribution(generator); - real_t y_hat = Evaluate(inputSet[outputIndex]); + real_t y_hat = evaluate(inputSet[outputIndex]); auto [z2, a2] = propagate(inputSet[outputIndex]); - cost_prev = Cost({ y_hat }, { outputSet[outputIndex] }); + cost_prev = cost({ y_hat }, { outputSet[outputIndex] }); real_t error = y_hat - outputSet[outputIndex]; // Weight updation for layer 2 @@ -131,9 +120,9 @@ void MLPPMLP::SGD(real_t learning_rate, int max_epoch, bool UI) { bias1 = alg.subtraction(bias1, alg.scalarMultiply(learning_rate, D1_2)); - y_hat = Evaluate(inputSet[outputIndex]); + y_hat = evaluate(inputSet[outputIndex]); if (UI) { - MLPPUtilities::CostInfo(epoch, cost_prev, Cost({ y_hat }, { outputSet[outputIndex] })); + MLPPUtilities::CostInfo(epoch, cost_prev, cost({ y_hat }, { outputSet[outputIndex] })); std::cout << "Layer 1:" << std::endl; MLPPUtilities::UI(weights1, bias1); std::cout << "Layer 2:" << std::endl; @@ -145,10 +134,11 @@ void MLPPMLP::SGD(real_t learning_rate, int max_epoch, bool UI) { break; } } - forwardPass(); + + forward_pass(); } -void MLPPMLP::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI) { +void MLPPMLP::mbgd(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI) { MLPPActivation avn; MLPPLinAlg alg; MLPPReg regularization; @@ -161,9 +151,9 @@ void MLPPMLP::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, boo while (true) { for (int i = 0; i < n_mini_batch; i++) { - std::vector y_hat = Evaluate(inputMiniBatches[i]); + std::vector y_hat = evaluate(inputMiniBatches[i]); auto [z2, a2] = propagate(inputMiniBatches[i]); - cost_prev = Cost(y_hat, outputMiniBatches[i]); + cost_prev = cost(y_hat, outputMiniBatches[i]); // Calculating the errors std::vector error = alg.subtraction(y_hat, outputMiniBatches[i]); @@ -196,42 +186,45 @@ void MLPPMLP::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, boo bias1 = alg.subtractMatrixRows(bias1, alg.scalarMultiply(learning_rate / outputMiniBatches[i].size(), D1_2)); - y_hat = Evaluate(inputMiniBatches[i]); + y_hat = evaluate(inputMiniBatches[i]); if (UI) { - MLPPUtilities::CostInfo(epoch, cost_prev, Cost(y_hat, outputMiniBatches[i])); + MLPPUtilities::CostInfo(epoch, cost_prev, cost(y_hat, outputMiniBatches[i])); std::cout << "Layer 1:" << std::endl; MLPPUtilities::UI(weights1, bias1); std::cout << "Layer 2:" << std::endl; MLPPUtilities::UI(weights2, bias2); } } + epoch++; + if (epoch > max_epoch) { break; } } - forwardPass(); + + forward_pass(); } real_t MLPPMLP::score() { - MLPPUtilities util; + MLPPUtilities util; return util.performance(y_hat, outputSet); } void MLPPMLP::save(std::string fileName) { - MLPPUtilities util; + MLPPUtilities util; util.saveParameters(fileName, weights1, bias1, 0, 1); util.saveParameters(fileName, weights2, bias2, 1, 2); } -real_t MLPPMLP::Cost(std::vector y_hat, std::vector y) { +real_t MLPPMLP::cost(std::vector y_hat, std::vector y) { MLPPReg regularization; class MLPPCost cost; return cost.LogLoss(y_hat, y) + regularization.regTerm(weights2, lambda, alpha, reg) + regularization.regTerm(weights1, lambda, alpha, reg); } -std::vector MLPPMLP::Evaluate(std::vector> X) { +std::vector MLPPMLP::evaluate(std::vector> X) { MLPPLinAlg alg; MLPPActivation avn; std::vector> z2 = alg.mat_vec_add(alg.matmult(X, weights1), bias1); @@ -247,7 +240,7 @@ std::tuple>, std::vector>> M return { z2, a2 }; } -real_t MLPPMLP::Evaluate(std::vector x) { +real_t MLPPMLP::evaluate(std::vector x) { MLPPLinAlg alg; MLPPActivation avn; std::vector z2 = alg.addition(alg.mat_vec_mult(alg.transpose(weights1), x), bias1); @@ -263,7 +256,7 @@ std::tuple, std::vector> MLPPMLP::propagate(std::vec return { z2, a2 }; } -void MLPPMLP::forwardPass() { +void MLPPMLP::forward_pass() { MLPPLinAlg alg; MLPPActivation avn; z2 = alg.mat_vec_add(alg.matmult(inputSet, weights1), bias1); @@ -271,7 +264,21 @@ void MLPPMLP::forwardPass() { y_hat = avn.sigmoid(alg.scalarAdd(bias2, alg.mat_vec_mult(a2, weights2))); } +MLPPMLP::MLPPMLP(std::vector> inputSet, std::vector outputSet, int n_hidden, std::string reg, real_t lambda, real_t alpha) : + inputSet(inputSet), outputSet(outputSet), n_hidden(n_hidden), n(inputSet.size()), k(inputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) { + MLPPActivation avn; + y_hat.resize(n); + weights1 = MLPPUtilities::weightInitialization(k, n_hidden); + weights2 = MLPPUtilities::weightInitialization(n_hidden); + bias1 = MLPPUtilities::biasInitialization(n_hidden); + bias2 = MLPPUtilities::biasInitialization(); +} + +MLPPMLP::MLPPMLP() { +} +MLPPMLP::~MLPPMLP() { +} // ======= OLD ======= @@ -474,12 +481,12 @@ void MLPPMLPOld::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, } real_t MLPPMLPOld::score() { - MLPPUtilities util; + MLPPUtilities util; return util.performance(y_hat, outputSet); } void MLPPMLPOld::save(std::string fileName) { - MLPPUtilities util; + MLPPUtilities util; util.saveParameters(fileName, weights1, bias1, 0, 1); util.saveParameters(fileName, weights2, bias2, 1, 2); } @@ -529,4 +536,3 @@ void MLPPMLPOld::forwardPass() { a2 = avn.sigmoid(z2); y_hat = avn.sigmoid(alg.scalarAdd(bias2, alg.mat_vec_mult(a2, weights2))); } - diff --git a/mlpp/mlp/mlp.h b/mlpp/mlp/mlp.h index 1ae5242..6216d2a 100644 --- a/mlpp/mlp/mlp.h +++ b/mlpp/mlp/mlp.h @@ -8,31 +8,48 @@ // Created by Marc Melikyan on 11/4/20. // +#include "core/containers/vector.h" #include "core/math/math_defs.h" +#include "core/string/ustring.h" +#include "core/variant/variant.h" + +#include "core/object/reference.h" + +#include "../lin_alg/mlpp_matrix.h" +#include "../lin_alg/mlpp_vector.h" #include #include #include -class MLPPMLP { +class MLPPMLP : public Reference { + GDCLASS(MLPPMLP, Reference); + public: - MLPPMLP(std::vector> inputSet, std::vector outputSet, int n_hidden, std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5); - std::vector modelSetTest(std::vector> X); - real_t modelTest(std::vector x); - void gradientDescent(real_t learning_rate, int max_epoch, bool UI = false); - void SGD(real_t learning_rate, int max_epoch, bool UI = false); - void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = false); + std::vector model_set_test(std::vector> X); + real_t model_test(std::vector x); + + void gradient_descent(real_t learning_rate, int max_epoch, bool UI = false); + void sgd(real_t learning_rate, int max_epoch, bool UI = false); + void mbgd(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = false); + real_t score(); void save(std::string fileName); -private: - real_t Cost(std::vector y_hat, std::vector y); + MLPPMLP(std::vector> inputSet, std::vector outputSet, int n_hidden, std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5); - std::vector Evaluate(std::vector> X); + MLPPMLP(); + ~MLPPMLP(); + +private: + real_t cost(std::vector y_hat, std::vector y); + + std::vector evaluate(std::vector> X); std::tuple>, std::vector>> propagate(std::vector> X); - real_t Evaluate(std::vector x); + real_t evaluate(std::vector x); std::tuple, std::vector> propagate(std::vector x); - void forwardPass(); + + void forward_pass(); std::vector> inputSet; std::vector outputSet;