From 0a1d42f627ad7f296ddf38a6d447be3d8f82af4d Mon Sep 17 00:00:00 2001 From: Relintai Date: Thu, 16 Feb 2023 19:15:36 +0100 Subject: [PATCH] Now MLPPDualSVC uses engine classes. --- mlpp/dual_svc/dual_svc.cpp | 158 ++++++++++++++++++++++++------------- mlpp/dual_svc/dual_svc.h | 45 ++++++----- test/mlpp_tests.cpp | 4 +- 3 files changed, 130 insertions(+), 77 deletions(-) diff --git a/mlpp/dual_svc/dual_svc.cpp b/mlpp/dual_svc/dual_svc.cpp index 34a091f..ad20ca2 100644 --- a/mlpp/dual_svc/dual_svc.cpp +++ b/mlpp/dual_svc/dual_svc.cpp @@ -11,14 +11,13 @@ #include "../regularization/reg.h" #include "../utilities/utilities.h" -#include #include -std::vector MLPPDualSVC::model_set_test(std::vector> X) { +Ref MLPPDualSVC::model_set_test(const Ref &X) { return evaluatem(X); } -real_t MLPPDualSVC::model_test(std::vector x) { +real_t MLPPDualSVC::model_test(const Ref &x) { return evaluatev(x); } @@ -32,25 +31,38 @@ void MLPPDualSVC::gradient_descent(real_t learning_rate, int max_epoch, bool ui) forward_pass(); + Ref input_set_i_row_tmp; + input_set_i_row_tmp.instance(); + input_set_i_row_tmp->resize(_input_set->size().x); + + Ref input_set_j_row_tmp; + input_set_j_row_tmp.instance(); + input_set_j_row_tmp->resize(_input_set->size().x); + while (true) { cost_prev = cost(_alpha, _input_set, _output_set); - _alpha = alg.subtraction(_alpha, alg.scalarMultiply(learning_rate, mlpp_cost.dualFormSVMDeriv(_alpha, _input_set, _output_set))); + _alpha = alg.subtractionnv(_alpha, alg.scalar_multiplynv(learning_rate, mlpp_cost.dual_form_svm_deriv(_alpha, _input_set, _output_set))); alpha_projection(); // Calculating the bias real_t biasGradient = 0; - for (uint32_t i = 0; i < _alpha.size(); i++) { + for (int i = 0; i < _alpha->size(); i++) { real_t sum = 0; - if (_alpha[i] < _C && _alpha[i] > 0) { - for (uint32_t j = 0; j < _alpha.size(); j++) { - if (_alpha[j] > 0) { - sum += _alpha[j] * _output_set[j] * alg.dot(_input_set[j], _input_set[i]); // TO DO: DON'T forget to add non-linear kernelizations. + if (_alpha->get_element(i) < _C && _alpha->get_element(i) > 0) { + for (int j = 0; j < _alpha->size(); j++) { + if (_alpha->get_element(j) > 0) { + _input_set->get_row_into_mlpp_vector(i, input_set_i_row_tmp); + _input_set->get_row_into_mlpp_vector(j, input_set_j_row_tmp); + + sum += _alpha->get_element(j) * _output_set->get_element(j) * alg.dotv(input_set_j_row_tmp, input_set_i_row_tmp); // TO DO: DON'T forget to add non-linear kernelizations. } } } - biasGradient = (1 - _output_set[i] * sum) / _output_set[i]; + + biasGradient = (1 - _output_set->get_element(i) * sum) / _output_set->get_element(i); + break; } @@ -60,9 +72,8 @@ void MLPPDualSVC::gradient_descent(real_t learning_rate, int max_epoch, bool ui) // UI PORTION if (ui) { - MLPPUtilities::CostInfo(epoch, cost_prev, cost(_alpha, _input_set, _output_set)); - MLPPUtilities::UI(_alpha, _bias); - std::cout << score() << std::endl; // TO DO: DELETE THIS. + MLPPUtilities::cost_info(epoch, cost_prev, cost(_alpha, _input_set, _output_set)); + MLPPUtilities::print_ui_vb(_alpha, _bias); } epoch++; @@ -148,20 +159,34 @@ void MLPPDualSVC::gradient_descent(real_t learning_rate, int max_epoch, bool ui) real_t MLPPDualSVC::score() { MLPPUtilities util; - return util.performance(_y_hat, _output_set); + return util.performance_vec(_y_hat, _output_set); } -MLPPDualSVC::MLPPDualSVC(std::vector> p_input_set, std::vector p_output_set, real_t p_C, std::string p_kernel) { +void MLPPDualSVC::save(const String &file_name) { + MLPPUtilities util; + + //util.saveParameters(file_name, _alpha, _bias); +} + +MLPPDualSVC::MLPPDualSVC(const Ref &p_input_set, const Ref &p_output_set, real_t p_C, KernelMethod p_kernel) { _input_set = p_input_set; _output_set = p_output_set; - _n = p_input_set.size(); - _k = p_input_set[0].size(); + _n = p_input_set->size().y; + _k = p_input_set->size().x; _C = p_C; _kernel = p_kernel; - _y_hat.resize(_n); - _bias = MLPPUtilities::biasInitialization(); - _alpha = MLPPUtilities::weightInitialization(_n); // One alpha for all training examples, as per the lagrangian multipliers. + _y_hat.instance(); + _y_hat->resize(_n); + + MLPPUtilities utils; + + _bias = utils.bias_initializationr(); + + _alpha.instance(); + _alpha->resize(_n); + + utils.weight_initializationv(_alpha); // One alpha for all training examples, as per the lagrangian multipliers. _K = kernel_functionm(_input_set, _input_set, _kernel); // For now this is unused. When non-linear kernels are added, the K will be manipulated. } @@ -170,51 +195,70 @@ MLPPDualSVC::MLPPDualSVC() { MLPPDualSVC::~MLPPDualSVC() { } -void MLPPDualSVC::save(std::string file_name) { - MLPPUtilities util; - - util.saveParameters(file_name, _alpha, _bias); -} - -real_t MLPPDualSVC::cost(std::vector alpha, std::vector> X, std::vector y) { +real_t MLPPDualSVC::cost(const Ref &alpha, const Ref &X, const Ref &y) { class MLPPCost cost; - return cost.dualFormSVM(alpha, X, y); + + return cost.dual_form_svm(alpha, X, y); } -real_t MLPPDualSVC::evaluatev(std::vector x) { +real_t MLPPDualSVC::evaluatev(const Ref &x) { MLPPActivation avn; return avn.sign(propagatev(x)); } -real_t MLPPDualSVC::propagatev(std::vector x) { +real_t MLPPDualSVC::propagatev(const Ref &x) { MLPPLinAlg alg; real_t z = 0; - for (uint32_t j = 0; j < _alpha.size(); j++) { - if (_alpha[j] != 0) { - z += _alpha[j] * _output_set[j] * alg.dot(_input_set[j], x); // TO DO: DON'T forget to add non-linear kernelizations. + + Ref input_set_row_tmp; + input_set_row_tmp.instance(); + input_set_row_tmp->resize(_input_set->size().x); + + for (int j = 0; j < _alpha->size(); j++) { + if (_alpha->get_element(j) != 0) { + _input_set->get_row_into_mlpp_vector(j, input_set_row_tmp); + z += _alpha->get_element(j) * _output_set->get_element(j) * alg.dotv(input_set_row_tmp, x); // TO DO: DON'T forget to add non-linear kernelizations. } } z += _bias; return z; } -std::vector MLPPDualSVC::evaluatem(std::vector> X) { +Ref MLPPDualSVC::evaluatem(const Ref &X) { MLPPActivation avn; - return avn.sign(propagatem(X)); + + return avn.sign_normv(propagatem(X)); } -std::vector MLPPDualSVC::propagatem(std::vector> X) { +Ref MLPPDualSVC::propagatem(const Ref &X) { MLPPLinAlg alg; - std::vector z; - for (uint32_t i = 0; i < X.size(); i++) { + Ref z; + z.instance(); + z->resize(X->size().y); + + Ref input_set_row_tmp; + input_set_row_tmp.instance(); + input_set_row_tmp->resize(_input_set->size().x); + + Ref x_row_tmp; + x_row_tmp.instance(); + x_row_tmp->resize(X->size().x); + + for (int i = 0; i < X->size().y; i++) { real_t sum = 0; - for (uint32_t j = 0; j < _alpha.size(); j++) { - if (_alpha[j] != 0) { - sum += _alpha[j] * _output_set[j] * alg.dot(_input_set[j], X[i]); // TO DO: DON'T forget to add non-linear kernelizations. + + for (int j = 0; j < _alpha->size(); j++) { + if (_alpha->get_element(j) != 0) { + _input_set->get_row_into_mlpp_vector(j, input_set_row_tmp); + X->get_row_into_mlpp_vector(i, x_row_tmp); + + sum += _alpha->get_element(j) * _output_set->get_element(j) * alg.dotv(input_set_row_tmp, x_row_tmp); // TO DO: DON'T forget to add non-linear kernelizations. } } + sum += _bias; - z.push_back(sum); + + z->set_element(i, sum); } return z; } @@ -223,36 +267,40 @@ void MLPPDualSVC::forward_pass() { MLPPActivation avn; _z = propagatem(_input_set); - _y_hat = avn.sign(_z); + _y_hat = avn.sign_normv(_z); } void MLPPDualSVC::alpha_projection() { - for (uint32_t i = 0; i < _alpha.size(); i++) { - if (_alpha[i] > _C) { - _alpha[i] = _C; - } else if (_alpha[i] < 0) { - _alpha[i] = 0; + for (int i = 0; i < _alpha->size(); i++) { + if (_alpha->get_element(i) > _C) { + _alpha->set_element(i, _C); + } else if (_alpha->get_element(i) < 0) { + _alpha->set_element(i, 0); } } } -real_t MLPPDualSVC::kernel_functionv(std::vector u, std::vector v, std::string kernel) { +real_t MLPPDualSVC::kernel_functionv(const Ref &v, const Ref &u, KernelMethod kernel) { MLPPLinAlg alg; - if (kernel == "Linear") { - return alg.dot(u, v); + if (kernel == KERNEL_METHOD_LINEAR) { + return alg.dotv(u, v); } return 0; } -std::vector> MLPPDualSVC::kernel_functionm(std::vector> A, std::vector> B, std::string kernel) { +Ref MLPPDualSVC::kernel_functionm(const Ref &U, const Ref &V, KernelMethod kernel) { MLPPLinAlg alg; - if (kernel == "Linear") { - return alg.matmult(_input_set, alg.transpose(_input_set)); + + if (kernel == KERNEL_METHOD_LINEAR) { + return alg.matmultm(_input_set, alg.transposem(_input_set)); } - return std::vector>(); + Ref m; + m.instance(); + + return m; } void MLPPDualSVC::_bind_methods() { diff --git a/mlpp/dual_svc/dual_svc.h b/mlpp/dual_svc/dual_svc.h index 02ea5e8..2ff6427 100644 --- a/mlpp/dual_svc/dual_svc.h +++ b/mlpp/dual_svc/dual_svc.h @@ -15,24 +15,29 @@ #include "core/object/reference.h" -#include -#include +#include "../lin_alg/mlpp_matrix.h" +#include "../lin_alg/mlpp_vector.h" class MLPPDualSVC : public Reference { GDCLASS(MLPPDualSVC, Reference); public: - std::vector model_set_test(std::vector> X); - real_t model_test(std::vector x); + enum KernelMethod { + KERNEL_METHOD_LINEAR = 0, + }; + +public: + Ref model_set_test(const Ref &X); + real_t model_test(const Ref &x); void gradient_descent(real_t learning_rate, int max_epoch, bool ui = false); //void SGD(real_t learning_rate, int max_epoch, bool ui = false); //void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool ui = false); real_t score(); - void save(std::string file_name); + void save(const String &file_name); - MLPPDualSVC(std::vector> p_input_set, std::vector p_output_set, real_t p_C, std::string p_kernel = "Linear"); + MLPPDualSVC(const Ref &p_input_set, const Ref &p_output_set, real_t p_C, KernelMethod p_kernel = KERNEL_METHOD_LINEAR); MLPPDualSVC(); ~MLPPDualSVC(); @@ -40,37 +45,37 @@ public: protected: void init(); - real_t cost(std::vector alpha, std::vector> X, std::vector y); + real_t cost(const Ref &alpha, const Ref &X, const Ref &y); - real_t evaluatev(std::vector x); - real_t propagatev(std::vector x); + real_t evaluatev(const Ref &x); + real_t propagatev(const Ref &x); - std::vector evaluatem(std::vector> X); - std::vector propagatem(std::vector> X); + Ref evaluatem(const Ref &X); + Ref propagatem(const Ref &X); void forward_pass(); void alpha_projection(); - real_t kernel_functionv(std::vector v, std::vector u, std::string kernel); - std::vector> kernel_functionm(std::vector> U, std::vector> V, std::string kernel); + real_t kernel_functionv(const Ref &v, const Ref &u, KernelMethod kernel); + Ref kernel_functionm(const Ref &U, const Ref &V, KernelMethod kernel); static void _bind_methods(); - std::vector> _input_set; - std::vector _output_set; - std::vector _z; - std::vector _y_hat; + Ref _input_set; + Ref _output_set; + Ref _z; + Ref _y_hat; real_t _bias; - std::vector _alpha; - std::vector> _K; + Ref _alpha; + Ref _K; real_t _C; int _n; int _k; - std::string _kernel; + KernelMethod _kernel; real_t _p; // Poly real_t _c; // Poly }; diff --git a/test/mlpp_tests.cpp b/test/mlpp_tests.cpp index 3ea5fa7..8068204 100644 --- a/test/mlpp_tests.cpp +++ b/test/mlpp_tests.cpp @@ -1247,9 +1247,9 @@ void MLPPTests::test_support_vector_classification_kernel(bool ui) { kernelSVMOld.gradientDescent(0.0001, 20, ui); std::cout << "SCORE: " << kernelSVMOld.score() << std::endl; - MLPPDualSVC kernelSVM(dt->get_input()->to_std_vector(), dt->get_output()->to_std_vector(), 1000); + MLPPDualSVC kernelSVM(dt->get_input(), dt->get_output(), 1000); kernelSVM.gradient_descent(0.0001, 20, ui); - std::cout << "SCORE: " << kernelSVM.score() << std::endl; + PLOG_MSG("SCORE: " + String::num(kernelSVM.score())); std::vector> linearlyIndependentMat = { { 1, 2, 3, 4 },