mirror of https://github.com/Relintai/pmlpp.git
326 lines
10 KiB
C++
326 lines
10 KiB
C++
/*************************************************************************/
|
|
/* dual_svc.cpp */
|
|
/*************************************************************************/
|
|
/* This file is part of: */
|
|
/* PMLPP Machine Learning Library */
|
|
/* https://github.com/Relintai/pmlpp */
|
|
/*************************************************************************/
|
|
/* Copyright (c) 2023-present Péter Magyar. */
|
|
/* Copyright (c) 2022-2023 Marc Melikyan */
|
|
/* */
|
|
/* Permission is hereby granted, free of charge, to any person obtaining */
|
|
/* a copy of this software and associated documentation files (the */
|
|
/* "Software"), to deal in the Software without restriction, including */
|
|
/* without limitation the rights to use, copy, modify, merge, publish, */
|
|
/* distribute, sublicense, and/or sell copies of the Software, and to */
|
|
/* permit persons to whom the Software is furnished to do so, subject to */
|
|
/* the following conditions: */
|
|
/* */
|
|
/* The above copyright notice and this permission notice shall be */
|
|
/* included in all copies or substantial portions of the Software. */
|
|
/* */
|
|
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
|
|
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
|
|
/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
|
|
/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
|
|
/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
|
|
/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
|
|
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
|
|
/*************************************************************************/
|
|
|
|
#include "dual_svc.h"
|
|
#include "../activation/activation.h"
|
|
#include "../cost/cost.h"
|
|
#include "../regularization/reg.h"
|
|
#include "../utilities/utilities.h"
|
|
|
|
#include <random>
|
|
|
|
Ref<MLPPVector> MLPPDualSVC::model_set_test(const Ref<MLPPMatrix> &X) {
|
|
return evaluatem(X);
|
|
}
|
|
|
|
real_t MLPPDualSVC::model_test(const Ref<MLPPVector> &x) {
|
|
return evaluatev(x);
|
|
}
|
|
|
|
void MLPPDualSVC::gradient_descent(real_t learning_rate, int max_epoch, bool ui) {
|
|
MLPPCost mlpp_cost;
|
|
MLPPActivation avn;
|
|
MLPPReg regularization;
|
|
real_t cost_prev = 0;
|
|
int epoch = 1;
|
|
|
|
forward_pass();
|
|
|
|
Ref<MLPPVector> input_set_i_row_tmp;
|
|
input_set_i_row_tmp.instance();
|
|
input_set_i_row_tmp->resize(_input_set->size().x);
|
|
|
|
Ref<MLPPVector> input_set_j_row_tmp;
|
|
input_set_j_row_tmp.instance();
|
|
input_set_j_row_tmp->resize(_input_set->size().x);
|
|
|
|
while (true) {
|
|
cost_prev = cost(_alpha, _input_set, _output_set);
|
|
|
|
_alpha->sub(mlpp_cost.dual_form_svm_deriv(_alpha, _input_set, _output_set)->scalar_multiplyn(learning_rate));
|
|
|
|
alpha_projection();
|
|
|
|
// Calculating the bias
|
|
real_t biasGradient = 0;
|
|
for (int i = 0; i < _alpha->size(); i++) {
|
|
real_t sum = 0;
|
|
if (_alpha->element_get(i) < _C && _alpha->element_get(i) > 0) {
|
|
for (int j = 0; j < _alpha->size(); j++) {
|
|
if (_alpha->element_get(j) > 0) {
|
|
_input_set->row_get_into_mlpp_vector(i, input_set_i_row_tmp);
|
|
_input_set->row_get_into_mlpp_vector(j, input_set_j_row_tmp);
|
|
|
|
sum += _alpha->element_get(j) * _output_set->element_get(j) * input_set_j_row_tmp->dot(input_set_i_row_tmp); // TO DO: DON'T forget to add non-linear kernelizations.
|
|
}
|
|
}
|
|
}
|
|
|
|
biasGradient = (1 - _output_set->element_get(i) * sum) / _output_set->element_get(i);
|
|
|
|
break;
|
|
}
|
|
|
|
_bias -= biasGradient * learning_rate;
|
|
|
|
forward_pass();
|
|
|
|
// UI PORTION
|
|
if (ui) {
|
|
MLPPUtilities::cost_info(epoch, cost_prev, cost(_alpha, _input_set, _output_set));
|
|
MLPPUtilities::print_ui_vb(_alpha, _bias);
|
|
}
|
|
|
|
epoch++;
|
|
|
|
if (epoch > max_epoch) {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
// void MLPPDualSVC::SGD(real_t learning_rate, int max_epoch, bool UI){
|
|
// class MLPPCost cost;
|
|
// MLPPActivation avn;
|
|
// MLPPLinAlg alg;
|
|
// MLPPReg regularization;
|
|
|
|
// real_t cost_prev = 0;
|
|
// int epoch = 1;
|
|
|
|
// while(true){
|
|
// std::random_device rd;
|
|
// std::default_random_engine generator(rd());
|
|
// std::uniform_int_distribution<int> distribution(0, int(n - 1));
|
|
// int outputIndex = distribution(generator);
|
|
|
|
// cost_prev = Cost(alpha, _input_set[outputIndex], _output_set[outputIndex]);
|
|
|
|
// // Bias updation
|
|
// bias -= learning_rate * costDeriv;
|
|
|
|
// y_hat = Evaluate({_input_set[outputIndex]});
|
|
|
|
// if(UI) {
|
|
// MLPPUtilities::CostInfo(epoch, cost_prev, Cost(alpha));
|
|
// MLPPUtilities::UI(weights, bias);
|
|
// }
|
|
// epoch++;
|
|
|
|
// if(epoch > max_epoch) { break; }
|
|
// }
|
|
// forwardPass();
|
|
// }
|
|
|
|
// void MLPPDualSVC::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI){
|
|
// class MLPPCost cost;
|
|
// MLPPActivation avn;
|
|
// MLPPLinAlg alg;
|
|
// MLPPReg regularization;
|
|
// real_t cost_prev = 0;
|
|
// int epoch = 1;
|
|
|
|
// // Creating the mini-batches
|
|
// int n_mini_batch = n/mini_batch_size;
|
|
// auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(_input_set, _output_set, n_mini_batch);
|
|
|
|
// while(true){
|
|
// for(int i = 0; i < n_mini_batch; i++){
|
|
// std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]);
|
|
// std::vector<real_t> z = propagate(inputMiniBatches[i]);
|
|
// cost_prev = Cost(z, outputMiniBatches[i], weights, C);
|
|
|
|
// // Calculating the weight gradients
|
|
// weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate/n, alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), cost.HingeLossDeriv(z, outputMiniBatches[i], C))));
|
|
// weights = regularization.regWeights(weights, learning_rate/n, 0, "Ridge");
|
|
|
|
// // Calculating the bias gradients
|
|
// bias -= learning_rate * alg.sum_elements(cost.HingeLossDeriv(y_hat, outputMiniBatches[i], C)) / n;
|
|
|
|
// forwardPass();
|
|
|
|
// y_hat = Evaluate(inputMiniBatches[i]);
|
|
|
|
// if(UI) {
|
|
// MLPPUtilities::CostInfo(epoch, cost_prev, Cost(z, outputMiniBatches[i], weights, C));
|
|
// MLPPUtilities::UI(weights, bias);
|
|
// }
|
|
// }
|
|
// epoch++;
|
|
// if(epoch > max_epoch) { break; }
|
|
// }
|
|
// forwardPass();
|
|
// }
|
|
|
|
real_t MLPPDualSVC::score() {
|
|
MLPPUtilities util;
|
|
return util.performance_vec(_y_hat, _output_set);
|
|
}
|
|
|
|
void MLPPDualSVC::save(const String &file_name) {
|
|
MLPPUtilities util;
|
|
|
|
//util.saveParameters(file_name, _alpha, _bias);
|
|
}
|
|
|
|
MLPPDualSVC::MLPPDualSVC(const Ref<MLPPMatrix> &p_input_set, const Ref<MLPPVector> &p_output_set, real_t p_C, KernelMethod p_kernel) {
|
|
_input_set = p_input_set;
|
|
_output_set = p_output_set;
|
|
_n = p_input_set->size().y;
|
|
_k = p_input_set->size().x;
|
|
_C = p_C;
|
|
_kernel = p_kernel;
|
|
|
|
_z.instance();
|
|
_y_hat.instance();
|
|
_alpha.instance();
|
|
|
|
_y_hat->resize(_n);
|
|
|
|
MLPPUtilities utils;
|
|
|
|
_bias = utils.bias_initializationr();
|
|
|
|
_alpha->resize(_n);
|
|
|
|
utils.weight_initializationv(_alpha); // One alpha for all training examples, as per the lagrangian multipliers.
|
|
_K = kernel_functionm(_input_set, _input_set, _kernel); // For now this is unused. When non-linear kernels are added, the K will be manipulated.
|
|
}
|
|
|
|
MLPPDualSVC::MLPPDualSVC() {
|
|
}
|
|
MLPPDualSVC::~MLPPDualSVC() {
|
|
}
|
|
|
|
real_t MLPPDualSVC::cost(const Ref<MLPPVector> &alpha, const Ref<MLPPMatrix> &X, const Ref<MLPPVector> &y) {
|
|
class MLPPCost cost;
|
|
|
|
return cost.dual_form_svm(alpha, X, y);
|
|
}
|
|
|
|
real_t MLPPDualSVC::evaluatev(const Ref<MLPPVector> &x) {
|
|
MLPPActivation avn;
|
|
return avn.sign_normr(propagatev(x));
|
|
}
|
|
|
|
real_t MLPPDualSVC::propagatev(const Ref<MLPPVector> &x) {
|
|
real_t z = 0;
|
|
|
|
Ref<MLPPVector> input_set_row_tmp;
|
|
input_set_row_tmp.instance();
|
|
input_set_row_tmp->resize(_input_set->size().x);
|
|
|
|
for (int j = 0; j < _alpha->size(); j++) {
|
|
if (_alpha->element_get(j) != 0) {
|
|
_input_set->row_get_into_mlpp_vector(j, input_set_row_tmp);
|
|
z += _alpha->element_get(j) * _output_set->element_get(j) * input_set_row_tmp->dot(x); // TO DO: DON'T forget to add non-linear kernelizations.
|
|
}
|
|
}
|
|
z += _bias;
|
|
return z;
|
|
}
|
|
|
|
Ref<MLPPVector> MLPPDualSVC::evaluatem(const Ref<MLPPMatrix> &X) {
|
|
MLPPActivation avn;
|
|
|
|
return avn.sign_normv(propagatem(X));
|
|
}
|
|
|
|
Ref<MLPPVector> MLPPDualSVC::propagatem(const Ref<MLPPMatrix> &X) {
|
|
Ref<MLPPVector> z;
|
|
z.instance();
|
|
z->resize(X->size().y);
|
|
|
|
Ref<MLPPVector> input_set_row_tmp;
|
|
input_set_row_tmp.instance();
|
|
input_set_row_tmp->resize(_input_set->size().x);
|
|
|
|
Ref<MLPPVector> x_row_tmp;
|
|
x_row_tmp.instance();
|
|
x_row_tmp->resize(X->size().x);
|
|
|
|
for (int i = 0; i < X->size().y; i++) {
|
|
real_t sum = 0;
|
|
|
|
for (int j = 0; j < _alpha->size(); j++) {
|
|
if (_alpha->element_get(j) != 0) {
|
|
_input_set->row_get_into_mlpp_vector(j, input_set_row_tmp);
|
|
X->row_get_into_mlpp_vector(i, x_row_tmp);
|
|
|
|
sum += _alpha->element_get(j) * _output_set->element_get(j) * input_set_row_tmp->dot(x_row_tmp); // TO DO: DON'T forget to add non-linear kernelizations.
|
|
}
|
|
}
|
|
|
|
sum += _bias;
|
|
|
|
z->element_set(i, sum);
|
|
}
|
|
return z;
|
|
}
|
|
|
|
void MLPPDualSVC::forward_pass() {
|
|
MLPPActivation avn;
|
|
|
|
_z = propagatem(_input_set);
|
|
_y_hat = avn.sign_normv(_z);
|
|
}
|
|
|
|
void MLPPDualSVC::alpha_projection() {
|
|
for (int i = 0; i < _alpha->size(); i++) {
|
|
if (_alpha->element_get(i) > _C) {
|
|
_alpha->element_set(i, _C);
|
|
} else if (_alpha->element_get(i) < 0) {
|
|
_alpha->element_set(i, 0);
|
|
}
|
|
}
|
|
}
|
|
|
|
real_t MLPPDualSVC::kernel_functionv(const Ref<MLPPVector> &v, const Ref<MLPPVector> &u, KernelMethod kernel) {
|
|
if (kernel == KERNEL_METHOD_LINEAR) {
|
|
return u->dot(v);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
Ref<MLPPMatrix> MLPPDualSVC::kernel_functionm(const Ref<MLPPMatrix> &U, const Ref<MLPPMatrix> &V, KernelMethod kernel) {
|
|
if (kernel == KERNEL_METHOD_LINEAR) {
|
|
return _input_set->multn(_input_set->transposen());
|
|
}
|
|
|
|
Ref<MLPPMatrix> m;
|
|
m.instance();
|
|
|
|
return m;
|
|
}
|
|
|
|
void MLPPDualSVC::_bind_methods() {
|
|
}
|