Relintai/pmlpp
1
0
Fork 0
mirror of https://github.com/Relintai/pmlpp.git synced 2025-01-02 16:29:35 +01:00
Code Issues Packages Projects Releases Wiki Activity

Initial cleanup pass on MLPPDualSVC.

Browse Source
This commit is contained in:
Relintai 2023-02-12 12:36:52 +01:00
parent 9a529c572d
commit cdb0e47d16
3 changed files with 128 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

166
mlpp/dual_svc/dual_svc.cpp
View File

@ -14,68 +14,57 @@
#include <iostream> #include <iostream>
#include <random> #include <random>
MLPPDualSVC::MLPPDualSVC(std::vector<std::vector<real_t>> p_inputSet, std::vector<real_t> p_outputSet, real_t p_C, std::string p_kernel) { std::vector<real_t> MLPPDualSVC::model_set_test(std::vector<std::vector<real_t>> X) {
inputSet = p_inputSet; return evaluatem(X);
outputSet = p_outputSet;
n = p_inputSet.size();
k = p_inputSet[0].size();
C = p_C;
kernel = p_kernel;
y_hat.resize(n);
bias = MLPPUtilities::biasInitialization();
alpha = MLPPUtilities::weightInitialization(n); // One alpha for all training examples, as per the lagrangian multipliers.
K = kernelFunction(inputSet, inputSet, kernel); // For now this is unused. When non-linear kernels are added, the K will be manipulated.
} }
std::vector<real_t> MLPPDualSVC::modelSetTest(std::vector<std::vector<real_t>> X) { real_t MLPPDualSVC::model_test(std::vector<real_t> x) {
return Evaluate(X); return evaluatev(x);
} }
real_t MLPPDualSVC::modelTest(std::vector<real_t> x) { void MLPPDualSVC::gradient_descent(real_t learning_rate, int max_epoch, bool ui) {
return Evaluate(x); MLPPCost mlpp_cost;
}
void MLPPDualSVC::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
class MLPPCost cost;
MLPPActivation avn; MLPPActivation avn;
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
real_t cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
forwardPass();
forward_pass();
while (true) { while (true) {
cost_prev = Cost(alpha, inputSet, outputSet); cost_prev = cost(_alpha, _input_set, _output_set);
alpha = alg.subtraction(alpha, alg.scalarMultiply(learning_rate, cost.dualFormSVMDeriv(alpha, inputSet, outputSet))); _alpha = alg.subtraction(_alpha, alg.scalarMultiply(learning_rate, mlpp_cost.dualFormSVMDeriv(_alpha, _input_set, _output_set)));
alphaProjection(); alpha_projection();
// Calculating the bias // Calculating the bias
real_t biasGradient = 0; real_t biasGradient = 0;
for (uint32_t i = 0; i < alpha.size(); i++) { for (uint32_t i = 0; i < _alpha.size(); i++) {
real_t sum = 0; real_t sum = 0;
if (alpha[i] < C && alpha[i] > 0) { if (_alpha[i] < _C && _alpha[i] > 0) {
for (uint32_t j = 0; j < alpha.size(); j++) { for (uint32_t j = 0; j < _alpha.size(); j++) {
if (alpha[j] > 0) { if (_alpha[j] > 0) {
sum += alpha[j] * outputSet[j] * alg.dot(inputSet[j], inputSet[i]); // TO DO: DON'T forget to add non-linear kernelizations. sum += _alpha[j] * _output_set[j] * alg.dot(_input_set[j], _input_set[i]); // TO DO: DON'T forget to add non-linear kernelizations.
} }
} }
} }
biasGradient = (1 - outputSet[i] * sum) / outputSet[i]; biasGradient = (1 - _output_set[i] * sum) / _output_set[i];
break; break;
} }
bias -= biasGradient * learning_rate;
forwardPass(); _bias -= biasGradient * learning_rate;
forward_pass();
// UI PORTION // UI PORTION
if (UI) { if (ui) {
MLPPUtilities::CostInfo(epoch, cost_prev, Cost(alpha, inputSet, outputSet)); MLPPUtilities::CostInfo(epoch, cost_prev, cost(_alpha, _input_set, _output_set));
MLPPUtilities::UI(alpha, bias); MLPPUtilities::UI(_alpha, _bias);
std::cout << score() << std::endl; // TO DO: DELETE THIS. std::cout << score() << std::endl; // TO DO: DELETE THIS.
} }
epoch++; epoch++;
if (epoch > max_epoch) { if (epoch > max_epoch) {
@ -99,12 +88,12 @@ void MLPPDualSVC::gradientDescent(real_t learning_rate, int max_epoch, bool UI)
// std::uniform_int_distribution<int> distribution(0, int(n - 1)); // std::uniform_int_distribution<int> distribution(0, int(n - 1));
// int outputIndex = distribution(generator); // int outputIndex = distribution(generator);
// cost_prev = Cost(alpha, inputSet[outputIndex], outputSet[outputIndex]); // cost_prev = Cost(alpha, _input_set[outputIndex], _output_set[outputIndex]);
// // Bias updation // // Bias updation
// bias -= learning_rate * costDeriv; // bias -= learning_rate * costDeriv;
// y_hat = Evaluate({inputSet[outputIndex]}); // y_hat = Evaluate({_input_set[outputIndex]});
// if(UI) { // if(UI) {
// MLPPUtilities::CostInfo(epoch, cost_prev, Cost(alpha)); // MLPPUtilities::CostInfo(epoch, cost_prev, Cost(alpha));
@ -127,7 +116,7 @@ void MLPPDualSVC::gradientDescent(real_t learning_rate, int max_epoch, bool UI)
// // Creating the mini-batches // // Creating the mini-batches
// int n_mini_batch = n/mini_batch_size; // int n_mini_batch = n/mini_batch_size;
// auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch); // auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(_input_set, _output_set, n_mini_batch);
// while(true){ // while(true){
// for(int i = 0; i < n_mini_batch; i++){ // for(int i = 0; i < n_mini_batch; i++){
@ -159,76 +148,97 @@ void MLPPDualSVC::gradientDescent(real_t learning_rate, int max_epoch, bool UI)
real_t MLPPDualSVC::score() { real_t MLPPDualSVC::score() {
MLPPUtilities util; MLPPUtilities util;
return util.performance(y_hat, outputSet); return util.performance(_y_hat, _output_set);
} }
void MLPPDualSVC::save(std::string fileName) { MLPPDualSVC::MLPPDualSVC(std::vector<std::vector<real_t>> p_input_set, std::vector<real_t> p_output_set, real_t p_C, std::string p_kernel) {
_input_set = p_input_set;
_output_set = p_output_set;
_n = p_input_set.size();
_k = p_input_set[0].size();
_C = p_C;
_kernel = p_kernel;
_y_hat.resize(_n);
_bias = MLPPUtilities::biasInitialization();
_alpha = MLPPUtilities::weightInitialization(_n); // One alpha for all training examples, as per the lagrangian multipliers.
_K = kernel_functionm(_input_set, _input_set, _kernel); // For now this is unused. When non-linear kernels are added, the K will be manipulated.
}
MLPPDualSVC::MLPPDualSVC() {
}
MLPPDualSVC::~MLPPDualSVC() {
}
void MLPPDualSVC::save(std::string file_name) {
MLPPUtilities util; MLPPUtilities util;
util.saveParameters(fileName, alpha, bias);
util.saveParameters(file_name, _alpha, _bias);
} }
real_t MLPPDualSVC::Cost(std::vector<real_t> alpha, std::vector<std::vector<real_t>> X, std::vector<real_t> y) { real_t MLPPDualSVC::cost(std::vector<real_t> alpha, std::vector<std::vector<real_t>> X, std::vector<real_t> y) {
class MLPPCost cost; class MLPPCost cost;
return cost.dualFormSVM(alpha, X, y); return cost.dualFormSVM(alpha, X, y);
} }
std::vector<real_t> MLPPDualSVC::Evaluate(std::vector<std::vector<real_t>> X) { real_t MLPPDualSVC::evaluatev(std::vector<real_t> x) {
MLPPActivation avn; MLPPActivation avn;
return avn.sign(propagate(X)); return avn.sign(propagatev(x));
} }
std::vector<real_t> MLPPDualSVC::propagate(std::vector<std::vector<real_t>> X) { real_t MLPPDualSVC::propagatev(std::vector<real_t> x) {
MLPPLinAlg alg;
real_t z = 0;
for (uint32_t j = 0; j < _alpha.size(); j++) {
if (_alpha[j] != 0) {
z += _alpha[j] * _output_set[j] * alg.dot(_input_set[j], x); // TO DO: DON'T forget to add non-linear kernelizations.
}
}
z += _bias;
return z;
}
std::vector<real_t> MLPPDualSVC::evaluatem(std::vector<std::vector<real_t>> X) {
MLPPActivation avn;
return avn.sign(propagatem(X));
}
std::vector<real_t> MLPPDualSVC::propagatem(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg; MLPPLinAlg alg;
std::vector<real_t> z; std::vector<real_t> z;
for (uint32_t i = 0; i < X.size(); i++) { for (uint32_t i = 0; i < X.size(); i++) {
real_t sum = 0; real_t sum = 0;
for (uint32_t j = 0; j < alpha.size(); j++) { for (uint32_t j = 0; j < _alpha.size(); j++) {
if (alpha[j] != 0) { if (_alpha[j] != 0) {
sum += alpha[j] * outputSet[j] * alg.dot(inputSet[j], X[i]); // TO DO: DON'T forget to add non-linear kernelizations. sum += _alpha[j] * _output_set[j] * alg.dot(_input_set[j], X[i]); // TO DO: DON'T forget to add non-linear kernelizations.
} }
} }
sum += bias; sum += _bias;
z.push_back(sum); z.push_back(sum);
} }
return z; return z;
} }
real_t MLPPDualSVC::Evaluate(std::vector<real_t> x) { void MLPPDualSVC::forward_pass() {
MLPPActivation avn;
return avn.sign(propagate(x));
}
real_t MLPPDualSVC::propagate(std::vector<real_t> x) {
MLPPLinAlg alg;
real_t z = 0;
for (uint32_t j = 0; j < alpha.size(); j++) {
if (alpha[j] != 0) {
z += alpha[j] * outputSet[j] * alg.dot(inputSet[j], x); // TO DO: DON'T forget to add non-linear kernelizations.
}
}
z += bias;
return z;
}
void MLPPDualSVC::forwardPass() {
MLPPActivation avn; MLPPActivation avn;
z = propagate(inputSet); _z = propagatem(_input_set);
y_hat = avn.sign(z); _y_hat = avn.sign(_z);
} }
void MLPPDualSVC::alphaProjection() { void MLPPDualSVC::alpha_projection() {
for (uint32_t i = 0; i < alpha.size(); i++) { for (uint32_t i = 0; i < _alpha.size(); i++) {
if (alpha[i] > C) { if (_alpha[i] > _C) {
alpha[i] = C; _alpha[i] = _C;
} else if (alpha[i] < 0) { } else if (_alpha[i] < 0) {
alpha[i] = 0; _alpha[i] = 0;
} }
} }
} }
real_t MLPPDualSVC::kernelFunction(std::vector<real_t> u, std::vector<real_t> v, std::string kernel) { real_t MLPPDualSVC::kernel_functionv(std::vector<real_t> u, std::vector<real_t> v, std::string kernel) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (kernel == "Linear") { if (kernel == "Linear") {
return alg.dot(u, v); return alg.dot(u, v);
} }
@ -236,10 +246,10 @@ real_t MLPPDualSVC::kernelFunction(std::vector<real_t> u, std::vector<real_t> v,
return 0; return 0;
} }
std::vector<std::vector<real_t>> MLPPDualSVC::kernelFunction(std::vector<std::vector<real_t>> A, std::vector<std::vector<real_t>> B, std::string kernel) { std::vector<std::vector<real_t>> MLPPDualSVC::kernel_functionm(std::vector<std::vector<real_t>> A, std::vector<std::vector<real_t>> B, std::string kernel) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (kernel == "Linear") { if (kernel == "Linear") {
return alg.matmult(inputSet, alg.transpose(inputSet)); return alg.matmult(_input_set, alg.transpose(_input_set));
} }
return std::vector<std::vector<real_t>>(); return std::vector<std::vector<real_t>>();

67
mlpp/dual_svc/dual_svc.h
View File

@ -18,52 +18,55 @@
class MLPPDualSVC { class MLPPDualSVC {
public: public:
MLPPDualSVC(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, real_t C, std::string kernel = "Linear"); std::vector<real_t> model_set_test(std::vector<std::vector<real_t>> X);
MLPPDualSVC(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, real_t C, std::string kernel, real_t p, real_t c); real_t model_test(std::vector<real_t> x);
void gradient_descent(real_t learning_rate, int max_epoch, bool ui = false);
//void SGD(real_t learning_rate, int max_epoch, bool ui = false);
//void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool ui = false);
std::vector<real_t> modelSetTest(std::vector<std::vector<real_t>> X);
real_t modelTest(std::vector<real_t> x);
void gradientDescent(real_t learning_rate, int max_epoch, bool UI = false);
void SGD(real_t learning_rate, int max_epoch, bool UI = false);
void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = false);
real_t score(); real_t score();
void save(std::string fileName); void save(std::string file_name);
MLPPDualSVC(std::vector<std::vector<real_t>> p_input_set, std::vector<real_t> p_output_set, real_t p_C, std::string p_kernel = "Linear");
MLPPDualSVC();
~MLPPDualSVC();
private: private:
void init(); void init();
real_t Cost(std::vector<real_t> alpha, std::vector<std::vector<real_t>> X, std::vector<real_t> y); real_t cost(std::vector<real_t> alpha, std::vector<std::vector<real_t>> X, std::vector<real_t> y);
std::vector<real_t> Evaluate(std::vector<std::vector<real_t>> X); real_t evaluatev(std::vector<real_t> x);
std::vector<real_t> propagate(std::vector<std::vector<real_t>> X); real_t propagatev(std::vector<real_t> x);
real_t Evaluate(std::vector<real_t> x);
real_t propagate(std::vector<real_t> x);
void forwardPass();
void alphaProjection(); std::vector<real_t> evaluatem(std::vector<std::vector<real_t>> X);
std::vector<real_t> propagatem(std::vector<std::vector<real_t>> X);
real_t kernelFunction(std::vector<real_t> v, std::vector<real_t> u, std::string kernel); void forward_pass();
std::vector<std::vector<real_t>> kernelFunction(std::vector<std::vector<real_t>> U, std::vector<std::vector<real_t>> V, std::string kernel);
std::vector<std::vector<real_t>> inputSet; void alpha_projection();
std::vector<real_t> outputSet;
std::vector<real_t> z;
std::vector<real_t> y_hat;
real_t bias;
std::vector<real_t> alpha; real_t kernel_functionv(std::vector<real_t> v, std::vector<real_t> u, std::string kernel);
std::vector<std::vector<real_t>> K; std::vector<std::vector<real_t>> kernel_functionm(std::vector<std::vector<real_t>> U, std::vector<std::vector<real_t>> V, std::string kernel);
real_t C; std::vector<std::vector<real_t>> _input_set;
int n; std::vector<real_t> _output_set;
int k; std::vector<real_t> _z;
std::vector<real_t> _y_hat;
real_t _bias;
std::string kernel; std::vector<real_t> _alpha;
real_t p; // Poly std::vector<std::vector<real_t>> _K;
real_t c; // Poly
// UI Portion real_t _C;
void UI(int epoch, real_t cost_prev); int _n;
int _k;
std::string _kernel;
real_t _p; // Poly
real_t _c; // Poly
}; };
#endif /* DualSVC_hpp */ #endif /* DualSVC_hpp */

6
test/mlpp_tests.cpp
View File

@ -1193,8 +1193,12 @@ void MLPPTests::test_support_vector_classification_kernel(bool ui) {
//SUPPORT VECTOR CLASSIFICATION (kernel method) //SUPPORT VECTOR CLASSIFICATION (kernel method)
Ref<MLPPDataSimple> dt = data.load_breast_cancer_svc(_breast_cancer_svm_data_path); Ref<MLPPDataSimple> dt = data.load_breast_cancer_svc(_breast_cancer_svm_data_path);
MLPPDualSVCOld kernelSVMOld(dt->get_input()->to_std_vector(), dt->get_output()->to_std_vector(), 1000);
kernelSVMOld.gradientDescent(0.0001, 20, ui);
std::cout << "SCORE: " << kernelSVMOld.score() << std::endl;
MLPPDualSVC kernelSVM(dt->get_input()->to_std_vector(), dt->get_output()->to_std_vector(), 1000); MLPPDualSVC kernelSVM(dt->get_input()->to_std_vector(), dt->get_output()->to_std_vector(), 1000);
kernelSVM.gradientDescent(0.0001, 20, ui); kernelSVM.gradient_descent(0.0001, 20, ui);
std::cout << "SCORE: " << kernelSVM.score() << std::endl; std::cout << "SCORE: " << kernelSVM.score() << std::endl;
std::vector<std::vector<real_t>> linearlyIndependentMat = { std::vector<std::vector<real_t>> linearlyIndependentMat = {