Relintai/pmlpp
1
0
Fork 0
mirror of https://github.com/Relintai/pmlpp.git synced 2025-04-03 02:12:44 +02:00
Code Issues Packages Projects Releases Wiki Activity

Use real_t instead of doubles.

Browse Source
This commit is contained in:
Relintai 2023-01-27 13:01:16 +01:00
parent 7a291b451a
commit 4deb34c852
84 changed files with 2663 additions and 2579 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README_ORIG.md
View File

@ -30,7 +30,7 @@ g++ main.cpp /usr/local/lib/MLPP.so --std=c++17
``` ```
## Usage ## Usage
Please note that ML++ uses the ```std::vector<double>``` data type for emulating vectors, and the ```std::vector<std::vector<double>>``` data type for emulating matrices. Please note that ML++ uses the ```std::vector<real_t>``` data type for emulating vectors, and the ```std::vector<std::vector<real_t>>``` data type for emulating matrices.
Begin by including the respective header file of your choice. Begin by including the respective header file of your choice.
```cpp ```cpp

162
main.cpp
View File

@ -52,19 +52,19 @@
#include "MLPP/Transforms/Transforms.hpp" #include "MLPP/Transforms/Transforms.hpp"
// double f(double x){ // real_t f(real_t x){
// return x*x*x + 2*x - 2; // return x*x*x + 2*x - 2;
// } // }
double f(double x){ real_t f(real_t x){
return sin(x); return sin(x);
} }
double f_prime(double x){ real_t f_prime(real_t x){
return 2 * x; return 2 * x;
} }
double f_prime_2var(std::vector<double> x){ real_t f_prime_2var(std::vector<real_t> x){
return 2 * x[0] + x[1]; return 2 * x[0] + x[1];
} }
/* /*
@ -74,7 +74,7 @@ double f_prime_2var(std::vector<double> x){
y''(2) = 12 y''(2) = 12
*/ */
// double f_mv(std::vector<double> x){ // real_t f_mv(std::vector<real_t> x){
// return x[0] * x[0] + x[0] * x[1] * x[1] + x[1] + 5; // return x[0] * x[0] + x[0] * x[1] * x[1] + x[1] + 5;
// } // }
@ -85,7 +85,7 @@ double f_prime_2var(std::vector<double> x){
∂^2f/∂x∂y = 2 ∂^2f/∂x∂y = 2
*/ */
double f_mv(std::vector<double> x){ real_t f_mv(std::vector<real_t> x){
return x[0] * x[0] * x[0] + x[0] + x[1] * x[1] * x[1] * x[0] + x[2] * x[2] * x[1]; return x[0] * x[0] * x[0] + x[0] + x[1] * x[1] * x[1] * x[0] + x[2] * x[2] * x[1];
} }
@ -128,30 +128,30 @@ int main() {
MLPPConvolutions conv; MLPPConvolutions conv;
// DATA SETS // DATA SETS
// std::vector<std::vector<double>> inputSet = {{1,2,3,4,5,6,7,8,9,10}, {3,5,9,12,15,18,21,24,27,30}}; // std::vector<std::vector<real_t>> inputSet = {{1,2,3,4,5,6,7,8,9,10}, {3,5,9,12,15,18,21,24,27,30}};
// std::vector<double> outputSet = {2,4,6,8,10,12,14,16,18,20}; // std::vector<real_t> outputSet = {2,4,6,8,10,12,14,16,18,20};
// std::vector<std::vector<double>> inputSet = {{1,2,3,4,5,6,7,8}, {0,0,0,0,1,1,1,1}}; // std::vector<std::vector<real_t>> inputSet = {{1,2,3,4,5,6,7,8}, {0,0,0,0,1,1,1,1}};
// std::vector<double> outputSet = {0,0,0,0,1,1,1,1}; // std::vector<real_t> outputSet = {0,0,0,0,1,1,1,1};
// std::vector<std::vector<double>> inputSet = {{4,3,0,-3,-4}, {0,0,0,1,1}}; // std::vector<std::vector<real_t>> inputSet = {{4,3,0,-3,-4}, {0,0,0,1,1}};
// std::vector<double> outputSet = {1,1,0,-1,-1}; // std::vector<real_t> outputSet = {1,1,0,-1,-1};
// std::vector<std::vector<double>> inputSet = {{0,1,2,3,4}}; // std::vector<std::vector<real_t>> inputSet = {{0,1,2,3,4}};
// std::vector<double> outputSet = {1,2,4,8,16}; // std::vector<real_t> outputSet = {1,2,4,8,16};
//std::vector<std::vector<double>> inputSet = {{32, 0, 7}, {2, 28, 17}, {0, 9, 23}}; //std::vector<std::vector<real_t>> inputSet = {{32, 0, 7}, {2, 28, 17}, {0, 9, 23}};
// std::vector<std::vector<double>> inputSet = {{1,1,0,0,1}, {0,0,1,1,1}, {0,1,1,0,1}}; // std::vector<std::vector<real_t>> inputSet = {{1,1,0,0,1}, {0,0,1,1,1}, {0,1,1,0,1}};
// std::vector<double> outputSet = {0,1,0,1,1}; // std::vector<real_t> outputSet = {0,1,0,1,1};
// std::vector<std::vector<double>> inputSet = {{0,0,1,1}, {0,1,0,1}}; // std::vector<std::vector<real_t>> inputSet = {{0,0,1,1}, {0,1,0,1}};
// std::vector<double> outputSet = {0,1,1,0}; // std::vector<real_t> outputSet = {0,1,1,0};
// // STATISTICS // // STATISTICS
// std::vector<double> x = {1,2,3,4,5,6,7,8,9,10}; // std::vector<real_t> x = {1,2,3,4,5,6,7,8,9,10};
// std::vector<double> y = {10,9,8,7,6,5,4,3,2,1}; // std::vector<real_t> y = {10,9,8,7,6,5,4,3,2,1};
// std::vector<double> w = {0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1}; // std::vector<real_t> w = {0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1};
// std::cout << "Arithmetic Mean: " << stat.mean(x) << std::endl; // std::cout << "Arithmetic Mean: " << stat.mean(x) << std::endl;
// std::cout << "Median: " << stat.median(x) << std::endl; // std::cout << "Median: " << stat.median(x) << std::endl;
@ -184,16 +184,16 @@ int main() {
// std::cout << "Absolute Average Deviation: " << stat.absAvgDeviation(x) << std::endl; // std::cout << "Absolute Average Deviation: " << stat.absAvgDeviation(x) << std::endl;
// LINEAR ALGEBRA // LINEAR ALGEBRA
// std::vector<std::vector<double>> square = {{1, 1}, {-1, 1}, {1, -1}, {-1, -1}}; // std::vector<std::vector<real_t>> square = {{1, 1}, {-1, 1}, {1, -1}, {-1, -1}};
// alg.printMatrix(alg.rotate(square, M_PI/4)); // alg.printMatrix(alg.rotate(square, M_PI/4));
// std::vector<std::vector<double>> A = { // std::vector<std::vector<real_t>> A = {
// {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, // {1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
// {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, // {1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
// }; // };
// std::vector<double> a = {4, 3, 1, 3}; // std::vector<real_t> a = {4, 3, 1, 3};
// std::vector<double> b = {3, 5, 6, 1}; // std::vector<real_t> b = {3, 5, 6, 1};
// alg.printMatrix(alg.matmult(alg.transpose(A), A)); // alg.printMatrix(alg.matmult(alg.transpose(A), A));
// std::cout << std::endl; // std::cout << std::endl;
@ -226,8 +226,8 @@ int main() {
// const int TRIAL_NUM = 1000; // const int TRIAL_NUM = 1000;
// double scoreSGD = 0; // real_t scoreSGD = 0;
// double scoreADAM = 0; // real_t scoreADAM = 0;
// for(int i = 0; i < TRIAL_NUM; i++){ // for(int i = 0; i < TRIAL_NUM; i++){
// LinReg model(alg.transpose(inputSet), outputSet); // LinReg model(alg.transpose(inputSet), outputSet);
// model.MBGD(0.001, 5, 1, 0); // model.MBGD(0.001, 5, 1, 0);
@ -269,8 +269,8 @@ int main() {
// std::cout << "ACCURACY: " << 100 * model.score() << "%" << std::endl; // std::cout << "ACCURACY: " << 100 * model.score() << "%" << std::endl;
// // PROBIT REGRESSION // // PROBIT REGRESSION
// std::vector<std::vector<double>> inputSet; // std::vector<std::vector<real_t>> inputSet;
// std::vector<double> outputSet; // std::vector<real_t> outputSet;
// data.setData(30, "/Users/marcmelikyan/Desktop/Data/BreastCancer.csv", inputSet, outputSet); // data.setData(30, "/Users/marcmelikyan/Desktop/Data/BreastCancer.csv", inputSet, outputSet);
// ProbitReg model(inputSet, outputSet); // ProbitReg model(inputSet, outputSet);
// model.SGD(0.001, 10000, 1); // model.SGD(0.001, 10000, 1);
@ -278,24 +278,24 @@ int main() {
// std::cout << "ACCURACY: " << 100 * model.score() << "%" << std::endl; // std::cout << "ACCURACY: " << 100 * model.score() << "%" << std::endl;
// // CLOGLOG REGRESSION // // CLOGLOG REGRESSION
// std::vector<std::vector<double>> inputSet = {{1,2,3,4,5,6,7,8}, {0,0,0,0,1,1,1,1}}; // std::vector<std::vector<real_t>> inputSet = {{1,2,3,4,5,6,7,8}, {0,0,0,0,1,1,1,1}};
// std::vector<double> outputSet = {0,0,0,0,1,1,1,1}; // std::vector<real_t> outputSet = {0,0,0,0,1,1,1,1};
// CLogLogReg model(alg.transpose(inputSet), outputSet); // CLogLogReg model(alg.transpose(inputSet), outputSet);
// model.SGD(0.1, 10000, 0); // model.SGD(0.1, 10000, 0);
// alg.printVector(model.modelSetTest(alg.transpose(inputSet))); // alg.printVector(model.modelSetTest(alg.transpose(inputSet)));
// std::cout << "ACCURACY: " << 100 * model.score() << "%" << std::endl; // std::cout << "ACCURACY: " << 100 * model.score() << "%" << std::endl;
// // EXPREG REGRESSION // // EXPREG REGRESSION
// std::vector<std::vector<double>> inputSet = {{0,1,2,3,4}}; // std::vector<std::vector<real_t>> inputSet = {{0,1,2,3,4}};
// std::vector<double> outputSet = {1,2,4,8,16}; // std::vector<real_t> outputSet = {1,2,4,8,16};
// ExpReg model(alg.transpose(inputSet), outputSet); // ExpReg model(alg.transpose(inputSet), outputSet);
// model.SGD(0.001, 10000, 0); // model.SGD(0.001, 10000, 0);
// alg.printVector(model.modelSetTest(alg.transpose(inputSet))); // alg.printVector(model.modelSetTest(alg.transpose(inputSet)));
// std::cout << "ACCURACY: " << 100 * model.score() << "%" << std::endl; // std::cout << "ACCURACY: " << 100 * model.score() << "%" << std::endl;
// // TANH REGRESSION // // TANH REGRESSION
// std::vector<std::vector<double>> inputSet = {{4,3,0,-3,-4}, {0,0,0,1,1}}; // std::vector<std::vector<real_t>> inputSet = {{4,3,0,-3,-4}, {0,0,0,1,1}};
// std::vector<double> outputSet = {1,1,0,-1,-1}; // std::vector<real_t> outputSet = {1,1,0,-1,-1};
// TanhReg model(alg.transpose(inputSet), outputSet); // TanhReg model(alg.transpose(inputSet), outputSet);
// model.SGD(0.1, 10000, 0); // model.SGD(0.1, 10000, 0);
// alg.printVector(model.modelSetTest(alg.transpose(inputSet))); // alg.printVector(model.modelSetTest(alg.transpose(inputSet)));
@ -320,9 +320,9 @@ int main() {
// alg.printMatrix(model.modelSetTest(inputSet)); // alg.printMatrix(model.modelSetTest(inputSet));
// // MLP // // MLP
// std::vector<std::vector<double>> inputSet = {{0,0,1,1}, {0,1,0,1}}; // std::vector<std::vector<real_t>> inputSet = {{0,0,1,1}, {0,1,0,1}};
// inputSet = alg.transpose(inputSet); // inputSet = alg.transpose(inputSet);
// std::vector<double> outputSet = {0,1,1,0}; // std::vector<real_t> outputSet = {0,1,1,0};
// MLP model(inputSet, outputSet, 2); // MLP model(inputSet, outputSet, 2);
// model.gradientDescent(0.1, 10000, 0); // model.gradientDescent(0.1, 10000, 0);
@ -337,7 +337,7 @@ int main() {
// std::cout << "ACCURACY: " << 100 * model.score() << "%" << std::endl; // std::cout << "ACCURACY: " << 100 * model.score() << "%" << std::endl;
// // AUTOENCODER // // AUTOENCODER
// std::vector<std::vector<double>> inputSet = {{1,2,3,4,5,6,7,8,9,10}, {3,5,9,12,15,18,21,24,27,30}}; // std::vector<std::vector<real_t>> inputSet = {{1,2,3,4,5,6,7,8,9,10}, {3,5,9,12,15,18,21,24,27,30}};
// AutoEncoder model(alg.transpose(inputSet), 5); // AutoEncoder model(alg.transpose(inputSet), 5);
// model.SGD(0.001, 300000, 0); // model.SGD(0.001, 300000, 0);
// alg.printMatrix(model.modelSetTest(alg.transpose(inputSet))); // alg.printMatrix(model.modelSetTest(alg.transpose(inputSet)));
@ -347,8 +347,8 @@ int main() {
// Possible Weight Init Methods: Default, Uniform, HeNormal, HeUniform, XavierNormal, XavierUniform // Possible Weight Init Methods: Default, Uniform, HeNormal, HeUniform, XavierNormal, XavierUniform
// Possible Activations: Linear, Sigmoid, Swish, Softplus, Softsign, CLogLog, Ar{Sinh, Cosh, Tanh, Csch, Sech, Coth}, GaussianCDF, GELU, UnitStep // Possible Activations: Linear, Sigmoid, Swish, Softplus, Softsign, CLogLog, Ar{Sinh, Cosh, Tanh, Csch, Sech, Coth}, GaussianCDF, GELU, UnitStep
// Possible Loss Functions: MSE, RMSE, MBE, LogLoss, CrossEntropy, HingeLoss // Possible Loss Functions: MSE, RMSE, MBE, LogLoss, CrossEntropy, HingeLoss
// std::vector<std::vector<double>> inputSet = {{0,0,1,1}, {0,1,0,1}}; // std::vector<std::vector<real_t>> inputSet = {{0,0,1,1}, {0,1,0,1}};
// std::vector<double> outputSet = {0,1,1,0}; // std::vector<real_t> outputSet = {0,1,1,0};
// ANN ann(alg.transpose(inputSet), outputSet); // ANN ann(alg.transpose(inputSet), outputSet);
// ann.addLayer(2, "Cosh"); // ann.addLayer(2, "Cosh");
// ann.addOutputLayer("Sigmoid", "LogLoss"); // ann.addOutputLayer("Sigmoid", "LogLoss");
@ -363,7 +363,7 @@ int main() {
// alg.printVector(ann.modelSetTest(alg.transpose(inputSet))); // alg.printVector(ann.modelSetTest(alg.transpose(inputSet)));
// std::cout << "ACCURACY: " << 100 * ann.score() << "%" << std::endl; // std::cout << "ACCURACY: " << 100 * ann.score() << "%" << std::endl;
std::vector<std::vector<double>> outputSet = {{1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20}, std::vector<std::vector<real_t>> outputSet = {{1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20},
{2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40}}; {2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40}};
WGAN gan(2, alg.transpose(outputSet)); // our gan is a wasserstein gan (wgan) WGAN gan(2, alg.transpose(outputSet)); // our gan is a wasserstein gan (wgan)
@ -376,8 +376,8 @@ int main() {
alg.printMatrix(gan.generateExample(100)); alg.printMatrix(gan.generateExample(100));
// typedef std::vector<std::vector<double>> Matrix; // typedef std::vector<std::vector<real_t>> Matrix;
// typedef std::vector<double> Vector; // typedef std::vector<real_t> Vector;
// Matrix inputSet = {{0,0}, {0,1}, {1,0}, {1,1}}; // XOR // Matrix inputSet = {{0,0}, {0,1}, {1,0}, {1,1}}; // XOR
// Vector outputSet = {0,1,1,0}; // Vector outputSet = {0,1,1,0};
@ -393,8 +393,8 @@ int main() {
// std::cout << "ACCURACY: " << 100 * ann.score() << "%" << std::endl; // Accuracy. // std::cout << "ACCURACY: " << 100 * ann.score() << "%" << std::endl; // Accuracy.
// // DYNAMICALLY SIZED MANN (Multidimensional Output ANN) // // DYNAMICALLY SIZED MANN (Multidimensional Output ANN)
// std::vector<std::vector<double>> inputSet = {{1,2,3},{2,4,6},{3,6,9},{4,8,12}}; // std::vector<std::vector<real_t>> inputSet = {{1,2,3},{2,4,6},{3,6,9},{4,8,12}};
// std::vector<std::vector<double>> outputSet = {{1,5}, {2,10}, {3,15}, {4,20}}; // std::vector<std::vector<real_t>> outputSet = {{1,5}, {2,10}, {3,15}, {4,20}};
// MANN mann(inputSet, outputSet); // MANN mann(inputSet, outputSet);
// mann.addOutputLayer("Linear", "MSE"); // mann.addOutputLayer("Linear", "MSE");
@ -402,14 +402,14 @@ int main() {
// alg.printMatrix(mann.modelSetTest(inputSet)); // alg.printMatrix(mann.modelSetTest(inputSet));
// std::cout << "ACCURACY: " << 100 * mann.score() << "%" << std::endl; // std::cout << "ACCURACY: " << 100 * mann.score() << "%" << std::endl;
// std::vector<std::vector<double>> inputSet; // std::vector<std::vector<real_t>> inputSet;
// std::vector<double> tempOutputSet; // std::vector<real_t> tempOutputSet;
// data.setData(4, "/Users/marcmelikyan/Desktop/Data/Iris.csv", inputSet, tempOutputSet); // data.setData(4, "/Users/marcmelikyan/Desktop/Data/Iris.csv", inputSet, tempOutputSet);
// std::vector<std::vector<double>> outputSet = data.oneHotRep(tempOutputSet, 3); // std::vector<std::vector<real_t>> outputSet = data.oneHotRep(tempOutputSet, 3);
// TRAIN TEST SPLIT CHECK // TRAIN TEST SPLIT CHECK
// std::vector<std::vector<double>> inputSet1 = {{1,2,3,4,5,6,7,8,9,10}, {3,5,9,12,15,18,21,24,27,30}}; // std::vector<std::vector<real_t>> inputSet1 = {{1,2,3,4,5,6,7,8,9,10}, {3,5,9,12,15,18,21,24,27,30}};
// std::vector<std::vector<double>> outputSet1 = {{2,4,6,8,10,12,14,16,18,20}}; // std::vector<std::vector<real_t>> outputSet1 = {{2,4,6,8,10,12,14,16,18,20}};
// auto [inputSet, outputSet, inputTestSet, outputTestSet] = data.trainTestSplit(alg.transpose(inputSet1), alg.transpose(outputSet1), 0.2); // auto [inputSet, outputSet, inputTestSet, outputTestSet] = data.trainTestSplit(alg.transpose(inputSet1), alg.transpose(outputSet1), 0.2);
// alg.printMatrix(inputSet); // alg.printMatrix(inputSet);
// alg.printMatrix(outputSet); // alg.printMatrix(outputSet);
@ -428,8 +428,8 @@ int main() {
// std::cout << "ACCURACY: " << 100 * mann.score() << "%" << std::endl; // std::cout << "ACCURACY: " << 100 * mann.score() << "%" << std::endl;
// // NAIVE BAYES // // NAIVE BAYES
// std::vector<std::vector<double>> inputSet = {{1,1,1,1,1}, {0,0,1,1,1}, {0,0,1,0,1}}; // std::vector<std::vector<real_t>> inputSet = {{1,1,1,1,1}, {0,0,1,1,1}, {0,0,1,0,1}};
// std::vector<double> outputSet = {0,1,0,1,1}; // std::vector<real_t> outputSet = {0,1,0,1,1};
// MultinomialNB MNB(alg.transpose(inputSet), outputSet, 2); // MultinomialNB MNB(alg.transpose(inputSet), outputSet, 2);
// alg.printVector(MNB.modelSetTest(alg.transpose(inputSet))); // alg.printVector(MNB.modelSetTest(alg.transpose(inputSet)));
@ -441,7 +441,7 @@ int main() {
// alg.printVector(GNB.modelSetTest(alg.transpose(inputSet))); // alg.printVector(GNB.modelSetTest(alg.transpose(inputSet)));
// // KMeans // // KMeans
// std::vector<std::vector<double>> inputSet = {{32, 0, 7}, {2, 28, 17}, {0, 9, 23}}; // std::vector<std::vector<real_t>> inputSet = {{32, 0, 7}, {2, 28, 17}, {0, 9, 23}};
// KMeans kmeans(inputSet, 3, "KMeans++"); // KMeans kmeans(inputSet, 3, "KMeans++");
// kmeans.train(3, 1); // kmeans.train(3, 1);
// std::cout << std::endl; // std::cout << std::endl;
@ -450,26 +450,26 @@ int main() {
// alg.printVector(kmeans.silhouette_scores()); // alg.printVector(kmeans.silhouette_scores());
// // kNN // // kNN
// std::vector<std::vector<double>> inputSet = {{1,2,3,4,5,6,7,8}, {0,0,0,0,1,1,1,1}}; // std::vector<std::vector<real_t>> inputSet = {{1,2,3,4,5,6,7,8}, {0,0,0,0,1,1,1,1}};
// std::vector<double> outputSet = {0,0,0,0,1,1,1,1}; // std::vector<real_t> outputSet = {0,0,0,0,1,1,1,1};
// kNN knn(alg.transpose(inputSet), outputSet, 8); // kNN knn(alg.transpose(inputSet), outputSet, 8);
// alg.printVector(knn.modelSetTest(alg.transpose(inputSet))); // alg.printVector(knn.modelSetTest(alg.transpose(inputSet)));
// std::cout << "ACCURACY: " << 100 * knn.score() << "%" << std::endl; // std::cout << "ACCURACY: " << 100 * knn.score() << "%" << std::endl;
// // CONVOLUTION, POOLING, ETC.. // // CONVOLUTION, POOLING, ETC..
// std::vector<std::vector<double>> input = { // std::vector<std::vector<real_t>> input = {
// {1}, // {1},
// }; // };
// std::vector<std::vector<std::vector<double>>> tensorSet; // std::vector<std::vector<std::vector<real_t>>> tensorSet;
// tensorSet.push_back(input); // tensorSet.push_back(input);
// tensorSet.push_back(input); // tensorSet.push_back(input);
// tensorSet.push_back(input); // tensorSet.push_back(input);
// alg.printTensor(data.rgb2xyz(tensorSet)); // alg.printTensor(data.rgb2xyz(tensorSet));
// std::vector<std::vector<double>> input = { // std::vector<std::vector<real_t>> input = {
// {62,55,55,54,49,48,47,55}, // {62,55,55,54,49,48,47,55},
// {62,57,54,52,48,47,48,53}, // {62,57,54,52,48,47,48,53},
// {61,60,52,49,48,47,49,54}, // {61,60,52,49,48,47,49,54},
@ -487,17 +487,17 @@ int main() {
// alg.printMatrix(conv.convolve(input, conv.getPrewittVertical(), 1)); // Can use padding // alg.printMatrix(conv.convolve(input, conv.getPrewittVertical(), 1)); // Can use padding
// alg.printMatrix(conv.pool(input, 4, 4, "Max")); // Can use Max, Min, or Average pooling. // alg.printMatrix(conv.pool(input, 4, 4, "Max")); // Can use Max, Min, or Average pooling.
// std::vector<std::vector<std::vector<double>>> tensorSet; // std::vector<std::vector<std::vector<real_t>>> tensorSet;
// tensorSet.push_back(input); // tensorSet.push_back(input);
// tensorSet.push_back(input); // tensorSet.push_back(input);
// alg.printVector(conv.globalPool(tensorSet, "Average")); // Can use Max, Min, or Average global pooling. // alg.printVector(conv.globalPool(tensorSet, "Average")); // Can use Max, Min, or Average global pooling.
// std::vector<std::vector<double>> laplacian = {{1, 1, 1}, {1, -4, 1}, {1, 1, 1}}; // std::vector<std::vector<real_t>> laplacian = {{1, 1, 1}, {1, -4, 1}, {1, 1, 1}};
// alg.printMatrix(conv.convolve(conv.gaussianFilter2D(5, 1), laplacian, 1)); // alg.printMatrix(conv.convolve(conv.gaussianFilter2D(5, 1), laplacian, 1));
// // PCA, SVD, eigenvalues & eigenvectors // // PCA, SVD, eigenvalues & eigenvectors
// std::vector<std::vector<double>> inputSet = {{1,1}, {1,1}}; // std::vector<std::vector<real_t>> inputSet = {{1,1}, {1,1}};
// auto [Eigenvectors, Eigenvalues] = alg.eig(inputSet); // auto [Eigenvectors, Eigenvalues] = alg.eig(inputSet);
// std::cout << "Eigenvectors:" << std::endl; // std::cout << "Eigenvectors:" << std::endl;
// alg.printMatrix(Eigenvectors); // alg.printMatrix(Eigenvectors);
@ -547,7 +547,7 @@ int main() {
// std::cout << std::endl; // std::cout << std::endl;
// std::vector<std::vector<double>> inputSet = {{1,2},{2,3},{3,4},{4,5},{5,6}}; // std::vector<std::vector<real_t>> inputSet = {{1,2},{2,3},{3,4},{4,5},{5,6}};
// std::cout << "Feature Scaling Example:" << std::endl; // std::cout << "Feature Scaling Example:" << std::endl;
// alg.printMatrix(data.featureScaling(inputSet)); // alg.printMatrix(data.featureScaling(inputSet));
// std::cout << std::endl; // std::cout << std::endl;
@ -561,20 +561,20 @@ int main() {
// std::cout << std::endl; // std::cout << std::endl;
// // Outlier Finder // // Outlier Finder
// std::vector<double> inputSet = {1,2,3,4,5,6,7,8,9,23554332523523}; // std::vector<real_t> inputSet = {1,2,3,4,5,6,7,8,9,23554332523523};
// OutlierFinder outlierFinder(2); // Any datapoint outside of 2 stds from the mean is marked as an outlier. // OutlierFinder outlierFinder(2); // Any datapoint outside of 2 stds from the mean is marked as an outlier.
// alg.printVector(outlierFinder.modelTest(inputSet)); // alg.printVector(outlierFinder.modelTest(inputSet));
// // Testing new Functions // // Testing new Functions
// double z_s = 0.001; // real_t z_s = 0.001;
// std::cout << avn.logit(z_s) << std::endl; // std::cout << avn.logit(z_s) << std::endl;
// std::cout << avn.logit(z_s, 1) << std::endl; // std::cout << avn.logit(z_s, 1) << std::endl;
// std::vector<double> z_v = {0.001}; // std::vector<real_t> z_v = {0.001};
// alg.printVector(avn.logit(z_v)); // alg.printVector(avn.logit(z_v));
// alg.printVector(avn.logit(z_v, 1)); // alg.printVector(avn.logit(z_v, 1));
// std::vector<std::vector<double>> Z_m = {{0.001}}; // std::vector<std::vector<real_t>> Z_m = {{0.001}};
// alg.printMatrix(avn.logit(Z_m)); // alg.printMatrix(avn.logit(Z_m));
// alg.printMatrix(avn.logit(Z_m, 1)); // alg.printMatrix(avn.logit(Z_m, 1));
@ -585,18 +585,18 @@ int main() {
// alg.printMatrix(alg.matrixPower({{5,5},{5,5}}, 2)); // alg.printMatrix(alg.matrixPower({{5,5},{5,5}}, 2));
// alg.printVector(alg.solve({{1,1}, {1.5, 4.0}}, {2200, 5050})); // alg.printVector(alg.solve({{1,1}, {1.5, 4.0}}, {2200, 5050}));
// std::vector<std::vector<double>> matrixOfCubes = {{1,2,64,27}}; // std::vector<std::vector<real_t>> matrixOfCubes = {{1,2,64,27}};
// std::vector<double> vectorOfCubes = {1,2,64,27}; // std::vector<real_t> vectorOfCubes = {1,2,64,27};
// alg.printMatrix(alg.cbrt(matrixOfCubes)); // alg.printMatrix(alg.cbrt(matrixOfCubes));
// alg.printVector(alg.cbrt(vectorOfCubes)); // alg.printVector(alg.cbrt(vectorOfCubes));
// std::cout << alg.max({{1,2,3,4,5}, {6,5,3,4,1}, {9,9,9,9,9}}) << std::endl; // std::cout << alg.max({{1,2,3,4,5}, {6,5,3,4,1}, {9,9,9,9,9}}) << std::endl;
// std::cout << alg.min({{1,2,3,4,5}, {6,5,3,4,1}, {9,9,9,9,9}}) << std::endl; // std::cout << alg.min({{1,2,3,4,5}, {6,5,3,4,1}, {9,9,9,9,9}}) << std::endl;
// std::vector<double> chicken; // std::vector<real_t> chicken;
// data.getImage("../../Data/apple.jpeg", chicken); // data.getImage("../../Data/apple.jpeg", chicken);
// alg.printVector(chicken); // alg.printVector(chicken);
// std::vector<std::vector<double>> P = {{12, -51, 4}, {6, 167, -68}, {-4, 24, -41}}; // std::vector<std::vector<real_t>> P = {{12, -51, 4}, {6, 167, -68}, {-4, 24, -41}};
// alg.printMatrix(P); // alg.printMatrix(P);
// alg.printMatrix(alg.gramSchmidtProcess(P)); // alg.printMatrix(alg.gramSchmidtProcess(P));
@ -608,7 +608,7 @@ int main() {
// alg.printMatrix(R); // alg.printMatrix(R);
// // Checking positive-definiteness checker. For Cholesky Decomp. // // Checking positive-definiteness checker. For Cholesky Decomp.
// std::vector<std::vector<double>> A = // std::vector<std::vector<real_t>> A =
// { // {
// {1,-1,-1,-1}, // {1,-1,-1,-1},
// {-1,2,2,2}, // {-1,2,2,2},
@ -653,7 +653,7 @@ int main() {
// std::cout << numAn.laplacian(f_mv, {1,1,1}) << std::endl; // std::cout << numAn.laplacian(f_mv, {1,1,1}) << std::endl;
// std::vector<std::vector<std::vector<double>>> tensor; // std::vector<std::vector<std::vector<real_t>>> tensor;
// tensor.push_back({{1,2}, {1,2}, {1,2}}); // tensor.push_back({{1,2}, {1,2}, {1,2}});
// tensor.push_back({{1,2}, {1,2}, {1,2}}); // tensor.push_back({{1,2}, {1,2}, {1,2}});
@ -672,7 +672,7 @@ int main() {
// alg.printMatrix(conv.gradOrientation(A)); // alg.printMatrix(conv.gradOrientation(A));
// std::vector<std::vector<double>> A = // std::vector<std::vector<real_t>> A =
// { // {
// {1,0,0,0}, // {1,0,0,0},
// {0,0,0,0}, // {0,0,0,0},
@ -689,23 +689,23 @@ int main() {
// std::cout << std::endl; // std::cout << std::endl;
// } // Harris detector works. Life is good! // } // Harris detector works. Life is good!
// std::vector<double> a = {3,4,4}; // std::vector<real_t> a = {3,4,4};
// std::vector<double> b = {4,4,4}; // std::vector<real_t> b = {4,4,4};
// alg.printVector(alg.cross(a,b)); // alg.printVector(alg.cross(a,b));
//SUPPORT VECTOR CLASSIFICATION (kernel method) //SUPPORT VECTOR CLASSIFICATION (kernel method)
// std::vector<std::vector<double>> inputSet; // std::vector<std::vector<real_t>> inputSet;
// std::vector<double> outputSet; // std::vector<real_t> outputSet;
// data.setData(30, "/Users/marcmelikyan/Desktop/Data/BreastCancerSVM.csv", inputSet, outputSet); // data.setData(30, "/Users/marcmelikyan/Desktop/Data/BreastCancerSVM.csv", inputSet, outputSet);
// std::vector<std::vector<double>> inputSet; // std::vector<std::vector<real_t>> inputSet;
// std::vector<double> outputSet; // std::vector<real_t> outputSet;
// data.setData(4, "/Users/marcmelikyan/Desktop/Data/IrisSVM.csv", inputSet, outputSet); // data.setData(4, "/Users/marcmelikyan/Desktop/Data/IrisSVM.csv", inputSet, outputSet);
// DualSVC kernelSVM(inputSet, outputSet, 1000); // DualSVC kernelSVM(inputSet, outputSet, 1000);
// kernelSVM.gradientDescent(0.0001, 20, 1); // kernelSVM.gradientDescent(0.0001, 20, 1);
// std::vector<std::vector<double>> linearlyIndependentMat = // std::vector<std::vector<real_t>> linearlyIndependentMat =
// { // {
// {1,2,3,4}, // {1,2,3,4},

274
mlpp/activation/activation.cpp
View File

@ -10,14 +10,14 @@
#include <cmath> #include <cmath>
#include <iostream> #include <iostream>
double MLPPActivation::linear(double z, bool deriv) { real_t MLPPActivation::linear(real_t z, bool deriv) {
if (deriv) { if (deriv) {
return 1; return 1;
} }
return z; return z;
} }
std::vector<double> MLPPActivation::linear(std::vector<double> z, bool deriv) { std::vector<real_t> MLPPActivation::linear(std::vector<real_t> z, bool deriv) {
if (deriv) { if (deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.onevec(z.size()); return alg.onevec(z.size());
@ -25,7 +25,7 @@ std::vector<double> MLPPActivation::linear(std::vector<double> z, bool deriv) {
return z; return z;
} }
std::vector<std::vector<double>> MLPPActivation::linear(std::vector<std::vector<double>> z, bool deriv) { std::vector<std::vector<real_t>> MLPPActivation::linear(std::vector<std::vector<real_t>> z, bool deriv) {
if (deriv) { if (deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.onemat(z.size(), z[0].size()); return alg.onemat(z.size(), z[0].size());
@ -33,14 +33,14 @@ std::vector<std::vector<double>> MLPPActivation::linear(std::vector<std::vector<
return z; return z;
} }
double MLPPActivation::sigmoid(double z, bool deriv) { real_t MLPPActivation::sigmoid(real_t z, bool deriv) {
if (deriv) { if (deriv) {
return sigmoid(z) * (1 - sigmoid(z)); return sigmoid(z) * (1 - sigmoid(z));
} }
return 1 / (1 + exp(-z)); return 1 / (1 + exp(-z));
} }
std::vector<double> MLPPActivation::sigmoid(std::vector<double> z, bool deriv) { std::vector<real_t> MLPPActivation::sigmoid(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.subtraction(sigmoid(z), alg.hadamard_product(sigmoid(z), sigmoid(z))); return alg.subtraction(sigmoid(z), alg.hadamard_product(sigmoid(z), sigmoid(z)));
@ -48,7 +48,7 @@ std::vector<double> MLPPActivation::sigmoid(std::vector<double> z, bool deriv) {
return alg.elementWiseDivision(alg.onevec(z.size()), alg.addition(alg.onevec(z.size()), alg.exp(alg.scalarMultiply(-1, z)))); return alg.elementWiseDivision(alg.onevec(z.size()), alg.addition(alg.onevec(z.size()), alg.exp(alg.scalarMultiply(-1, z))));
} }
std::vector<std::vector<double>> MLPPActivation::sigmoid(std::vector<std::vector<double>> z, bool deriv) { std::vector<std::vector<real_t>> MLPPActivation::sigmoid(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.subtraction(sigmoid(z), alg.hadamard_product(sigmoid(z), sigmoid(z))); return alg.subtraction(sigmoid(z), alg.hadamard_product(sigmoid(z), sigmoid(z)));
@ -56,12 +56,12 @@ std::vector<std::vector<double>> MLPPActivation::sigmoid(std::vector<std::vector
return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.addition(alg.onemat(z.size(), z[0].size()), alg.exp(alg.scalarMultiply(-1, z)))); return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.addition(alg.onemat(z.size(), z[0].size()), alg.exp(alg.scalarMultiply(-1, z))));
} }
std::vector<double> MLPPActivation::softmax(std::vector<double> z, bool deriv) { std::vector<real_t> MLPPActivation::softmax(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
std::vector<double> a; std::vector<real_t> a;
a.resize(z.size()); a.resize(z.size());
std::vector<double> expZ = alg.exp(z); std::vector<real_t> expZ = alg.exp(z);
double sum = 0; real_t sum = 0;
for (int i = 0; i < z.size(); i++) { for (int i = 0; i < z.size(); i++) {
sum += expZ[i]; sum += expZ[i];
@ -72,9 +72,9 @@ std::vector<double> MLPPActivation::softmax(std::vector<double> z, bool deriv) {
return a; return a;
} }
std::vector<std::vector<double>> MLPPActivation::softmax(std::vector<std::vector<double>> z, bool deriv) { std::vector<std::vector<real_t>> MLPPActivation::softmax(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
std::vector<std::vector<double>> a; std::vector<std::vector<real_t>> a;
a.resize(z.size()); a.resize(z.size());
for (int i = 0; i < z.size(); i++) { for (int i = 0; i < z.size(); i++) {
@ -83,18 +83,18 @@ std::vector<std::vector<double>> MLPPActivation::softmax(std::vector<std::vector
return a; return a;
} }
std::vector<double> MLPPActivation::adjSoftmax(std::vector<double> z) { std::vector<real_t> MLPPActivation::adjSoftmax(std::vector<real_t> z) {
MLPPLinAlg alg; MLPPLinAlg alg;
std::vector<double> a; std::vector<real_t> a;
double C = -*std::max_element(z.begin(), z.end()); real_t C = -*std::max_element(z.begin(), z.end());
z = alg.scalarAdd(C, z); z = alg.scalarAdd(C, z);
return softmax(z); return softmax(z);
} }
std::vector<std::vector<double>> MLPPActivation::adjSoftmax(std::vector<std::vector<double>> z) { std::vector<std::vector<real_t>> MLPPActivation::adjSoftmax(std::vector<std::vector<real_t>> z) {
MLPPLinAlg alg; MLPPLinAlg alg;
std::vector<std::vector<double>> a; std::vector<std::vector<real_t>> a;
a.resize(z.size()); a.resize(z.size());
for (int i = 0; i < z.size(); i++) { for (int i = 0; i < z.size(); i++) {
@ -103,10 +103,10 @@ std::vector<std::vector<double>> MLPPActivation::adjSoftmax(std::vector<std::vec
return a; return a;
} }
std::vector<std::vector<double>> MLPPActivation::softmaxDeriv(std::vector<double> z) { std::vector<std::vector<real_t>> MLPPActivation::softmaxDeriv(std::vector<real_t> z) {
MLPPLinAlg alg; MLPPLinAlg alg;
std::vector<std::vector<double>> deriv; std::vector<std::vector<real_t>> deriv;
std::vector<double> a = softmax(z); std::vector<real_t> a = softmax(z);
deriv.resize(a.size()); deriv.resize(a.size());
for (int i = 0; i < deriv.size(); i++) { for (int i = 0; i < deriv.size(); i++) {
deriv[i].resize(a.size()); deriv[i].resize(a.size());
@ -123,10 +123,10 @@ std::vector<std::vector<double>> MLPPActivation::softmaxDeriv(std::vector<double
return deriv; return deriv;
} }
std::vector<std::vector<std::vector<double>>> MLPPActivation::softmaxDeriv(std::vector<std::vector<double>> z) { std::vector<std::vector<std::vector<real_t>>> MLPPActivation::softmaxDeriv(std::vector<std::vector<real_t>> z) {
MLPPLinAlg alg; MLPPLinAlg alg;
std::vector<std::vector<std::vector<double>>> deriv; std::vector<std::vector<std::vector<real_t>>> deriv;
std::vector<std::vector<double>> a = softmax(z); std::vector<std::vector<real_t>> a = softmax(z);
deriv.resize(a.size()); deriv.resize(a.size());
for (int i = 0; i < deriv.size(); i++) { for (int i = 0; i < deriv.size(); i++) {
@ -144,14 +144,14 @@ std::vector<std::vector<std::vector<double>>> MLPPActivation::softmaxDeriv(std::
return deriv; return deriv;
} }
double MLPPActivation::softplus(double z, bool deriv) { real_t MLPPActivation::softplus(real_t z, bool deriv) {
if (deriv) { if (deriv) {
return sigmoid(z); return sigmoid(z);
} }
return std::log(1 + exp(z)); return std::log(1 + exp(z));
} }
std::vector<double> MLPPActivation::softplus(std::vector<double> z, bool deriv) { std::vector<real_t> MLPPActivation::softplus(std::vector<real_t> z, bool deriv) {
if (deriv) { if (deriv) {
return sigmoid(z); return sigmoid(z);
} }
@ -159,7 +159,7 @@ std::vector<double> MLPPActivation::softplus(std::vector<double> z, bool deriv)
return alg.log(alg.addition(alg.onevec(z.size()), alg.exp(z))); return alg.log(alg.addition(alg.onevec(z.size()), alg.exp(z)));
} }
std::vector<std::vector<double>> MLPPActivation::softplus(std::vector<std::vector<double>> z, bool deriv) { std::vector<std::vector<real_t>> MLPPActivation::softplus(std::vector<std::vector<real_t>> z, bool deriv) {
if (deriv) { if (deriv) {
return sigmoid(z); return sigmoid(z);
} }
@ -167,14 +167,14 @@ std::vector<std::vector<double>> MLPPActivation::softplus(std::vector<std::vecto
return alg.log(alg.addition(alg.onemat(z.size(), z[0].size()), alg.exp(z))); return alg.log(alg.addition(alg.onemat(z.size(), z[0].size()), alg.exp(z)));
} }
double MLPPActivation::softsign(double z, bool deriv) { real_t MLPPActivation::softsign(real_t z, bool deriv) {
if (deriv) { if (deriv) {
return 1 / ((1 + abs(z)) * (1 + abs(z))); return 1 / ((1 + abs(z)) * (1 + abs(z)));
} }
return z / (1 + abs(z)); return z / (1 + abs(z));
} }
std::vector<double> MLPPActivation::softsign(std::vector<double> z, bool deriv) { std::vector<real_t> MLPPActivation::softsign(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.elementWiseDivision(alg.onevec(z.size()), alg.exponentiate(alg.addition(alg.onevec(z.size()), alg.abs(z)), 2)); return alg.elementWiseDivision(alg.onevec(z.size()), alg.exponentiate(alg.addition(alg.onevec(z.size()), alg.abs(z)), 2));
@ -182,7 +182,7 @@ std::vector<double> MLPPActivation::softsign(std::vector<double> z, bool deriv)
return alg.elementWiseDivision(z, alg.addition(alg.onevec(z.size()), alg.abs(z))); return alg.elementWiseDivision(z, alg.addition(alg.onevec(z.size()), alg.abs(z)));
} }
std::vector<std::vector<double>> MLPPActivation::softsign(std::vector<std::vector<double>> z, bool deriv) { std::vector<std::vector<real_t>> MLPPActivation::softsign(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.exponentiate(alg.addition(alg.onemat(z.size(), z[0].size()), alg.abs(z)), 2)); return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.exponentiate(alg.addition(alg.onemat(z.size(), z[0].size()), alg.abs(z)), 2));
@ -190,14 +190,14 @@ std::vector<std::vector<double>> MLPPActivation::softsign(std::vector<std::vecto
return alg.elementWiseDivision(z, alg.addition(alg.onemat(z.size(), z[0].size()), alg.abs(z))); return alg.elementWiseDivision(z, alg.addition(alg.onemat(z.size(), z[0].size()), alg.abs(z)));
} }
double MLPPActivation::gaussianCDF(double z, bool deriv) { real_t MLPPActivation::gaussianCDF(real_t z, bool deriv) {
if (deriv) { if (deriv) {
return (1 / sqrt(2 * M_PI)) * exp(-z * z / 2); return (1 / sqrt(2 * M_PI)) * exp(-z * z / 2);
} }
return 0.5 * (1 + erf(z / sqrt(2))); return 0.5 * (1 + erf(z / sqrt(2)));
} }
std::vector<double> MLPPActivation::gaussianCDF(std::vector<double> z, bool deriv) { std::vector<real_t> MLPPActivation::gaussianCDF(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.scalarMultiply(1 / sqrt(2 * M_PI), alg.exp(alg.scalarMultiply(-1 / 2, alg.hadamard_product(z, z)))); return alg.scalarMultiply(1 / sqrt(2 * M_PI), alg.exp(alg.scalarMultiply(-1 / 2, alg.hadamard_product(z, z))));
@ -205,7 +205,7 @@ std::vector<double> MLPPActivation::gaussianCDF(std::vector<double> z, bool deri
return alg.scalarMultiply(0.5, alg.addition(alg.onevec(z.size()), alg.erf(alg.scalarMultiply(1 / sqrt(2), z)))); return alg.scalarMultiply(0.5, alg.addition(alg.onevec(z.size()), alg.erf(alg.scalarMultiply(1 / sqrt(2), z))));
} }
std::vector<std::vector<double>> MLPPActivation::gaussianCDF(std::vector<std::vector<double>> z, bool deriv) { std::vector<std::vector<real_t>> MLPPActivation::gaussianCDF(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.scalarMultiply(1 / sqrt(2 * M_PI), alg.exp(alg.scalarMultiply(-1 / 2, alg.hadamard_product(z, z)))); return alg.scalarMultiply(1 / sqrt(2 * M_PI), alg.exp(alg.scalarMultiply(-1 / 2, alg.hadamard_product(z, z))));
@ -213,14 +213,14 @@ std::vector<std::vector<double>> MLPPActivation::gaussianCDF(std::vector<std::ve
return alg.scalarMultiply(0.5, alg.addition(alg.onemat(z.size(), z[0].size()), alg.erf(alg.scalarMultiply(1 / sqrt(2), z)))); return alg.scalarMultiply(0.5, alg.addition(alg.onemat(z.size(), z[0].size()), alg.erf(alg.scalarMultiply(1 / sqrt(2), z))));
} }
double MLPPActivation::cloglog(double z, bool deriv) { real_t MLPPActivation::cloglog(real_t z, bool deriv) {
if (deriv) { if (deriv) {
return exp(z - exp(z)); return exp(z - exp(z));
} }
return 1 - exp(-exp(z)); return 1 - exp(-exp(z));
} }
std::vector<double> MLPPActivation::cloglog(std::vector<double> z, bool deriv) { std::vector<real_t> MLPPActivation::cloglog(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.exp(alg.scalarMultiply(-1, alg.exp(z))); return alg.exp(alg.scalarMultiply(-1, alg.exp(z)));
@ -228,7 +228,7 @@ std::vector<double> MLPPActivation::cloglog(std::vector<double> z, bool deriv) {
return alg.scalarMultiply(-1, alg.scalarAdd(-1, alg.exp(alg.scalarMultiply(-1, alg.exp(z))))); return alg.scalarMultiply(-1, alg.scalarAdd(-1, alg.exp(alg.scalarMultiply(-1, alg.exp(z)))));
} }
std::vector<std::vector<double>> MLPPActivation::cloglog(std::vector<std::vector<double>> z, bool deriv) { std::vector<std::vector<real_t>> MLPPActivation::cloglog(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.exp(alg.scalarMultiply(-1, alg.exp(z))); return alg.exp(alg.scalarMultiply(-1, alg.exp(z)));
@ -236,14 +236,14 @@ std::vector<std::vector<double>> MLPPActivation::cloglog(std::vector<std::vector
return alg.scalarMultiply(-1, alg.scalarAdd(-1, alg.exp(alg.scalarMultiply(-1, alg.exp(z))))); return alg.scalarMultiply(-1, alg.scalarAdd(-1, alg.exp(alg.scalarMultiply(-1, alg.exp(z)))));
} }
double MLPPActivation::logit(double z, bool deriv) { real_t MLPPActivation::logit(real_t z, bool deriv) {
if (deriv) { if (deriv) {
return 1 / z - 1 / (z - 1); return 1 / z - 1 / (z - 1);
} }
return std::log(z / (1 - z)); return std::log(z / (1 - z));
} }
std::vector<double> MLPPActivation::logit(std::vector<double> z, bool deriv) { std::vector<real_t> MLPPActivation::logit(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.subtraction(alg.elementWiseDivision(alg.onevec(z.size()), z), alg.elementWiseDivision(alg.onevec(z.size()), alg.subtraction(z, alg.onevec(z.size())))); return alg.subtraction(alg.elementWiseDivision(alg.onevec(z.size()), z), alg.elementWiseDivision(alg.onevec(z.size()), alg.subtraction(z, alg.onevec(z.size()))));
@ -251,7 +251,7 @@ std::vector<double> MLPPActivation::logit(std::vector<double> z, bool deriv) {
return alg.log(alg.elementWiseDivision(z, alg.subtraction(alg.onevec(z.size()), z))); return alg.log(alg.elementWiseDivision(z, alg.subtraction(alg.onevec(z.size()), z)));
} }
std::vector<std::vector<double>> MLPPActivation::logit(std::vector<std::vector<double>> z, bool deriv) { std::vector<std::vector<real_t>> MLPPActivation::logit(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.subtraction(alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), z), alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.subtraction(z, alg.onemat(z.size(), z[0].size())))); return alg.subtraction(alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), z), alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.subtraction(z, alg.onemat(z.size(), z[0].size()))));
@ -259,23 +259,23 @@ std::vector<std::vector<double>> MLPPActivation::logit(std::vector<std::vector<d
return alg.log(alg.elementWiseDivision(z, alg.subtraction(alg.onemat(z.size(), z[0].size()), z))); return alg.log(alg.elementWiseDivision(z, alg.subtraction(alg.onemat(z.size(), z[0].size()), z)));
} }
double MLPPActivation::unitStep(double z, bool deriv) { real_t MLPPActivation::unitStep(real_t z, bool deriv) {
if (deriv) { if (deriv) {
return 0; return 0;
} }
return z < 0 ? 0 : 1; return z < 0 ? 0 : 1;
} }
std::vector<double> MLPPActivation::unitStep(std::vector<double> z, bool deriv) { std::vector<real_t> MLPPActivation::unitStep(std::vector<real_t> z, bool deriv) {
if (deriv) { if (deriv) {
std::vector<double> deriv; std::vector<real_t> deriv;
deriv.resize(z.size()); deriv.resize(z.size());
for (int i = 0; i < z.size(); i++) { for (int i = 0; i < z.size(); i++) {
deriv[i] = unitStep(z[i], 1); deriv[i] = unitStep(z[i], 1);
} }
return deriv; return deriv;
} }
std::vector<double> a; std::vector<real_t> a;
a.resize(z.size()); a.resize(z.size());
for (int i = 0; i < a.size(); i++) { for (int i = 0; i < a.size(); i++) {
@ -284,16 +284,16 @@ std::vector<double> MLPPActivation::unitStep(std::vector<double> z, bool deriv)
return a; return a;
} }
std::vector<std::vector<double>> MLPPActivation::unitStep(std::vector<std::vector<double>> z, bool deriv) { std::vector<std::vector<real_t>> MLPPActivation::unitStep(std::vector<std::vector<real_t>> z, bool deriv) {
if (deriv) { if (deriv) {
std::vector<std::vector<double>> deriv; std::vector<std::vector<real_t>> deriv;
deriv.resize(z.size()); deriv.resize(z.size());
for (int i = 0; i < z.size(); i++) { for (int i = 0; i < z.size(); i++) {
deriv[i] = unitStep(z[i], 1); deriv[i] = unitStep(z[i], 1);
} }
return deriv; return deriv;
} }
std::vector<std::vector<double>> a; std::vector<std::vector<real_t>> a;
a.resize(z.size()); a.resize(z.size());
for (int i = 0; i < a.size(); i++) { for (int i = 0; i < a.size(); i++) {
@ -302,14 +302,14 @@ std::vector<std::vector<double>> MLPPActivation::unitStep(std::vector<std::vecto
return a; return a;
} }
double MLPPActivation::swish(double z, bool deriv) { real_t MLPPActivation::swish(real_t z, bool deriv) {
if (deriv) { if (deriv) {
return swish(z) + sigmoid(z) * (1 - swish(z)); return swish(z) + sigmoid(z) * (1 - swish(z));
} }
return z * sigmoid(z); return z * sigmoid(z);
} }
std::vector<double> MLPPActivation::swish(std::vector<double> z, bool deriv) { std::vector<real_t> MLPPActivation::swish(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
alg.addition(swish(z), alg.subtraction(sigmoid(z), alg.hadamard_product(sigmoid(z), swish(z)))); alg.addition(swish(z), alg.subtraction(sigmoid(z), alg.hadamard_product(sigmoid(z), swish(z))));
@ -317,7 +317,7 @@ std::vector<double> MLPPActivation::swish(std::vector<double> z, bool deriv) {
return alg.hadamard_product(z, sigmoid(z)); return alg.hadamard_product(z, sigmoid(z));
} }
std::vector<std::vector<double>> MLPPActivation::swish(std::vector<std::vector<double>> z, bool deriv) { std::vector<std::vector<real_t>> MLPPActivation::swish(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
alg.addition(swish(z), alg.subtraction(sigmoid(z), alg.hadamard_product(sigmoid(z), swish(z)))); alg.addition(swish(z), alg.subtraction(sigmoid(z), alg.hadamard_product(sigmoid(z), swish(z))));
@ -325,14 +325,14 @@ std::vector<std::vector<double>> MLPPActivation::swish(std::vector<std::vector<d
return alg.hadamard_product(z, sigmoid(z)); return alg.hadamard_product(z, sigmoid(z));
} }
double MLPPActivation::mish(double z, bool deriv) { real_t MLPPActivation::mish(real_t z, bool deriv) {
if (deriv) { if (deriv) {
return sech(softplus(z)) * sech(softplus(z)) * z * sigmoid(z) + mish(z) / z; return sech(softplus(z)) * sech(softplus(z)) * z * sigmoid(z) + mish(z) / z;
} }
return z * tanh(softplus(z)); return z * tanh(softplus(z));
} }
std::vector<double> MLPPActivation::mish(std::vector<double> z, bool deriv) { std::vector<real_t> MLPPActivation::mish(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.addition(alg.hadamard_product(alg.hadamard_product(alg.hadamard_product(sech(softplus(z)), sech(softplus(z))), z), sigmoid(z)), alg.elementWiseDivision(mish(z), z)); return alg.addition(alg.hadamard_product(alg.hadamard_product(alg.hadamard_product(sech(softplus(z)), sech(softplus(z))), z), sigmoid(z)), alg.elementWiseDivision(mish(z), z));
@ -340,7 +340,7 @@ std::vector<double> MLPPActivation::mish(std::vector<double> z, bool deriv) {
return alg.hadamard_product(z, tanh(softplus(z))); return alg.hadamard_product(z, tanh(softplus(z)));
} }
std::vector<std::vector<double>> MLPPActivation::mish(std::vector<std::vector<double>> z, bool deriv) { std::vector<std::vector<real_t>> MLPPActivation::mish(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.addition(alg.hadamard_product(alg.hadamard_product(alg.hadamard_product(sech(softplus(z)), sech(softplus(z))), z), sigmoid(z)), alg.elementWiseDivision(mish(z), z)); return alg.addition(alg.hadamard_product(alg.hadamard_product(alg.hadamard_product(sech(softplus(z)), sech(softplus(z))), z), sigmoid(z)), alg.elementWiseDivision(mish(z), z));
@ -348,14 +348,14 @@ std::vector<std::vector<double>> MLPPActivation::mish(std::vector<std::vector<do
return alg.hadamard_product(z, tanh(softplus(z))); return alg.hadamard_product(z, tanh(softplus(z)));
} }
double MLPPActivation::sinc(double z, bool deriv) { real_t MLPPActivation::sinc(real_t z, bool deriv) {
if (deriv) { if (deriv) {
return (z * std::cos(z) - std::sin(z)) / (z * z); return (z * std::cos(z) - std::sin(z)) / (z * z);
} }
return std::sin(z) / z; return std::sin(z) / z;
} }
std::vector<double> MLPPActivation::sinc(std::vector<double> z, bool deriv) { std::vector<real_t> MLPPActivation::sinc(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.elementWiseDivision(alg.subtraction(alg.hadamard_product(z, alg.cos(z)), alg.sin(z)), alg.hadamard_product(z, z)); return alg.elementWiseDivision(alg.subtraction(alg.hadamard_product(z, alg.cos(z)), alg.sin(z)), alg.hadamard_product(z, z));
@ -363,7 +363,7 @@ std::vector<double> MLPPActivation::sinc(std::vector<double> z, bool deriv) {
return alg.elementWiseDivision(alg.sin(z), z); return alg.elementWiseDivision(alg.sin(z), z);
} }
std::vector<std::vector<double>> MLPPActivation::sinc(std::vector<std::vector<double>> z, bool deriv) { std::vector<std::vector<real_t>> MLPPActivation::sinc(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.elementWiseDivision(alg.subtraction(alg.hadamard_product(z, alg.cos(z)), alg.sin(z)), alg.hadamard_product(z, z)); return alg.elementWiseDivision(alg.subtraction(alg.hadamard_product(z, alg.cos(z)), alg.sin(z)), alg.hadamard_product(z, z));
@ -371,7 +371,7 @@ std::vector<std::vector<double>> MLPPActivation::sinc(std::vector<std::vector<do
return alg.elementWiseDivision(alg.sin(z), z); return alg.elementWiseDivision(alg.sin(z), z);
} }
double MLPPActivation::RELU(double z, bool deriv) { real_t MLPPActivation::RELU(real_t z, bool deriv) {
if (deriv) { if (deriv) {
if (z <= 0) { if (z <= 0) {
return 0; return 0;
@ -382,16 +382,16 @@ double MLPPActivation::RELU(double z, bool deriv) {
return fmax(0, z); return fmax(0, z);
} }
std::vector<double> MLPPActivation::RELU(std::vector<double> z, bool deriv) { std::vector<real_t> MLPPActivation::RELU(std::vector<real_t> z, bool deriv) {
if (deriv) { if (deriv) {
std::vector<double> deriv; std::vector<real_t> deriv;
deriv.resize(z.size()); deriv.resize(z.size());
for (int i = 0; i < z.size(); i++) { for (int i = 0; i < z.size(); i++) {
deriv[i] = RELU(z[i], 1); deriv[i] = RELU(z[i], 1);
} }
return deriv; return deriv;
} }
std::vector<double> a; std::vector<real_t> a;
a.resize(z.size()); a.resize(z.size());
for (int i = 0; i < a.size(); i++) { for (int i = 0; i < a.size(); i++) {
@ -400,16 +400,16 @@ std::vector<double> MLPPActivation::RELU(std::vector<double> z, bool deriv) {
return a; return a;
} }
std::vector<std::vector<double>> MLPPActivation::RELU(std::vector<std::vector<double>> z, bool deriv) { std::vector<std::vector<real_t>> MLPPActivation::RELU(std::vector<std::vector<real_t>> z, bool deriv) {
if (deriv) { if (deriv) {
std::vector<std::vector<double>> deriv; std::vector<std::vector<real_t>> deriv;
deriv.resize(z.size()); deriv.resize(z.size());
for (int i = 0; i < z.size(); i++) { for (int i = 0; i < z.size(); i++) {
deriv[i] = RELU(z[i], 1); deriv[i] = RELU(z[i], 1);
} }
return deriv; return deriv;
} }
std::vector<std::vector<double>> a; std::vector<std::vector<real_t>> a;
a.resize(z.size()); a.resize(z.size());
for (int i = 0; i < a.size(); i++) { for (int i = 0; i < a.size(); i++) {
@ -418,7 +418,7 @@ std::vector<std::vector<double>> MLPPActivation::RELU(std::vector<std::vector<do
return a; return a;
} }
double MLPPActivation::leakyReLU(double z, double c, bool deriv) { real_t MLPPActivation::leakyReLU(real_t z, real_t c, bool deriv) {
if (deriv) { if (deriv) {
if (z <= 0) { if (z <= 0) {
return c; return c;
@ -429,16 +429,16 @@ double MLPPActivation::leakyReLU(double z, double c, bool deriv) {
return fmax(c * z, z); return fmax(c * z, z);
} }
std::vector<double> MLPPActivation::leakyReLU(std::vector<double> z, double c, bool deriv) { std::vector<real_t> MLPPActivation::leakyReLU(std::vector<real_t> z, real_t c, bool deriv) {
if (deriv) { if (deriv) {
std::vector<double> deriv; std::vector<real_t> deriv;
deriv.resize(z.size()); deriv.resize(z.size());
for (int i = 0; i < z.size(); i++) { for (int i = 0; i < z.size(); i++) {
deriv[i] = leakyReLU(z[i], c, 1); deriv[i] = leakyReLU(z[i], c, 1);
} }
return deriv; return deriv;
} }
std::vector<double> a; std::vector<real_t> a;
a.resize(z.size()); a.resize(z.size());
for (int i = 0; i < a.size(); i++) { for (int i = 0; i < a.size(); i++) {
@ -447,16 +447,16 @@ std::vector<double> MLPPActivation::leakyReLU(std::vector<double> z, double c, b
return a; return a;
} }
std::vector<std::vector<double>> MLPPActivation::leakyReLU(std::vector<std::vector<double>> z, double c, bool deriv) { std::vector<std::vector<real_t>> MLPPActivation::leakyReLU(std::vector<std::vector<real_t>> z, real_t c, bool deriv) {
if (deriv) { if (deriv) {
std::vector<std::vector<double>> deriv; std::vector<std::vector<real_t>> deriv;
deriv.resize(z.size()); deriv.resize(z.size());
for (int i = 0; i < z.size(); i++) { for (int i = 0; i < z.size(); i++) {
deriv[i] = leakyReLU(z[i], c, 1); deriv[i] = leakyReLU(z[i], c, 1);
} }
return deriv; return deriv;
} }
std::vector<std::vector<double>> a; std::vector<std::vector<real_t>> a;
a.resize(z.size()); a.resize(z.size());
for (int i = 0; i < a.size(); i++) { for (int i = 0; i < a.size(); i++) {
@ -465,7 +465,7 @@ std::vector<std::vector<double>> MLPPActivation::leakyReLU(std::vector<std::vect
return a; return a;
} }
double MLPPActivation::ELU(double z, double c, bool deriv) { real_t MLPPActivation::ELU(real_t z, real_t c, bool deriv) {
if (deriv) { if (deriv) {
if (z <= 0) { if (z <= 0) {
return c * exp(z); return c * exp(z);
@ -480,16 +480,16 @@ double MLPPActivation::ELU(double z, double c, bool deriv) {
} }
} }
std::vector<double> MLPPActivation::ELU(std::vector<double> z, double c, bool deriv) { std::vector<real_t> MLPPActivation::ELU(std::vector<real_t> z, real_t c, bool deriv) {
if (deriv) { if (deriv) {
std::vector<double> deriv; std::vector<real_t> deriv;
deriv.resize(z.size()); deriv.resize(z.size());
for (int i = 0; i < z.size(); i++) { for (int i = 0; i < z.size(); i++) {
deriv[i] = ELU(z[i], c, 1); deriv[i] = ELU(z[i], c, 1);
} }
return deriv; return deriv;
} }
std::vector<double> a; std::vector<real_t> a;
a.resize(z.size()); a.resize(z.size());
for (int i = 0; i < a.size(); i++) { for (int i = 0; i < a.size(); i++) {
@ -498,16 +498,16 @@ std::vector<double> MLPPActivation::ELU(std::vector<double> z, double c, bool de
return a; return a;
} }
std::vector<std::vector<double>> MLPPActivation::ELU(std::vector<std::vector<double>> z, double c, bool deriv) { std::vector<std::vector<real_t>> MLPPActivation::ELU(std::vector<std::vector<real_t>> z, real_t c, bool deriv) {
if (deriv) { if (deriv) {
std::vector<std::vector<double>> deriv; std::vector<std::vector<real_t>> deriv;
deriv.resize(z.size()); deriv.resize(z.size());
for (int i = 0; i < z.size(); i++) { for (int i = 0; i < z.size(); i++) {
deriv[i] = ELU(z[i], c, 1); deriv[i] = ELU(z[i], c, 1);
} }
return deriv; return deriv;
} }
std::vector<std::vector<double>> a; std::vector<std::vector<real_t>> a;
a.resize(z.size()); a.resize(z.size());
for (int i = 0; i < a.size(); i++) { for (int i = 0; i < a.size(); i++) {
@ -516,23 +516,23 @@ std::vector<std::vector<double>> MLPPActivation::ELU(std::vector<std::vector<dou
return a; return a;
} }
double MLPPActivation::SELU(double z, double lambda, double c, bool deriv) { real_t MLPPActivation::SELU(real_t z, real_t lambda, real_t c, bool deriv) {
if (deriv) { if (deriv) {
return ELU(z, c, 1); return ELU(z, c, 1);
} }
return lambda * ELU(z, c); return lambda * ELU(z, c);
} }
std::vector<double> MLPPActivation::SELU(std::vector<double> z, double lambda, double c, bool deriv) { std::vector<real_t> MLPPActivation::SELU(std::vector<real_t> z, real_t lambda, real_t c, bool deriv) {
if (deriv) { if (deriv) {
std::vector<double> deriv; std::vector<real_t> deriv;
deriv.resize(z.size()); deriv.resize(z.size());
for (int i = 0; i < z.size(); i++) { for (int i = 0; i < z.size(); i++) {
deriv[i] = SELU(z[i], lambda, c, 1); deriv[i] = SELU(z[i], lambda, c, 1);
} }
return deriv; return deriv;
} }
std::vector<double> a; std::vector<real_t> a;
a.resize(z.size()); a.resize(z.size());
for (int i = 0; i < a.size(); i++) { for (int i = 0; i < a.size(); i++) {
@ -541,16 +541,16 @@ std::vector<double> MLPPActivation::SELU(std::vector<double> z, double lambda, d
return a; return a;
} }
std::vector<std::vector<double>> MLPPActivation::SELU(std::vector<std::vector<double>> z, double lambda, double c, bool deriv) { std::vector<std::vector<real_t>> MLPPActivation::SELU(std::vector<std::vector<real_t>> z, real_t lambda, real_t c, bool deriv) {
if (deriv) { if (deriv) {
std::vector<std::vector<double>> deriv; std::vector<std::vector<real_t>> deriv;
deriv.resize(z.size()); deriv.resize(z.size());
for (int i = 0; i < z.size(); i++) { for (int i = 0; i < z.size(); i++) {
deriv[i] = SELU(z[i], lambda, c, 1); deriv[i] = SELU(z[i], lambda, c, 1);
} }
return deriv; return deriv;
} }
std::vector<std::vector<double>> a; std::vector<std::vector<real_t>> a;
a.resize(z.size()); a.resize(z.size());
for (int i = 0; i < a.size(); i++) { for (int i = 0; i < a.size(); i++) {
@ -559,23 +559,23 @@ std::vector<std::vector<double>> MLPPActivation::SELU(std::vector<std::vector<do
return a; return a;
} }
double MLPPActivation::GELU(double z, bool deriv) { real_t MLPPActivation::GELU(real_t z, bool deriv) {
if (deriv) { if (deriv) {
return 0.5 * tanh(0.0356774 * std::pow(z, 3) + 0.797885 * z) + (0.0535161 * std::pow(z, 3) + 0.398942 * z) * std::pow(sech(0.0356774 * std::pow(z, 3) + 0.797885 * z), 2) + 0.5; return 0.5 * tanh(0.0356774 * std::pow(z, 3) + 0.797885 * z) + (0.0535161 * std::pow(z, 3) + 0.398942 * z) * std::pow(sech(0.0356774 * std::pow(z, 3) + 0.797885 * z), 2) + 0.5;
} }
return 0.5 * z * (1 + tanh(sqrt(2 / M_PI) * (z + 0.044715 * std::pow(z, 3)))); return 0.5 * z * (1 + tanh(sqrt(2 / M_PI) * (z + 0.044715 * std::pow(z, 3))));
} }
std::vector<double> MLPPActivation::GELU(std::vector<double> z, bool deriv) { std::vector<real_t> MLPPActivation::GELU(std::vector<real_t> z, bool deriv) {
if (deriv) { if (deriv) {
std::vector<double> deriv; std::vector<real_t> deriv;
deriv.resize(z.size()); deriv.resize(z.size());
for (int i = 0; i < z.size(); i++) { for (int i = 0; i < z.size(); i++) {
deriv[i] = GELU(z[i], 1); deriv[i] = GELU(z[i], 1);
} }
return deriv; return deriv;
} }
std::vector<double> a; std::vector<real_t> a;
a.resize(z.size()); a.resize(z.size());
for (int i = 0; i < a.size(); i++) { for (int i = 0; i < a.size(); i++) {
@ -584,16 +584,16 @@ std::vector<double> MLPPActivation::GELU(std::vector<double> z, bool deriv) {
return a; return a;
} }
std::vector<std::vector<double>> MLPPActivation::GELU(std::vector<std::vector<double>> z, bool deriv) { std::vector<std::vector<real_t>> MLPPActivation::GELU(std::vector<std::vector<real_t>> z, bool deriv) {
if (deriv) { if (deriv) {
std::vector<std::vector<double>> deriv; std::vector<std::vector<real_t>> deriv;
deriv.resize(z.size()); deriv.resize(z.size());
for (int i = 0; i < z.size(); i++) { for (int i = 0; i < z.size(); i++) {
deriv[i] = GELU(z[i], 1); deriv[i] = GELU(z[i], 1);
} }
return deriv; return deriv;
} }
std::vector<std::vector<double>> a; std::vector<std::vector<real_t>> a;
a.resize(z.size()); a.resize(z.size());
for (int i = 0; i < a.size(); i++) { for (int i = 0; i < a.size(); i++) {
@ -602,7 +602,7 @@ std::vector<std::vector<double>> MLPPActivation::GELU(std::vector<std::vector<do
return a; return a;
} }
double MLPPActivation::sign(double z, bool deriv) { real_t MLPPActivation::sign(real_t z, bool deriv) {
if (deriv) { if (deriv) {
return 0; return 0;
} }
@ -615,16 +615,16 @@ double MLPPActivation::sign(double z, bool deriv) {
} }
} }
std::vector<double> MLPPActivation::sign(std::vector<double> z, bool deriv) { std::vector<real_t> MLPPActivation::sign(std::vector<real_t> z, bool deriv) {
if (deriv) { if (deriv) {
std::vector<double> deriv; std::vector<real_t> deriv;
deriv.resize(z.size()); deriv.resize(z.size());
for (int i = 0; i < z.size(); i++) { for (int i = 0; i < z.size(); i++) {
deriv[i] = sign(z[i], 1); deriv[i] = sign(z[i], 1);
} }
return deriv; return deriv;
} }
std::vector<double> a; std::vector<real_t> a;
a.resize(z.size()); a.resize(z.size());
for (int i = 0; i < a.size(); i++) { for (int i = 0; i < a.size(); i++) {
@ -633,16 +633,16 @@ std::vector<double> MLPPActivation::sign(std::vector<double> z, bool deriv) {
return a; return a;
} }
std::vector<std::vector<double>> MLPPActivation::sign(std::vector<std::vector<double>> z, bool deriv) { std::vector<std::vector<real_t>> MLPPActivation::sign(std::vector<std::vector<real_t>> z, bool deriv) {
if (deriv) { if (deriv) {
std::vector<std::vector<double>> deriv; std::vector<std::vector<real_t>> deriv;
deriv.resize(z.size()); deriv.resize(z.size());
for (int i = 0; i < z.size(); i++) { for (int i = 0; i < z.size(); i++) {
deriv[i] = sign(z[i], 1); deriv[i] = sign(z[i], 1);
} }
return deriv; return deriv;
} }
std::vector<std::vector<double>> a; std::vector<std::vector<real_t>> a;
a.resize(z.size()); a.resize(z.size());
for (int i = 0; i < a.size(); i++) { for (int i = 0; i < a.size(); i++) {
@ -651,14 +651,14 @@ std::vector<std::vector<double>> MLPPActivation::sign(std::vector<std::vector<do
return a; return a;
} }
double MLPPActivation::sinh(double z, bool deriv) { real_t MLPPActivation::sinh(real_t z, bool deriv) {
if (deriv) { if (deriv) {
return cosh(z); return cosh(z);
} }
return 0.5 * (exp(z) - exp(-z)); return 0.5 * (exp(z) - exp(-z));
} }
std::vector<double> MLPPActivation::sinh(std::vector<double> z, bool deriv) { std::vector<real_t> MLPPActivation::sinh(std::vector<real_t> z, bool deriv) {
if (deriv) { if (deriv) {
return cosh(z); return cosh(z);
} }
@ -666,7 +666,7 @@ std::vector<double> MLPPActivation::sinh(std::vector<double> z, bool deriv) {
return alg.scalarMultiply(0.5, alg.subtraction(alg.exp(z), alg.exp(alg.scalarMultiply(-1, z)))); return alg.scalarMultiply(0.5, alg.subtraction(alg.exp(z), alg.exp(alg.scalarMultiply(-1, z))));
} }
std::vector<std::vector<double>> MLPPActivation::sinh(std::vector<std::vector<double>> z, bool deriv) { std::vector<std::vector<real_t>> MLPPActivation::sinh(std::vector<std::vector<real_t>> z, bool deriv) {
if (deriv) { if (deriv) {
return cosh(z); return cosh(z);
} }
@ -674,14 +674,14 @@ std::vector<std::vector<double>> MLPPActivation::sinh(std::vector<std::vector<do
return alg.scalarMultiply(0.5, alg.subtraction(alg.exp(z), alg.exp(alg.scalarMultiply(-1, z)))); return alg.scalarMultiply(0.5, alg.subtraction(alg.exp(z), alg.exp(alg.scalarMultiply(-1, z))));
} }
double MLPPActivation::cosh(double z, bool deriv) { real_t MLPPActivation::cosh(real_t z, bool deriv) {
if (deriv) { if (deriv) {
return sinh(z); return sinh(z);
} }
return 0.5 * (exp(z) + exp(-z)); return 0.5 * (exp(z) + exp(-z));
} }
std::vector<double> MLPPActivation::cosh(std::vector<double> z, bool deriv) { std::vector<real_t> MLPPActivation::cosh(std::vector<real_t> z, bool deriv) {
if (deriv) { if (deriv) {
return sinh(z); return sinh(z);
} }
@ -689,7 +689,7 @@ std::vector<double> MLPPActivation::cosh(std::vector<double> z, bool deriv) {
return alg.scalarMultiply(0.5, alg.addition(alg.exp(z), alg.exp(alg.scalarMultiply(-1, z)))); return alg.scalarMultiply(0.5, alg.addition(alg.exp(z), alg.exp(alg.scalarMultiply(-1, z))));
} }
std::vector<std::vector<double>> MLPPActivation::cosh(std::vector<std::vector<double>> z, bool deriv) { std::vector<std::vector<real_t>> MLPPActivation::cosh(std::vector<std::vector<real_t>> z, bool deriv) {
if (deriv) { if (deriv) {
return sinh(z); return sinh(z);
} }
@ -697,14 +697,14 @@ std::vector<std::vector<double>> MLPPActivation::cosh(std::vector<std::vector<do
return alg.scalarMultiply(0.5, alg.addition(alg.exp(z), alg.exp(alg.scalarMultiply(-1, z)))); return alg.scalarMultiply(0.5, alg.addition(alg.exp(z), alg.exp(alg.scalarMultiply(-1, z))));
} }
double MLPPActivation::tanh(double z, bool deriv) { real_t MLPPActivation::tanh(real_t z, bool deriv) {
if (deriv) { if (deriv) {
return 1 - tanh(z) * tanh(z); return 1 - tanh(z) * tanh(z);
} }
return (exp(z) - exp(-z)) / (exp(z) + exp(-z)); return (exp(z) - exp(-z)) / (exp(z) + exp(-z));
} }
std::vector<double> MLPPActivation::tanh(std::vector<double> z, bool deriv) { std::vector<real_t> MLPPActivation::tanh(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.scalarMultiply(-1, alg.scalarAdd(-1, alg.hadamard_product(tanh(z), tanh(z)))); return alg.scalarMultiply(-1, alg.scalarAdd(-1, alg.hadamard_product(tanh(z), tanh(z))));
@ -712,7 +712,7 @@ std::vector<double> MLPPActivation::tanh(std::vector<double> z, bool deriv) {
return alg.elementWiseDivision(alg.subtraction(alg.exp(z), alg.exp(alg.scalarMultiply(-1, z))), alg.addition(alg.exp(z), alg.exp(alg.scalarMultiply(-1, z)))); return alg.elementWiseDivision(alg.subtraction(alg.exp(z), alg.exp(alg.scalarMultiply(-1, z))), alg.addition(alg.exp(z), alg.exp(alg.scalarMultiply(-1, z))));
} }
std::vector<std::vector<double>> MLPPActivation::tanh(std::vector<std::vector<double>> z, bool deriv) { std::vector<std::vector<real_t>> MLPPActivation::tanh(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.scalarMultiply(-1, alg.scalarAdd(-1, alg.hadamard_product(tanh(z), tanh(z)))); return alg.scalarMultiply(-1, alg.scalarAdd(-1, alg.hadamard_product(tanh(z), tanh(z))));
@ -721,14 +721,14 @@ std::vector<std::vector<double>> MLPPActivation::tanh(std::vector<std::vector<do
return alg.elementWiseDivision(alg.subtraction(alg.exp(z), alg.exp(alg.scalarMultiply(-1, z))), alg.addition(alg.exp(z), alg.exp(alg.scalarMultiply(-1, z)))); return alg.elementWiseDivision(alg.subtraction(alg.exp(z), alg.exp(alg.scalarMultiply(-1, z))), alg.addition(alg.exp(z), alg.exp(alg.scalarMultiply(-1, z))));
} }
double MLPPActivation::csch(double z, bool deriv) { real_t MLPPActivation::csch(real_t z, bool deriv) {
if (deriv) { if (deriv) {
return -csch(z) * coth(z); return -csch(z) * coth(z);
} }
return 1 / sinh(z); return 1 / sinh(z);
} }
std::vector<double> MLPPActivation::csch(std::vector<double> z, bool deriv) { std::vector<real_t> MLPPActivation::csch(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.hadamard_product(alg.scalarMultiply(-1, csch(z)), coth(z)); return alg.hadamard_product(alg.scalarMultiply(-1, csch(z)), coth(z));
@ -736,7 +736,7 @@ std::vector<double> MLPPActivation::csch(std::vector<double> z, bool deriv) {
return alg.elementWiseDivision(alg.onevec(z.size()), sinh(z)); return alg.elementWiseDivision(alg.onevec(z.size()), sinh(z));
} }
std::vector<std::vector<double>> MLPPActivation::csch(std::vector<std::vector<double>> z, bool deriv) { std::vector<std::vector<real_t>> MLPPActivation::csch(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.hadamard_product(alg.scalarMultiply(-1, csch(z)), coth(z)); return alg.hadamard_product(alg.scalarMultiply(-1, csch(z)), coth(z));
@ -744,41 +744,41 @@ std::vector<std::vector<double>> MLPPActivation::csch(std::vector<std::vector<do
return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), sinh(z)); return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), sinh(z));
} }
double MLPPActivation::sech(double z, bool deriv) { real_t MLPPActivation::sech(real_t z, bool deriv) {
if (deriv) { if (deriv) {
return -sech(z) * tanh(z); return -sech(z) * tanh(z);
} }
return 1 / cosh(z); return 1 / cosh(z);
} }
std::vector<double> MLPPActivation::sech(std::vector<double> z, bool deriv) { std::vector<real_t> MLPPActivation::sech(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.hadamard_product(alg.scalarMultiply(-1, sech(z)), tanh(z)); return alg.hadamard_product(alg.scalarMultiply(-1, sech(z)), tanh(z));
} }
return alg.elementWiseDivision(alg.onevec(z.size()), cosh(z)); return alg.elementWiseDivision(alg.onevec(z.size()), cosh(z));
// return activation(z, deriv, static_cast<void (*)(double, bool)>(&sech)); // return activation(z, deriv, static_cast<void (*)(real_t, bool)>(&sech));
} }
std::vector<std::vector<double>> MLPPActivation::sech(std::vector<std::vector<double>> z, bool deriv) { std::vector<std::vector<real_t>> MLPPActivation::sech(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.hadamard_product(alg.scalarMultiply(-1, sech(z)), tanh(z)); return alg.hadamard_product(alg.scalarMultiply(-1, sech(z)), tanh(z));
} }
return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), cosh(z)); return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), cosh(z));
// return activation(z, deriv, static_cast<void (*)(double, bool)>(&sech)); // return activation(z, deriv, static_cast<void (*)(real_t, bool)>(&sech));
} }
double MLPPActivation::coth(double z, bool deriv) { real_t MLPPActivation::coth(real_t z, bool deriv) {
if (deriv) { if (deriv) {
return -csch(z) * csch(z); return -csch(z) * csch(z);
} }
return 1 / tanh(z); return 1 / tanh(z);
} }
std::vector<double> MLPPActivation::coth(std::vector<double> z, bool deriv) { std::vector<real_t> MLPPActivation::coth(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.hadamard_product(alg.scalarMultiply(-1, csch(z)), csch(z)); return alg.hadamard_product(alg.scalarMultiply(-1, csch(z)), csch(z));
@ -786,7 +786,7 @@ std::vector<double> MLPPActivation::coth(std::vector<double> z, bool deriv) {
return alg.elementWiseDivision(alg.onevec(z.size()), tanh(z)); return alg.elementWiseDivision(alg.onevec(z.size()), tanh(z));
} }
std::vector<std::vector<double>> MLPPActivation::coth(std::vector<std::vector<double>> z, bool deriv) { std::vector<std::vector<real_t>> MLPPActivation::coth(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.hadamard_product(alg.scalarMultiply(-1, csch(z)), csch(z)); return alg.hadamard_product(alg.scalarMultiply(-1, csch(z)), csch(z));
@ -794,14 +794,14 @@ std::vector<std::vector<double>> MLPPActivation::coth(std::vector<std::vector<do
return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), tanh(z)); return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), tanh(z));
} }
double MLPPActivation::arsinh(double z, bool deriv) { real_t MLPPActivation::arsinh(real_t z, bool deriv) {
if (deriv) { if (deriv) {
return 1 / sqrt(z * z + 1); return 1 / sqrt(z * z + 1);
} }
return std::log(z + sqrt(z * z + 1)); return std::log(z + sqrt(z * z + 1));
} }
std::vector<double> MLPPActivation::arsinh(std::vector<double> z, bool deriv) { std::vector<real_t> MLPPActivation::arsinh(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.elementWiseDivision(alg.onevec(z.size()), alg.sqrt(alg.addition(alg.hadamard_product(z, z), alg.onevec(z.size())))); return alg.elementWiseDivision(alg.onevec(z.size()), alg.sqrt(alg.addition(alg.hadamard_product(z, z), alg.onevec(z.size()))));
@ -809,7 +809,7 @@ std::vector<double> MLPPActivation::arsinh(std::vector<double> z, bool deriv) {
return alg.log(alg.addition(z, alg.sqrt(alg.addition(alg.hadamard_product(z, z), alg.onevec(z.size()))))); return alg.log(alg.addition(z, alg.sqrt(alg.addition(alg.hadamard_product(z, z), alg.onevec(z.size())))));
} }
std::vector<std::vector<double>> MLPPActivation::arsinh(std::vector<std::vector<double>> z, bool deriv) { std::vector<std::vector<real_t>> MLPPActivation::arsinh(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.sqrt(alg.addition(alg.hadamard_product(z, z), alg.onemat(z.size(), z[0].size())))); return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.sqrt(alg.addition(alg.hadamard_product(z, z), alg.onemat(z.size(), z[0].size()))));
@ -817,14 +817,14 @@ std::vector<std::vector<double>> MLPPActivation::arsinh(std::vector<std::vector<
return alg.log(alg.addition(z, alg.sqrt(alg.addition(alg.hadamard_product(z, z), alg.onemat(z.size(), z[0].size()))))); return alg.log(alg.addition(z, alg.sqrt(alg.addition(alg.hadamard_product(z, z), alg.onemat(z.size(), z[0].size())))));
} }
double MLPPActivation::arcosh(double z, bool deriv) { real_t MLPPActivation::arcosh(real_t z, bool deriv) {
if (deriv) { if (deriv) {
return 1 / sqrt(z * z - 1); return 1 / sqrt(z * z - 1);
} }
return std::log(z + sqrt(z * z - 1)); return std::log(z + sqrt(z * z - 1));
} }
std::vector<double> MLPPActivation::arcosh(std::vector<double> z, bool deriv) { std::vector<real_t> MLPPActivation::arcosh(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.elementWiseDivision(alg.onevec(z.size()), alg.sqrt(alg.subtraction(alg.hadamard_product(z, z), alg.onevec(z.size())))); return alg.elementWiseDivision(alg.onevec(z.size()), alg.sqrt(alg.subtraction(alg.hadamard_product(z, z), alg.onevec(z.size()))));
@ -832,7 +832,7 @@ std::vector<double> MLPPActivation::arcosh(std::vector<double> z, bool deriv) {
return alg.log(alg.addition(z, alg.sqrt(alg.subtraction(alg.hadamard_product(z, z), alg.onevec(z.size()))))); return alg.log(alg.addition(z, alg.sqrt(alg.subtraction(alg.hadamard_product(z, z), alg.onevec(z.size())))));
} }
std::vector<std::vector<double>> MLPPActivation::arcosh(std::vector<std::vector<double>> z, bool deriv) { std::vector<std::vector<real_t>> MLPPActivation::arcosh(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.sqrt(alg.subtraction(alg.hadamard_product(z, z), alg.onemat(z.size(), z[0].size())))); return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.sqrt(alg.subtraction(alg.hadamard_product(z, z), alg.onemat(z.size(), z[0].size()))));
@ -840,14 +840,14 @@ std::vector<std::vector<double>> MLPPActivation::arcosh(std::vector<std::vector<
return alg.log(alg.addition(z, alg.sqrt(alg.subtraction(alg.hadamard_product(z, z), alg.onemat(z.size(), z[0].size()))))); return alg.log(alg.addition(z, alg.sqrt(alg.subtraction(alg.hadamard_product(z, z), alg.onemat(z.size(), z[0].size())))));
} }
double MLPPActivation::artanh(double z, bool deriv) { real_t MLPPActivation::artanh(real_t z, bool deriv) {
if (deriv) { if (deriv) {
return 1 / (1 - z * z); return 1 / (1 - z * z);
} }
return 0.5 * std::log((1 + z) / (1 - z)); return 0.5 * std::log((1 + z) / (1 - z));
} }
std::vector<double> MLPPActivation::artanh(std::vector<double> z, bool deriv) { std::vector<real_t> MLPPActivation::artanh(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.elementWiseDivision(alg.onevec(z.size()), alg.subtraction(alg.onevec(z.size()), alg.hadamard_product(z, z))); return alg.elementWiseDivision(alg.onevec(z.size()), alg.subtraction(alg.onevec(z.size()), alg.hadamard_product(z, z)));
@ -855,7 +855,7 @@ std::vector<double> MLPPActivation::artanh(std::vector<double> z, bool deriv) {
return alg.scalarMultiply(0.5, alg.log(alg.elementWiseDivision(alg.addition(alg.onevec(z.size()), z), alg.subtraction(alg.onevec(z.size()), z)))); return alg.scalarMultiply(0.5, alg.log(alg.elementWiseDivision(alg.addition(alg.onevec(z.size()), z), alg.subtraction(alg.onevec(z.size()), z))));
} }
std::vector<std::vector<double>> MLPPActivation::artanh(std::vector<std::vector<double>> z, bool deriv) { std::vector<std::vector<real_t>> MLPPActivation::artanh(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.subtraction(alg.onemat(z.size(), z[0].size()), alg.hadamard_product(z, z))); return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.subtraction(alg.onemat(z.size(), z[0].size()), alg.hadamard_product(z, z)));
@ -863,14 +863,14 @@ std::vector<std::vector<double>> MLPPActivation::artanh(std::vector<std::vector<
return alg.scalarMultiply(0.5, alg.log(alg.elementWiseDivision(alg.addition(alg.onemat(z.size(), z[0].size()), z), alg.subtraction(alg.onemat(z.size(), z[0].size()), z)))); return alg.scalarMultiply(0.5, alg.log(alg.elementWiseDivision(alg.addition(alg.onemat(z.size(), z[0].size()), z), alg.subtraction(alg.onemat(z.size(), z[0].size()), z))));
} }
double MLPPActivation::arcsch(double z, bool deriv) { real_t MLPPActivation::arcsch(real_t z, bool deriv) {
if (deriv) { if (deriv) {
return -1 / ((z * z) * sqrt(1 + (1 / (z * z)))); return -1 / ((z * z) * sqrt(1 + (1 / (z * z))));
} }
return std::log(sqrt(1 + (1 / (z * z))) + (1 / z)); return std::log(sqrt(1 + (1 / (z * z))) + (1 / z));
} }
std::vector<double> MLPPActivation::arcsch(std::vector<double> z, bool deriv) { std::vector<real_t> MLPPActivation::arcsch(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.elementWiseDivision(alg.full(z.size(), -1), alg.hadamard_product(alg.hadamard_product(z, z), alg.sqrt(alg.addition(alg.onevec(z.size()), alg.elementWiseDivision(alg.onevec(z.size()), alg.hadamard_product(z, z)))))); return alg.elementWiseDivision(alg.full(z.size(), -1), alg.hadamard_product(alg.hadamard_product(z, z), alg.sqrt(alg.addition(alg.onevec(z.size()), alg.elementWiseDivision(alg.onevec(z.size()), alg.hadamard_product(z, z))))));
@ -878,7 +878,7 @@ std::vector<double> MLPPActivation::arcsch(std::vector<double> z, bool deriv) {
return alg.log(alg.addition(alg.sqrt(alg.addition(alg.onevec(z.size()), alg.elementWiseDivision(alg.onevec(z.size()), alg.hadamard_product(z, z)))), alg.elementWiseDivision(alg.onevec(z.size()), z))); return alg.log(alg.addition(alg.sqrt(alg.addition(alg.onevec(z.size()), alg.elementWiseDivision(alg.onevec(z.size()), alg.hadamard_product(z, z)))), alg.elementWiseDivision(alg.onevec(z.size()), z)));
} }
std::vector<std::vector<double>> MLPPActivation::arcsch(std::vector<std::vector<double>> z, bool deriv) { std::vector<std::vector<real_t>> MLPPActivation::arcsch(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.elementWiseDivision(alg.full(z.size(), z[0].size(), -1), alg.hadamard_product(alg.hadamard_product(z, z), alg.sqrt(alg.addition(alg.onemat(z.size(), z[0].size()), alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.hadamard_product(z, z)))))); return alg.elementWiseDivision(alg.full(z.size(), z[0].size(), -1), alg.hadamard_product(alg.hadamard_product(z, z), alg.sqrt(alg.addition(alg.onemat(z.size(), z[0].size()), alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.hadamard_product(z, z))))));
@ -886,14 +886,14 @@ std::vector<std::vector<double>> MLPPActivation::arcsch(std::vector<std::vector<
return alg.log(alg.addition(alg.sqrt(alg.addition(alg.onemat(z.size(), z[0].size()), alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.hadamard_product(z, z)))), alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), z))); return alg.log(alg.addition(alg.sqrt(alg.addition(alg.onemat(z.size(), z[0].size()), alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.hadamard_product(z, z)))), alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), z)));
} }
double MLPPActivation::arsech(double z, bool deriv) { real_t MLPPActivation::arsech(real_t z, bool deriv) {
if (deriv) { if (deriv) {
return -1 / (z * sqrt(1 - z * z)); return -1 / (z * sqrt(1 - z * z));
} }
return std::log((1 / z) + ((1 / z) + 1) * ((1 / z) - 1)); return std::log((1 / z) + ((1 / z) + 1) * ((1 / z) - 1));
} }
std::vector<double> MLPPActivation::arsech(std::vector<double> z, bool deriv) { std::vector<real_t> MLPPActivation::arsech(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.elementWiseDivision(alg.full(z.size(), -1), alg.hadamard_product(z, alg.sqrt(alg.subtraction(alg.onevec(z.size()), alg.hadamard_product(z, z))))); return alg.elementWiseDivision(alg.full(z.size(), -1), alg.hadamard_product(z, alg.sqrt(alg.subtraction(alg.onevec(z.size()), alg.hadamard_product(z, z)))));
@ -901,7 +901,7 @@ std::vector<double> MLPPActivation::arsech(std::vector<double> z, bool deriv) {
return alg.log(alg.addition(alg.elementWiseDivision(alg.onevec(z.size()), z), alg.hadamard_product(alg.addition(alg.elementWiseDivision(alg.onevec(z.size()), z), alg.onevec(z.size())), alg.subtraction(alg.elementWiseDivision(alg.onevec(z.size()), z), alg.onevec(z.size()))))); return alg.log(alg.addition(alg.elementWiseDivision(alg.onevec(z.size()), z), alg.hadamard_product(alg.addition(alg.elementWiseDivision(alg.onevec(z.size()), z), alg.onevec(z.size())), alg.subtraction(alg.elementWiseDivision(alg.onevec(z.size()), z), alg.onevec(z.size())))));
} }
std::vector<std::vector<double>> MLPPActivation::arsech(std::vector<std::vector<double>> z, bool deriv) { std::vector<std::vector<real_t>> MLPPActivation::arsech(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.elementWiseDivision(alg.full(z.size(), z[0].size(), -1), alg.hadamard_product(z, alg.sqrt(alg.subtraction(alg.onemat(z.size(), z[0].size()), alg.hadamard_product(z, z))))); return alg.elementWiseDivision(alg.full(z.size(), z[0].size(), -1), alg.hadamard_product(z, alg.sqrt(alg.subtraction(alg.onemat(z.size(), z[0].size()), alg.hadamard_product(z, z)))));
@ -909,14 +909,14 @@ std::vector<std::vector<double>> MLPPActivation::arsech(std::vector<std::vector<
return alg.log(alg.addition(alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), z), alg.hadamard_product(alg.addition(alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), z), alg.onemat(z.size(), z[0].size())), alg.subtraction(alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), z), alg.onemat(z.size(), z[0].size()))))); return alg.log(alg.addition(alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), z), alg.hadamard_product(alg.addition(alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), z), alg.onemat(z.size(), z[0].size())), alg.subtraction(alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), z), alg.onemat(z.size(), z[0].size())))));
} }
double MLPPActivation::arcoth(double z, bool deriv) { real_t MLPPActivation::arcoth(real_t z, bool deriv) {
if (deriv) { if (deriv) {
return 1 / (1 - z * z); return 1 / (1 - z * z);
} }
return 0.5 * std::log((1 + z) / (z - 1)); return 0.5 * std::log((1 + z) / (z - 1));
} }
std::vector<double> MLPPActivation::arcoth(std::vector<double> z, bool deriv) { std::vector<real_t> MLPPActivation::arcoth(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.elementWiseDivision(alg.onevec(z.size()), alg.subtraction(alg.onevec(z.size()), alg.hadamard_product(z, z))); return alg.elementWiseDivision(alg.onevec(z.size()), alg.subtraction(alg.onevec(z.size()), alg.hadamard_product(z, z)));
@ -924,7 +924,7 @@ std::vector<double> MLPPActivation::arcoth(std::vector<double> z, bool deriv) {
return alg.scalarMultiply(0.5, alg.log(alg.elementWiseDivision(alg.addition(alg.onevec(z.size()), z), alg.subtraction(z, alg.onevec(z.size()))))); return alg.scalarMultiply(0.5, alg.log(alg.elementWiseDivision(alg.addition(alg.onevec(z.size()), z), alg.subtraction(z, alg.onevec(z.size())))));
} }
std::vector<std::vector<double>> MLPPActivation::arcoth(std::vector<std::vector<double>> z, bool deriv) { std::vector<std::vector<real_t>> MLPPActivation::arcoth(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (deriv) { if (deriv) {
return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.subtraction(alg.onemat(z.size(), z[0].size()), alg.hadamard_product(z, z))); return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.subtraction(alg.onemat(z.size(), z[0].size()), alg.hadamard_product(z, z)));
@ -933,16 +933,16 @@ std::vector<std::vector<double>> MLPPActivation::arcoth(std::vector<std::vector<
} }
// TO DO: Implement this template activation // TO DO: Implement this template activation
std::vector<double> MLPPActivation::activation(std::vector<double> z, bool deriv, double (*function)(double, bool)) { std::vector<real_t> MLPPActivation::activation(std::vector<real_t> z, bool deriv, real_t (*function)(real_t, bool)) {
if (deriv) { if (deriv) {
std::vector<double> deriv; std::vector<real_t> deriv;
deriv.resize(z.size()); deriv.resize(z.size());
for (int i = 0; i < z.size(); i++) { for (int i = 0; i < z.size(); i++) {
deriv[i] = function(z[i], 1); deriv[i] = function(z[i], 1);
} }
return deriv; return deriv;
} }
std::vector<double> a; std::vector<real_t> a;
a.resize(z.size()); a.resize(z.size());
for (int i = 0; i < z.size(); i++) { for (int i = 0; i < z.size(); i++) {
a[i] = function(z[i], deriv); a[i] = function(z[i], deriv);

190
mlpp/activation/activation.h
View File

@ -8,136 +8,138 @@
// Created by Marc Melikyan on 1/16/21. // Created by Marc Melikyan on 1/16/21.
// //
#include "core/math/math_defs.h"
#include <vector> #include <vector>
class MLPPActivation { class MLPPActivation {
public: public:
double linear(double z, bool deriv = 0); real_t linear(real_t z, bool deriv = 0);
std::vector<double> linear(std::vector<double> z, bool deriv = 0); std::vector<real_t> linear(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<double>> linear(std::vector<std::vector<double>> z, bool deriv = 0); std::vector<std::vector<real_t>> linear(std::vector<std::vector<real_t>> z, bool deriv = 0);
double sigmoid(double z, bool deriv = 0); real_t sigmoid(real_t z, bool deriv = 0);
std::vector<double> sigmoid(std::vector<double> z, bool deriv = 0); std::vector<real_t> sigmoid(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<double>> sigmoid(std::vector<std::vector<double>> z, bool deriv = 0); std::vector<std::vector<real_t>> sigmoid(std::vector<std::vector<real_t>> z, bool deriv = 0);
std::vector<double> softmax(std::vector<double> z, bool deriv = 0); std::vector<real_t> softmax(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<double>> softmax(std::vector<std::vector<double>> z, bool deriv = 0); std::vector<std::vector<real_t>> softmax(std::vector<std::vector<real_t>> z, bool deriv = 0);
std::vector<double> adjSoftmax(std::vector<double> z); std::vector<real_t> adjSoftmax(std::vector<real_t> z);
std::vector<std::vector<double>> adjSoftmax(std::vector<std::vector<double>> z); std::vector<std::vector<real_t>> adjSoftmax(std::vector<std::vector<real_t>> z);
std::vector<std::vector<double>> softmaxDeriv(std::vector<double> z); std::vector<std::vector<real_t>> softmaxDeriv(std::vector<real_t> z);
std::vector<std::vector<std::vector<double>>> softmaxDeriv(std::vector<std::vector<double>> z); std::vector<std::vector<std::vector<real_t>>> softmaxDeriv(std::vector<std::vector<real_t>> z);
double softplus(double z, bool deriv = 0); real_t softplus(real_t z, bool deriv = 0);
std::vector<double> softplus(std::vector<double> z, bool deriv = 0); std::vector<real_t> softplus(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<double>> softplus(std::vector<std::vector<double>> z, bool deriv = 0); std::vector<std::vector<real_t>> softplus(std::vector<std::vector<real_t>> z, bool deriv = 0);
double softsign(double z, bool deriv = 0); real_t softsign(real_t z, bool deriv = 0);
std::vector<double> softsign(std::vector<double> z, bool deriv = 0); std::vector<real_t> softsign(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<double>> softsign(std::vector<std::vector<double>> z, bool deriv = 0); std::vector<std::vector<real_t>> softsign(std::vector<std::vector<real_t>> z, bool deriv = 0);
double gaussianCDF(double z, bool deriv = 0); real_t gaussianCDF(real_t z, bool deriv = 0);
std::vector<double> gaussianCDF(std::vector<double> z, bool deriv = 0); std::vector<real_t> gaussianCDF(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<double>> gaussianCDF(std::vector<std::vector<double>> z, bool deriv = 0); std::vector<std::vector<real_t>> gaussianCDF(std::vector<std::vector<real_t>> z, bool deriv = 0);
double cloglog(double z, bool deriv = 0); real_t cloglog(real_t z, bool deriv = 0);
std::vector<double> cloglog(std::vector<double> z, bool deriv = 0); std::vector<real_t> cloglog(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<double>> cloglog(std::vector<std::vector<double>> z, bool deriv = 0); std::vector<std::vector<real_t>> cloglog(std::vector<std::vector<real_t>> z, bool deriv = 0);
double logit(double z, bool deriv = 0); real_t logit(real_t z, bool deriv = 0);
std::vector<double> logit(std::vector<double> z, bool deriv = 0); std::vector<real_t> logit(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<double>> logit(std::vector<std::vector<double>> z, bool deriv = 0); std::vector<std::vector<real_t>> logit(std::vector<std::vector<real_t>> z, bool deriv = 0);
double unitStep(double z, bool deriv = 0); real_t unitStep(real_t z, bool deriv = 0);
std::vector<double> unitStep(std::vector<double> z, bool deriv = 0); std::vector<real_t> unitStep(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<double>> unitStep(std::vector<std::vector<double>> z, bool deriv = 0); std::vector<std::vector<real_t>> unitStep(std::vector<std::vector<real_t>> z, bool deriv = 0);
double swish(double z, bool deriv = 0); real_t swish(real_t z, bool deriv = 0);
std::vector<double> swish(std::vector<double> z, bool deriv = 0); std::vector<real_t> swish(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<double>> swish(std::vector<std::vector<double>> z, bool deriv = 0); std::vector<std::vector<real_t>> swish(std::vector<std::vector<real_t>> z, bool deriv = 0);
double mish(double z, bool deriv = 0); real_t mish(real_t z, bool deriv = 0);
std::vector<double> mish(std::vector<double> z, bool deriv = 0); std::vector<real_t> mish(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<double>> mish(std::vector<std::vector<double>> z, bool deriv = 0); std::vector<std::vector<real_t>> mish(std::vector<std::vector<real_t>> z, bool deriv = 0);
double sinc(double z, bool deriv = 0); real_t sinc(real_t z, bool deriv = 0);
std::vector<double> sinc(std::vector<double> z, bool deriv = 0); std::vector<real_t> sinc(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<double>> sinc(std::vector<std::vector<double>> z, bool deriv = 0); std::vector<std::vector<real_t>> sinc(std::vector<std::vector<real_t>> z, bool deriv = 0);
double RELU(double z, bool deriv = 0); real_t RELU(real_t z, bool deriv = 0);
std::vector<double> RELU(std::vector<double> z, bool deriv = 0); std::vector<real_t> RELU(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<double>> RELU(std::vector<std::vector<double>> z, bool deriv = 0); std::vector<std::vector<real_t>> RELU(std::vector<std::vector<real_t>> z, bool deriv = 0);
double leakyReLU(double z, double c, bool deriv = 0); real_t leakyReLU(real_t z, real_t c, bool deriv = 0);
std::vector<double> leakyReLU(std::vector<double> z, double c, bool deriv = 0); std::vector<real_t> leakyReLU(std::vector<real_t> z, real_t c, bool deriv = 0);
std::vector<std::vector<double>> leakyReLU(std::vector<std::vector<double>> z, double c, bool deriv = 0); std::vector<std::vector<real_t>> leakyReLU(std::vector<std::vector<real_t>> z, real_t c, bool deriv = 0);
double ELU(double z, double c, bool deriv = 0); real_t ELU(real_t z, real_t c, bool deriv = 0);
std::vector<double> ELU(std::vector<double> z, double c, bool deriv = 0); std::vector<real_t> ELU(std::vector<real_t> z, real_t c, bool deriv = 0);
std::vector<std::vector<double>> ELU(std::vector<std::vector<double>> z, double c, bool deriv = 0); std::vector<std::vector<real_t>> ELU(std::vector<std::vector<real_t>> z, real_t c, bool deriv = 0);
double SELU(double z, double lambda, double c, bool deriv = 0); real_t SELU(real_t z, real_t lambda, real_t c, bool deriv = 0);
std::vector<double> SELU(std::vector<double> z, double lambda, double c, bool deriv = 0); std::vector<real_t> SELU(std::vector<real_t> z, real_t lambda, real_t c, bool deriv = 0);
std::vector<std::vector<double>> SELU(std::vector<std::vector<double>>, double lambda, double c, bool deriv = 0); std::vector<std::vector<real_t>> SELU(std::vector<std::vector<real_t>>, real_t lambda, real_t c, bool deriv = 0);
double GELU(double z, bool deriv = 0); real_t GELU(real_t z, bool deriv = 0);
std::vector<double> GELU(std::vector<double> z, bool deriv = 0); std::vector<real_t> GELU(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<double>> GELU(std::vector<std::vector<double>> z, bool deriv = 0); std::vector<std::vector<real_t>> GELU(std::vector<std::vector<real_t>> z, bool deriv = 0);
double sign(double z, bool deriv = 0); real_t sign(real_t z, bool deriv = 0);
std::vector<double> sign(std::vector<double> z, bool deriv = 0); std::vector<real_t> sign(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<double>> sign(std::vector<std::vector<double>> z, bool deriv = 0); std::vector<std::vector<real_t>> sign(std::vector<std::vector<real_t>> z, bool deriv = 0);
double sinh(double z, bool deriv = 0); real_t sinh(real_t z, bool deriv = 0);
std::vector<double> sinh(std::vector<double> z, bool deriv = 0); std::vector<real_t> sinh(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<double>> sinh(std::vector<std::vector<double>> z, bool deriv = 0); std::vector<std::vector<real_t>> sinh(std::vector<std::vector<real_t>> z, bool deriv = 0);
double cosh(double z, bool deriv = 0); real_t cosh(real_t z, bool deriv = 0);
std::vector<double> cosh(std::vector<double> z, bool deriv = 0); std::vector<real_t> cosh(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<double>> cosh(std::vector<std::vector<double>> z, bool deriv = 0); std::vector<std::vector<real_t>> cosh(std::vector<std::vector<real_t>> z, bool deriv = 0);
double tanh(double z, bool deriv = 0); real_t tanh(real_t z, bool deriv = 0);
std::vector<double> tanh(std::vector<double> z, bool deriv = 0); std::vector<real_t> tanh(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<double>> tanh(std::vector<std::vector<double>> z, bool deriv = 0); std::vector<std::vector<real_t>> tanh(std::vector<std::vector<real_t>> z, bool deriv = 0);
double csch(double z, bool deriv = 0); real_t csch(real_t z, bool deriv = 0);
std::vector<double> csch(std::vector<double> z, bool deriv = 0); std::vector<real_t> csch(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<double>> csch(std::vector<std::vector<double>> z, bool deriv = 0); std::vector<std::vector<real_t>> csch(std::vector<std::vector<real_t>> z, bool deriv = 0);
double sech(double z, bool deriv = 0); real_t sech(real_t z, bool deriv = 0);
std::vector<double> sech(std::vector<double> z, bool deriv = 0); std::vector<real_t> sech(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<double>> sech(std::vector<std::vector<double>> z, bool deriv = 0); std::vector<std::vector<real_t>> sech(std::vector<std::vector<real_t>> z, bool deriv = 0);
double coth(double z, bool deriv = 0); real_t coth(real_t z, bool deriv = 0);
std::vector<double> coth(std::vector<double> z, bool deriv = 0); std::vector<real_t> coth(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<double>> coth(std::vector<std::vector<double>> z, bool deriv = 0); std::vector<std::vector<real_t>> coth(std::vector<std::vector<real_t>> z, bool deriv = 0);
double arsinh(double z, bool deriv = 0); real_t arsinh(real_t z, bool deriv = 0);
std::vector<double> arsinh(std::vector<double> z, bool deriv = 0); std::vector<real_t> arsinh(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<double>> arsinh(std::vector<std::vector<double>> z, bool deriv = 0); std::vector<std::vector<real_t>> arsinh(std::vector<std::vector<real_t>> z, bool deriv = 0);
double arcosh(double z, bool deriv = 0); real_t arcosh(real_t z, bool deriv = 0);
std::vector<double> arcosh(std::vector<double> z, bool deriv = 0); std::vector<real_t> arcosh(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<double>> arcosh(std::vector<std::vector<double>> z, bool deriv = 0); std::vector<std::vector<real_t>> arcosh(std::vector<std::vector<real_t>> z, bool deriv = 0);
double artanh(double z, bool deriv = 0); real_t artanh(real_t z, bool deriv = 0);
std::vector<double> artanh(std::vector<double> z, bool deriv = 0); std::vector<real_t> artanh(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<double>> artanh(std::vector<std::vector<double>> z, bool deriv = 0); std::vector<std::vector<real_t>> artanh(std::vector<std::vector<real_t>> z, bool deriv = 0);
double arcsch(double z, bool deriv = 0); real_t arcsch(real_t z, bool deriv = 0);
std::vector<double> arcsch(std::vector<double> z, bool deriv = 0); std::vector<real_t> arcsch(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<double>> arcsch(std::vector<std::vector<double>> z, bool deriv = 0); std::vector<std::vector<real_t>> arcsch(std::vector<std::vector<real_t>> z, bool deriv = 0);
double arsech(double z, bool deriv = 0); real_t arsech(real_t z, bool deriv = 0);
std::vector<double> arsech(std::vector<double> z, bool deriv = 0); std::vector<real_t> arsech(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<double>> arsech(std::vector<std::vector<double>> z, bool deriv = 0); std::vector<std::vector<real_t>> arsech(std::vector<std::vector<real_t>> z, bool deriv = 0);
double arcoth(double z, bool deriv = 0); real_t arcoth(real_t z, bool deriv = 0);
std::vector<double> arcoth(std::vector<double> z, bool deriv = 0); std::vector<real_t> arcoth(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<double>> arcoth(std::vector<std::vector<double>> z, bool deriv = 0); std::vector<std::vector<real_t>> arcoth(std::vector<std::vector<real_t>> z, bool deriv = 0);
std::vector<double> activation(std::vector<double> z, bool deriv, double (*function)(double, bool)); std::vector<real_t> activation(std::vector<real_t> z, bool deriv, real_t (*function)(real_t, bool));
private: private:
}; };

212
mlpp/ann/ann.cpp
View File

@ -15,7 +15,7 @@
#include <iostream> #include <iostream>
#include <random> #include <random>
MLPPANN::MLPPANN(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet) : MLPPANN::MLPPANN(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet) :
inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), lrScheduler("None"), decayConstant(0), dropRate(0) { inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), lrScheduler("None"), decayConstant(0), dropRate(0) {
} }
@ -23,7 +23,7 @@ MLPPANN::~MLPPANN() {
delete outputLayer; delete outputLayer;
} }
std::vector<double> MLPPANN::modelSetTest(std::vector<std::vector<double>> X) { std::vector<real_t> MLPPANN::modelSetTest(std::vector<std::vector<real_t>> X) {
if (!network.empty()) { if (!network.empty()) {
network[0].input = X; network[0].input = X;
network[0].forwardPass(); network[0].forwardPass();
@ -40,7 +40,7 @@ std::vector<double> MLPPANN::modelSetTest(std::vector<std::vector<double>> X) {
return outputLayer->a; return outputLayer->a;
} }
double MLPPANN::modelTest(std::vector<double> x) { real_t MLPPANN::modelTest(std::vector<real_t> x) {
if (!network.empty()) { if (!network.empty()) {
network[0].Test(x); network[0].Test(x);
for (int i = 1; i < network.size(); i++) { for (int i = 1; i < network.size(); i++) {
@ -53,13 +53,13 @@ double MLPPANN::modelTest(std::vector<double> x) {
return outputLayer->a_test; return outputLayer->a_test;
} }
void MLPPANN::gradientDescent(double learning_rate, int max_epoch, bool UI) { void MLPPANN::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
class MLPPCost cost; class MLPPCost cost;
MLPPLinAlg alg; MLPPLinAlg alg;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
forwardPass(); forwardPass();
double initial_learning_rate = learning_rate; real_t initial_learning_rate = learning_rate;
alg.printMatrix(network[network.size() - 1].weights); alg.printMatrix(network[network.size() - 1].weights);
while (true) { while (true) {
@ -87,13 +87,13 @@ void MLPPANN::gradientDescent(double learning_rate, int max_epoch, bool UI) {
} }
} }
void MLPPANN::SGD(double learning_rate, int max_epoch, bool UI) { void MLPPANN::SGD(real_t learning_rate, int max_epoch, bool UI) {
class MLPPCost cost; class MLPPCost cost;
MLPPLinAlg alg; MLPPLinAlg alg;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
double initial_learning_rate = learning_rate; real_t initial_learning_rate = learning_rate;
while (true) { while (true) {
learning_rate = applyLearningRateScheduler(initial_learning_rate, decayConstant, epoch, dropRate); learning_rate = applyLearningRateScheduler(initial_learning_rate, decayConstant, epoch, dropRate);
@ -103,7 +103,7 @@ void MLPPANN::SGD(double learning_rate, int max_epoch, bool UI) {
std::uniform_int_distribution<int> distribution(0, int(n - 1)); std::uniform_int_distribution<int> distribution(0, int(n - 1));
int outputIndex = distribution(generator); int outputIndex = distribution(generator);
std::vector<double> y_hat = modelSetTest({ inputSet[outputIndex] }); std::vector<real_t> y_hat = modelSetTest({ inputSet[outputIndex] });
cost_prev = Cost({ y_hat }, { outputSet[outputIndex] }); cost_prev = Cost({ y_hat }, { outputSet[outputIndex] });
auto [cumulativeHiddenLayerWGrad, outputWGrad] = computeGradients(y_hat, { outputSet[outputIndex] }); auto [cumulativeHiddenLayerWGrad, outputWGrad] = computeGradients(y_hat, { outputSet[outputIndex] });
@ -125,13 +125,13 @@ void MLPPANN::SGD(double learning_rate, int max_epoch, bool UI) {
forwardPass(); forwardPass();
} }
void MLPPANN::MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI) { void MLPPANN::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI) {
class MLPPCost cost; class MLPPCost cost;
MLPPLinAlg alg; MLPPLinAlg alg;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
double initial_learning_rate = learning_rate; real_t initial_learning_rate = learning_rate;
// Creating the mini-batches // Creating the mini-batches
int n_mini_batch = n / mini_batch_size; int n_mini_batch = n / mini_batch_size;
@ -141,7 +141,7 @@ void MLPPANN::MBGD(double learning_rate, int max_epoch, int mini_batch_size, boo
while (true) { while (true) {
learning_rate = applyLearningRateScheduler(initial_learning_rate, decayConstant, epoch, dropRate); learning_rate = applyLearningRateScheduler(initial_learning_rate, decayConstant, epoch, dropRate);
for (int i = 0; i < n_mini_batch; i++) { for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = modelSetTest(inputMiniBatches[i]); std::vector<real_t> y_hat = modelSetTest(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]); cost_prev = Cost(y_hat, outputMiniBatches[i]);
auto [cumulativeHiddenLayerWGrad, outputWGrad] = computeGradients(y_hat, outputMiniBatches[i]); auto [cumulativeHiddenLayerWGrad, outputWGrad] = computeGradients(y_hat, outputMiniBatches[i]);
@ -163,13 +163,13 @@ void MLPPANN::MBGD(double learning_rate, int max_epoch, int mini_batch_size, boo
forwardPass(); forwardPass();
} }
void MLPPANN::Momentum(double learning_rate, int max_epoch, int mini_batch_size, double gamma, bool NAG, bool UI) { void MLPPANN::Momentum(real_t learning_rate, int max_epoch, int mini_batch_size, real_t gamma, bool NAG, bool UI) {
class MLPPCost cost; class MLPPCost cost;
MLPPLinAlg alg; MLPPLinAlg alg;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
double initial_learning_rate = learning_rate; real_t initial_learning_rate = learning_rate;
// Creating the mini-batches // Creating the mini-batches
int n_mini_batch = n / mini_batch_size; int n_mini_batch = n / mini_batch_size;
@ -178,13 +178,13 @@ void MLPPANN::Momentum(double learning_rate, int max_epoch, int mini_batch_size,
auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch); auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch);
// Initializing necessary components for Adam. // Initializing necessary components for Adam.
std::vector<std::vector<std::vector<double>>> v_hidden; std::vector<std::vector<std::vector<real_t>>> v_hidden;
std::vector<double> v_output; std::vector<real_t> v_output;
while (true) { while (true) {
learning_rate = applyLearningRateScheduler(initial_learning_rate, decayConstant, epoch, dropRate); learning_rate = applyLearningRateScheduler(initial_learning_rate, decayConstant, epoch, dropRate);
for (int i = 0; i < n_mini_batch; i++) { for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = modelSetTest(inputMiniBatches[i]); std::vector<real_t> y_hat = modelSetTest(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]); cost_prev = Cost(y_hat, outputMiniBatches[i]);
auto [cumulativeHiddenLayerWGrad, outputWGrad] = computeGradients(y_hat, outputMiniBatches[i]); auto [cumulativeHiddenLayerWGrad, outputWGrad] = computeGradients(y_hat, outputMiniBatches[i]);
@ -220,13 +220,13 @@ void MLPPANN::Momentum(double learning_rate, int max_epoch, int mini_batch_size,
forwardPass(); forwardPass();
} }
void MLPPANN::Adagrad(double learning_rate, int max_epoch, int mini_batch_size, double e, bool UI) { void MLPPANN::Adagrad(real_t learning_rate, int max_epoch, int mini_batch_size, real_t e, bool UI) {
class MLPPCost cost; class MLPPCost cost;
MLPPLinAlg alg; MLPPLinAlg alg;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
double initial_learning_rate = learning_rate; real_t initial_learning_rate = learning_rate;
// Creating the mini-batches // Creating the mini-batches
int n_mini_batch = n / mini_batch_size; int n_mini_batch = n / mini_batch_size;
@ -235,13 +235,13 @@ void MLPPANN::Adagrad(double learning_rate, int max_epoch, int mini_batch_size,
auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch); auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch);
// Initializing necessary components for Adam. // Initializing necessary components for Adam.
std::vector<std::vector<std::vector<double>>> v_hidden; std::vector<std::vector<std::vector<real_t>>> v_hidden;
std::vector<double> v_output; std::vector<real_t> v_output;
while (true) { while (true) {
learning_rate = applyLearningRateScheduler(initial_learning_rate, decayConstant, epoch, dropRate); learning_rate = applyLearningRateScheduler(initial_learning_rate, decayConstant, epoch, dropRate);
for (int i = 0; i < n_mini_batch; i++) { for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = modelSetTest(inputMiniBatches[i]); std::vector<real_t> y_hat = modelSetTest(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]); cost_prev = Cost(y_hat, outputMiniBatches[i]);
auto [cumulativeHiddenLayerWGrad, outputWGrad] = computeGradients(y_hat, outputMiniBatches[i]); auto [cumulativeHiddenLayerWGrad, outputWGrad] = computeGradients(y_hat, outputMiniBatches[i]);
@ -258,8 +258,8 @@ void MLPPANN::Adagrad(double learning_rate, int max_epoch, int mini_batch_size,
v_output = alg.addition(v_output, alg.exponentiate(outputWGrad, 2)); v_output = alg.addition(v_output, alg.exponentiate(outputWGrad, 2));
std::vector<std::vector<std::vector<double>>> hiddenLayerUpdations = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(cumulativeHiddenLayerWGrad, alg.scalarAdd(e, alg.sqrt(v_hidden)))); std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(cumulativeHiddenLayerWGrad, alg.scalarAdd(e, alg.sqrt(v_hidden))));
std::vector<double> outputLayerUpdation = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(outputWGrad, alg.scalarAdd(e, alg.sqrt(v_output)))); std::vector<real_t> outputLayerUpdation = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(outputWGrad, alg.scalarAdd(e, alg.sqrt(v_output))));
updateParameters(hiddenLayerUpdations, outputLayerUpdation, learning_rate); // subject to change. may want bias to have this matrix too. updateParameters(hiddenLayerUpdations, outputLayerUpdation, learning_rate); // subject to change. may want bias to have this matrix too.
y_hat = modelSetTest(inputMiniBatches[i]); y_hat = modelSetTest(inputMiniBatches[i]);
@ -276,13 +276,13 @@ void MLPPANN::Adagrad(double learning_rate, int max_epoch, int mini_batch_size,
forwardPass(); forwardPass();
} }
void MLPPANN::Adadelta(double learning_rate, int max_epoch, int mini_batch_size, double b1, double e, bool UI) { void MLPPANN::Adadelta(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t e, bool UI) {
class MLPPCost cost; class MLPPCost cost;
MLPPLinAlg alg; MLPPLinAlg alg;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
double initial_learning_rate = learning_rate; real_t initial_learning_rate = learning_rate;
// Creating the mini-batches // Creating the mini-batches
int n_mini_batch = n / mini_batch_size; int n_mini_batch = n / mini_batch_size;
@ -291,13 +291,13 @@ void MLPPANN::Adadelta(double learning_rate, int max_epoch, int mini_batch_size,
auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch); auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch);
// Initializing necessary components for Adam. // Initializing necessary components for Adam.
std::vector<std::vector<std::vector<double>>> v_hidden; std::vector<std::vector<std::vector<real_t>>> v_hidden;
std::vector<double> v_output; std::vector<real_t> v_output;
while (true) { while (true) {
learning_rate = applyLearningRateScheduler(initial_learning_rate, decayConstant, epoch, dropRate); learning_rate = applyLearningRateScheduler(initial_learning_rate, decayConstant, epoch, dropRate);
for (int i = 0; i < n_mini_batch; i++) { for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = modelSetTest(inputMiniBatches[i]); std::vector<real_t> y_hat = modelSetTest(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]); cost_prev = Cost(y_hat, outputMiniBatches[i]);
auto [cumulativeHiddenLayerWGrad, outputWGrad] = computeGradients(y_hat, outputMiniBatches[i]); auto [cumulativeHiddenLayerWGrad, outputWGrad] = computeGradients(y_hat, outputMiniBatches[i]);
@ -314,8 +314,8 @@ void MLPPANN::Adadelta(double learning_rate, int max_epoch, int mini_batch_size,
v_output = alg.addition(v_output, alg.exponentiate(outputWGrad, 2)); v_output = alg.addition(v_output, alg.exponentiate(outputWGrad, 2));
std::vector<std::vector<std::vector<double>>> hiddenLayerUpdations = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(cumulativeHiddenLayerWGrad, alg.scalarAdd(e, alg.sqrt(v_hidden)))); std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(cumulativeHiddenLayerWGrad, alg.scalarAdd(e, alg.sqrt(v_hidden))));
std::vector<double> outputLayerUpdation = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(outputWGrad, alg.scalarAdd(e, alg.sqrt(v_output)))); std::vector<real_t> outputLayerUpdation = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(outputWGrad, alg.scalarAdd(e, alg.sqrt(v_output))));
updateParameters(hiddenLayerUpdations, outputLayerUpdation, learning_rate); // subject to change. may want bias to have this matrix too. updateParameters(hiddenLayerUpdations, outputLayerUpdation, learning_rate); // subject to change. may want bias to have this matrix too.
y_hat = modelSetTest(inputMiniBatches[i]); y_hat = modelSetTest(inputMiniBatches[i]);
@ -332,13 +332,13 @@ void MLPPANN::Adadelta(double learning_rate, int max_epoch, int mini_batch_size,
forwardPass(); forwardPass();
} }
void MLPPANN::Adam(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI) { void MLPPANN::Adam(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t b2, real_t e, bool UI) {
class MLPPCost cost; class MLPPCost cost;
MLPPLinAlg alg; MLPPLinAlg alg;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
double initial_learning_rate = learning_rate; real_t initial_learning_rate = learning_rate;
// Creating the mini-batches // Creating the mini-batches
int n_mini_batch = n / mini_batch_size; int n_mini_batch = n / mini_batch_size;
@ -347,15 +347,15 @@ void MLPPANN::Adam(double learning_rate, int max_epoch, int mini_batch_size, dou
auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch); auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch);
// Initializing necessary components for Adam. // Initializing necessary components for Adam.
std::vector<std::vector<std::vector<double>>> m_hidden; std::vector<std::vector<std::vector<real_t>>> m_hidden;
std::vector<std::vector<std::vector<double>>> v_hidden; std::vector<std::vector<std::vector<real_t>>> v_hidden;
std::vector<double> m_output; std::vector<real_t> m_output;
std::vector<double> v_output; std::vector<real_t> v_output;
while (true) { while (true) {
learning_rate = applyLearningRateScheduler(initial_learning_rate, decayConstant, epoch, dropRate); learning_rate = applyLearningRateScheduler(initial_learning_rate, decayConstant, epoch, dropRate);
for (int i = 0; i < n_mini_batch; i++) { for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = modelSetTest(inputMiniBatches[i]); std::vector<real_t> y_hat = modelSetTest(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]); cost_prev = Cost(y_hat, outputMiniBatches[i]);
auto [cumulativeHiddenLayerWGrad, outputWGrad] = computeGradients(y_hat, outputMiniBatches[i]); auto [cumulativeHiddenLayerWGrad, outputWGrad] = computeGradients(y_hat, outputMiniBatches[i]);
@ -375,14 +375,14 @@ void MLPPANN::Adam(double learning_rate, int max_epoch, int mini_batch_size, dou
m_output = alg.addition(alg.scalarMultiply(b1, m_output), alg.scalarMultiply(1 - b1, outputWGrad)); m_output = alg.addition(alg.scalarMultiply(b1, m_output), alg.scalarMultiply(1 - b1, outputWGrad));
v_output = alg.addition(alg.scalarMultiply(b2, v_output), alg.scalarMultiply(1 - b2, alg.exponentiate(outputWGrad, 2))); v_output = alg.addition(alg.scalarMultiply(b2, v_output), alg.scalarMultiply(1 - b2, alg.exponentiate(outputWGrad, 2)));
std::vector<std::vector<std::vector<double>>> m_hidden_hat = alg.scalarMultiply(1 / (1 - std::pow(b1, epoch)), m_hidden); std::vector<std::vector<std::vector<real_t>>> m_hidden_hat = alg.scalarMultiply(1 / (1 - std::pow(b1, epoch)), m_hidden);
std::vector<std::vector<std::vector<double>>> v_hidden_hat = alg.scalarMultiply(1 / (1 - std::pow(b2, epoch)), v_hidden); std::vector<std::vector<std::vector<real_t>>> v_hidden_hat = alg.scalarMultiply(1 / (1 - std::pow(b2, epoch)), v_hidden);
std::vector<double> m_output_hat = alg.scalarMultiply(1 / (1 - std::pow(b1, epoch)), m_output); std::vector<real_t> m_output_hat = alg.scalarMultiply(1 / (1 - std::pow(b1, epoch)), m_output);
std::vector<double> v_output_hat = alg.scalarMultiply(1 / (1 - std::pow(b2, epoch)), v_output); std::vector<real_t> v_output_hat = alg.scalarMultiply(1 / (1 - std::pow(b2, epoch)), v_output);
std::vector<std::vector<std::vector<double>>> hiddenLayerUpdations = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(m_hidden_hat, alg.scalarAdd(e, alg.sqrt(v_hidden_hat)))); std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(m_hidden_hat, alg.scalarAdd(e, alg.sqrt(v_hidden_hat))));
std::vector<double> outputLayerUpdation = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(m_output_hat, alg.scalarAdd(e, alg.sqrt(v_output_hat)))); std::vector<real_t> outputLayerUpdation = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(m_output_hat, alg.scalarAdd(e, alg.sqrt(v_output_hat))));
updateParameters(hiddenLayerUpdations, outputLayerUpdation, learning_rate); // subject to change. may want bias to have this matrix too. updateParameters(hiddenLayerUpdations, outputLayerUpdation, learning_rate); // subject to change. may want bias to have this matrix too.
y_hat = modelSetTest(inputMiniBatches[i]); y_hat = modelSetTest(inputMiniBatches[i]);
@ -399,13 +399,13 @@ void MLPPANN::Adam(double learning_rate, int max_epoch, int mini_batch_size, dou
forwardPass(); forwardPass();
} }
void MLPPANN::Adamax(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI) { void MLPPANN::Adamax(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t b2, real_t e, bool UI) {
class MLPPCost cost; class MLPPCost cost;
MLPPLinAlg alg; MLPPLinAlg alg;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
double initial_learning_rate = learning_rate; real_t initial_learning_rate = learning_rate;
// Creating the mini-batches // Creating the mini-batches
int n_mini_batch = n / mini_batch_size; int n_mini_batch = n / mini_batch_size;
@ -414,15 +414,15 @@ void MLPPANN::Adamax(double learning_rate, int max_epoch, int mini_batch_size, d
auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch); auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch);
// Initializing necessary components for Adam. // Initializing necessary components for Adam.
std::vector<std::vector<std::vector<double>>> m_hidden; std::vector<std::vector<std::vector<real_t>>> m_hidden;
std::vector<std::vector<std::vector<double>>> u_hidden; std::vector<std::vector<std::vector<real_t>>> u_hidden;
std::vector<double> m_output; std::vector<real_t> m_output;
std::vector<double> u_output; std::vector<real_t> u_output;
while (true) { while (true) {
learning_rate = applyLearningRateScheduler(initial_learning_rate, decayConstant, epoch, dropRate); learning_rate = applyLearningRateScheduler(initial_learning_rate, decayConstant, epoch, dropRate);
for (int i = 0; i < n_mini_batch; i++) { for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = modelSetTest(inputMiniBatches[i]); std::vector<real_t> y_hat = modelSetTest(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]); cost_prev = Cost(y_hat, outputMiniBatches[i]);
auto [cumulativeHiddenLayerWGrad, outputWGrad] = computeGradients(y_hat, outputMiniBatches[i]); auto [cumulativeHiddenLayerWGrad, outputWGrad] = computeGradients(y_hat, outputMiniBatches[i]);
@ -442,12 +442,12 @@ void MLPPANN::Adamax(double learning_rate, int max_epoch, int mini_batch_size, d
m_output = alg.addition(alg.scalarMultiply(b1, m_output), alg.scalarMultiply(1 - b1, outputWGrad)); m_output = alg.addition(alg.scalarMultiply(b1, m_output), alg.scalarMultiply(1 - b1, outputWGrad));
u_output = alg.max(alg.scalarMultiply(b2, u_output), alg.abs(outputWGrad)); u_output = alg.max(alg.scalarMultiply(b2, u_output), alg.abs(outputWGrad));
std::vector<std::vector<std::vector<double>>> m_hidden_hat = alg.scalarMultiply(1 / (1 - std::pow(b1, epoch)), m_hidden); std::vector<std::vector<std::vector<real_t>>> m_hidden_hat = alg.scalarMultiply(1 / (1 - std::pow(b1, epoch)), m_hidden);
std::vector<double> m_output_hat = alg.scalarMultiply(1 / (1 - std::pow(b1, epoch)), m_output); std::vector<real_t> m_output_hat = alg.scalarMultiply(1 / (1 - std::pow(b1, epoch)), m_output);
std::vector<std::vector<std::vector<double>>> hiddenLayerUpdations = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(m_hidden_hat, alg.scalarAdd(e, u_hidden))); std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(m_hidden_hat, alg.scalarAdd(e, u_hidden)));
std::vector<double> outputLayerUpdation = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(m_output_hat, alg.scalarAdd(e, u_output))); std::vector<real_t> outputLayerUpdation = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(m_output_hat, alg.scalarAdd(e, u_output)));
updateParameters(hiddenLayerUpdations, outputLayerUpdation, learning_rate); // subject to change. may want bias to have this matrix too. updateParameters(hiddenLayerUpdations, outputLayerUpdation, learning_rate); // subject to change. may want bias to have this matrix too.
y_hat = modelSetTest(inputMiniBatches[i]); y_hat = modelSetTest(inputMiniBatches[i]);
@ -464,13 +464,13 @@ void MLPPANN::Adamax(double learning_rate, int max_epoch, int mini_batch_size, d
forwardPass(); forwardPass();
} }
void MLPPANN::Nadam(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI) { void MLPPANN::Nadam(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t b2, real_t e, bool UI) {
class MLPPCost cost; class MLPPCost cost;
MLPPLinAlg alg; MLPPLinAlg alg;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
double initial_learning_rate = learning_rate; real_t initial_learning_rate = learning_rate;
// Creating the mini-batches // Creating the mini-batches
int n_mini_batch = n / mini_batch_size; int n_mini_batch = n / mini_batch_size;
@ -479,16 +479,16 @@ void MLPPANN::Nadam(double learning_rate, int max_epoch, int mini_batch_size, do
auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch); auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch);
// Initializing necessary components for Adam. // Initializing necessary components for Adam.
std::vector<std::vector<std::vector<double>>> m_hidden; std::vector<std::vector<std::vector<real_t>>> m_hidden;
std::vector<std::vector<std::vector<double>>> v_hidden; std::vector<std::vector<std::vector<real_t>>> v_hidden;
std::vector<std::vector<std::vector<double>>> m_hidden_final; std::vector<std::vector<std::vector<real_t>>> m_hidden_final;
std::vector<double> m_output; std::vector<real_t> m_output;
std::vector<double> v_output; std::vector<real_t> v_output;
while (true) { while (true) {
learning_rate = applyLearningRateScheduler(initial_learning_rate, decayConstant, epoch, dropRate); learning_rate = applyLearningRateScheduler(initial_learning_rate, decayConstant, epoch, dropRate);
for (int i = 0; i < n_mini_batch; i++) { for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = modelSetTest(inputMiniBatches[i]); std::vector<real_t> y_hat = modelSetTest(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]); cost_prev = Cost(y_hat, outputMiniBatches[i]);
auto [cumulativeHiddenLayerWGrad, outputWGrad] = computeGradients(y_hat, outputMiniBatches[i]); auto [cumulativeHiddenLayerWGrad, outputWGrad] = computeGradients(y_hat, outputMiniBatches[i]);
@ -508,16 +508,16 @@ void MLPPANN::Nadam(double learning_rate, int max_epoch, int mini_batch_size, do
m_output = alg.addition(alg.scalarMultiply(b1, m_output), alg.scalarMultiply(1 - b1, outputWGrad)); m_output = alg.addition(alg.scalarMultiply(b1, m_output), alg.scalarMultiply(1 - b1, outputWGrad));
v_output = alg.addition(alg.scalarMultiply(b2, v_output), alg.scalarMultiply(1 - b2, alg.exponentiate(outputWGrad, 2))); v_output = alg.addition(alg.scalarMultiply(b2, v_output), alg.scalarMultiply(1 - b2, alg.exponentiate(outputWGrad, 2)));
std::vector<std::vector<std::vector<double>>> m_hidden_hat = alg.scalarMultiply(1 / (1 - std::pow(b1, epoch)), m_hidden); std::vector<std::vector<std::vector<real_t>>> m_hidden_hat = alg.scalarMultiply(1 / (1 - std::pow(b1, epoch)), m_hidden);
std::vector<std::vector<std::vector<double>>> v_hidden_hat = alg.scalarMultiply(1 / (1 - std::pow(b2, epoch)), v_hidden); std::vector<std::vector<std::vector<real_t>>> v_hidden_hat = alg.scalarMultiply(1 / (1 - std::pow(b2, epoch)), v_hidden);
std::vector<std::vector<std::vector<double>>> m_hidden_final = alg.addition(alg.scalarMultiply(b1, m_hidden_hat), alg.scalarMultiply((1 - b1) / (1 - std::pow(b1, epoch)), cumulativeHiddenLayerWGrad)); std::vector<std::vector<std::vector<real_t>>> m_hidden_final = alg.addition(alg.scalarMultiply(b1, m_hidden_hat), alg.scalarMultiply((1 - b1) / (1 - std::pow(b1, epoch)), cumulativeHiddenLayerWGrad));
std::vector<double> m_output_hat = alg.scalarMultiply(1 / (1 - std::pow(b1, epoch)), m_output); std::vector<real_t> m_output_hat = alg.scalarMultiply(1 / (1 - std::pow(b1, epoch)), m_output);
std::vector<double> v_output_hat = alg.scalarMultiply(1 / (1 - std::pow(b2, epoch)), v_output); std::vector<real_t> v_output_hat = alg.scalarMultiply(1 / (1 - std::pow(b2, epoch)), v_output);
std::vector<double> m_output_final = alg.addition(alg.scalarMultiply(b1, m_output_hat), alg.scalarMultiply((1 - b1) / (1 - std::pow(b1, epoch)), outputWGrad)); std::vector<real_t> m_output_final = alg.addition(alg.scalarMultiply(b1, m_output_hat), alg.scalarMultiply((1 - b1) / (1 - std::pow(b1, epoch)), outputWGrad));
std::vector<std::vector<std::vector<double>>> hiddenLayerUpdations = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(m_hidden_final, alg.scalarAdd(e, alg.sqrt(v_hidden_hat)))); std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(m_hidden_final, alg.scalarAdd(e, alg.sqrt(v_hidden_hat))));
std::vector<double> outputLayerUpdation = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(m_output_final, alg.scalarAdd(e, alg.sqrt(v_output_hat)))); std::vector<real_t> outputLayerUpdation = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(m_output_final, alg.scalarAdd(e, alg.sqrt(v_output_hat))));
updateParameters(hiddenLayerUpdations, outputLayerUpdation, learning_rate); // subject to change. may want bias to have this matrix too. updateParameters(hiddenLayerUpdations, outputLayerUpdation, learning_rate); // subject to change. may want bias to have this matrix too.
y_hat = modelSetTest(inputMiniBatches[i]); y_hat = modelSetTest(inputMiniBatches[i]);
@ -534,13 +534,13 @@ void MLPPANN::Nadam(double learning_rate, int max_epoch, int mini_batch_size, do
forwardPass(); forwardPass();
} }
void MLPPANN::AMSGrad(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI) { void MLPPANN::AMSGrad(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t b2, real_t e, bool UI) {
class MLPPCost cost; class MLPPCost cost;
MLPPLinAlg alg; MLPPLinAlg alg;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
double initial_learning_rate = learning_rate; real_t initial_learning_rate = learning_rate;
// Creating the mini-batches // Creating the mini-batches
int n_mini_batch = n / mini_batch_size; int n_mini_batch = n / mini_batch_size;
@ -549,19 +549,19 @@ void MLPPANN::AMSGrad(double learning_rate, int max_epoch, int mini_batch_size,
auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch); auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch);
// Initializing necessary components for Adam. // Initializing necessary components for Adam.
std::vector<std::vector<std::vector<double>>> m_hidden; std::vector<std::vector<std::vector<real_t>>> m_hidden;
std::vector<std::vector<std::vector<double>>> v_hidden; std::vector<std::vector<std::vector<real_t>>> v_hidden;
std::vector<std::vector<std::vector<double>>> v_hidden_hat; std::vector<std::vector<std::vector<real_t>>> v_hidden_hat;
std::vector<double> m_output; std::vector<real_t> m_output;
std::vector<double> v_output; std::vector<real_t> v_output;
std::vector<double> v_output_hat; std::vector<real_t> v_output_hat;
while (true) { while (true) {
learning_rate = applyLearningRateScheduler(initial_learning_rate, decayConstant, epoch, dropRate); learning_rate = applyLearningRateScheduler(initial_learning_rate, decayConstant, epoch, dropRate);
for (int i = 0; i < n_mini_batch; i++) { for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = modelSetTest(inputMiniBatches[i]); std::vector<real_t> y_hat = modelSetTest(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]); cost_prev = Cost(y_hat, outputMiniBatches[i]);
auto [cumulativeHiddenLayerWGrad, outputWGrad] = computeGradients(y_hat, outputMiniBatches[i]); auto [cumulativeHiddenLayerWGrad, outputWGrad] = computeGradients(y_hat, outputMiniBatches[i]);
@ -587,8 +587,8 @@ void MLPPANN::AMSGrad(double learning_rate, int max_epoch, int mini_batch_size,
v_output_hat = alg.max(v_output_hat, v_output); v_output_hat = alg.max(v_output_hat, v_output);
std::vector<std::vector<std::vector<double>>> hiddenLayerUpdations = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(m_hidden, alg.scalarAdd(e, alg.sqrt(v_hidden_hat)))); std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(m_hidden, alg.scalarAdd(e, alg.sqrt(v_hidden_hat))));
std::vector<double> outputLayerUpdation = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(m_output, alg.scalarAdd(e, alg.sqrt(v_output_hat)))); std::vector<real_t> outputLayerUpdation = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(m_output, alg.scalarAdd(e, alg.sqrt(v_output_hat))));
updateParameters(hiddenLayerUpdations, outputLayerUpdation, learning_rate); // subject to change. may want bias to have this matrix too. updateParameters(hiddenLayerUpdations, outputLayerUpdation, learning_rate); // subject to change. may want bias to have this matrix too.
y_hat = modelSetTest(inputMiniBatches[i]); y_hat = modelSetTest(inputMiniBatches[i]);
@ -605,7 +605,7 @@ void MLPPANN::AMSGrad(double learning_rate, int max_epoch, int mini_batch_size,
forwardPass(); forwardPass();
} }
double MLPPANN::score() { real_t MLPPANN::score() {
MLPPUtilities util; MLPPUtilities util;
forwardPass(); forwardPass();
return util.performance(y_hat, outputSet); return util.performance(y_hat, outputSet);
@ -624,12 +624,12 @@ void MLPPANN::save(std::string fileName) {
} }
} }
void MLPPANN::setLearningRateScheduler(std::string type, double decayConstant) { void MLPPANN::setLearningRateScheduler(std::string type, real_t decayConstant) {
lrScheduler = type; lrScheduler = type;
MLPPANN::decayConstant = decayConstant; MLPPANN::decayConstant = decayConstant;
} }
void MLPPANN::setLearningRateScheduler(std::string type, double decayConstant, double dropRate) { void MLPPANN::setLearningRateScheduler(std::string type, real_t decayConstant, real_t dropRate) {
lrScheduler = type; lrScheduler = type;
MLPPANN::decayConstant = decayConstant; MLPPANN::decayConstant = decayConstant;
MLPPANN::dropRate = dropRate; MLPPANN::dropRate = dropRate;
@ -637,7 +637,7 @@ void MLPPANN::setLearningRateScheduler(std::string type, double decayConstant, d
// https://en.wikipedia.org/wiki/Learning_rate // https://en.wikipedia.org/wiki/Learning_rate
// Learning Rate Decay (C2W2L09) - Andrew Ng - Deep Learning Specialization // Learning Rate Decay (C2W2L09) - Andrew Ng - Deep Learning Specialization
double MLPPANN::applyLearningRateScheduler(double learningRate, double decayConstant, double epoch, double dropRate) { real_t MLPPANN::applyLearningRateScheduler(real_t learningRate, real_t decayConstant, real_t epoch, real_t dropRate) {
if (lrScheduler == "Time") { if (lrScheduler == "Time") {
return learningRate / (1 + decayConstant * epoch); return learningRate / (1 + decayConstant * epoch);
} else if (lrScheduler == "Epoch") { } else if (lrScheduler == "Epoch") {
@ -650,7 +650,7 @@ double MLPPANN::applyLearningRateScheduler(double learningRate, double decayCons
return learningRate; return learningRate;
} }
void MLPPANN::addLayer(int n_hidden, std::string activation, std::string weightInit, std::string reg, double lambda, double alpha) { void MLPPANN::addLayer(int n_hidden, std::string activation, std::string weightInit, std::string reg, real_t lambda, real_t alpha) {
if (network.empty()) { if (network.empty()) {
network.push_back(MLPPHiddenLayer(n_hidden, activation, inputSet, weightInit, reg, lambda, alpha)); network.push_back(MLPPHiddenLayer(n_hidden, activation, inputSet, weightInit, reg, lambda, alpha));
network[0].forwardPass(); network[0].forwardPass();
@ -660,7 +660,7 @@ void MLPPANN::addLayer(int n_hidden, std::string activation, std::string weightI
} }
} }
void MLPPANN::addOutputLayer(std::string activation, std::string loss, std::string weightInit, std::string reg, double lambda, double alpha) { void MLPPANN::addOutputLayer(std::string activation, std::string loss, std::string weightInit, std::string reg, real_t lambda, real_t alpha) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (!network.empty()) { if (!network.empty()) {
outputLayer = new MLPPOutputLayer(network[network.size() - 1].n_hidden, activation, loss, network[network.size() - 1].a, weightInit, reg, lambda, alpha); outputLayer = new MLPPOutputLayer(network[network.size() - 1].n_hidden, activation, loss, network[network.size() - 1].a, weightInit, reg, lambda, alpha);
@ -669,10 +669,10 @@ void MLPPANN::addOutputLayer(std::string activation, std::string loss, std::stri
} }
} }
double MLPPANN::Cost(std::vector<double> y_hat, std::vector<double> y) { real_t MLPPANN::Cost(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPReg regularization; MLPPReg regularization;
class MLPPCost cost; class MLPPCost cost;
double totalRegTerm = 0; real_t totalRegTerm = 0;
auto cost_function = outputLayer->cost_map[outputLayer->cost]; auto cost_function = outputLayer->cost_map[outputLayer->cost];
if (!network.empty()) { if (!network.empty()) {
@ -700,7 +700,7 @@ void MLPPANN::forwardPass() {
y_hat = outputLayer->a; y_hat = outputLayer->a;
} }
void MLPPANN::updateParameters(std::vector<std::vector<std::vector<double>>> hiddenLayerUpdations, std::vector<double> outputLayerUpdation, double learning_rate) { void MLPPANN::updateParameters(std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations, std::vector<real_t> outputLayerUpdation, real_t learning_rate) {
MLPPLinAlg alg; MLPPLinAlg alg;
outputLayer->weights = alg.subtraction(outputLayer->weights, outputLayerUpdation); outputLayer->weights = alg.subtraction(outputLayer->weights, outputLayerUpdation);
@ -717,39 +717,39 @@ void MLPPANN::updateParameters(std::vector<std::vector<std::vector<double>>> hid
} }
} }
std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<double>> MLPPANN::computeGradients(std::vector<double> y_hat, std::vector<double> outputSet) { std::tuple<std::vector<std::vector<std::vector<real_t>>>, std::vector<real_t>> MLPPANN::computeGradients(std::vector<real_t> y_hat, std::vector<real_t> outputSet) {
// std::cout << "BEGIN" << std::endl; // std::cout << "BEGIN" << std::endl;
class MLPPCost cost; class MLPPCost cost;
MLPPActivation avn; MLPPActivation avn;
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
std::vector<std::vector<std::vector<double>>> cumulativeHiddenLayerWGrad; // Tensor containing ALL hidden grads. std::vector<std::vector<std::vector<real_t>>> cumulativeHiddenLayerWGrad; // Tensor containing ALL hidden grads.
auto costDeriv = outputLayer->costDeriv_map[outputLayer->cost]; auto costDeriv = outputLayer->costDeriv_map[outputLayer->cost];
auto outputAvn = outputLayer->activation_map[outputLayer->activation]; auto outputAvn = outputLayer->activation_map[outputLayer->activation];
outputLayer->delta = alg.hadamard_product((cost.*costDeriv)(y_hat, outputSet), (avn.*outputAvn)(outputLayer->z, 1)); outputLayer->delta = alg.hadamard_product((cost.*costDeriv)(y_hat, outputSet), (avn.*outputAvn)(outputLayer->z, 1));
std::vector<double> outputWGrad = alg.mat_vec_mult(alg.transpose(outputLayer->input), outputLayer->delta); std::vector<real_t> outputWGrad = alg.mat_vec_mult(alg.transpose(outputLayer->input), outputLayer->delta);
outputWGrad = alg.addition(outputWGrad, regularization.regDerivTerm(outputLayer->weights, outputLayer->lambda, outputLayer->alpha, outputLayer->reg)); outputWGrad = alg.addition(outputWGrad, regularization.regDerivTerm(outputLayer->weights, outputLayer->lambda, outputLayer->alpha, outputLayer->reg));
if (!network.empty()) { if (!network.empty()) {
auto hiddenLayerAvn = network[network.size() - 1].activation_map[network[network.size() - 1].activation]; auto hiddenLayerAvn = network[network.size() - 1].activation_map[network[network.size() - 1].activation];
network[network.size() - 1].delta = alg.hadamard_product(alg.outerProduct(outputLayer->delta, outputLayer->weights), (avn.*hiddenLayerAvn)(network[network.size() - 1].z, 1)); network[network.size() - 1].delta = alg.hadamard_product(alg.outerProduct(outputLayer->delta, outputLayer->weights), (avn.*hiddenLayerAvn)(network[network.size() - 1].z, 1));
std::vector<std::vector<double>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[network.size() - 1].input), network[network.size() - 1].delta); std::vector<std::vector<real_t>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[network.size() - 1].input), network[network.size() - 1].delta);
cumulativeHiddenLayerWGrad.push_back(alg.addition(hiddenLayerWGrad, regularization.regDerivTerm(network[network.size() - 1].weights, network[network.size() - 1].lambda, network[network.size() - 1].alpha, network[network.size() - 1].reg))); // Adding to our cumulative hidden layer grads. Maintain reg terms as well. cumulativeHiddenLayerWGrad.push_back(alg.addition(hiddenLayerWGrad, regularization.regDerivTerm(network[network.size() - 1].weights, network[network.size() - 1].lambda, network[network.size() - 1].alpha, network[network.size() - 1].reg))); // Adding to our cumulative hidden layer grads. Maintain reg terms as well.
for (int i = network.size() - 2; i >= 0; i--) { for (int i = network.size() - 2; i >= 0; i--) {
auto hiddenLayerAvn = network[i].activation_map[network[i].activation]; auto hiddenLayerAvn = network[i].activation_map[network[i].activation];
network[i].delta = alg.hadamard_product(alg.matmult(network[i + 1].delta, alg.transpose(network[i + 1].weights)), (avn.*hiddenLayerAvn)(network[i].z, 1)); network[i].delta = alg.hadamard_product(alg.matmult(network[i + 1].delta, alg.transpose(network[i + 1].weights)), (avn.*hiddenLayerAvn)(network[i].z, 1));
std::vector<std::vector<double>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[i].input), network[i].delta); std::vector<std::vector<real_t>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[i].input), network[i].delta);
cumulativeHiddenLayerWGrad.push_back(alg.addition(hiddenLayerWGrad, regularization.regDerivTerm(network[i].weights, network[i].lambda, network[i].alpha, network[i].reg))); // Adding to our cumulative hidden layer grads. Maintain reg terms as well. cumulativeHiddenLayerWGrad.push_back(alg.addition(hiddenLayerWGrad, regularization.regDerivTerm(network[i].weights, network[i].lambda, network[i].alpha, network[i].reg))); // Adding to our cumulative hidden layer grads. Maintain reg terms as well.
} }
} }
return { cumulativeHiddenLayerWGrad, outputWGrad }; return { cumulativeHiddenLayerWGrad, outputWGrad };
} }
void MLPPANN::UI(int epoch, double cost_prev, std::vector<double> y_hat, std::vector<double> outputSet) { void MLPPANN::UI(int epoch, real_t cost_prev, std::vector<real_t> y_hat, std::vector<real_t> outputSet) {
MLPPUtilities::CostInfo(epoch, cost_prev, Cost(y_hat, outputSet)); MLPPUtilities::CostInfo(epoch, cost_prev, Cost(y_hat, outputSet));
std::cout << "Layer " << network.size() + 1 << ": " << std::endl; std::cout << "Layer " << network.size() + 1 << ": " << std::endl;
MLPPUtilities::UI(outputLayer->weights, outputLayer->bias); MLPPUtilities::UI(outputLayer->weights, outputLayer->bias);

58
mlpp/ann/ann.h
View File

@ -7,6 +7,8 @@
// Created by Marc Melikyan on 11/4/20. // Created by Marc Melikyan on 11/4/20.
// //
#include "core/math/math_defs.h"
#include "../hidden_layer/hidden_layer.h" #include "../hidden_layer/hidden_layer.h"
#include "../output_layer/output_layer.h" #include "../output_layer/output_layer.h"
@ -16,43 +18,43 @@
class MLPPANN { class MLPPANN {
public: public:
MLPPANN(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet); MLPPANN(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet);
~MLPPANN(); ~MLPPANN();
std::vector<double> modelSetTest(std::vector<std::vector<double>> X); std::vector<real_t> modelSetTest(std::vector<std::vector<real_t>> X);
double modelTest(std::vector<double> x); real_t modelTest(std::vector<real_t> x);
void gradientDescent(double learning_rate, int max_epoch, bool UI = 1); void gradientDescent(real_t learning_rate, int max_epoch, bool UI = 1);
void SGD(double learning_rate, int max_epoch, bool UI = 1); void SGD(real_t learning_rate, int max_epoch, bool UI = 1);
void MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI = 1); void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
void Momentum(double learning_rate, int max_epoch, int mini_batch_size, double gamma, bool NAG, bool UI = 1); void Momentum(real_t learning_rate, int max_epoch, int mini_batch_size, real_t gamma, bool NAG, bool UI = 1);
void Adagrad(double learning_rate, int max_epoch, int mini_batch_size, double e, bool UI = 1); void Adagrad(real_t learning_rate, int max_epoch, int mini_batch_size, real_t e, bool UI = 1);
void Adadelta(double learning_rate, int max_epoch, int mini_batch_size, double b1, double e, bool UI = 1); void Adadelta(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t e, bool UI = 1);
void Adam(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI = 1); void Adam(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t b2, real_t e, bool UI = 1);
void Adamax(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI = 1); void Adamax(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t b2, real_t e, bool UI = 1);
void Nadam(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI = 1); void Nadam(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t b2, real_t e, bool UI = 1);
void AMSGrad(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI = 1); void AMSGrad(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t b2, real_t e, bool UI = 1);
double score(); real_t score();
void save(std::string fileName); void save(std::string fileName);
void setLearningRateScheduler(std::string type, double decayConstant); void setLearningRateScheduler(std::string type, real_t decayConstant);
void setLearningRateScheduler(std::string type, double decayConstant, double dropRate); void setLearningRateScheduler(std::string type, real_t decayConstant, real_t dropRate);
void addLayer(int n_hidden, std::string activation, std::string weightInit = "Default", std::string reg = "None", double lambda = 0.5, double alpha = 0.5); void addLayer(int n_hidden, std::string activation, std::string weightInit = "Default", std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
void addOutputLayer(std::string activation, std::string loss, std::string weightInit = "Default", std::string reg = "None", double lambda = 0.5, double alpha = 0.5); void addOutputLayer(std::string activation, std::string loss, std::string weightInit = "Default", std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
private: private:
double applyLearningRateScheduler(double learningRate, double decayConstant, double epoch, double dropRate); real_t applyLearningRateScheduler(real_t learningRate, real_t decayConstant, real_t epoch, real_t dropRate);
double Cost(std::vector<double> y_hat, std::vector<double> y); real_t Cost(std::vector<real_t> y_hat, std::vector<real_t> y);
void forwardPass(); void forwardPass();
void updateParameters(std::vector<std::vector<std::vector<double>>> hiddenLayerUpdations, std::vector<double> outputLayerUpdation, double learning_rate); void updateParameters(std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations, std::vector<real_t> outputLayerUpdation, real_t learning_rate);
std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<double>> computeGradients(std::vector<double> y_hat, std::vector<double> outputSet); std::tuple<std::vector<std::vector<std::vector<real_t>>>, std::vector<real_t>> computeGradients(std::vector<real_t> y_hat, std::vector<real_t> outputSet);
void UI(int epoch, double cost_prev, std::vector<double> y_hat, std::vector<double> outputSet); void UI(int epoch, real_t cost_prev, std::vector<real_t> y_hat, std::vector<real_t> outputSet);
std::vector<std::vector<double>> inputSet; std::vector<std::vector<real_t>> inputSet;
std::vector<double> outputSet; std::vector<real_t> outputSet;
std::vector<double> y_hat; std::vector<real_t> y_hat;
std::vector<MLPPHiddenLayer> network; std::vector<MLPPHiddenLayer> network;
MLPPOutputLayer *outputLayer; MLPPOutputLayer *outputLayer;
@ -61,8 +63,8 @@ private:
int k; int k;
std::string lrScheduler; std::string lrScheduler;
double decayConstant; real_t decayConstant;
double dropRate; real_t dropRate;
}; };
#endif /* ANN_hpp */ #endif /* ANN_hpp */

82
mlpp/auto_encoder/auto_encoder.cpp
View File

@ -13,7 +13,7 @@
#include <iostream> #include <iostream>
#include <random> #include <random>
MLPPAutoEncoder::MLPPAutoEncoder(std::vector<std::vector<double>> inputSet, int n_hidden) : MLPPAutoEncoder::MLPPAutoEncoder(std::vector<std::vector<real_t>> inputSet, int n_hidden) :
inputSet(inputSet), n_hidden(n_hidden), n(inputSet.size()), k(inputSet[0].size()) { inputSet(inputSet), n_hidden(n_hidden), n(inputSet.size()), k(inputSet[0].size()) {
MLPPActivation avn; MLPPActivation avn;
y_hat.resize(inputSet.size()); y_hat.resize(inputSet.size());
@ -24,18 +24,18 @@ MLPPAutoEncoder::MLPPAutoEncoder(std::vector<std::vector<double>> inputSet, int
bias2 = MLPPUtilities::biasInitialization(k); bias2 = MLPPUtilities::biasInitialization(k);
} }
std::vector<std::vector<double>> MLPPAutoEncoder::modelSetTest(std::vector<std::vector<double>> X) { std::vector<std::vector<real_t>> MLPPAutoEncoder::modelSetTest(std::vector<std::vector<real_t>> X) {
return Evaluate(X); return Evaluate(X);
} }
std::vector<double> MLPPAutoEncoder::modelTest(std::vector<double> x) { std::vector<real_t> MLPPAutoEncoder::modelTest(std::vector<real_t> x) {
return Evaluate(x); return Evaluate(x);
} }
void MLPPAutoEncoder::gradientDescent(double learning_rate, int max_epoch, bool UI) { void MLPPAutoEncoder::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
MLPPActivation avn; MLPPActivation avn;
MLPPLinAlg alg; MLPPLinAlg alg;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
forwardPass(); forwardPass();
@ -43,10 +43,10 @@ void MLPPAutoEncoder::gradientDescent(double learning_rate, int max_epoch, bool
cost_prev = Cost(y_hat, inputSet); cost_prev = Cost(y_hat, inputSet);
// Calculating the errors // Calculating the errors
std::vector<std::vector<double>> error = alg.subtraction(y_hat, inputSet); std::vector<std::vector<real_t>> error = alg.subtraction(y_hat, inputSet);
// Calculating the weight/bias gradients for layer 2 // Calculating the weight/bias gradients for layer 2
std::vector<std::vector<double>> D2_1 = alg.matmult(alg.transpose(a2), error); std::vector<std::vector<real_t>> D2_1 = alg.matmult(alg.transpose(a2), error);
// weights and bias updation for layer 2 // weights and bias updation for layer 2
weights2 = alg.subtraction(weights2, alg.scalarMultiply(learning_rate / n, D2_1)); weights2 = alg.subtraction(weights2, alg.scalarMultiply(learning_rate / n, D2_1));
@ -56,11 +56,11 @@ void MLPPAutoEncoder::gradientDescent(double learning_rate, int max_epoch, bool
//Calculating the weight/bias for layer 1 //Calculating the weight/bias for layer 1
std::vector<std::vector<double>> D1_1 = alg.matmult(error, alg.transpose(weights2)); std::vector<std::vector<real_t>> D1_1 = alg.matmult(error, alg.transpose(weights2));
std::vector<std::vector<double>> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1)); std::vector<std::vector<real_t>> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1));
std::vector<std::vector<double>> D1_3 = alg.matmult(alg.transpose(inputSet), D1_2); std::vector<std::vector<real_t>> D1_3 = alg.matmult(alg.transpose(inputSet), D1_2);
// weight an bias updation for layer 1 // weight an bias updation for layer 1
weights1 = alg.subtraction(weights1, alg.scalarMultiply(learning_rate / n, D1_3)); weights1 = alg.subtraction(weights1, alg.scalarMultiply(learning_rate / n, D1_3));
@ -85,10 +85,10 @@ void MLPPAutoEncoder::gradientDescent(double learning_rate, int max_epoch, bool
} }
} }
void MLPPAutoEncoder::SGD(double learning_rate, int max_epoch, bool UI) { void MLPPAutoEncoder::SGD(real_t learning_rate, int max_epoch, bool UI) {
MLPPActivation avn; MLPPActivation avn;
MLPPLinAlg alg; MLPPLinAlg alg;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
while (true) { while (true) {
@ -97,22 +97,22 @@ void MLPPAutoEncoder::SGD(double learning_rate, int max_epoch, bool UI) {
std::uniform_int_distribution<int> distribution(0, int(n - 1)); std::uniform_int_distribution<int> distribution(0, int(n - 1));
int outputIndex = distribution(generator); int outputIndex = distribution(generator);
std::vector<double> y_hat = Evaluate(inputSet[outputIndex]); std::vector<real_t> y_hat = Evaluate(inputSet[outputIndex]);
auto [z2, a2] = propagate(inputSet[outputIndex]); auto [z2, a2] = propagate(inputSet[outputIndex]);
cost_prev = Cost({ y_hat }, { inputSet[outputIndex] }); cost_prev = Cost({ y_hat }, { inputSet[outputIndex] });
std::vector<double> error = alg.subtraction(y_hat, inputSet[outputIndex]); std::vector<real_t> error = alg.subtraction(y_hat, inputSet[outputIndex]);
// Weight updation for layer 2 // Weight updation for layer 2
std::vector<std::vector<double>> D2_1 = alg.outerProduct(error, a2); std::vector<std::vector<real_t>> D2_1 = alg.outerProduct(error, a2);
weights2 = alg.subtraction(weights2, alg.scalarMultiply(learning_rate, alg.transpose(D2_1))); weights2 = alg.subtraction(weights2, alg.scalarMultiply(learning_rate, alg.transpose(D2_1)));
// Bias updation for layer 2 // Bias updation for layer 2
bias2 = alg.subtraction(bias2, alg.scalarMultiply(learning_rate, error)); bias2 = alg.subtraction(bias2, alg.scalarMultiply(learning_rate, error));
// Weight updation for layer 1 // Weight updation for layer 1
std::vector<double> D1_1 = alg.mat_vec_mult(weights2, error); std::vector<real_t> D1_1 = alg.mat_vec_mult(weights2, error);
std::vector<double> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1)); std::vector<real_t> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1));
std::vector<std::vector<double>> D1_3 = alg.outerProduct(inputSet[outputIndex], D1_2); std::vector<std::vector<real_t>> D1_3 = alg.outerProduct(inputSet[outputIndex], D1_2);
weights1 = alg.subtraction(weights1, alg.scalarMultiply(learning_rate, D1_3)); weights1 = alg.subtraction(weights1, alg.scalarMultiply(learning_rate, D1_3));
// Bias updation for layer 1 // Bias updation for layer 1
@ -136,28 +136,28 @@ void MLPPAutoEncoder::SGD(double learning_rate, int max_epoch, bool UI) {
forwardPass(); forwardPass();
} }
void MLPPAutoEncoder::MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI) { void MLPPAutoEncoder::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI) {
MLPPActivation avn; MLPPActivation avn;
MLPPLinAlg alg; MLPPLinAlg alg;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
// Creating the mini-batches // Creating the mini-batches
int n_mini_batch = n / mini_batch_size; int n_mini_batch = n / mini_batch_size;
std::vector<std::vector<std::vector<double>>> inputMiniBatches = MLPPUtilities::createMiniBatches(inputSet, n_mini_batch); std::vector<std::vector<std::vector<real_t>>> inputMiniBatches = MLPPUtilities::createMiniBatches(inputSet, n_mini_batch);
while (true) { while (true) {
for (int i = 0; i < n_mini_batch; i++) { for (int i = 0; i < n_mini_batch; i++) {
std::vector<std::vector<double>> y_hat = Evaluate(inputMiniBatches[i]); std::vector<std::vector<real_t>> y_hat = Evaluate(inputMiniBatches[i]);
auto [z2, a2] = propagate(inputMiniBatches[i]); auto [z2, a2] = propagate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, inputMiniBatches[i]); cost_prev = Cost(y_hat, inputMiniBatches[i]);
// Calculating the errors // Calculating the errors
std::vector<std::vector<double>> error = alg.subtraction(y_hat, inputMiniBatches[i]); std::vector<std::vector<real_t>> error = alg.subtraction(y_hat, inputMiniBatches[i]);
// Calculating the weight/bias gradients for layer 2 // Calculating the weight/bias gradients for layer 2
std::vector<std::vector<double>> D2_1 = alg.matmult(alg.transpose(a2), error); std::vector<std::vector<real_t>> D2_1 = alg.matmult(alg.transpose(a2), error);
// weights and bias updation for layer 2 // weights and bias updation for layer 2
weights2 = alg.subtraction(weights2, alg.scalarMultiply(learning_rate / inputMiniBatches[i].size(), D2_1)); weights2 = alg.subtraction(weights2, alg.scalarMultiply(learning_rate / inputMiniBatches[i].size(), D2_1));
@ -167,11 +167,11 @@ void MLPPAutoEncoder::MBGD(double learning_rate, int max_epoch, int mini_batch_s
//Calculating the weight/bias for layer 1 //Calculating the weight/bias for layer 1
std::vector<std::vector<double>> D1_1 = alg.matmult(error, alg.transpose(weights2)); std::vector<std::vector<real_t>> D1_1 = alg.matmult(error, alg.transpose(weights2));
std::vector<std::vector<double>> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1)); std::vector<std::vector<real_t>> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1));
std::vector<std::vector<double>> D1_3 = alg.matmult(alg.transpose(inputMiniBatches[i]), D1_2); std::vector<std::vector<real_t>> D1_3 = alg.matmult(alg.transpose(inputMiniBatches[i]), D1_2);
// weight an bias updation for layer 1 // weight an bias updation for layer 1
weights1 = alg.subtraction(weights1, alg.scalarMultiply(learning_rate / inputMiniBatches[i].size(), D1_3)); weights1 = alg.subtraction(weights1, alg.scalarMultiply(learning_rate / inputMiniBatches[i].size(), D1_3));
@ -196,7 +196,7 @@ void MLPPAutoEncoder::MBGD(double learning_rate, int max_epoch, int mini_batch_s
forwardPass(); forwardPass();
} }
double MLPPAutoEncoder::score() { real_t MLPPAutoEncoder::score() {
MLPPUtilities util; MLPPUtilities util;
return util.performance(y_hat, inputSet); return util.performance(y_hat, inputSet);
} }
@ -207,40 +207,40 @@ void MLPPAutoEncoder::save(std::string fileName) {
util.saveParameters(fileName, weights2, bias2, 1, 2); util.saveParameters(fileName, weights2, bias2, 1, 2);
} }
double MLPPAutoEncoder::Cost(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) { real_t MLPPAutoEncoder::Cost(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
class MLPPCost cost; class MLPPCost cost;
return cost.MSE(y_hat, inputSet); return cost.MSE(y_hat, inputSet);
} }
std::vector<std::vector<double>> MLPPAutoEncoder::Evaluate(std::vector<std::vector<double>> X) { std::vector<std::vector<real_t>> MLPPAutoEncoder::Evaluate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPActivation avn; MLPPActivation avn;
std::vector<std::vector<double>> z2 = alg.mat_vec_add(alg.matmult(X, weights1), bias1); std::vector<std::vector<real_t>> z2 = alg.mat_vec_add(alg.matmult(X, weights1), bias1);
std::vector<std::vector<double>> a2 = avn.sigmoid(z2); std::vector<std::vector<real_t>> a2 = avn.sigmoid(z2);
return alg.mat_vec_add(alg.matmult(a2, weights2), bias2); return alg.mat_vec_add(alg.matmult(a2, weights2), bias2);
} }
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> MLPPAutoEncoder::propagate(std::vector<std::vector<double>> X) { std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> MLPPAutoEncoder::propagate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPActivation avn; MLPPActivation avn;
std::vector<std::vector<double>> z2 = alg.mat_vec_add(alg.matmult(X, weights1), bias1); std::vector<std::vector<real_t>> z2 = alg.mat_vec_add(alg.matmult(X, weights1), bias1);
std::vector<std::vector<double>> a2 = avn.sigmoid(z2); std::vector<std::vector<real_t>> a2 = avn.sigmoid(z2);
return { z2, a2 }; return { z2, a2 };
} }
std::vector<double> MLPPAutoEncoder::Evaluate(std::vector<double> x) { std::vector<real_t> MLPPAutoEncoder::Evaluate(std::vector<real_t> x) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPActivation avn; MLPPActivation avn;
std::vector<double> z2 = alg.addition(alg.mat_vec_mult(alg.transpose(weights1), x), bias1); std::vector<real_t> z2 = alg.addition(alg.mat_vec_mult(alg.transpose(weights1), x), bias1);
std::vector<double> a2 = avn.sigmoid(z2); std::vector<real_t> a2 = avn.sigmoid(z2);
return alg.addition(alg.mat_vec_mult(alg.transpose(weights2), a2), bias2); return alg.addition(alg.mat_vec_mult(alg.transpose(weights2), a2), bias2);
} }
std::tuple<std::vector<double>, std::vector<double>> MLPPAutoEncoder::propagate(std::vector<double> x) { std::tuple<std::vector<real_t>, std::vector<real_t>> MLPPAutoEncoder::propagate(std::vector<real_t> x) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPActivation avn; MLPPActivation avn;
std::vector<double> z2 = alg.addition(alg.mat_vec_mult(alg.transpose(weights1), x), bias1); std::vector<real_t> z2 = alg.addition(alg.mat_vec_mult(alg.transpose(weights1), x), bias1);
std::vector<double> a2 = avn.sigmoid(z2); std::vector<real_t> a2 = avn.sigmoid(z2);
return { z2, a2 }; return { z2, a2 };
} }

42
mlpp/auto_encoder/auto_encoder.h
View File

@ -8,41 +8,43 @@
// Created by Marc Melikyan on 11/4/20. // Created by Marc Melikyan on 11/4/20.
// //
#include "core/math/math_defs.h"
#include <string> #include <string>
#include <tuple> #include <tuple>
#include <vector> #include <vector>
class MLPPAutoEncoder { class MLPPAutoEncoder {
public: public:
MLPPAutoEncoder(std::vector<std::vector<double>> inputSet, int n_hidden); MLPPAutoEncoder(std::vector<std::vector<real_t>> inputSet, int n_hidden);
std::vector<std::vector<double>> modelSetTest(std::vector<std::vector<double>> X); std::vector<std::vector<real_t>> modelSetTest(std::vector<std::vector<real_t>> X);
std::vector<double> modelTest(std::vector<double> x); std::vector<real_t> modelTest(std::vector<real_t> x);
void gradientDescent(double learning_rate, int max_epoch, bool UI = 1); void gradientDescent(real_t learning_rate, int max_epoch, bool UI = 1);
void SGD(double learning_rate, int max_epoch, bool UI = 1); void SGD(real_t learning_rate, int max_epoch, bool UI = 1);
void MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI = 1); void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
double score(); real_t score();
void save(std::string fileName); void save(std::string fileName);
private: private:
double Cost(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y); real_t Cost(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
std::vector<std::vector<double>> Evaluate(std::vector<std::vector<double>> X); std::vector<std::vector<real_t>> Evaluate(std::vector<std::vector<real_t>> X);
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> propagate(std::vector<std::vector<double>> X); std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> propagate(std::vector<std::vector<real_t>> X);
std::vector<double> Evaluate(std::vector<double> x); std::vector<real_t> Evaluate(std::vector<real_t> x);
std::tuple<std::vector<double>, std::vector<double>> propagate(std::vector<double> x); std::tuple<std::vector<real_t>, std::vector<real_t>> propagate(std::vector<real_t> x);
void forwardPass(); void forwardPass();
std::vector<std::vector<double>> inputSet; std::vector<std::vector<real_t>> inputSet;
std::vector<std::vector<double>> y_hat; std::vector<std::vector<real_t>> y_hat;
std::vector<std::vector<double>> weights1; std::vector<std::vector<real_t>> weights1;
std::vector<std::vector<double>> weights2; std::vector<std::vector<real_t>> weights2;
std::vector<double> bias1; std::vector<real_t> bias1;
std::vector<double> bias2; std::vector<real_t> bias2;
std::vector<std::vector<double>> z2; std::vector<std::vector<real_t>> z2;
std::vector<std::vector<double>> a2; std::vector<std::vector<real_t>> a2;
int n; int n;
int k; int k;

22
mlpp/bernoulli_nb/bernoulli_nb.cpp
View File

@ -12,23 +12,23 @@
#include <iostream> #include <iostream>
#include <random> #include <random>
MLPPBernoulliNB::MLPPBernoulliNB(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet) : MLPPBernoulliNB::MLPPBernoulliNB(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet) :
inputSet(inputSet), outputSet(outputSet), class_num(2) { inputSet(inputSet), outputSet(outputSet), class_num(2) {
y_hat.resize(outputSet.size()); y_hat.resize(outputSet.size());
Evaluate(); Evaluate();
} }
std::vector<double> MLPPBernoulliNB::modelSetTest(std::vector<std::vector<double>> X) { std::vector<real_t> MLPPBernoulliNB::modelSetTest(std::vector<std::vector<real_t>> X) {
std::vector<double> y_hat; std::vector<real_t> y_hat;
for (int i = 0; i < X.size(); i++) { for (int i = 0; i < X.size(); i++) {
y_hat.push_back(modelTest(X[i])); y_hat.push_back(modelTest(X[i]));
} }
return y_hat; return y_hat;
} }
double MLPPBernoulliNB::modelTest(std::vector<double> x) { real_t MLPPBernoulliNB::modelTest(std::vector<real_t> x) {
double score_0 = 1; real_t score_0 = 1;
double score_1 = 1; real_t score_1 = 1;
std::vector<int> foundIndices; std::vector<int> foundIndices;
@ -68,7 +68,7 @@ double MLPPBernoulliNB::modelTest(std::vector<double> x) {
} }
} }
double MLPPBernoulliNB::score() { real_t MLPPBernoulliNB::score() {
MLPPUtilities util; MLPPUtilities util;
return util.performance(y_hat, outputSet); return util.performance(y_hat, outputSet);
} }
@ -76,7 +76,7 @@ double MLPPBernoulliNB::score() {
void MLPPBernoulliNB::computeVocab() { void MLPPBernoulliNB::computeVocab() {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPData data; MLPPData data;
vocab = data.vecToSet<double>(alg.flatten(inputSet)); vocab = data.vecToSet<real_t>(alg.flatten(inputSet));
} }
void MLPPBernoulliNB::computeTheta() { void MLPPBernoulliNB::computeTheta() {
@ -110,10 +110,10 @@ void MLPPBernoulliNB::computeTheta() {
void MLPPBernoulliNB::Evaluate() { void MLPPBernoulliNB::Evaluate() {
for (int i = 0; i < outputSet.size(); i++) { for (int i = 0; i < outputSet.size(); i++) {
// Pr(B | A) * Pr(A) // Pr(B | A) * Pr(A)
double score_0 = 1; real_t score_0 = 1;
double score_1 = 1; real_t score_1 = 1;
double sum = 0; real_t sum = 0;
for (int i = 0; i < outputSet.size(); i++) { for (int i = 0; i < outputSet.size(); i++) {
if (outputSet[i] == 1) { if (outputSet[i] == 1) {
sum += outputSet[i]; sum += outputSet[i];

24
mlpp/bernoulli_nb/bernoulli_nb.h
View File

@ -8,15 +8,17 @@
// Created by Marc Melikyan on 1/17/21. // Created by Marc Melikyan on 1/17/21.
// //
#include "core/math/math_defs.h"
#include <map> #include <map>
#include <vector> #include <vector>
class MLPPBernoulliNB { class MLPPBernoulliNB {
public: public:
MLPPBernoulliNB(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet); MLPPBernoulliNB(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet);
std::vector<double> modelSetTest(std::vector<std::vector<double>> X); std::vector<real_t> modelSetTest(std::vector<std::vector<real_t>> X);
double modelTest(std::vector<double> x); real_t modelTest(std::vector<real_t> x);
double score(); real_t score();
private: private:
void computeVocab(); void computeVocab();
@ -24,17 +26,17 @@ private:
void Evaluate(); void Evaluate();
// Model Params // Model Params
double prior_1 = 0; real_t prior_1 = 0;
double prior_0 = 0; real_t prior_0 = 0;
std::vector<std::map<double, int>> theta; std::vector<std::map<real_t, int>> theta;
std::vector<double> vocab; std::vector<real_t> vocab;
int class_num; int class_num;
// Datasets // Datasets
std::vector<std::vector<double>> inputSet; std::vector<std::vector<real_t>> inputSet;
std::vector<double> outputSet; std::vector<real_t> outputSet;
std::vector<double> y_hat; std::vector<real_t> y_hat;
}; };
#endif /* BernoulliNB_hpp */ #endif /* BernoulliNB_hpp */

50
mlpp/c_log_log_reg/c_log_log_reg.cpp
View File

@ -14,33 +14,33 @@
#include <iostream> #include <iostream>
#include <random> #include <random>
MLPPCLogLogReg::MLPPCLogLogReg(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, std::string reg, double lambda, double alpha) : MLPPCLogLogReg::MLPPCLogLogReg(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, std::string reg, real_t lambda, real_t alpha) :
inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) { inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) {
y_hat.resize(n); y_hat.resize(n);
weights = MLPPUtilities::weightInitialization(k); weights = MLPPUtilities::weightInitialization(k);
bias = MLPPUtilities::biasInitialization(); bias = MLPPUtilities::biasInitialization();
} }
std::vector<double> MLPPCLogLogReg::modelSetTest(std::vector<std::vector<double>> X) { std::vector<real_t> MLPPCLogLogReg::modelSetTest(std::vector<std::vector<real_t>> X) {
return Evaluate(X); return Evaluate(X);
} }
double MLPPCLogLogReg::modelTest(std::vector<double> x) { real_t MLPPCLogLogReg::modelTest(std::vector<real_t> x) {
return Evaluate(x); return Evaluate(x);
} }
void MLPPCLogLogReg::gradientDescent(double learning_rate, int max_epoch, bool UI) { void MLPPCLogLogReg::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
MLPPActivation avn; MLPPActivation avn;
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
forwardPass(); forwardPass();
while (true) { while (true) {
cost_prev = Cost(y_hat, outputSet); cost_prev = Cost(y_hat, outputSet);
std::vector<double> error = alg.subtraction(y_hat, outputSet); std::vector<real_t> error = alg.subtraction(y_hat, outputSet);
// Calculating the weight gradients // Calculating the weight gradients
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputSet), alg.hadamard_product(error, avn.cloglog(z, 1))))); weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputSet), alg.hadamard_product(error, avn.cloglog(z, 1)))));
@ -63,18 +63,18 @@ void MLPPCLogLogReg::gradientDescent(double learning_rate, int max_epoch, bool U
} }
} }
void MLPPCLogLogReg::MLE(double learning_rate, int max_epoch, bool UI) { void MLPPCLogLogReg::MLE(real_t learning_rate, int max_epoch, bool UI) {
MLPPActivation avn; MLPPActivation avn;
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
forwardPass(); forwardPass();
while (true) { while (true) {
cost_prev = Cost(y_hat, outputSet); cost_prev = Cost(y_hat, outputSet);
std::vector<double> error = alg.subtraction(y_hat, outputSet); std::vector<real_t> error = alg.subtraction(y_hat, outputSet);
weights = alg.addition(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputSet), alg.hadamard_product(error, avn.cloglog(z, 1))))); weights = alg.addition(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputSet), alg.hadamard_product(error, avn.cloglog(z, 1)))));
weights = regularization.regWeights(weights, lambda, alpha, reg); weights = regularization.regWeights(weights, lambda, alpha, reg);
@ -95,10 +95,10 @@ void MLPPCLogLogReg::MLE(double learning_rate, int max_epoch, bool UI) {
} }
} }
void MLPPCLogLogReg::SGD(double learning_rate, int max_epoch, bool UI) { void MLPPCLogLogReg::SGD(real_t learning_rate, int max_epoch, bool UI) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
forwardPass(); forwardPass();
@ -108,11 +108,11 @@ void MLPPCLogLogReg::SGD(double learning_rate, int max_epoch, bool UI) {
std::uniform_int_distribution<int> distribution(0, int(n - 1)); std::uniform_int_distribution<int> distribution(0, int(n - 1));
int outputIndex = distribution(generator); int outputIndex = distribution(generator);
double y_hat = Evaluate(inputSet[outputIndex]); real_t y_hat = Evaluate(inputSet[outputIndex]);
double z = propagate(inputSet[outputIndex]); real_t z = propagate(inputSet[outputIndex]);
cost_prev = Cost({ y_hat }, { outputSet[outputIndex] }); cost_prev = Cost({ y_hat }, { outputSet[outputIndex] });
double error = y_hat - outputSet[outputIndex]; real_t error = y_hat - outputSet[outputIndex];
// Weight Updation // Weight Updation
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate * error * exp(z - exp(z)), inputSet[outputIndex])); weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate * error * exp(z - exp(z)), inputSet[outputIndex]));
@ -136,11 +136,11 @@ void MLPPCLogLogReg::SGD(double learning_rate, int max_epoch, bool UI) {
forwardPass(); forwardPass();
} }
void MLPPCLogLogReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI) { void MLPPCLogLogReg::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI) {
MLPPActivation avn; MLPPActivation avn;
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
// Creating the mini-batches // Creating the mini-batches
@ -149,11 +149,11 @@ void MLPPCLogLogReg::MBGD(double learning_rate, int max_epoch, int mini_batch_si
while (true) { while (true) {
for (int i = 0; i < n_mini_batch; i++) { for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = Evaluate(inputMiniBatches[i]); std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]);
std::vector<double> z = propagate(inputMiniBatches[i]); std::vector<real_t> z = propagate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]); cost_prev = Cost(y_hat, outputMiniBatches[i]);
std::vector<double> error = alg.subtraction(y_hat, outputMiniBatches[i]); std::vector<real_t> error = alg.subtraction(y_hat, outputMiniBatches[i]);
// Calculating the weight gradients // Calculating the weight gradients
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), alg.hadamard_product(error, avn.cloglog(z, 1))))); weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), alg.hadamard_product(error, avn.cloglog(z, 1)))));
@ -179,35 +179,35 @@ void MLPPCLogLogReg::MBGD(double learning_rate, int max_epoch, int mini_batch_si
forwardPass(); forwardPass();
} }
double MLPPCLogLogReg::score() { real_t MLPPCLogLogReg::score() {
MLPPUtilities util; MLPPUtilities util;
return util.performance(y_hat, outputSet); return util.performance(y_hat, outputSet);
} }
double MLPPCLogLogReg::Cost(std::vector<double> y_hat, std::vector<double> y) { real_t MLPPCLogLogReg::Cost(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPReg regularization; MLPPReg regularization;
class MLPPCost cost; class MLPPCost cost;
return cost.MSE(y_hat, y) + regularization.regTerm(weights, lambda, alpha, reg); return cost.MSE(y_hat, y) + regularization.regTerm(weights, lambda, alpha, reg);
} }
std::vector<double> MLPPCLogLogReg::Evaluate(std::vector<std::vector<double>> X) { std::vector<real_t> MLPPCLogLogReg::Evaluate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPActivation avn; MLPPActivation avn;
return avn.cloglog(alg.scalarAdd(bias, alg.mat_vec_mult(X, weights))); return avn.cloglog(alg.scalarAdd(bias, alg.mat_vec_mult(X, weights)));
} }
std::vector<double> MLPPCLogLogReg::propagate(std::vector<std::vector<double>> X) { std::vector<real_t> MLPPCLogLogReg::propagate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.scalarAdd(bias, alg.mat_vec_mult(X, weights)); return alg.scalarAdd(bias, alg.mat_vec_mult(X, weights));
} }
double MLPPCLogLogReg::Evaluate(std::vector<double> x) { real_t MLPPCLogLogReg::Evaluate(std::vector<real_t> x) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPActivation avn; MLPPActivation avn;
return avn.cloglog(alg.dot(weights, x) + bias); return avn.cloglog(alg.dot(weights, x) + bias);
} }
double MLPPCLogLogReg::propagate(std::vector<double> x) { real_t MLPPCLogLogReg::propagate(std::vector<real_t> x) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.dot(weights, x) + bias; return alg.dot(weights, x) + bias;
} }

44
mlpp/c_log_log_reg/c_log_log_reg.h
View File

@ -8,45 +8,47 @@
// Created by Marc Melikyan on 10/2/20. // Created by Marc Melikyan on 10/2/20.
// //
#include "core/math/math_defs.h"
#include <string> #include <string>
#include <vector> #include <vector>
class MLPPCLogLogReg { class MLPPCLogLogReg {
public: public:
MLPPCLogLogReg(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, std::string reg = "None", double lambda = 0.5, double alpha = 0.5); MLPPCLogLogReg(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
std::vector<double> modelSetTest(std::vector<std::vector<double>> X); std::vector<real_t> modelSetTest(std::vector<std::vector<real_t>> X);
double modelTest(std::vector<double> x); real_t modelTest(std::vector<real_t> x);
void gradientDescent(double learning_rate, int max_epoch, bool UI = 1); void gradientDescent(real_t learning_rate, int max_epoch, bool UI = 1);
void MLE(double learning_rate, int max_epoch, bool UI = 1); void MLE(real_t learning_rate, int max_epoch, bool UI = 1);
void SGD(double learning_rate, int max_epoch, bool UI = 1); void SGD(real_t learning_rate, int max_epoch, bool UI = 1);
void MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI = 1); void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
double score(); real_t score();
private: private:
void weightInitialization(int k); void weightInitialization(int k);
void biasInitialization(); void biasInitialization();
double Cost(std::vector<double> y_hat, std::vector<double> y); real_t Cost(std::vector<real_t> y_hat, std::vector<real_t> y);
std::vector<double> Evaluate(std::vector<std::vector<double>> X); std::vector<real_t> Evaluate(std::vector<std::vector<real_t>> X);
std::vector<double> propagate(std::vector<std::vector<double>> X); std::vector<real_t> propagate(std::vector<std::vector<real_t>> X);
double Evaluate(std::vector<double> x); real_t Evaluate(std::vector<real_t> x);
double propagate(std::vector<double> x); real_t propagate(std::vector<real_t> x);
void forwardPass(); void forwardPass();
std::vector<std::vector<double>> inputSet; std::vector<std::vector<real_t>> inputSet;
std::vector<double> outputSet; std::vector<real_t> outputSet;
std::vector<double> y_hat; std::vector<real_t> y_hat;
std::vector<double> z; std::vector<real_t> z;
std::vector<double> weights; std::vector<real_t> weights;
double bias; real_t bias;
int n; int n;
int k; int k;
// Regularization Params // Regularization Params
std::string reg; std::string reg;
double lambda; real_t lambda;
double alpha; /* This is the controlling param for Elastic Net*/ real_t alpha; /* This is the controlling param for Elastic Net*/
}; };
#endif /* CLogLogReg_hpp */ #endif /* CLogLogReg_hpp */

112
mlpp/convolutions/convolutions.cpp
View File

@ -14,15 +14,15 @@ MLPPConvolutions::MLPPConvolutions() :
prewittHorizontal({ { 1, 1, 1 }, { 0, 0, 0 }, { -1, -1, -1 } }), prewittVertical({ { 1, 0, -1 }, { 1, 0, -1 }, { 1, 0, -1 } }), sobelHorizontal({ { 1, 2, 1 }, { 0, 0, 0 }, { -1, -2, -1 } }), sobelVertical({ { -1, 0, 1 }, { -2, 0, 2 }, { -1, 0, 1 } }), scharrHorizontal({ { 3, 10, 3 }, { 0, 0, 0 }, { -3, -10, -3 } }), scharrVertical({ { 3, 0, -3 }, { 10, 0, -10 }, { 3, 0, -3 } }), robertsHorizontal({ { 0, 1 }, { -1, 0 } }), robertsVertical({ { 1, 0 }, { 0, -1 } }) { prewittHorizontal({ { 1, 1, 1 }, { 0, 0, 0 }, { -1, -1, -1 } }), prewittVertical({ { 1, 0, -1 }, { 1, 0, -1 }, { 1, 0, -1 } }), sobelHorizontal({ { 1, 2, 1 }, { 0, 0, 0 }, { -1, -2, -1 } }), sobelVertical({ { -1, 0, 1 }, { -2, 0, 2 }, { -1, 0, 1 } }), scharrHorizontal({ { 3, 10, 3 }, { 0, 0, 0 }, { -3, -10, -3 } }), scharrVertical({ { 3, 0, -3 }, { 10, 0, -10 }, { 3, 0, -3 } }), robertsHorizontal({ { 0, 1 }, { -1, 0 } }), robertsVertical({ { 1, 0 }, { 0, -1 } }) {
} }
std::vector<std::vector<double>> MLPPConvolutions::convolve(std::vector<std::vector<double>> input, std::vector<std::vector<double>> filter, int S, int P) { std::vector<std::vector<real_t>> MLPPConvolutions::convolve(std::vector<std::vector<real_t>> input, std::vector<std::vector<real_t>> filter, int S, int P) {
MLPPLinAlg alg; MLPPLinAlg alg;
std::vector<std::vector<double>> featureMap; std::vector<std::vector<real_t>> featureMap;
int N = input.size(); int N = input.size();
int F = filter.size(); int F = filter.size();
int mapSize = (N - F + 2 * P) / S + 1; // This is computed as ⌊mapSize⌋ by def- thanks C++! int mapSize = (N - F + 2 * P) / S + 1; // This is computed as ⌊mapSize⌋ by def- thanks C++!
if (P != 0) { if (P != 0) {
std::vector<std::vector<double>> paddedInput; std::vector<std::vector<real_t>> paddedInput;
paddedInput.resize(N + 2 * P); paddedInput.resize(N + 2 * P);
for (int i = 0; i < paddedInput.size(); i++) { for (int i = 0; i < paddedInput.size(); i++) {
paddedInput[i].resize(N + 2 * P); paddedInput[i].resize(N + 2 * P);
@ -50,7 +50,7 @@ std::vector<std::vector<double>> MLPPConvolutions::convolve(std::vector<std::vec
for (int i = 0; i < mapSize; i++) { for (int i = 0; i < mapSize; i++) {
for (int j = 0; j < mapSize; j++) { for (int j = 0; j < mapSize; j++) {
std::vector<double> convolvingInput; std::vector<real_t> convolvingInput;
for (int k = 0; k < F; k++) { for (int k = 0; k < F; k++) {
for (int p = 0; p < F; p++) { for (int p = 0; p < F; p++) {
if (i == 0 && j == 0) { if (i == 0 && j == 0) {
@ -70,9 +70,9 @@ std::vector<std::vector<double>> MLPPConvolutions::convolve(std::vector<std::vec
return featureMap; return featureMap;
} }
std::vector<std::vector<std::vector<double>>> MLPPConvolutions::convolve(std::vector<std::vector<std::vector<double>>> input, std::vector<std::vector<std::vector<double>>> filter, int S, int P) { std::vector<std::vector<std::vector<real_t>>> MLPPConvolutions::convolve(std::vector<std::vector<std::vector<real_t>>> input, std::vector<std::vector<std::vector<real_t>>> filter, int S, int P) {
MLPPLinAlg alg; MLPPLinAlg alg;
std::vector<std::vector<std::vector<double>>> featureMap; std::vector<std::vector<std::vector<real_t>>> featureMap;
int N = input[0].size(); int N = input[0].size();
int F = filter[0].size(); int F = filter[0].size();
int C = filter.size() / input.size(); int C = filter.size() / input.size();
@ -80,7 +80,7 @@ std::vector<std::vector<std::vector<double>>> MLPPConvolutions::convolve(std::ve
if (P != 0) { if (P != 0) {
for (int c = 0; c < input.size(); c++) { for (int c = 0; c < input.size(); c++) {
std::vector<std::vector<double>> paddedInput; std::vector<std::vector<real_t>> paddedInput;
paddedInput.resize(N + 2 * P); paddedInput.resize(N + 2 * P);
for (int i = 0; i < paddedInput.size(); i++) { for (int i = 0; i < paddedInput.size(); i++) {
paddedInput[i].resize(N + 2 * P); paddedInput[i].resize(N + 2 * P);
@ -113,7 +113,7 @@ std::vector<std::vector<std::vector<double>>> MLPPConvolutions::convolve(std::ve
for (int c = 0; c < C; c++) { for (int c = 0; c < C; c++) {
for (int i = 0; i < mapSize; i++) { for (int i = 0; i < mapSize; i++) {
for (int j = 0; j < mapSize; j++) { for (int j = 0; j < mapSize; j++) {
std::vector<double> convolvingInput; std::vector<real_t> convolvingInput;
for (int t = 0; t < input.size(); t++) { for (int t = 0; t < input.size(); t++) {
for (int k = 0; k < F; k++) { for (int k = 0; k < F; k++) {
for (int p = 0; p < F; p++) { for (int p = 0; p < F; p++) {
@ -136,9 +136,9 @@ std::vector<std::vector<std::vector<double>>> MLPPConvolutions::convolve(std::ve
return featureMap; return featureMap;
} }
std::vector<std::vector<double>> MLPPConvolutions::pool(std::vector<std::vector<double>> input, int F, int S, std::string type) { std::vector<std::vector<real_t>> MLPPConvolutions::pool(std::vector<std::vector<real_t>> input, int F, int S, std::string type) {
MLPPLinAlg alg; MLPPLinAlg alg;
std::vector<std::vector<double>> pooledMap; std::vector<std::vector<real_t>> pooledMap;
int N = input.size(); int N = input.size();
int mapSize = floor((N - F) / S + 1); int mapSize = floor((N - F) / S + 1);
@ -149,7 +149,7 @@ std::vector<std::vector<double>> MLPPConvolutions::pool(std::vector<std::vector<
for (int i = 0; i < mapSize; i++) { for (int i = 0; i < mapSize; i++) {
for (int j = 0; j < mapSize; j++) { for (int j = 0; j < mapSize; j++) {
std::vector<double> poolingInput; std::vector<real_t> poolingInput;
for (int k = 0; k < F; k++) { for (int k = 0; k < F; k++) {
for (int p = 0; p < F; p++) { for (int p = 0; p < F; p++) {
if (i == 0 && j == 0) { if (i == 0 && j == 0) {
@ -176,15 +176,15 @@ std::vector<std::vector<double>> MLPPConvolutions::pool(std::vector<std::vector<
return pooledMap; return pooledMap;
} }
std::vector<std::vector<std::vector<double>>> MLPPConvolutions::pool(std::vector<std::vector<std::vector<double>>> input, int F, int S, std::string type) { std::vector<std::vector<std::vector<real_t>>> MLPPConvolutions::pool(std::vector<std::vector<std::vector<real_t>>> input, int F, int S, std::string type) {
std::vector<std::vector<std::vector<double>>> pooledMap; std::vector<std::vector<std::vector<real_t>>> pooledMap;
for (int i = 0; i < input.size(); i++) { for (int i = 0; i < input.size(); i++) {
pooledMap.push_back(pool(input[i], F, S, type)); pooledMap.push_back(pool(input[i], F, S, type));
} }
return pooledMap; return pooledMap;
} }
double MLPPConvolutions::globalPool(std::vector<std::vector<double>> input, std::string type) { real_t MLPPConvolutions::globalPool(std::vector<std::vector<real_t>> input, std::string type) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (type == "Average") { if (type == "Average") {
MLPPStat stat; MLPPStat stat;
@ -196,21 +196,21 @@ double MLPPConvolutions::globalPool(std::vector<std::vector<double>> input, std:
} }
} }
std::vector<double> MLPPConvolutions::globalPool(std::vector<std::vector<std::vector<double>>> input, std::string type) { std::vector<real_t> MLPPConvolutions::globalPool(std::vector<std::vector<std::vector<real_t>>> input, std::string type) {
std::vector<double> pooledMap; std::vector<real_t> pooledMap;
for (int i = 0; i < input.size(); i++) { for (int i = 0; i < input.size(); i++) {
pooledMap.push_back(globalPool(input[i], type)); pooledMap.push_back(globalPool(input[i], type));
} }
return pooledMap; return pooledMap;
} }
double MLPPConvolutions::gaussian2D(double x, double y, double std) { real_t MLPPConvolutions::gaussian2D(real_t x, real_t y, real_t std) {
double std_sq = std * std; real_t std_sq = std * std;
return 1 / (2 * M_PI * std_sq) * std::exp(-(x * x + y * y) / 2 * std_sq); return 1 / (2 * M_PI * std_sq) * std::exp(-(x * x + y * y) / 2 * std_sq);
} }
std::vector<std::vector<double>> MLPPConvolutions::gaussianFilter2D(int size, double std) { std::vector<std::vector<real_t>> MLPPConvolutions::gaussianFilter2D(int size, real_t std) {
std::vector<std::vector<double>> filter; std::vector<std::vector<real_t>> filter;
filter.resize(size); filter.resize(size);
for (int i = 0; i < filter.size(); i++) { for (int i = 0; i < filter.size(); i++) {
filter[i].resize(size); filter[i].resize(size);
@ -229,8 +229,8 @@ been easier to carry out the calculation explicitly, mainly because it is more i
and also because my convolution algorithm is only built for filters with equally sized and also because my convolution algorithm is only built for filters with equally sized
heights and widths. heights and widths.
*/ */
std::vector<std::vector<double>> MLPPConvolutions::dx(std::vector<std::vector<double>> input) { std::vector<std::vector<real_t>> MLPPConvolutions::dx(std::vector<std::vector<real_t>> input) {
std::vector<std::vector<double>> deriv; // We assume a gray scale image. std::vector<std::vector<real_t>> deriv; // We assume a gray scale image.
deriv.resize(input.size()); deriv.resize(input.size());
for (int i = 0; i < deriv.size(); i++) { for (int i = 0; i < deriv.size(); i++) {
deriv[i].resize(input[i].size()); deriv[i].resize(input[i].size());
@ -250,8 +250,8 @@ std::vector<std::vector<double>> MLPPConvolutions::dx(std::vector<std::vector<do
return deriv; return deriv;
} }
std::vector<std::vector<double>> MLPPConvolutions::dy(std::vector<std::vector<double>> input) { std::vector<std::vector<real_t>> MLPPConvolutions::dy(std::vector<std::vector<real_t>> input) {
std::vector<std::vector<double>> deriv; std::vector<std::vector<real_t>> deriv;
deriv.resize(input.size()); deriv.resize(input.size());
for (int i = 0; i < deriv.size(); i++) { for (int i = 0; i < deriv.size(); i++) {
deriv[i].resize(input[i].size()); deriv[i].resize(input[i].size());
@ -271,22 +271,22 @@ std::vector<std::vector<double>> MLPPConvolutions::dy(std::vector<std::vector<do
return deriv; return deriv;
} }
std::vector<std::vector<double>> MLPPConvolutions::gradMagnitude(std::vector<std::vector<double>> input) { std::vector<std::vector<real_t>> MLPPConvolutions::gradMagnitude(std::vector<std::vector<real_t>> input) {
MLPPLinAlg alg; MLPPLinAlg alg;
std::vector<std::vector<double>> xDeriv_2 = alg.hadamard_product(dx(input), dx(input)); std::vector<std::vector<real_t>> xDeriv_2 = alg.hadamard_product(dx(input), dx(input));
std::vector<std::vector<double>> yDeriv_2 = alg.hadamard_product(dy(input), dy(input)); std::vector<std::vector<real_t>> yDeriv_2 = alg.hadamard_product(dy(input), dy(input));
return alg.sqrt(alg.addition(xDeriv_2, yDeriv_2)); return alg.sqrt(alg.addition(xDeriv_2, yDeriv_2));
} }
std::vector<std::vector<double>> MLPPConvolutions::gradOrientation(std::vector<std::vector<double>> input) { std::vector<std::vector<real_t>> MLPPConvolutions::gradOrientation(std::vector<std::vector<real_t>> input) {
std::vector<std::vector<double>> deriv; std::vector<std::vector<real_t>> deriv;
deriv.resize(input.size()); deriv.resize(input.size());
for (int i = 0; i < deriv.size(); i++) { for (int i = 0; i < deriv.size(); i++) {
deriv[i].resize(input[i].size()); deriv[i].resize(input[i].size());
} }
std::vector<std::vector<double>> xDeriv = dx(input); std::vector<std::vector<real_t>> xDeriv = dx(input);
std::vector<std::vector<double>> yDeriv = dy(input); std::vector<std::vector<real_t>> yDeriv = dy(input);
for (int i = 0; i < deriv.size(); i++) { for (int i = 0; i < deriv.size(); i++) {
for (int j = 0; j < deriv[i].size(); j++) { for (int j = 0; j < deriv[i].size(); j++) {
deriv[i][j] = std::atan2(yDeriv[i][j], xDeriv[i][j]); deriv[i][j] = std::atan2(yDeriv[i][j], xDeriv[i][j]);
@ -295,33 +295,33 @@ std::vector<std::vector<double>> MLPPConvolutions::gradOrientation(std::vector<s
return deriv; return deriv;
} }
std::vector<std::vector<std::vector<double>>> MLPPConvolutions::computeM(std::vector<std::vector<double>> input) { std::vector<std::vector<std::vector<real_t>>> MLPPConvolutions::computeM(std::vector<std::vector<real_t>> input) {
double const SIGMA = 1; real_t const SIGMA = 1;
double const GAUSSIAN_SIZE = 3; real_t const GAUSSIAN_SIZE = 3;
double const GAUSSIAN_PADDING = ((input.size() - 1) + GAUSSIAN_SIZE - input.size()) / 2; // Convs must be same. real_t const GAUSSIAN_PADDING = ((input.size() - 1) + GAUSSIAN_SIZE - input.size()) / 2; // Convs must be same.
std::cout << GAUSSIAN_PADDING << std::endl; std::cout << GAUSSIAN_PADDING << std::endl;
MLPPLinAlg alg; MLPPLinAlg alg;
std::vector<std::vector<double>> xDeriv = dx(input); std::vector<std::vector<real_t>> xDeriv = dx(input);
std::vector<std::vector<double>> yDeriv = dy(input); std::vector<std::vector<real_t>> yDeriv = dy(input);
std::vector<std::vector<double>> gaussianFilter = gaussianFilter2D(GAUSSIAN_SIZE, SIGMA); // Sigma of 1, size of 3. std::vector<std::vector<real_t>> gaussianFilter = gaussianFilter2D(GAUSSIAN_SIZE, SIGMA); // Sigma of 1, size of 3.
std::vector<std::vector<double>> xxDeriv = convolve(alg.hadamard_product(xDeriv, xDeriv), gaussianFilter, 1, GAUSSIAN_PADDING); std::vector<std::vector<real_t>> xxDeriv = convolve(alg.hadamard_product(xDeriv, xDeriv), gaussianFilter, 1, GAUSSIAN_PADDING);
std::vector<std::vector<double>> yyDeriv = convolve(alg.hadamard_product(yDeriv, yDeriv), gaussianFilter, 1, GAUSSIAN_PADDING); std::vector<std::vector<real_t>> yyDeriv = convolve(alg.hadamard_product(yDeriv, yDeriv), gaussianFilter, 1, GAUSSIAN_PADDING);
std::vector<std::vector<double>> xyDeriv = convolve(alg.hadamard_product(xDeriv, yDeriv), gaussianFilter, 1, GAUSSIAN_PADDING); std::vector<std::vector<real_t>> xyDeriv = convolve(alg.hadamard_product(xDeriv, yDeriv), gaussianFilter, 1, GAUSSIAN_PADDING);
std::vector<std::vector<std::vector<double>>> M = { xxDeriv, yyDeriv, xyDeriv }; std::vector<std::vector<std::vector<real_t>>> M = { xxDeriv, yyDeriv, xyDeriv };
return M; return M;
} }
std::vector<std::vector<std::string>> MLPPConvolutions::harrisCornerDetection(std::vector<std::vector<double>> input) { std::vector<std::vector<std::string>> MLPPConvolutions::harrisCornerDetection(std::vector<std::vector<real_t>> input) {
double const k = 0.05; // Empirically determined wherein k -> [0.04, 0.06], though conventionally 0.05 is typically used as well. real_t const k = 0.05; // Empirically determined wherein k -> [0.04, 0.06], though conventionally 0.05 is typically used as well.
MLPPLinAlg alg; MLPPLinAlg alg;
std::vector<std::vector<std::vector<double>>> M = computeM(input); std::vector<std::vector<std::vector<real_t>>> M = computeM(input);
std::vector<std::vector<double>> det = alg.subtraction(alg.hadamard_product(M[0], M[1]), alg.hadamard_product(M[2], M[2])); std::vector<std::vector<real_t>> det = alg.subtraction(alg.hadamard_product(M[0], M[1]), alg.hadamard_product(M[2], M[2]));
std::vector<std::vector<double>> trace = alg.addition(M[0], M[1]); std::vector<std::vector<real_t>> trace = alg.addition(M[0], M[1]);
// The reason this is not a scalar is because xxDeriv, xyDeriv, yxDeriv, and yyDeriv are not scalars. // The reason this is not a scalar is because xxDeriv, xyDeriv, yxDeriv, and yyDeriv are not scalars.
std::vector<std::vector<double>> r = alg.subtraction(det, alg.scalarMultiply(k, alg.hadamard_product(trace, trace))); std::vector<std::vector<real_t>> r = alg.subtraction(det, alg.scalarMultiply(k, alg.hadamard_product(trace, trace)));
std::vector<std::vector<std::string>> imageTypes; std::vector<std::vector<std::string>> imageTypes;
imageTypes.resize(r.size()); imageTypes.resize(r.size());
alg.printMatrix(r); alg.printMatrix(r);
@ -340,34 +340,34 @@ std::vector<std::vector<std::string>> MLPPConvolutions::harrisCornerDetection(st
return imageTypes; return imageTypes;
} }
std::vector<std::vector<double>> MLPPConvolutions::getPrewittHorizontal() { std::vector<std::vector<real_t>> MLPPConvolutions::getPrewittHorizontal() {
return prewittHorizontal; return prewittHorizontal;
} }
std::vector<std::vector<double>> MLPPConvolutions::getPrewittVertical() { std::vector<std::vector<real_t>> MLPPConvolutions::getPrewittVertical() {
return prewittVertical; return prewittVertical;
} }
std::vector<std::vector<double>> MLPPConvolutions::getSobelHorizontal() { std::vector<std::vector<real_t>> MLPPConvolutions::getSobelHorizontal() {
return sobelHorizontal; return sobelHorizontal;
} }
std::vector<std::vector<double>> MLPPConvolutions::getSobelVertical() { std::vector<std::vector<real_t>> MLPPConvolutions::getSobelVertical() {
return sobelVertical; return sobelVertical;
} }
std::vector<std::vector<double>> MLPPConvolutions::getScharrHorizontal() { std::vector<std::vector<real_t>> MLPPConvolutions::getScharrHorizontal() {
return scharrHorizontal; return scharrHorizontal;
} }
std::vector<std::vector<double>> MLPPConvolutions::getScharrVertical() { std::vector<std::vector<real_t>> MLPPConvolutions::getScharrVertical() {
return scharrVertical; return scharrVertical;
} }
std::vector<std::vector<double>> MLPPConvolutions::getRobertsHorizontal() { std::vector<std::vector<real_t>> MLPPConvolutions::getRobertsHorizontal() {
return robertsHorizontal; return robertsHorizontal;
} }
std::vector<std::vector<double>> MLPPConvolutions::getRobertsVertical() { std::vector<std::vector<real_t>> MLPPConvolutions::getRobertsVertical() {
return robertsVertical; return robertsVertical;
} }

62
mlpp/convolutions/convolutions.h
View File

@ -5,46 +5,48 @@
#include <vector> #include <vector>
#include <string> #include <string>
#include "core/math/math_defs.h"
class MLPPConvolutions { class MLPPConvolutions {
public: public:
MLPPConvolutions(); MLPPConvolutions();
std::vector<std::vector<double>> convolve(std::vector<std::vector<double>> input, std::vector<std::vector<double>> filter, int S, int P = 0); std::vector<std::vector<real_t>> convolve(std::vector<std::vector<real_t>> input, std::vector<std::vector<real_t>> filter, int S, int P = 0);
std::vector<std::vector<std::vector<double>>> convolve(std::vector<std::vector<std::vector<double>>> input, std::vector<std::vector<std::vector<double>>> filter, int S, int P = 0); std::vector<std::vector<std::vector<real_t>>> convolve(std::vector<std::vector<std::vector<real_t>>> input, std::vector<std::vector<std::vector<real_t>>> filter, int S, int P = 0);
std::vector<std::vector<double>> pool(std::vector<std::vector<double>> input, int F, int S, std::string type); std::vector<std::vector<real_t>> pool(std::vector<std::vector<real_t>> input, int F, int S, std::string type);
std::vector<std::vector<std::vector<double>>> pool(std::vector<std::vector<std::vector<double>>> input, int F, int S, std::string type); std::vector<std::vector<std::vector<real_t>>> pool(std::vector<std::vector<std::vector<real_t>>> input, int F, int S, std::string type);
double globalPool(std::vector<std::vector<double>> input, std::string type); real_t globalPool(std::vector<std::vector<real_t>> input, std::string type);
std::vector<double> globalPool(std::vector<std::vector<std::vector<double>>> input, std::string type); std::vector<real_t> globalPool(std::vector<std::vector<std::vector<real_t>>> input, std::string type);
double gaussian2D(double x, double y, double std); real_t gaussian2D(real_t x, real_t y, real_t std);
std::vector<std::vector<double>> gaussianFilter2D(int size, double std); std::vector<std::vector<real_t>> gaussianFilter2D(int size, real_t std);
std::vector<std::vector<double>> dx(std::vector<std::vector<double>> input); std::vector<std::vector<real_t>> dx(std::vector<std::vector<real_t>> input);
std::vector<std::vector<double>> dy(std::vector<std::vector<double>> input); std::vector<std::vector<real_t>> dy(std::vector<std::vector<real_t>> input);
std::vector<std::vector<double>> gradMagnitude(std::vector<std::vector<double>> input); std::vector<std::vector<real_t>> gradMagnitude(std::vector<std::vector<real_t>> input);
std::vector<std::vector<double>> gradOrientation(std::vector<std::vector<double>> input); std::vector<std::vector<real_t>> gradOrientation(std::vector<std::vector<real_t>> input);
std::vector<std::vector<std::vector<double>>> computeM(std::vector<std::vector<double>> input); std::vector<std::vector<std::vector<real_t>>> computeM(std::vector<std::vector<real_t>> input);
std::vector<std::vector<std::string>> harrisCornerDetection(std::vector<std::vector<double>> input); std::vector<std::vector<std::string>> harrisCornerDetection(std::vector<std::vector<real_t>> input);
std::vector<std::vector<double>> getPrewittHorizontal(); std::vector<std::vector<real_t>> getPrewittHorizontal();
std::vector<std::vector<double>> getPrewittVertical(); std::vector<std::vector<real_t>> getPrewittVertical();
std::vector<std::vector<double>> getSobelHorizontal(); std::vector<std::vector<real_t>> getSobelHorizontal();
std::vector<std::vector<double>> getSobelVertical(); std::vector<std::vector<real_t>> getSobelVertical();
std::vector<std::vector<double>> getScharrHorizontal(); std::vector<std::vector<real_t>> getScharrHorizontal();
std::vector<std::vector<double>> getScharrVertical(); std::vector<std::vector<real_t>> getScharrVertical();
std::vector<std::vector<double>> getRobertsHorizontal(); std::vector<std::vector<real_t>> getRobertsHorizontal();
std::vector<std::vector<double>> getRobertsVertical(); std::vector<std::vector<real_t>> getRobertsVertical();
private: private:
std::vector<std::vector<double>> prewittHorizontal; std::vector<std::vector<real_t>> prewittHorizontal;
std::vector<std::vector<double>> prewittVertical; std::vector<std::vector<real_t>> prewittVertical;
std::vector<std::vector<double>> sobelHorizontal; std::vector<std::vector<real_t>> sobelHorizontal;
std::vector<std::vector<double>> sobelVertical; std::vector<std::vector<real_t>> sobelVertical;
std::vector<std::vector<double>> scharrHorizontal; std::vector<std::vector<real_t>> scharrHorizontal;
std::vector<std::vector<double>> scharrVertical; std::vector<std::vector<real_t>> scharrVertical;
std::vector<std::vector<double>> robertsHorizontal; std::vector<std::vector<real_t>> robertsHorizontal;
std::vector<std::vector<double>> robertsVertical; std::vector<std::vector<real_t>> robertsVertical;
}; };
#endif // Convolutions_hpp #endif // Convolutions_hpp

160
mlpp/cost/cost.cpp
View File

@ -11,16 +11,16 @@
#include <iostream> #include <iostream>
double MLPPCost::MSE(std::vector<double> y_hat, std::vector<double> y) { real_t MLPPCost::MSE(std::vector<real_t> y_hat, std::vector<real_t> y) {
double sum = 0; real_t sum = 0;
for (int i = 0; i < y_hat.size(); i++) { for (int i = 0; i < y_hat.size(); i++) {
sum += (y_hat[i] - y[i]) * (y_hat[i] - y[i]); sum += (y_hat[i] - y[i]) * (y_hat[i] - y[i]);
} }
return sum / 2 * y_hat.size(); return sum / 2 * y_hat.size();
} }
double MLPPCost::MSE(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) { real_t MLPPCost::MSE(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
double sum = 0; real_t sum = 0;
for (int i = 0; i < y_hat.size(); i++) { for (int i = 0; i < y_hat.size(); i++) {
for (int j = 0; j < y_hat[i].size(); j++) { for (int j = 0; j < y_hat[i].size(); j++) {
sum += (y_hat[i][j] - y[i][j]) * (y_hat[i][j] - y[i][j]); sum += (y_hat[i][j] - y[i][j]) * (y_hat[i][j] - y[i][j]);
@ -29,26 +29,26 @@ double MLPPCost::MSE(std::vector<std::vector<double>> y_hat, std::vector<std::ve
return sum / 2 * y_hat.size(); return sum / 2 * y_hat.size();
} }
std::vector<double> MLPPCost::MSEDeriv(std::vector<double> y_hat, std::vector<double> y) { std::vector<real_t> MLPPCost::MSEDeriv(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.subtraction(y_hat, y); return alg.subtraction(y_hat, y);
} }
std::vector<std::vector<double>> MLPPCost::MSEDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) { std::vector<std::vector<real_t>> MLPPCost::MSEDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.subtraction(y_hat, y); return alg.subtraction(y_hat, y);
} }
double MLPPCost::RMSE(std::vector<double> y_hat, std::vector<double> y) { real_t MLPPCost::RMSE(std::vector<real_t> y_hat, std::vector<real_t> y) {
double sum = 0; real_t sum = 0;
for (int i = 0; i < y_hat.size(); i++) { for (int i = 0; i < y_hat.size(); i++) {
sum += (y_hat[i] - y[i]) * (y_hat[i] - y[i]); sum += (y_hat[i] - y[i]) * (y_hat[i] - y[i]);
} }
return sqrt(sum / y_hat.size()); return sqrt(sum / y_hat.size());
} }
double MLPPCost::RMSE(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) { real_t MLPPCost::RMSE(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
double sum = 0; real_t sum = 0;
for (int i = 0; i < y_hat.size(); i++) { for (int i = 0; i < y_hat.size(); i++) {
for (int j = 0; j < y_hat[i].size(); j++) { for (int j = 0; j < y_hat[i].size(); j++) {
sum += (y_hat[i][j] - y[i][j]) * (y_hat[i][j] - y[i][j]); sum += (y_hat[i][j] - y[i][j]) * (y_hat[i][j] - y[i][j]);
@ -57,26 +57,26 @@ double MLPPCost::RMSE(std::vector<std::vector<double>> y_hat, std::vector<std::v
return sqrt(sum / y_hat.size()); return sqrt(sum / y_hat.size());
} }
std::vector<double> MLPPCost::RMSEDeriv(std::vector<double> y_hat, std::vector<double> y) { std::vector<real_t> MLPPCost::RMSEDeriv(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.scalarMultiply(1 / (2 * sqrt(MSE(y_hat, y))), MSEDeriv(y_hat, y)); return alg.scalarMultiply(1 / (2 * sqrt(MSE(y_hat, y))), MSEDeriv(y_hat, y));
} }
std::vector<std::vector<double>> MLPPCost::RMSEDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) { std::vector<std::vector<real_t>> MLPPCost::RMSEDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.scalarMultiply(1 / (2 / sqrt(MSE(y_hat, y))), MSEDeriv(y_hat, y)); return alg.scalarMultiply(1 / (2 / sqrt(MSE(y_hat, y))), MSEDeriv(y_hat, y));
} }
double MLPPCost::MAE(std::vector<double> y_hat, std::vector<double> y) { real_t MLPPCost::MAE(std::vector<real_t> y_hat, std::vector<real_t> y) {
double sum = 0; real_t sum = 0;
for (int i = 0; i < y_hat.size(); i++) { for (int i = 0; i < y_hat.size(); i++) {
sum += abs((y_hat[i] - y[i])); sum += abs((y_hat[i] - y[i]));
} }
return sum / y_hat.size(); return sum / y_hat.size();
} }
double MLPPCost::MAE(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) { real_t MLPPCost::MAE(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
double sum = 0; real_t sum = 0;
for (int i = 0; i < y_hat.size(); i++) { for (int i = 0; i < y_hat.size(); i++) {
for (int j = 0; j < y_hat[i].size(); j++) { for (int j = 0; j < y_hat[i].size(); j++) {
sum += abs((y_hat[i][j] - y[i][j])); sum += abs((y_hat[i][j] - y[i][j]));
@ -85,8 +85,8 @@ double MLPPCost::MAE(std::vector<std::vector<double>> y_hat, std::vector<std::ve
return sum / y_hat.size(); return sum / y_hat.size();
} }
std::vector<double> MLPPCost::MAEDeriv(std::vector<double> y_hat, std::vector<double> y) { std::vector<real_t> MLPPCost::MAEDeriv(std::vector<real_t> y_hat, std::vector<real_t> y) {
std::vector<double> deriv; std::vector<real_t> deriv;
deriv.resize(y_hat.size()); deriv.resize(y_hat.size());
for (int i = 0; i < deriv.size(); i++) { for (int i = 0; i < deriv.size(); i++) {
if (y_hat[i] < 0) { if (y_hat[i] < 0) {
@ -100,8 +100,8 @@ std::vector<double> MLPPCost::MAEDeriv(std::vector<double> y_hat, std::vector<do
return deriv; return deriv;
} }
std::vector<std::vector<double>> MLPPCost::MAEDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) { std::vector<std::vector<real_t>> MLPPCost::MAEDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
std::vector<std::vector<double>> deriv; std::vector<std::vector<real_t>> deriv;
deriv.resize(y_hat.size()); deriv.resize(y_hat.size());
for (int i = 0; i < deriv.size(); i++) { for (int i = 0; i < deriv.size(); i++) {
deriv.resize(y_hat[i].size()); deriv.resize(y_hat[i].size());
@ -120,16 +120,16 @@ std::vector<std::vector<double>> MLPPCost::MAEDeriv(std::vector<std::vector<doub
return deriv; return deriv;
} }
double MLPPCost::MBE(std::vector<double> y_hat, std::vector<double> y) { real_t MLPPCost::MBE(std::vector<real_t> y_hat, std::vector<real_t> y) {
double sum = 0; real_t sum = 0;
for (int i = 0; i < y_hat.size(); i++) { for (int i = 0; i < y_hat.size(); i++) {
sum += (y_hat[i] - y[i]); sum += (y_hat[i] - y[i]);
} }
return sum / y_hat.size(); return sum / y_hat.size();
} }
double MLPPCost::MBE(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) { real_t MLPPCost::MBE(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
double sum = 0; real_t sum = 0;
for (int i = 0; i < y_hat.size(); i++) { for (int i = 0; i < y_hat.size(); i++) {
for (int j = 0; j < y_hat[i].size(); j++) { for (int j = 0; j < y_hat[i].size(); j++) {
sum += (y_hat[i][j] - y[i][j]); sum += (y_hat[i][j] - y[i][j]);
@ -138,19 +138,19 @@ double MLPPCost::MBE(std::vector<std::vector<double>> y_hat, std::vector<std::ve
return sum / y_hat.size(); return sum / y_hat.size();
} }
std::vector<double> MLPPCost::MBEDeriv(std::vector<double> y_hat, std::vector<double> y) { std::vector<real_t> MLPPCost::MBEDeriv(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.onevec(y_hat.size()); return alg.onevec(y_hat.size());
} }
std::vector<std::vector<double>> MLPPCost::MBEDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) { std::vector<std::vector<real_t>> MLPPCost::MBEDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.onemat(y_hat.size(), y_hat[0].size()); return alg.onemat(y_hat.size(), y_hat[0].size());
} }
double MLPPCost::LogLoss(std::vector<double> y_hat, std::vector<double> y) { real_t MLPPCost::LogLoss(std::vector<real_t> y_hat, std::vector<real_t> y) {
double sum = 0; real_t sum = 0;
double eps = 1e-8; real_t eps = 1e-8;
for (int i = 0; i < y_hat.size(); i++) { for (int i = 0; i < y_hat.size(); i++) {
sum += -(y[i] * std::log(y_hat[i] + eps) + (1 - y[i]) * std::log(1 - y_hat[i] + eps)); sum += -(y[i] * std::log(y_hat[i] + eps) + (1 - y[i]) * std::log(1 - y_hat[i] + eps));
} }
@ -158,9 +158,9 @@ double MLPPCost::LogLoss(std::vector<double> y_hat, std::vector<double> y) {
return sum / y_hat.size(); return sum / y_hat.size();
} }
double MLPPCost::LogLoss(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) { real_t MLPPCost::LogLoss(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
double sum = 0; real_t sum = 0;
double eps = 1e-8; real_t eps = 1e-8;
for (int i = 0; i < y_hat.size(); i++) { for (int i = 0; i < y_hat.size(); i++) {
for (int j = 0; j < y_hat[i].size(); j++) { for (int j = 0; j < y_hat[i].size(); j++) {
sum += -(y[i][j] * std::log(y_hat[i][j] + eps) + (1 - y[i][j]) * std::log(1 - y_hat[i][j] + eps)); sum += -(y[i][j] * std::log(y_hat[i][j] + eps) + (1 - y[i][j]) * std::log(1 - y_hat[i][j] + eps));
@ -170,18 +170,18 @@ double MLPPCost::LogLoss(std::vector<std::vector<double>> y_hat, std::vector<std
return sum / y_hat.size(); return sum / y_hat.size();
} }
std::vector<double> MLPPCost::LogLossDeriv(std::vector<double> y_hat, std::vector<double> y) { std::vector<real_t> MLPPCost::LogLossDeriv(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.addition(alg.scalarMultiply(-1, alg.elementWiseDivision(y, y_hat)), alg.elementWiseDivision(alg.scalarMultiply(-1, alg.scalarAdd(-1, y)), alg.scalarMultiply(-1, alg.scalarAdd(-1, y_hat)))); return alg.addition(alg.scalarMultiply(-1, alg.elementWiseDivision(y, y_hat)), alg.elementWiseDivision(alg.scalarMultiply(-1, alg.scalarAdd(-1, y)), alg.scalarMultiply(-1, alg.scalarAdd(-1, y_hat))));
} }
std::vector<std::vector<double>> MLPPCost::LogLossDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) { std::vector<std::vector<real_t>> MLPPCost::LogLossDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.addition(alg.scalarMultiply(-1, alg.elementWiseDivision(y, y_hat)), alg.elementWiseDivision(alg.scalarMultiply(-1, alg.scalarAdd(-1, y)), alg.scalarMultiply(-1, alg.scalarAdd(-1, y_hat)))); return alg.addition(alg.scalarMultiply(-1, alg.elementWiseDivision(y, y_hat)), alg.elementWiseDivision(alg.scalarMultiply(-1, alg.scalarAdd(-1, y)), alg.scalarMultiply(-1, alg.scalarAdd(-1, y_hat))));
} }
double MLPPCost::CrossEntropy(std::vector<double> y_hat, std::vector<double> y) { real_t MLPPCost::CrossEntropy(std::vector<real_t> y_hat, std::vector<real_t> y) {
double sum = 0; real_t sum = 0;
for (int i = 0; i < y_hat.size(); i++) { for (int i = 0; i < y_hat.size(); i++) {
sum += y[i] * std::log(y_hat[i]); sum += y[i] * std::log(y_hat[i]);
} }
@ -189,8 +189,8 @@ double MLPPCost::CrossEntropy(std::vector<double> y_hat, std::vector<double> y)
return -1 * sum; return -1 * sum;
} }
double MLPPCost::CrossEntropy(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) { real_t MLPPCost::CrossEntropy(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
double sum = 0; real_t sum = 0;
for (int i = 0; i < y_hat.size(); i++) { for (int i = 0; i < y_hat.size(); i++) {
for (int j = 0; j < y_hat[i].size(); j++) { for (int j = 0; j < y_hat[i].size(); j++) {
sum += y[i][j] * std::log(y_hat[i][j]); sum += y[i][j] * std::log(y_hat[i][j]);
@ -200,19 +200,19 @@ double MLPPCost::CrossEntropy(std::vector<std::vector<double>> y_hat, std::vecto
return -1 * sum; return -1 * sum;
} }
std::vector<double> MLPPCost::CrossEntropyDeriv(std::vector<double> y_hat, std::vector<double> y) { std::vector<real_t> MLPPCost::CrossEntropyDeriv(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.scalarMultiply(-1, alg.elementWiseDivision(y, y_hat)); return alg.scalarMultiply(-1, alg.elementWiseDivision(y, y_hat));
} }
std::vector<std::vector<double>> MLPPCost::CrossEntropyDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) { std::vector<std::vector<real_t>> MLPPCost::CrossEntropyDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.scalarMultiply(-1, alg.elementWiseDivision(y, y_hat)); return alg.scalarMultiply(-1, alg.elementWiseDivision(y, y_hat));
} }
double MLPPCost::HuberLoss(std::vector<double> y_hat, std::vector<double> y, double delta) { real_t MLPPCost::HuberLoss(std::vector<real_t> y_hat, std::vector<real_t> y, real_t delta) {
MLPPLinAlg alg; MLPPLinAlg alg;
double sum = 0; real_t sum = 0;
for (int i = 0; i < y_hat.size(); i++) { for (int i = 0; i < y_hat.size(); i++) {
if (abs(y[i] - y_hat[i]) <= delta) { if (abs(y[i] - y_hat[i]) <= delta) {
sum += (y[i] - y_hat[i]) * (y[i] - y_hat[i]); sum += (y[i] - y_hat[i]) * (y[i] - y_hat[i]);
@ -223,9 +223,9 @@ double MLPPCost::HuberLoss(std::vector<double> y_hat, std::vector<double> y, dou
return sum; return sum;
} }
double MLPPCost::HuberLoss(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y, double delta) { real_t MLPPCost::HuberLoss(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y, real_t delta) {
MLPPLinAlg alg; MLPPLinAlg alg;
double sum = 0; real_t sum = 0;
for (int i = 0; i < y_hat.size(); i++) { for (int i = 0; i < y_hat.size(); i++) {
for (int j = 0; j < y_hat[i].size(); j++) { for (int j = 0; j < y_hat[i].size(); j++) {
if (abs(y[i][j] - y_hat[i][j]) <= delta) { if (abs(y[i][j] - y_hat[i][j]) <= delta) {
@ -238,10 +238,10 @@ double MLPPCost::HuberLoss(std::vector<std::vector<double>> y_hat, std::vector<s
return sum; return sum;
} }
std::vector<double> MLPPCost::HuberLossDeriv(std::vector<double> y_hat, std::vector<double> y, double delta) { std::vector<real_t> MLPPCost::HuberLossDeriv(std::vector<real_t> y_hat, std::vector<real_t> y, real_t delta) {
MLPPLinAlg alg; MLPPLinAlg alg;
double sum = 0; real_t sum = 0;
std::vector<double> deriv; std::vector<real_t> deriv;
deriv.resize(y_hat.size()); deriv.resize(y_hat.size());
for (int i = 0; i < y_hat.size(); i++) { for (int i = 0; i < y_hat.size(); i++) {
@ -258,10 +258,10 @@ std::vector<double> MLPPCost::HuberLossDeriv(std::vector<double> y_hat, std::vec
return deriv; return deriv;
} }
std::vector<std::vector<double>> MLPPCost::HuberLossDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y, double delta) { std::vector<std::vector<real_t>> MLPPCost::HuberLossDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y, real_t delta) {
MLPPLinAlg alg; MLPPLinAlg alg;
double sum = 0; real_t sum = 0;
std::vector<std::vector<double>> deriv; std::vector<std::vector<real_t>> deriv;
deriv.resize(y_hat.size()); deriv.resize(y_hat.size());
for (int i = 0; i < deriv.size(); i++) { for (int i = 0; i < deriv.size(); i++) {
deriv[i].resize(y_hat[i].size()); deriv[i].resize(y_hat[i].size());
@ -283,8 +283,8 @@ std::vector<std::vector<double>> MLPPCost::HuberLossDeriv(std::vector<std::vecto
return deriv; return deriv;
} }
double MLPPCost::HingeLoss(std::vector<double> y_hat, std::vector<double> y) { real_t MLPPCost::HingeLoss(std::vector<real_t> y_hat, std::vector<real_t> y) {
double sum = 0; real_t sum = 0;
for (int i = 0; i < y_hat.size(); i++) { for (int i = 0; i < y_hat.size(); i++) {
sum += fmax(0, 1 - y[i] * y_hat[i]); sum += fmax(0, 1 - y[i] * y_hat[i]);
} }
@ -292,8 +292,8 @@ double MLPPCost::HingeLoss(std::vector<double> y_hat, std::vector<double> y) {
return sum / y_hat.size(); return sum / y_hat.size();
} }
double MLPPCost::HingeLoss(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) { real_t MLPPCost::HingeLoss(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
double sum = 0; real_t sum = 0;
for (int i = 0; i < y_hat.size(); i++) { for (int i = 0; i < y_hat.size(); i++) {
for (int j = 0; j < y_hat[i].size(); j++) { for (int j = 0; j < y_hat[i].size(); j++) {
sum += fmax(0, 1 - y[i][j] * y_hat[i][j]); sum += fmax(0, 1 - y[i][j] * y_hat[i][j]);
@ -303,8 +303,8 @@ double MLPPCost::HingeLoss(std::vector<std::vector<double>> y_hat, std::vector<s
return sum / y_hat.size(); return sum / y_hat.size();
} }
std::vector<double> MLPPCost::HingeLossDeriv(std::vector<double> y_hat, std::vector<double> y) { std::vector<real_t> MLPPCost::HingeLossDeriv(std::vector<real_t> y_hat, std::vector<real_t> y) {
std::vector<double> deriv; std::vector<real_t> deriv;
deriv.resize(y_hat.size()); deriv.resize(y_hat.size());
for (int i = 0; i < y_hat.size(); i++) { for (int i = 0; i < y_hat.size(); i++) {
if (1 - y[i] * y_hat[i] > 0) { if (1 - y[i] * y_hat[i] > 0) {
@ -316,8 +316,8 @@ std::vector<double> MLPPCost::HingeLossDeriv(std::vector<double> y_hat, std::vec
return deriv; return deriv;
} }
std::vector<std::vector<double>> MLPPCost::HingeLossDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) { std::vector<std::vector<real_t>> MLPPCost::HingeLossDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
std::vector<std::vector<double>> deriv; std::vector<std::vector<real_t>> deriv;
for (int i = 0; i < y_hat.size(); i++) { for (int i = 0; i < y_hat.size(); i++) {
for (int j = 0; j < y_hat[i].size(); j++) { for (int j = 0; j < y_hat[i].size(); j++) {
if (1 - y[i][j] * y_hat[i][j] > 0) { if (1 - y[i][j] * y_hat[i][j] > 0) {
@ -330,16 +330,16 @@ std::vector<std::vector<double>> MLPPCost::HingeLossDeriv(std::vector<std::vecto
return deriv; return deriv;
} }
double MLPPCost::WassersteinLoss(std::vector<double> y_hat, std::vector<double> y) { real_t MLPPCost::WassersteinLoss(std::vector<real_t> y_hat, std::vector<real_t> y) {
double sum = 0; real_t sum = 0;
for (int i = 0; i < y_hat.size(); i++) { for (int i = 0; i < y_hat.size(); i++) {
sum += y_hat[i] * y[i]; sum += y_hat[i] * y[i];
} }
return -sum / y_hat.size(); return -sum / y_hat.size();
} }
double MLPPCost::WassersteinLoss(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) { real_t MLPPCost::WassersteinLoss(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
double sum = 0; real_t sum = 0;
for (int i = 0; i < y_hat.size(); i++) { for (int i = 0; i < y_hat.size(); i++) {
for (int j = 0; j < y_hat[i].size(); j++) { for (int j = 0; j < y_hat[i].size(); j++) {
sum += y_hat[i][j] * y[i][j]; sum += y_hat[i][j] * y[i][j];
@ -348,59 +348,59 @@ double MLPPCost::WassersteinLoss(std::vector<std::vector<double>> y_hat, std::ve
return -sum / y_hat.size(); return -sum / y_hat.size();
} }
std::vector<double> MLPPCost::WassersteinLossDeriv(std::vector<double> y_hat, std::vector<double> y) { std::vector<real_t> MLPPCost::WassersteinLossDeriv(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.scalarMultiply(-1, y); // Simple. return alg.scalarMultiply(-1, y); // Simple.
} }
std::vector<std::vector<double>> MLPPCost::WassersteinLossDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) { std::vector<std::vector<real_t>> MLPPCost::WassersteinLossDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.scalarMultiply(-1, y); // Simple. return alg.scalarMultiply(-1, y); // Simple.
} }
double MLPPCost::HingeLoss(std::vector<double> y_hat, std::vector<double> y, std::vector<double> weights, double C) { real_t MLPPCost::HingeLoss(std::vector<real_t> y_hat, std::vector<real_t> y, std::vector<real_t> weights, real_t C) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
return C * HingeLoss(y_hat, y) + regularization.regTerm(weights, 1, 0, "Ridge"); return C * HingeLoss(y_hat, y) + regularization.regTerm(weights, 1, 0, "Ridge");
} }
double MLPPCost::HingeLoss(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y, std::vector<std::vector<double>> weights, double C) { real_t MLPPCost::HingeLoss(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y, std::vector<std::vector<real_t>> weights, real_t C) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
return C * HingeLoss(y_hat, y) + regularization.regTerm(weights, 1, 0, "Ridge"); return C * HingeLoss(y_hat, y) + regularization.regTerm(weights, 1, 0, "Ridge");
} }
std::vector<double> MLPPCost::HingeLossDeriv(std::vector<double> y_hat, std::vector<double> y, double C) { std::vector<real_t> MLPPCost::HingeLossDeriv(std::vector<real_t> y_hat, std::vector<real_t> y, real_t C) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
return alg.scalarMultiply(C, HingeLossDeriv(y_hat, y)); return alg.scalarMultiply(C, HingeLossDeriv(y_hat, y));
} }
std::vector<std::vector<double>> MLPPCost::HingeLossDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y, double C) { std::vector<std::vector<real_t>> MLPPCost::HingeLossDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y, real_t C) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
return alg.scalarMultiply(C, HingeLossDeriv(y_hat, y)); return alg.scalarMultiply(C, HingeLossDeriv(y_hat, y));
} }
double MLPPCost::dualFormSVM(std::vector<double> alpha, std::vector<std::vector<double>> X, std::vector<double> y) { real_t MLPPCost::dualFormSVM(std::vector<real_t> alpha, std::vector<std::vector<real_t>> X, std::vector<real_t> y) {
MLPPLinAlg alg; MLPPLinAlg alg;
std::vector<std::vector<double>> Y = alg.diag(y); // Y is a diagnoal matrix. Y[i][j] = y[i] if i = i, else Y[i][j] = 0. Yt = Y. std::vector<std::vector<real_t>> Y = alg.diag(y); // Y is a diagnoal matrix. Y[i][j] = y[i] if i = i, else Y[i][j] = 0. Yt = Y.
std::vector<std::vector<double>> K = alg.matmult(X, alg.transpose(X)); // TO DO: DON'T forget to add non-linear kernelizations. std::vector<std::vector<real_t>> K = alg.matmult(X, alg.transpose(X)); // TO DO: DON'T forget to add non-linear kernelizations.
std::vector<std::vector<double>> Q = alg.matmult(alg.matmult(alg.transpose(Y), K), Y); std::vector<std::vector<real_t>> Q = alg.matmult(alg.matmult(alg.transpose(Y), K), Y);
double alphaQ = alg.matmult(alg.matmult({ alpha }, Q), alg.transpose({ alpha }))[0][0]; real_t alphaQ = alg.matmult(alg.matmult({ alpha }, Q), alg.transpose({ alpha }))[0][0];
std::vector<double> one = alg.onevec(alpha.size()); std::vector<real_t> one = alg.onevec(alpha.size());
return -alg.dot(one, alpha) + 0.5 * alphaQ; return -alg.dot(one, alpha) + 0.5 * alphaQ;
} }
std::vector<double> MLPPCost::dualFormSVMDeriv(std::vector<double> alpha, std::vector<std::vector<double>> X, std::vector<double> y) { std::vector<real_t> MLPPCost::dualFormSVMDeriv(std::vector<real_t> alpha, std::vector<std::vector<real_t>> X, std::vector<real_t> y) {
MLPPLinAlg alg; MLPPLinAlg alg;
std::vector<std::vector<double>> Y = alg.zeromat(y.size(), y.size()); std::vector<std::vector<real_t>> Y = alg.zeromat(y.size(), y.size());
for (int i = 0; i < y.size(); i++) { for (int i = 0; i < y.size(); i++) {
Y[i][i] = y[i]; // Y is a diagnoal matrix. Y[i][j] = y[i] if i = i, else Y[i][j] = 0. Yt = Y. Y[i][i] = y[i]; // Y is a diagnoal matrix. Y[i][j] = y[i] if i = i, else Y[i][j] = 0. Yt = Y.
} }
std::vector<std::vector<double>> K = alg.matmult(X, alg.transpose(X)); // TO DO: DON'T forget to add non-linear kernelizations. std::vector<std::vector<real_t>> K = alg.matmult(X, alg.transpose(X)); // TO DO: DON'T forget to add non-linear kernelizations.
std::vector<std::vector<double>> Q = alg.matmult(alg.matmult(alg.transpose(Y), K), Y); std::vector<std::vector<real_t>> Q = alg.matmult(alg.matmult(alg.transpose(Y), K), Y);
std::vector<double> alphaQDeriv = alg.mat_vec_mult(Q, alpha); std::vector<real_t> alphaQDeriv = alg.mat_vec_mult(Q, alpha);
std::vector<double> one = alg.onevec(alpha.size()); std::vector<real_t> one = alg.onevec(alpha.size());
return alg.subtraction(alphaQDeriv, one); return alg.subtraction(alphaQDeriv, one);
} }

86
mlpp/cost/cost.h
View File

@ -8,76 +8,78 @@
// Created by Marc Melikyan on 1/16/21. // Created by Marc Melikyan on 1/16/21.
// //
#include "core/math/math_defs.h"
#include <vector> #include <vector>
class MLPPCost { class MLPPCost {
public: public:
// Regression Costs // Regression Costs
double MSE(std::vector<double> y_hat, std::vector<double> y); real_t MSE(std::vector<real_t> y_hat, std::vector<real_t> y);
double MSE(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y); real_t MSE(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
std::vector<double> MSEDeriv(std::vector<double> y_hat, std::vector<double> y); std::vector<real_t> MSEDeriv(std::vector<real_t> y_hat, std::vector<real_t> y);
std::vector<std::vector<double>> MSEDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y); std::vector<std::vector<real_t>> MSEDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
double RMSE(std::vector<double> y_hat, std::vector<double> y); real_t RMSE(std::vector<real_t> y_hat, std::vector<real_t> y);
double RMSE(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y); real_t RMSE(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
std::vector<double> RMSEDeriv(std::vector<double> y_hat, std::vector<double> y); std::vector<real_t> RMSEDeriv(std::vector<real_t> y_hat, std::vector<real_t> y);
std::vector<std::vector<double>> RMSEDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y); std::vector<std::vector<real_t>> RMSEDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
double MAE(std::vector<double> y_hat, std::vector<double> y); real_t MAE(std::vector<real_t> y_hat, std::vector<real_t> y);
double MAE(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y); real_t MAE(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
std::vector<double> MAEDeriv(std::vector<double> y_hat, std::vector<double> y); std::vector<real_t> MAEDeriv(std::vector<real_t> y_hat, std::vector<real_t> y);
std::vector<std::vector<double>> MAEDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y); std::vector<std::vector<real_t>> MAEDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
double MBE(std::vector<double> y_hat, std::vector<double> y); real_t MBE(std::vector<real_t> y_hat, std::vector<real_t> y);
double MBE(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y); real_t MBE(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
std::vector<double> MBEDeriv(std::vector<double> y_hat, std::vector<double> y); std::vector<real_t> MBEDeriv(std::vector<real_t> y_hat, std::vector<real_t> y);
std::vector<std::vector<double>> MBEDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y); std::vector<std::vector<real_t>> MBEDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
// Classification Costs // Classification Costs
double LogLoss(std::vector<double> y_hat, std::vector<double> y); real_t LogLoss(std::vector<real_t> y_hat, std::vector<real_t> y);
double LogLoss(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y); real_t LogLoss(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
std::vector<double> LogLossDeriv(std::vector<double> y_hat, std::vector<double> y); std::vector<real_t> LogLossDeriv(std::vector<real_t> y_hat, std::vector<real_t> y);
std::vector<std::vector<double>> LogLossDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y); std::vector<std::vector<real_t>> LogLossDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
double CrossEntropy(std::vector<double> y_hat, std::vector<double> y); real_t CrossEntropy(std::vector<real_t> y_hat, std::vector<real_t> y);
double CrossEntropy(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y); real_t CrossEntropy(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
std::vector<double> CrossEntropyDeriv(std::vector<double> y_hat, std::vector<double> y); std::vector<real_t> CrossEntropyDeriv(std::vector<real_t> y_hat, std::vector<real_t> y);
std::vector<std::vector<double>> CrossEntropyDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y); std::vector<std::vector<real_t>> CrossEntropyDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
double HuberLoss(std::vector<double> y_hat, std::vector<double> y, double delta); real_t HuberLoss(std::vector<real_t> y_hat, std::vector<real_t> y, real_t delta);
double HuberLoss(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y, double delta); real_t HuberLoss(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y, real_t delta);
std::vector<double> HuberLossDeriv(std::vector<double> y_hat, std::vector<double> y, double delta); std::vector<real_t> HuberLossDeriv(std::vector<real_t> y_hat, std::vector<real_t> y, real_t delta);
std::vector<std::vector<double>> HuberLossDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y, double delta); std::vector<std::vector<real_t>> HuberLossDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y, real_t delta);
double HingeLoss(std::vector<double> y_hat, std::vector<double> y); real_t HingeLoss(std::vector<real_t> y_hat, std::vector<real_t> y);
double HingeLoss(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y); real_t HingeLoss(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
std::vector<double> HingeLossDeriv(std::vector<double> y_hat, std::vector<double> y); std::vector<real_t> HingeLossDeriv(std::vector<real_t> y_hat, std::vector<real_t> y);
std::vector<std::vector<double>> HingeLossDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y); std::vector<std::vector<real_t>> HingeLossDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
double HingeLoss(std::vector<double> y_hat, std::vector<double> y, std::vector<double> weights, double C); real_t HingeLoss(std::vector<real_t> y_hat, std::vector<real_t> y, std::vector<real_t> weights, real_t C);
double HingeLoss(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y, std::vector<std::vector<double>> weights, double C); real_t HingeLoss(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y, std::vector<std::vector<real_t>> weights, real_t C);
std::vector<double> HingeLossDeriv(std::vector<double> y_hat, std::vector<double> y, double C); std::vector<real_t> HingeLossDeriv(std::vector<real_t> y_hat, std::vector<real_t> y, real_t C);
std::vector<std::vector<double>> HingeLossDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y, double C); std::vector<std::vector<real_t>> HingeLossDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y, real_t C);
double WassersteinLoss(std::vector<double> y_hat, std::vector<double> y); real_t WassersteinLoss(std::vector<real_t> y_hat, std::vector<real_t> y);
double WassersteinLoss(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y); real_t WassersteinLoss(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
std::vector<double> WassersteinLossDeriv(std::vector<double> y_hat, std::vector<double> y); std::vector<real_t> WassersteinLossDeriv(std::vector<real_t> y_hat, std::vector<real_t> y);
std::vector<std::vector<double>> WassersteinLossDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y); std::vector<std::vector<real_t>> WassersteinLossDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
double dualFormSVM(std::vector<double> alpha, std::vector<std::vector<double>> X, std::vector<double> y); // TO DO: DON'T forget to add non-linear kernelizations. real_t dualFormSVM(std::vector<real_t> alpha, std::vector<std::vector<real_t>> X, std::vector<real_t> y); // TO DO: DON'T forget to add non-linear kernelizations.
std::vector<double> dualFormSVMDeriv(std::vector<double> alpha, std::vector<std::vector<double>> X, std::vector<double> y); std::vector<real_t> dualFormSVMDeriv(std::vector<real_t> alpha, std::vector<std::vector<real_t>> X, std::vector<real_t> y);
private: private:
}; };

210
mlpp/data/data.cpp
View File

@ -56,7 +56,7 @@ Ref<MLPPDataComplex> MLPPData::load_iris(const String &path) {
const int IRIS_SIZE = 4; const int IRIS_SIZE = 4;
const int ONE_HOT_NUM = 3; const int ONE_HOT_NUM = 3;
std::vector<double> tempOutputSet; std::vector<real_t> tempOutputSet;
Ref<MLPPDataComplex> data; Ref<MLPPDataComplex> data;
data.instance(); data.instance();
@ -71,7 +71,7 @@ Ref<MLPPDataComplex> MLPPData::load_wine(const String &path) {
const int WINE_SIZE = 4; const int WINE_SIZE = 4;
const int ONE_HOT_NUM = 3; const int ONE_HOT_NUM = 3;
std::vector<double> tempOutputSet; std::vector<real_t> tempOutputSet;
Ref<MLPPDataComplex> data; Ref<MLPPDataComplex> data;
data.instance(); data.instance();
@ -86,8 +86,8 @@ Ref<MLPPDataComplex> MLPPData::load_mnist_train(const String &path) {
const int MNIST_SIZE = 784; const int MNIST_SIZE = 784;
const int ONE_HOT_NUM = 10; const int ONE_HOT_NUM = 10;
std::vector<std::vector<double>> inputSet; std::vector<std::vector<real_t>> inputSet;
std::vector<double> tempOutputSet; std::vector<real_t> tempOutputSet;
Ref<MLPPDataComplex> data; Ref<MLPPDataComplex> data;
data.instance(); data.instance();
@ -101,8 +101,8 @@ Ref<MLPPDataComplex> MLPPData::load_mnist_train(const String &path) {
Ref<MLPPDataComplex> MLPPData::load_mnist_test(const String &path) { Ref<MLPPDataComplex> MLPPData::load_mnist_test(const String &path) {
const int MNIST_SIZE = 784; const int MNIST_SIZE = 784;
const int ONE_HOT_NUM = 10; const int ONE_HOT_NUM = 10;
std::vector<std::vector<double>> inputSet; std::vector<std::vector<real_t>> inputSet;
std::vector<double> tempOutputSet; std::vector<real_t> tempOutputSet;
Ref<MLPPDataComplex> data; Ref<MLPPDataComplex> data;
data.instance(); data.instance();
@ -137,7 +137,7 @@ Ref<MLPPDataESimple> MLPPData::load_fires_and_crime(const String &path) {
// MULTIVARIATE SUPERVISED // MULTIVARIATE SUPERVISED
void MLPPData::set_data_supervised(int k, const String &file_name, std::vector<std::vector<double>> &inputSet, std::vector<double> &outputSet) { void MLPPData::set_data_supervised(int k, const String &file_name, std::vector<std::vector<real_t>> &inputSet, std::vector<real_t> &outputSet) {
MLPPLinAlg alg; MLPPLinAlg alg;
inputSet.resize(k); inputSet.resize(k);
@ -150,10 +150,10 @@ void MLPPData::set_data_supervised(int k, const String &file_name, std::vector<s
Vector<String> ll = file->get_csv_line(); Vector<String> ll = file->get_csv_line();
for (int i = 0; i < k; ++i) { for (int i = 0; i < k; ++i) {
inputSet[i].push_back(ll[i].to_double()); inputSet[i].push_back(static_cast<real_t>(ll[i].to_double()));
} }
outputSet.push_back(ll[k].to_double()); outputSet.push_back(static_cast<real_t>(ll[k].to_double()));
} }
inputSet = alg.transpose(inputSet); inputSet = alg.transpose(inputSet);
@ -161,7 +161,7 @@ void MLPPData::set_data_supervised(int k, const String &file_name, std::vector<s
memdelete(file); memdelete(file);
} }
void MLPPData::set_data_unsupervised(int k, const String &file_name, std::vector<std::vector<double>> &inputSet) { void MLPPData::set_data_unsupervised(int k, const String &file_name, std::vector<std::vector<real_t>> &inputSet) {
MLPPLinAlg alg; MLPPLinAlg alg;
inputSet.resize(k); inputSet.resize(k);
@ -174,7 +174,7 @@ void MLPPData::set_data_unsupervised(int k, const String &file_name, std::vector
Vector<String> ll = file->get_csv_line(); Vector<String> ll = file->get_csv_line();
for (int i = 0; i < k; ++i) { for (int i = 0; i < k; ++i) {
inputSet[i].push_back(ll[i].to_double()); inputSet[i].push_back(static_cast<real_t>(ll[i].to_double()));
} }
} }
@ -183,7 +183,7 @@ void MLPPData::set_data_unsupervised(int k, const String &file_name, std::vector
memdelete(file); memdelete(file);
} }
void MLPPData::set_data_simple(const String &file_name, std::vector<double> &inputSet, std::vector<double> &outputSet) { void MLPPData::set_data_simple(const String &file_name, std::vector<real_t> &inputSet, std::vector<real_t> &outputSet) {
FileAccess *file = FileAccess::open(file_name, FileAccess::READ); FileAccess *file = FileAccess::open(file_name, FileAccess::READ);
ERR_FAIL_COND(!file); ERR_FAIL_COND(!file);
@ -192,15 +192,15 @@ void MLPPData::set_data_simple(const String &file_name, std::vector<double> &inp
Vector<String> ll = file->get_csv_line(); Vector<String> ll = file->get_csv_line();
for (int i = 0; i < ll.size(); i += 2) { for (int i = 0; i < ll.size(); i += 2) {
inputSet.push_back(ll[i].to_double()); inputSet.push_back(static_cast<real_t>(ll[i].to_double()));
outputSet.push_back(ll[i + 1].to_double()); outputSet.push_back(static_cast<real_t>(ll[i + 1].to_double()));
} }
} }
memdelete(file); memdelete(file);
} }
MLPPData::SplitComplexData MLPPData::train_test_split(const Ref<MLPPDataComplex> &data, double test_size) { MLPPData::SplitComplexData MLPPData::train_test_split(const Ref<MLPPDataComplex> &data, real_t test_size) {
SplitComplexData res; SplitComplexData res;
res.train.instance(); res.train.instance();
@ -237,7 +237,7 @@ MLPPData::SplitComplexData MLPPData::train_test_split(const Ref<MLPPDataComplex>
return res; return res;
} }
Array MLPPData::train_test_split_bind(const Ref<MLPPDataComplex> &data, double test_size) { Array MLPPData::train_test_split_bind(const Ref<MLPPDataComplex> &data, real_t test_size) {
SplitComplexData res = train_test_split(data, test_size); SplitComplexData res = train_test_split(data, test_size);
Array arr; Array arr;
@ -248,80 +248,80 @@ Array MLPPData::train_test_split_bind(const Ref<MLPPDataComplex> &data, double t
} }
// Loading Datasets // Loading Datasets
std::tuple<std::vector<std::vector<double>>, std::vector<double>> MLPPData::loadBreastCancer() { std::tuple<std::vector<std::vector<real_t>>, std::vector<real_t>> MLPPData::loadBreastCancer() {
const int BREAST_CANCER_SIZE = 30; // k = 30 const int BREAST_CANCER_SIZE = 30; // k = 30
std::vector<std::vector<double>> inputSet; std::vector<std::vector<real_t>> inputSet;
std::vector<double> outputSet; std::vector<real_t> outputSet;
setData(BREAST_CANCER_SIZE, "MLPP/Data/Datasets/BreastCancer.csv", inputSet, outputSet); setData(BREAST_CANCER_SIZE, "MLPP/Data/Datasets/BreastCancer.csv", inputSet, outputSet);
return { inputSet, outputSet }; return { inputSet, outputSet };
} }
std::tuple<std::vector<std::vector<double>>, std::vector<double>> MLPPData::loadBreastCancerSVC() { std::tuple<std::vector<std::vector<real_t>>, std::vector<real_t>> MLPPData::loadBreastCancerSVC() {
const int BREAST_CANCER_SIZE = 30; // k = 30 const int BREAST_CANCER_SIZE = 30; // k = 30
std::vector<std::vector<double>> inputSet; std::vector<std::vector<real_t>> inputSet;
std::vector<double> outputSet; std::vector<real_t> outputSet;
setData(BREAST_CANCER_SIZE, "MLPP/Data/Datasets/BreastCancerSVM.csv", inputSet, outputSet); setData(BREAST_CANCER_SIZE, "MLPP/Data/Datasets/BreastCancerSVM.csv", inputSet, outputSet);
return { inputSet, outputSet }; return { inputSet, outputSet };
} }
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> MLPPData::loadIris() { std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> MLPPData::loadIris() {
const int IRIS_SIZE = 4; const int IRIS_SIZE = 4;
const int ONE_HOT_NUM = 3; const int ONE_HOT_NUM = 3;
std::vector<std::vector<double>> inputSet; std::vector<std::vector<real_t>> inputSet;
std::vector<double> tempOutputSet; std::vector<real_t> tempOutputSet;
setData(IRIS_SIZE, "/Users/marcmelikyan/Desktop/Data/Iris.csv", inputSet, tempOutputSet); setData(IRIS_SIZE, "/Users/marcmelikyan/Desktop/Data/Iris.csv", inputSet, tempOutputSet);
std::vector<std::vector<double>> outputSet = oneHotRep(tempOutputSet, ONE_HOT_NUM); std::vector<std::vector<real_t>> outputSet = oneHotRep(tempOutputSet, ONE_HOT_NUM);
return { inputSet, outputSet }; return { inputSet, outputSet };
} }
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> MLPPData::loadWine() { std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> MLPPData::loadWine() {
const int WINE_SIZE = 4; const int WINE_SIZE = 4;
const int ONE_HOT_NUM = 3; const int ONE_HOT_NUM = 3;
std::vector<std::vector<double>> inputSet; std::vector<std::vector<real_t>> inputSet;
std::vector<double> tempOutputSet; std::vector<real_t> tempOutputSet;
setData(WINE_SIZE, "MLPP/Data/Datasets/Iris.csv", inputSet, tempOutputSet); setData(WINE_SIZE, "MLPP/Data/Datasets/Iris.csv", inputSet, tempOutputSet);
std::vector<std::vector<double>> outputSet = oneHotRep(tempOutputSet, ONE_HOT_NUM); std::vector<std::vector<real_t>> outputSet = oneHotRep(tempOutputSet, ONE_HOT_NUM);
return { inputSet, outputSet }; return { inputSet, outputSet };
} }
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> MLPPData::loadMnistTrain() { std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> MLPPData::loadMnistTrain() {
const int MNIST_SIZE = 784; const int MNIST_SIZE = 784;
const int ONE_HOT_NUM = 10; const int ONE_HOT_NUM = 10;
std::vector<std::vector<double>> inputSet; std::vector<std::vector<real_t>> inputSet;
std::vector<double> tempOutputSet; std::vector<real_t> tempOutputSet;
setData(MNIST_SIZE, "MLPP/Data/Datasets/MnistTrain.csv", inputSet, tempOutputSet); setData(MNIST_SIZE, "MLPP/Data/Datasets/MnistTrain.csv", inputSet, tempOutputSet);
std::vector<std::vector<double>> outputSet = oneHotRep(tempOutputSet, ONE_HOT_NUM); std::vector<std::vector<real_t>> outputSet = oneHotRep(tempOutputSet, ONE_HOT_NUM);
return { inputSet, outputSet }; return { inputSet, outputSet };
} }
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> MLPPData::loadMnistTest() { std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> MLPPData::loadMnistTest() {
const int MNIST_SIZE = 784; const int MNIST_SIZE = 784;
const int ONE_HOT_NUM = 10; const int ONE_HOT_NUM = 10;
std::vector<std::vector<double>> inputSet; std::vector<std::vector<real_t>> inputSet;
std::vector<double> tempOutputSet; std::vector<real_t> tempOutputSet;
setData(MNIST_SIZE, "MLPP/Data/Datasets/MnistTest.csv", inputSet, tempOutputSet); setData(MNIST_SIZE, "MLPP/Data/Datasets/MnistTest.csv", inputSet, tempOutputSet);
std::vector<std::vector<double>> outputSet = oneHotRep(tempOutputSet, ONE_HOT_NUM); std::vector<std::vector<real_t>> outputSet = oneHotRep(tempOutputSet, ONE_HOT_NUM);
return { inputSet, outputSet }; return { inputSet, outputSet };
} }
std::tuple<std::vector<std::vector<double>>, std::vector<double>> MLPPData::loadCaliforniaHousing() { std::tuple<std::vector<std::vector<real_t>>, std::vector<real_t>> MLPPData::loadCaliforniaHousing() {
const int CALIFORNIA_HOUSING_SIZE = 13; // k = 30 const int CALIFORNIA_HOUSING_SIZE = 13; // k = 30
std::vector<std::vector<double>> inputSet; std::vector<std::vector<real_t>> inputSet;
std::vector<double> outputSet; std::vector<real_t> outputSet;
setData(CALIFORNIA_HOUSING_SIZE, "MLPP/Data/Datasets/CaliforniaHousing.csv", inputSet, outputSet); setData(CALIFORNIA_HOUSING_SIZE, "MLPP/Data/Datasets/CaliforniaHousing.csv", inputSet, outputSet);
return { inputSet, outputSet }; return { inputSet, outputSet };
} }
std::tuple<std::vector<double>, std::vector<double>> MLPPData::loadFiresAndCrime() { std::tuple<std::vector<real_t>, std::vector<real_t>> MLPPData::loadFiresAndCrime() {
std::vector<double> inputSet; // k is implicitly 1. std::vector<real_t> inputSet; // k is implicitly 1.
std::vector<double> outputSet; std::vector<real_t> outputSet;
setData("MLPP/Data/Datasets/FiresAndCrime.csv", inputSet, outputSet); setData("MLPP/Data/Datasets/FiresAndCrime.csv", inputSet, outputSet);
return { inputSet, outputSet }; return { inputSet, outputSet };
@ -330,15 +330,15 @@ std::tuple<std::vector<double>, std::vector<double>> MLPPData::loadFiresAndCrime
// Note that inputs and outputs should be pairs (technically), but this // Note that inputs and outputs should be pairs (technically), but this
// implementation will separate them. (My implementation keeps them tied together.) // implementation will separate them. (My implementation keeps them tied together.)
// Not yet sure whether this is intentional or not (or it's something like a compiler specific difference) // Not yet sure whether this is intentional or not (or it's something like a compiler specific difference)
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>, std::vector<std::vector<double>>, std::vector<std::vector<double>>> MLPPData::trainTestSplit(std::vector<std::vector<double>> inputSet, std::vector<std::vector<double>> outputSet, double testSize) { std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> MLPPData::trainTestSplit(std::vector<std::vector<real_t>> inputSet, std::vector<std::vector<real_t>> outputSet, real_t testSize) {
std::random_device rd; std::random_device rd;
std::default_random_engine generator(rd()); std::default_random_engine generator(rd());
std::shuffle(inputSet.begin(), inputSet.end(), generator); // inputSet random shuffle std::shuffle(inputSet.begin(), inputSet.end(), generator); // inputSet random shuffle
std::shuffle(outputSet.begin(), outputSet.end(), generator); // outputSet random shuffle) std::shuffle(outputSet.begin(), outputSet.end(), generator); // outputSet random shuffle)
std::vector<std::vector<double>> inputTestSet; std::vector<std::vector<real_t>> inputTestSet;
std::vector<std::vector<double>> outputTestSet; std::vector<std::vector<real_t>> outputTestSet;
int testInputNumber = testSize * inputSet.size(); // implicit usage of floor int testInputNumber = testSize * inputSet.size(); // implicit usage of floor
int testOutputNumber = testSize * outputSet.size(); // implicit usage of floor int testOutputNumber = testSize * outputSet.size(); // implicit usage of floor
@ -358,7 +358,7 @@ std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>, s
// MULTIVARIATE SUPERVISED // MULTIVARIATE SUPERVISED
void MLPPData::setData(int k, std::string fileName, std::vector<std::vector<double>> &inputSet, std::vector<double> &outputSet) { void MLPPData::setData(int k, std::string fileName, std::vector<std::vector<real_t>> &inputSet, std::vector<real_t> &outputSet) {
MLPPLinAlg alg; MLPPLinAlg alg;
std::string inputTemp; std::string inputTemp;
std::string outputTemp; std::string outputTemp;
@ -386,7 +386,7 @@ void MLPPData::setData(int k, std::string fileName, std::vector<std::vector<doub
dataFile.close(); dataFile.close();
} }
void MLPPData::printData(std::vector<std::string> inputName, std::string outputName, std::vector<std::vector<double>> inputSet, std::vector<double> outputSet) { void MLPPData::printData(std::vector<std::string> inputName, std::string outputName, std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet) {
MLPPLinAlg alg; MLPPLinAlg alg;
inputSet = alg.transpose(inputSet); inputSet = alg.transpose(inputSet);
for (int i = 0; i < inputSet.size(); i++) { for (int i = 0; i < inputSet.size(); i++) {
@ -404,7 +404,7 @@ void MLPPData::printData(std::vector<std::string> inputName, std::string outputN
// UNSUPERVISED // UNSUPERVISED
void MLPPData::setData(int k, std::string fileName, std::vector<std::vector<double>> &inputSet) { void MLPPData::setData(int k, std::string fileName, std::vector<std::vector<real_t>> &inputSet) {
MLPPLinAlg alg; MLPPLinAlg alg;
std::string inputTemp; std::string inputTemp;
@ -428,7 +428,7 @@ void MLPPData::setData(int k, std::string fileName, std::vector<std::vector<doub
dataFile.close(); dataFile.close();
} }
void MLPPData::printData(std::vector<std::string> inputName, std::vector<std::vector<double>> inputSet) { void MLPPData::printData(std::vector<std::string> inputName, std::vector<std::vector<real_t>> inputSet) {
MLPPLinAlg alg; MLPPLinAlg alg;
inputSet = alg.transpose(inputSet); inputSet = alg.transpose(inputSet);
for (int i = 0; i < inputSet.size(); i++) { for (int i = 0; i < inputSet.size(); i++) {
@ -441,7 +441,7 @@ void MLPPData::printData(std::vector<std::string> inputName, std::vector<std::ve
// SIMPLE // SIMPLE
void MLPPData::setData(std::string fileName, std::vector<double> &inputSet, std::vector<double> &outputSet) { void MLPPData::setData(std::string fileName, std::vector<real_t> &inputSet, std::vector<real_t> &outputSet) {
std::string inputTemp, outputTemp; std::string inputTemp, outputTemp;
std::ifstream dataFile(fileName); std::ifstream dataFile(fileName);
@ -464,7 +464,7 @@ void MLPPData::setData(std::string fileName, std::vector<double> &inputSet, std:
dataFile.close(); dataFile.close();
} }
void MLPPData::printData(std::string &inputName, std::string &outputName, std::vector<double> &inputSet, std::vector<double> &outputSet) { void MLPPData::printData(std::string &inputName, std::string &outputName, std::vector<real_t> &inputSet, std::vector<real_t> &outputSet) {
std::cout << inputName << std::endl; std::cout << inputName << std::endl;
for (int i = 0; i < inputSet.size(); i++) { for (int i = 0; i < inputSet.size(); i++) {
std::cout << inputSet[i] << std::endl; std::cout << inputSet[i] << std::endl;
@ -477,8 +477,8 @@ void MLPPData::printData(std::string &inputName, std::string &outputName, std::v
} }
// Images // Images
std::vector<std::vector<double>> MLPPData::rgb2gray(std::vector<std::vector<std::vector<double>>> input) { std::vector<std::vector<real_t>> MLPPData::rgb2gray(std::vector<std::vector<std::vector<real_t>>> input) {
std::vector<std::vector<double>> grayScale; std::vector<std::vector<real_t>> grayScale;
grayScale.resize(input[0].size()); grayScale.resize(input[0].size());
for (int i = 0; i < grayScale.size(); i++) { for (int i = 0; i < grayScale.size(); i++) {
grayScale[i].resize(input[0][i].size()); grayScale[i].resize(input[0][i].size());
@ -491,9 +491,9 @@ std::vector<std::vector<double>> MLPPData::rgb2gray(std::vector<std::vector<std:
return grayScale; return grayScale;
} }
std::vector<std::vector<std::vector<double>>> MLPPData::rgb2ycbcr(std::vector<std::vector<std::vector<double>>> input) { std::vector<std::vector<std::vector<real_t>>> MLPPData::rgb2ycbcr(std::vector<std::vector<std::vector<real_t>>> input) {
MLPPLinAlg alg; MLPPLinAlg alg;
std::vector<std::vector<std::vector<double>>> YCbCr; std::vector<std::vector<std::vector<real_t>>> YCbCr;
YCbCr = alg.resize(YCbCr, input); YCbCr = alg.resize(YCbCr, input);
for (int i = 0; i < YCbCr[0].size(); i++) { for (int i = 0; i < YCbCr[0].size(); i++) {
for (int j = 0; j < YCbCr[0][i].size(); j++) { for (int j = 0; j < YCbCr[0][i].size(); j++) {
@ -507,19 +507,19 @@ std::vector<std::vector<std::vector<double>>> MLPPData::rgb2ycbcr(std::vector<st
// Conversion formulas available here: // Conversion formulas available here:
// https://www.rapidtables.com/convert/color/rgb-to-hsv.html // https://www.rapidtables.com/convert/color/rgb-to-hsv.html
std::vector<std::vector<std::vector<double>>> MLPPData::rgb2hsv(std::vector<std::vector<std::vector<double>>> input) { std::vector<std::vector<std::vector<real_t>>> MLPPData::rgb2hsv(std::vector<std::vector<std::vector<real_t>>> input) {
MLPPLinAlg alg; MLPPLinAlg alg;
std::vector<std::vector<std::vector<double>>> HSV; std::vector<std::vector<std::vector<real_t>>> HSV;
HSV = alg.resize(HSV, input); HSV = alg.resize(HSV, input);
for (int i = 0; i < HSV[0].size(); i++) { for (int i = 0; i < HSV[0].size(); i++) {
for (int j = 0; j < HSV[0][i].size(); j++) { for (int j = 0; j < HSV[0][i].size(); j++) {
double rPrime = input[0][i][j] / 255; real_t rPrime = input[0][i][j] / 255;
double gPrime = input[1][i][j] / 255; real_t gPrime = input[1][i][j] / 255;
double bPrime = input[2][i][j] / 255; real_t bPrime = input[2][i][j] / 255;
double cMax = alg.max({ rPrime, gPrime, bPrime }); real_t cMax = alg.max({ rPrime, gPrime, bPrime });
double cMin = alg.min({ rPrime, gPrime, bPrime }); real_t cMin = alg.min({ rPrime, gPrime, bPrime });
double delta = cMax - cMin; real_t delta = cMax - cMin;
// H calculation. // H calculation.
if (delta == 0) { if (delta == 0) {
@ -549,19 +549,19 @@ std::vector<std::vector<std::vector<double>>> MLPPData::rgb2hsv(std::vector<std:
} }
// http://machinethatsees.blogspot.com/2013/07/how-to-convert-rgb-to-xyz-or-vice-versa.html // http://machinethatsees.blogspot.com/2013/07/how-to-convert-rgb-to-xyz-or-vice-versa.html
std::vector<std::vector<std::vector<double>>> MLPPData::rgb2xyz(std::vector<std::vector<std::vector<double>>> input) { std::vector<std::vector<std::vector<real_t>>> MLPPData::rgb2xyz(std::vector<std::vector<std::vector<real_t>>> input) {
MLPPLinAlg alg; MLPPLinAlg alg;
std::vector<std::vector<std::vector<double>>> XYZ; std::vector<std::vector<std::vector<real_t>>> XYZ;
XYZ = alg.resize(XYZ, input); XYZ = alg.resize(XYZ, input);
std::vector<std::vector<double>> RGB2XYZ = { { 0.4124564, 0.3575761, 0.1804375 }, { 0.2126726, 0.7151522, 0.0721750 }, { 0.0193339, 0.1191920, 0.9503041 } }; std::vector<std::vector<real_t>> RGB2XYZ = { { 0.4124564, 0.3575761, 0.1804375 }, { 0.2126726, 0.7151522, 0.0721750 }, { 0.0193339, 0.1191920, 0.9503041 } };
return alg.vector_wise_tensor_product(input, RGB2XYZ); return alg.vector_wise_tensor_product(input, RGB2XYZ);
} }
std::vector<std::vector<std::vector<double>>> MLPPData::xyz2rgb(std::vector<std::vector<std::vector<double>>> input) { std::vector<std::vector<std::vector<real_t>>> MLPPData::xyz2rgb(std::vector<std::vector<std::vector<real_t>>> input) {
MLPPLinAlg alg; MLPPLinAlg alg;
std::vector<std::vector<std::vector<double>>> XYZ; std::vector<std::vector<std::vector<real_t>>> XYZ;
XYZ = alg.resize(XYZ, input); XYZ = alg.resize(XYZ, input);
std::vector<std::vector<double>> RGB2XYZ = alg.inverse({ { 0.4124564, 0.3575761, 0.1804375 }, { 0.2126726, 0.7151522, 0.0721750 }, { 0.0193339, 0.1191920, 0.9503041 } }); std::vector<std::vector<real_t>> RGB2XYZ = alg.inverse({ { 0.4124564, 0.3575761, 0.1804375 }, { 0.2126726, 0.7151522, 0.0721750 }, { 0.0193339, 0.1191920, 0.9503041 } });
return alg.vector_wise_tensor_product(input, RGB2XYZ); return alg.vector_wise_tensor_product(input, RGB2XYZ);
} }
@ -640,11 +640,11 @@ std::vector<std::string> MLPPData::segment(std::string text) {
return segmented_data; return segmented_data;
} }
std::vector<double> MLPPData::tokenize(std::string text) { std::vector<real_t> MLPPData::tokenize(std::string text) {
int max_num = 0; int max_num = 0;
bool new_num = true; bool new_num = true;
std::vector<std::string> segmented_data = segment(text); std::vector<std::string> segmented_data = segment(text);
std::vector<double> tokenized_data; std::vector<real_t> tokenized_data;
tokenized_data.resize(segmented_data.size()); tokenized_data.resize(segmented_data.size());
for (int i = 0; i < segmented_data.size(); i++) { for (int i = 0; i < segmented_data.size(); i++) {
for (int j = i - 1; j >= 0; j--) { for (int j = i - 1; j >= 0; j--) {
@ -710,7 +710,7 @@ std::string MLPPData::stemming(std::string text) {
return text; return text;
} }
std::vector<std::vector<double>> MLPPData::BOW(std::vector<std::string> sentences, std::string type) { std::vector<std::vector<real_t>> MLPPData::BOW(std::vector<std::string> sentences, std::string type) {
/* /*
STEPS OF BOW: STEPS OF BOW:
1) To lowercase (done by removeStopWords function by def) 1) To lowercase (done by removeStopWords function by def)
@ -729,7 +729,7 @@ std::vector<std::vector<double>> MLPPData::BOW(std::vector<std::string> sentence
segmented_sentences[i] = removeStopWords(sentences[i]); segmented_sentences[i] = removeStopWords(sentences[i]);
} }
std::vector<std::vector<double>> bow; std::vector<std::vector<real_t>> bow;
bow.resize(sentences.size()); bow.resize(sentences.size());
for (int i = 0; i < bow.size(); i++) { for (int i = 0; i < bow.size(); i++) {
@ -752,7 +752,7 @@ std::vector<std::vector<double>> MLPPData::BOW(std::vector<std::string> sentence
return bow; return bow;
} }
std::vector<std::vector<double>> MLPPData::TFIDF(std::vector<std::string> sentences) { std::vector<std::vector<real_t>> MLPPData::TFIDF(std::vector<std::string> sentences) {
MLPPLinAlg alg; MLPPLinAlg alg;
std::vector<std::string> wordList = removeNullByte(removeStopWords(createWordList(sentences))); std::vector<std::string> wordList = removeNullByte(removeStopWords(createWordList(sentences)));
@ -763,7 +763,7 @@ std::vector<std::vector<double>> MLPPData::TFIDF(std::vector<std::string> senten
segmented_sentences[i] = removeStopWords(sentences[i]); segmented_sentences[i] = removeStopWords(sentences[i]);
} }
std::vector<std::vector<double>> TF; std::vector<std::vector<real_t>> TF;
std::vector<int> frequency; std::vector<int> frequency;
frequency.resize(wordList.size()); frequency.resize(wordList.size());
TF.resize(segmented_sentences.size()); TF.resize(segmented_sentences.size());
@ -783,17 +783,17 @@ std::vector<std::vector<double>> MLPPData::TFIDF(std::vector<std::string> senten
} }
} }
} }
TF[i] = alg.scalarMultiply(double(1) / double(segmented_sentences[i].size()), TF[i]); TF[i] = alg.scalarMultiply(real_t(1) / real_t(segmented_sentences[i].size()), TF[i]);
} }
std::vector<double> IDF; std::vector<real_t> IDF;
IDF.resize(frequency.size()); IDF.resize(frequency.size());
for (int i = 0; i < IDF.size(); i++) { for (int i = 0; i < IDF.size(); i++) {
IDF[i] = std::log((double)segmented_sentences.size() / (double)frequency[i]); IDF[i] = std::log((real_t)segmented_sentences.size() / (real_t)frequency[i]);
} }
std::vector<std::vector<double>> TFIDF; std::vector<std::vector<real_t>> TFIDF;
TFIDF.resize(segmented_sentences.size()); TFIDF.resize(segmented_sentences.size());
for (int i = 0; i < TFIDF.size(); i++) { for (int i = 0; i < TFIDF.size(); i++) {
TFIDF[i].resize(wordList.size()); TFIDF[i].resize(wordList.size());
@ -808,7 +808,7 @@ std::vector<std::vector<double>> MLPPData::TFIDF(std::vector<std::string> senten
return TFIDF; return TFIDF;
} }
std::tuple<std::vector<std::vector<double>>, std::vector<std::string>> MLPPData::word2Vec(std::vector<std::string> sentences, std::string type, int windowSize, int dimension, double learning_rate, int max_epoch) { std::tuple<std::vector<std::vector<real_t>>, std::vector<std::string>> MLPPData::word2Vec(std::vector<std::string> sentences, std::string type, int windowSize, int dimension, real_t learning_rate, int max_epoch) {
std::vector<std::string> wordList = removeNullByte(removeStopWords(createWordList(sentences))); std::vector<std::string> wordList = removeNullByte(removeStopWords(createWordList(sentences)));
std::vector<std::vector<std::string>> segmented_sentences; std::vector<std::vector<std::string>> segmented_sentences;
@ -841,10 +841,10 @@ std::tuple<std::vector<std::vector<double>>, std::vector<std::string>> MLPPData:
inputStrings.insert(inputStrings.end(), outputStrings.begin(), outputStrings.end()); inputStrings.insert(inputStrings.end(), outputStrings.begin(), outputStrings.end());
std::vector<std::vector<double>> BOW = MLPPData::BOW(inputStrings, "Binary"); std::vector<std::vector<real_t>> BOW = MLPPData::BOW(inputStrings, "Binary");
std::vector<std::vector<double>> inputSet; std::vector<std::vector<real_t>> inputSet;
std::vector<std::vector<double>> outputSet; std::vector<std::vector<real_t>> outputSet;
for (int i = 0; i < inputSize; i++) { for (int i = 0; i < inputSize; i++) {
inputSet.push_back(BOW[i]); inputSet.push_back(BOW[i]);
@ -862,17 +862,17 @@ std::tuple<std::vector<std::vector<double>>, std::vector<std::string>> MLPPData:
} }
model->gradientDescent(learning_rate, max_epoch, 1); model->gradientDescent(learning_rate, max_epoch, 1);
std::vector<std::vector<double>> wordEmbeddings = model->getEmbeddings(); std::vector<std::vector<real_t>> wordEmbeddings = model->getEmbeddings();
delete model; delete model;
return { wordEmbeddings, wordList }; return { wordEmbeddings, wordList };
} }
struct WordsToVecResult { struct WordsToVecResult {
std::vector<std::vector<double>> word_embeddings; std::vector<std::vector<real_t>> word_embeddings;
std::vector<std::string> word_list; std::vector<std::string> word_list;
}; };
MLPPData::WordsToVecResult MLPPData::word_to_vec(std::vector<std::string> sentences, std::string type, int windowSize, int dimension, double learning_rate, int max_epoch) { MLPPData::WordsToVecResult MLPPData::word_to_vec(std::vector<std::string> sentences, std::string type, int windowSize, int dimension, real_t learning_rate, int max_epoch) {
WordsToVecResult res; WordsToVecResult res;
res.word_list = removeNullByte(removeStopWords(createWordList(sentences))); res.word_list = removeNullByte(removeStopWords(createWordList(sentences)));
@ -907,10 +907,10 @@ MLPPData::WordsToVecResult MLPPData::word_to_vec(std::vector<std::string> senten
inputStrings.insert(inputStrings.end(), outputStrings.begin(), outputStrings.end()); inputStrings.insert(inputStrings.end(), outputStrings.begin(), outputStrings.end());
std::vector<std::vector<double>> BOW = MLPPData::BOW(inputStrings, "Binary"); std::vector<std::vector<real_t>> BOW = MLPPData::BOW(inputStrings, "Binary");
std::vector<std::vector<double>> inputSet; std::vector<std::vector<real_t>> inputSet;
std::vector<std::vector<double>> outputSet; std::vector<std::vector<real_t>> outputSet;
for (int i = 0; i < inputSize; i++) { for (int i = 0; i < inputSize; i++) {
inputSet.push_back(BOW[i]); inputSet.push_back(BOW[i]);
@ -934,19 +934,19 @@ MLPPData::WordsToVecResult MLPPData::word_to_vec(std::vector<std::string> senten
return res; return res;
} }
std::vector<std::vector<double>> MLPPData::LSA(std::vector<std::string> sentences, int dim) { std::vector<std::vector<real_t>> MLPPData::LSA(std::vector<std::string> sentences, int dim) {
MLPPLinAlg alg; MLPPLinAlg alg;
std::vector<std::vector<double>> docWordData = BOW(sentences, "Binary"); std::vector<std::vector<real_t>> docWordData = BOW(sentences, "Binary");
auto [U, S, Vt] = alg.SVD(docWordData); auto [U, S, Vt] = alg.SVD(docWordData);
std::vector<std::vector<double>> S_trunc = alg.zeromat(dim, dim); std::vector<std::vector<real_t>> S_trunc = alg.zeromat(dim, dim);
std::vector<std::vector<double>> Vt_trunc; std::vector<std::vector<real_t>> Vt_trunc;
for (int i = 0; i < dim; i++) { for (int i = 0; i < dim; i++) {
S_trunc[i][i] = S[i][i]; S_trunc[i][i] = S[i][i];
Vt_trunc.push_back(Vt[i]); Vt_trunc.push_back(Vt[i]);
} }
std::vector<std::vector<double>> embeddings = alg.matmult(S_trunc, Vt_trunc); std::vector<std::vector<real_t>> embeddings = alg.matmult(S_trunc, Vt_trunc);
return embeddings; return embeddings;
} }
@ -977,10 +977,10 @@ void MLPPData::setInputNames(std::string fileName, std::vector<std::string> &inp
dataFile.close(); dataFile.close();
} }
std::vector<std::vector<double>> MLPPData::featureScaling(std::vector<std::vector<double>> X) { std::vector<std::vector<real_t>> MLPPData::featureScaling(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg; MLPPLinAlg alg;
X = alg.transpose(X); X = alg.transpose(X);
std::vector<double> max_elements, min_elements; std::vector<real_t> max_elements, min_elements;
max_elements.resize(X.size()); max_elements.resize(X.size());
min_elements.resize(X.size()); min_elements.resize(X.size());
@ -997,7 +997,7 @@ std::vector<std::vector<double>> MLPPData::featureScaling(std::vector<std::vecto
return alg.transpose(X); return alg.transpose(X);
} }
std::vector<std::vector<double>> MLPPData::meanNormalization(std::vector<std::vector<double>> X) { std::vector<std::vector<real_t>> MLPPData::meanNormalization(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPStat stat; MLPPStat stat;
// (X_j - mu_j) / std_j, for every j // (X_j - mu_j) / std_j, for every j
@ -1009,11 +1009,11 @@ std::vector<std::vector<double>> MLPPData::meanNormalization(std::vector<std::ve
return X; return X;
} }
std::vector<std::vector<double>> MLPPData::meanCentering(std::vector<std::vector<double>> X) { std::vector<std::vector<real_t>> MLPPData::meanCentering(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPStat stat; MLPPStat stat;
for (int i = 0; i < X.size(); i++) { for (int i = 0; i < X.size(); i++) {
double mean_i = stat.mean(X[i]); real_t mean_i = stat.mean(X[i]);
for (int j = 0; j < X[i].size(); j++) { for (int j = 0; j < X[i].size(); j++) {
X[i][j] -= mean_i; X[i][j] -= mean_i;
} }
@ -1021,8 +1021,8 @@ std::vector<std::vector<double>> MLPPData::meanCentering(std::vector<std::vector
return X; return X;
} }
std::vector<std::vector<double>> MLPPData::oneHotRep(std::vector<double> tempOutputSet, int n_class) { std::vector<std::vector<real_t>> MLPPData::oneHotRep(std::vector<real_t> tempOutputSet, int n_class) {
std::vector<std::vector<double>> outputSet; std::vector<std::vector<real_t>> outputSet;
outputSet.resize(tempOutputSet.size()); outputSet.resize(tempOutputSet.size());
for (int i = 0; i < tempOutputSet.size(); i++) { for (int i = 0; i < tempOutputSet.size(); i++) {
for (int j = 0; j <= n_class - 1; j++) { for (int j = 0; j <= n_class - 1; j++) {
@ -1036,8 +1036,8 @@ std::vector<std::vector<double>> MLPPData::oneHotRep(std::vector<double> tempOut
return outputSet; return outputSet;
} }
std::vector<double> MLPPData::reverseOneHot(std::vector<std::vector<double>> tempOutputSet) { std::vector<real_t> MLPPData::reverseOneHot(std::vector<std::vector<real_t>> tempOutputSet) {
std::vector<double> outputSet; std::vector<real_t> outputSet;
int n_class = tempOutputSet[0].size(); int n_class = tempOutputSet[0].size();
for (int i = 0; i < tempOutputSet.size(); i++) { for (int i = 0; i < tempOutputSet.size(); i++) {
int current_class = 1; int current_class = 1;

88
mlpp/data/data.h
View File

@ -9,6 +9,8 @@
// Created by Marc Melikyan on 11/4/20. // Created by Marc Melikyan on 11/4/20.
// //
#include "core/math/math_defs.h"
#include "core/string/ustring.h" #include "core/string/ustring.h"
#include "core/variant/array.h" #include "core/variant/array.h"
@ -22,8 +24,8 @@ class MLPPDataESimple : public Reference {
GDCLASS(MLPPDataESimple, Reference); GDCLASS(MLPPDataESimple, Reference);
public: public:
std::vector<double> input; std::vector<real_t> input;
std::vector<double> output; std::vector<real_t> output;
protected: protected:
static void _bind_methods(); static void _bind_methods();
@ -33,8 +35,8 @@ class MLPPDataSimple : public Reference {
GDCLASS(MLPPDataSimple, Reference); GDCLASS(MLPPDataSimple, Reference);
public: public:
std::vector<std::vector<double>> input; std::vector<std::vector<real_t>> input;
std::vector<double> output; std::vector<real_t> output;
protected: protected:
static void _bind_methods(); static void _bind_methods();
@ -44,8 +46,8 @@ class MLPPDataComplex : public Reference {
GDCLASS(MLPPDataComplex, Reference); GDCLASS(MLPPDataComplex, Reference);
public: public:
std::vector<std::vector<double>> input; std::vector<std::vector<real_t>> input;
std::vector<std::vector<double>> output; std::vector<std::vector<real_t>> output;
protected: protected:
static void _bind_methods(); static void _bind_methods();
@ -65,48 +67,48 @@ public:
Ref<MLPPDataSimple> load_california_housing(const String &path); Ref<MLPPDataSimple> load_california_housing(const String &path);
Ref<MLPPDataESimple> load_fires_and_crime(const String &path); Ref<MLPPDataESimple> load_fires_and_crime(const String &path);
void set_data_supervised(int k, const String &file_name, std::vector<std::vector<double>> &inputSet, std::vector<double> &outputSet); void set_data_supervised(int k, const String &file_name, std::vector<std::vector<real_t>> &inputSet, std::vector<real_t> &outputSet);
void set_data_unsupervised(int k, const String &file_name, std::vector<std::vector<double>> &inputSet); void set_data_unsupervised(int k, const String &file_name, std::vector<std::vector<real_t>> &inputSet);
void set_data_simple(const String &file_name, std::vector<double> &inputSet, std::vector<double> &outputSet); void set_data_simple(const String &file_name, std::vector<real_t> &inputSet, std::vector<real_t> &outputSet);
struct SplitComplexData { struct SplitComplexData {
Ref<MLPPDataComplex> train; Ref<MLPPDataComplex> train;
Ref<MLPPDataComplex> test; Ref<MLPPDataComplex> test;
}; };
SplitComplexData train_test_split(const Ref<MLPPDataComplex> &data, double test_size); SplitComplexData train_test_split(const Ref<MLPPDataComplex> &data, real_t test_size);
Array train_test_split_bind(const Ref<MLPPDataComplex> &data, double test_size); Array train_test_split_bind(const Ref<MLPPDataComplex> &data, real_t test_size);
// Load Datasets // Load Datasets
std::tuple<std::vector<std::vector<double>>, std::vector<double>> loadBreastCancer(); std::tuple<std::vector<std::vector<real_t>>, std::vector<real_t>> loadBreastCancer();
std::tuple<std::vector<std::vector<double>>, std::vector<double>> loadBreastCancerSVC(); std::tuple<std::vector<std::vector<real_t>>, std::vector<real_t>> loadBreastCancerSVC();
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> loadIris(); std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> loadIris();
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> loadWine(); std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> loadWine();
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> loadMnistTrain(); std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> loadMnistTrain();
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> loadMnistTest(); std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> loadMnistTest();
std::tuple<std::vector<std::vector<double>>, std::vector<double>> loadCaliforniaHousing(); std::tuple<std::vector<std::vector<real_t>>, std::vector<real_t>> loadCaliforniaHousing();
std::tuple<std::vector<double>, std::vector<double>> loadFiresAndCrime(); std::tuple<std::vector<real_t>, std::vector<real_t>> loadFiresAndCrime();
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>, std::vector<std::vector<double>>, std::vector<std::vector<double>>> trainTestSplit(std::vector<std::vector<double>> inputSet, std::vector<std::vector<double>> outputSet, double testSize); std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> trainTestSplit(std::vector<std::vector<real_t>> inputSet, std::vector<std::vector<real_t>> outputSet, real_t testSize);
// Supervised // Supervised
void setData(int k, std::string fileName, std::vector<std::vector<double>> &inputSet, std::vector<double> &outputSet); void setData(int k, std::string fileName, std::vector<std::vector<real_t>> &inputSet, std::vector<real_t> &outputSet);
void printData(std::vector<std::string> inputName, std::string outputName, std::vector<std::vector<double>> inputSet, std::vector<double> outputSet); void printData(std::vector<std::string> inputName, std::string outputName, std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet);
// Unsupervised // Unsupervised
void setData(int k, std::string fileName, std::vector<std::vector<double>> &inputSet); void setData(int k, std::string fileName, std::vector<std::vector<real_t>> &inputSet);
void printData(std::vector<std::string> inputName, std::vector<std::vector<double>> inputSet); void printData(std::vector<std::string> inputName, std::vector<std::vector<real_t>> inputSet);
// Simple // Simple
void setData(std::string fileName, std::vector<double> &inputSet, std::vector<double> &outputSet); void setData(std::string fileName, std::vector<real_t> &inputSet, std::vector<real_t> &outputSet);
void printData(std::string &inputName, std::string &outputName, std::vector<double> &inputSet, std::vector<double> &outputSet); void printData(std::string &inputName, std::string &outputName, std::vector<real_t> &inputSet, std::vector<real_t> &outputSet);
// Images // Images
std::vector<std::vector<double>> rgb2gray(std::vector<std::vector<std::vector<double>>> input); std::vector<std::vector<real_t>> rgb2gray(std::vector<std::vector<std::vector<real_t>>> input);
std::vector<std::vector<std::vector<double>>> rgb2ycbcr(std::vector<std::vector<std::vector<double>>> input); std::vector<std::vector<std::vector<real_t>>> rgb2ycbcr(std::vector<std::vector<std::vector<real_t>>> input);
std::vector<std::vector<std::vector<double>>> rgb2hsv(std::vector<std::vector<std::vector<double>>> input); std::vector<std::vector<std::vector<real_t>>> rgb2hsv(std::vector<std::vector<std::vector<real_t>>> input);
std::vector<std::vector<std::vector<double>>> rgb2xyz(std::vector<std::vector<std::vector<double>>> input); std::vector<std::vector<std::vector<real_t>>> rgb2xyz(std::vector<std::vector<std::vector<real_t>>> input);
std::vector<std::vector<std::vector<double>>> xyz2rgb(std::vector<std::vector<std::vector<double>>> input); std::vector<std::vector<std::vector<real_t>>> xyz2rgb(std::vector<std::vector<std::vector<real_t>>> input);
// Text-Based & NLP // Text-Based & NLP
std::string toLower(std::string text); std::string toLower(std::string text);
@ -115,35 +117,35 @@ public:
std::vector<std::string> removeSpaces(std::vector<std::string> data); std::vector<std::string> removeSpaces(std::vector<std::string> data);
std::vector<std::string> removeNullByte(std::vector<std::string> data); std::vector<std::string> removeNullByte(std::vector<std::string> data);
std::vector<std::string> segment(std::string text); std::vector<std::string> segment(std::string text);
std::vector<double> tokenize(std::string text); std::vector<real_t> tokenize(std::string text);
std::vector<std::string> removeStopWords(std::string text); std::vector<std::string> removeStopWords(std::string text);
std::vector<std::string> removeStopWords(std::vector<std::string> segmented_data); std::vector<std::string> removeStopWords(std::vector<std::string> segmented_data);
std::string stemming(std::string text); std::string stemming(std::string text);
std::vector<std::vector<double>> BOW(std::vector<std::string> sentences, std::string = "Default"); std::vector<std::vector<real_t>> BOW(std::vector<std::string> sentences, std::string = "Default");
std::vector<std::vector<double>> TFIDF(std::vector<std::string> sentences); std::vector<std::vector<real_t>> TFIDF(std::vector<std::string> sentences);
std::tuple<std::vector<std::vector<double>>, std::vector<std::string>> word2Vec(std::vector<std::string> sentences, std::string type, int windowSize, int dimension, double learning_rate, int max_epoch); std::tuple<std::vector<std::vector<real_t>>, std::vector<std::string>> word2Vec(std::vector<std::string> sentences, std::string type, int windowSize, int dimension, real_t learning_rate, int max_epoch);
struct WordsToVecResult { struct WordsToVecResult {
std::vector<std::vector<double>> word_embeddings; std::vector<std::vector<real_t>> word_embeddings;
std::vector<std::string> word_list; std::vector<std::string> word_list;
}; };
WordsToVecResult word_to_vec(std::vector<std::string> sentences, std::string type, int windowSize, int dimension, double learning_rate, int max_epoch); WordsToVecResult word_to_vec(std::vector<std::string> sentences, std::string type, int windowSize, int dimension, real_t learning_rate, int max_epoch);
std::vector<std::vector<double>> LSA(std::vector<std::string> sentences, int dim); std::vector<std::vector<real_t>> LSA(std::vector<std::string> sentences, int dim);
std::vector<std::string> createWordList(std::vector<std::string> sentences); std::vector<std::string> createWordList(std::vector<std::string> sentences);
// Extra // Extra
void setInputNames(std::string fileName, std::vector<std::string> &inputNames); void setInputNames(std::string fileName, std::vector<std::string> &inputNames);
std::vector<std::vector<double>> featureScaling(std::vector<std::vector<double>> X); std::vector<std::vector<real_t>> featureScaling(std::vector<std::vector<real_t>> X);
std::vector<std::vector<double>> meanNormalization(std::vector<std::vector<double>> X); std::vector<std::vector<real_t>> meanNormalization(std::vector<std::vector<real_t>> X);
std::vector<std::vector<double>> meanCentering(std::vector<std::vector<double>> X); std::vector<std::vector<real_t>> meanCentering(std::vector<std::vector<real_t>> X);
std::vector<std::vector<double>> oneHotRep(std::vector<double> tempOutputSet, int n_class); std::vector<std::vector<real_t>> oneHotRep(std::vector<real_t> tempOutputSet, int n_class);
std::vector<double> reverseOneHot(std::vector<std::vector<double>> tempOutputSet); std::vector<real_t> reverseOneHot(std::vector<std::vector<real_t>> tempOutputSet);
template <class T> template <class T>
std::vector<T> vecToSet(std::vector<T> inputSet) { std::vector<T> vecToSet(std::vector<T> inputSet) {

48
mlpp/dual_svc/dual_svc.cpp
View File

@ -15,7 +15,7 @@
#include <random> #include <random>
MLPPDualSVC::MLPPDualSVC(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, double C, std::string kernel) : MLPPDualSVC::MLPPDualSVC(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, real_t C, std::string kernel) :
inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), C(C), kernel(kernel) { inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), C(C), kernel(kernel) {
y_hat.resize(n); y_hat.resize(n);
bias = MLPPUtilities::biasInitialization(); bias = MLPPUtilities::biasInitialization();
@ -23,20 +23,20 @@ MLPPDualSVC::MLPPDualSVC(std::vector<std::vector<double>> inputSet, std::vector<
K = kernelFunction(inputSet, inputSet, kernel); // For now this is unused. When non-linear kernels are added, the K will be manipulated. K = kernelFunction(inputSet, inputSet, kernel); // For now this is unused. When non-linear kernels are added, the K will be manipulated.
} }
std::vector<double> MLPPDualSVC::modelSetTest(std::vector<std::vector<double>> X) { std::vector<real_t> MLPPDualSVC::modelSetTest(std::vector<std::vector<real_t>> X) {
return Evaluate(X); return Evaluate(X);
} }
double MLPPDualSVC::modelTest(std::vector<double> x) { real_t MLPPDualSVC::modelTest(std::vector<real_t> x) {
return Evaluate(x); return Evaluate(x);
} }
void MLPPDualSVC::gradientDescent(double learning_rate, int max_epoch, bool UI) { void MLPPDualSVC::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
class MLPPCost cost; class MLPPCost cost;
MLPPActivation avn; MLPPActivation avn;
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
forwardPass(); forwardPass();
@ -48,9 +48,9 @@ void MLPPDualSVC::gradientDescent(double learning_rate, int max_epoch, bool UI)
alphaProjection(); alphaProjection();
// Calculating the bias // Calculating the bias
double biasGradient = 0; real_t biasGradient = 0;
for (int i = 0; i < alpha.size(); i++) { for (int i = 0; i < alpha.size(); i++) {
double sum = 0; real_t sum = 0;
if (alpha[i] < C && alpha[i] > 0) { if (alpha[i] < C && alpha[i] > 0) {
for (int j = 0; j < alpha.size(); j++) { for (int j = 0; j < alpha.size(); j++) {
if (alpha[j] > 0) { if (alpha[j] > 0) {
@ -79,13 +79,13 @@ void MLPPDualSVC::gradientDescent(double learning_rate, int max_epoch, bool UI)
} }
} }
// void MLPPDualSVC::SGD(double learning_rate, int max_epoch, bool UI){ // void MLPPDualSVC::SGD(real_t learning_rate, int max_epoch, bool UI){
// class MLPPCost cost; // class MLPPCost cost;
// MLPPActivation avn; // MLPPActivation avn;
// MLPPLinAlg alg; // MLPPLinAlg alg;
// MLPPReg regularization; // MLPPReg regularization;
// double cost_prev = 0; // real_t cost_prev = 0;
// int epoch = 1; // int epoch = 1;
// while(true){ // while(true){
@ -112,12 +112,12 @@ void MLPPDualSVC::gradientDescent(double learning_rate, int max_epoch, bool UI)
// forwardPass(); // forwardPass();
// } // }
// void MLPPDualSVC::MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI){ // void MLPPDualSVC::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI){
// class MLPPCost cost; // class MLPPCost cost;
// MLPPActivation avn; // MLPPActivation avn;
// MLPPLinAlg alg; // MLPPLinAlg alg;
// MLPPReg regularization; // MLPPReg regularization;
// double cost_prev = 0; // real_t cost_prev = 0;
// int epoch = 1; // int epoch = 1;
// // Creating the mini-batches // // Creating the mini-batches
@ -126,8 +126,8 @@ void MLPPDualSVC::gradientDescent(double learning_rate, int max_epoch, bool UI)
// while(true){ // while(true){
// for(int i = 0; i < n_mini_batch; i++){ // for(int i = 0; i < n_mini_batch; i++){
// std::vector<double> y_hat = Evaluate(inputMiniBatches[i]); // std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]);
// std::vector<double> z = propagate(inputMiniBatches[i]); // std::vector<real_t> z = propagate(inputMiniBatches[i]);
// cost_prev = Cost(z, outputMiniBatches[i], weights, C); // cost_prev = Cost(z, outputMiniBatches[i], weights, C);
// // Calculating the weight gradients // // Calculating the weight gradients
@ -152,7 +152,7 @@ void MLPPDualSVC::gradientDescent(double learning_rate, int max_epoch, bool UI)
// forwardPass(); // forwardPass();
// } // }
double MLPPDualSVC::score() { real_t MLPPDualSVC::score() {
MLPPUtilities util; MLPPUtilities util;
return util.performance(y_hat, outputSet); return util.performance(y_hat, outputSet);
} }
@ -162,21 +162,21 @@ void MLPPDualSVC::save(std::string fileName) {
util.saveParameters(fileName, alpha, bias); util.saveParameters(fileName, alpha, bias);
} }
double MLPPDualSVC::Cost(std::vector<double> alpha, std::vector<std::vector<double>> X, std::vector<double> y) { real_t MLPPDualSVC::Cost(std::vector<real_t> alpha, std::vector<std::vector<real_t>> X, std::vector<real_t> y) {
class MLPPCost cost; class MLPPCost cost;
return cost.dualFormSVM(alpha, X, y); return cost.dualFormSVM(alpha, X, y);
} }
std::vector<double> MLPPDualSVC::Evaluate(std::vector<std::vector<double>> X) { std::vector<real_t> MLPPDualSVC::Evaluate(std::vector<std::vector<real_t>> X) {
MLPPActivation avn; MLPPActivation avn;
return avn.sign(propagate(X)); return avn.sign(propagate(X));
} }
std::vector<double> MLPPDualSVC::propagate(std::vector<std::vector<double>> X) { std::vector<real_t> MLPPDualSVC::propagate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg; MLPPLinAlg alg;
std::vector<double> z; std::vector<real_t> z;
for (int i = 0; i < X.size(); i++) { for (int i = 0; i < X.size(); i++) {
double sum = 0; real_t sum = 0;
for (int j = 0; j < alpha.size(); j++) { for (int j = 0; j < alpha.size(); j++) {
if (alpha[j] != 0) { if (alpha[j] != 0) {
sum += alpha[j] * outputSet[j] * alg.dot(inputSet[j], X[i]); // TO DO: DON'T forget to add non-linear kernelizations. sum += alpha[j] * outputSet[j] * alg.dot(inputSet[j], X[i]); // TO DO: DON'T forget to add non-linear kernelizations.
@ -188,14 +188,14 @@ std::vector<double> MLPPDualSVC::propagate(std::vector<std::vector<double>> X) {
return z; return z;
} }
double MLPPDualSVC::Evaluate(std::vector<double> x) { real_t MLPPDualSVC::Evaluate(std::vector<real_t> x) {
MLPPActivation avn; MLPPActivation avn;
return avn.sign(propagate(x)); return avn.sign(propagate(x));
} }
double MLPPDualSVC::propagate(std::vector<double> x) { real_t MLPPDualSVC::propagate(std::vector<real_t> x) {
MLPPLinAlg alg; MLPPLinAlg alg;
double z = 0; real_t z = 0;
for (int j = 0; j < alpha.size(); j++) { for (int j = 0; j < alpha.size(); j++) {
if (alpha[j] != 0) { if (alpha[j] != 0) {
z += alpha[j] * outputSet[j] * alg.dot(inputSet[j], x); // TO DO: DON'T forget to add non-linear kernelizations. z += alpha[j] * outputSet[j] * alg.dot(inputSet[j], x); // TO DO: DON'T forget to add non-linear kernelizations.
@ -223,14 +223,14 @@ void MLPPDualSVC::alphaProjection() {
} }
} }
double MLPPDualSVC::kernelFunction(std::vector<double> u, std::vector<double> v, std::string kernel) { real_t MLPPDualSVC::kernelFunction(std::vector<real_t> u, std::vector<real_t> v, std::string kernel) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (kernel == "Linear") { if (kernel == "Linear") {
return alg.dot(u, v); return alg.dot(u, v);
} // warning: non-void function does not return a value in all control paths [-Wreturn-type] } // warning: non-void function does not return a value in all control paths [-Wreturn-type]
} }
std::vector<std::vector<double>> MLPPDualSVC::kernelFunction(std::vector<std::vector<double>> A, std::vector<std::vector<double>> B, std::string kernel) { std::vector<std::vector<real_t>> MLPPDualSVC::kernelFunction(std::vector<std::vector<real_t>> A, std::vector<std::vector<real_t>> B, std::string kernel) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (kernel == "Linear") { if (kernel == "Linear") {
return alg.matmult(inputSet, alg.transpose(inputSet)); return alg.matmult(inputSet, alg.transpose(inputSet));

54
mlpp/dual_svc/dual_svc.h
View File

@ -11,6 +11,8 @@
// http://ciml.info/dl/v0_99/ciml-v0_99-ch11.pdf // http://ciml.info/dl/v0_99/ciml-v0_99-ch11.pdf
// Were excellent for the practical intution behind the dual formulation. // Were excellent for the practical intution behind the dual formulation.
#include "core/math/math_defs.h"
#include <string> #include <string>
#include <vector> #include <vector>
@ -18,52 +20,52 @@
class MLPPDualSVC { class MLPPDualSVC {
public: public:
MLPPDualSVC(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, double C, std::string kernel = "Linear"); MLPPDualSVC(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, real_t C, std::string kernel = "Linear");
MLPPDualSVC(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, double C, std::string kernel, double p, double c); MLPPDualSVC(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, real_t C, std::string kernel, real_t p, real_t c);
std::vector<double> modelSetTest(std::vector<std::vector<double>> X); std::vector<real_t> modelSetTest(std::vector<std::vector<real_t>> X);
double modelTest(std::vector<double> x); real_t modelTest(std::vector<real_t> x);
void gradientDescent(double learning_rate, int max_epoch, bool UI = 1); void gradientDescent(real_t learning_rate, int max_epoch, bool UI = 1);
void SGD(double learning_rate, int max_epoch, bool UI = 1); void SGD(real_t learning_rate, int max_epoch, bool UI = 1);
void MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI = 1); void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
double score(); real_t score();
void save(std::string fileName); void save(std::string fileName);
private: private:
void init(); void init();
double Cost(std::vector<double> alpha, std::vector<std::vector<double>> X, std::vector<double> y); real_t Cost(std::vector<real_t> alpha, std::vector<std::vector<real_t>> X, std::vector<real_t> y);
std::vector<double> Evaluate(std::vector<std::vector<double>> X); std::vector<real_t> Evaluate(std::vector<std::vector<real_t>> X);
std::vector<double> propagate(std::vector<std::vector<double>> X); std::vector<real_t> propagate(std::vector<std::vector<real_t>> X);
double Evaluate(std::vector<double> x); real_t Evaluate(std::vector<real_t> x);
double propagate(std::vector<double> x); real_t propagate(std::vector<real_t> x);
void forwardPass(); void forwardPass();
void alphaProjection(); void alphaProjection();
double kernelFunction(std::vector<double> v, std::vector<double> u, std::string kernel); real_t kernelFunction(std::vector<real_t> v, std::vector<real_t> u, std::string kernel);
std::vector<std::vector<double>> kernelFunction(std::vector<std::vector<double>> U, std::vector<std::vector<double>> V, std::string kernel); std::vector<std::vector<real_t>> kernelFunction(std::vector<std::vector<real_t>> U, std::vector<std::vector<real_t>> V, std::string kernel);
std::vector<std::vector<double>> inputSet; std::vector<std::vector<real_t>> inputSet;
std::vector<double> outputSet; std::vector<real_t> outputSet;
std::vector<double> z; std::vector<real_t> z;
std::vector<double> y_hat; std::vector<real_t> y_hat;
double bias; real_t bias;
std::vector<double> alpha; std::vector<real_t> alpha;
std::vector<std::vector<double>> K; std::vector<std::vector<real_t>> K;
double C; real_t C;
int n; int n;
int k; int k;
std::string kernel; std::string kernel;
double p; // Poly real_t p; // Poly
double c; // Poly real_t c; // Poly
// UI Portion // UI Portion
void UI(int epoch, double cost_prev); void UI(int epoch, real_t cost_prev);
}; };

68
mlpp/exp_reg/exp_reg.cpp
View File

@ -15,7 +15,7 @@
#include <random> #include <random>
MLPPExpReg::MLPPExpReg(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, std::string reg, double lambda, double alpha) : MLPPExpReg::MLPPExpReg(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, std::string reg, real_t lambda, real_t alpha) :
inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) { inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) {
y_hat.resize(n); y_hat.resize(n);
weights = MLPPUtilities::weightInitialization(k); weights = MLPPUtilities::weightInitialization(k);
@ -23,41 +23,41 @@ MLPPExpReg::MLPPExpReg(std::vector<std::vector<double>> inputSet, std::vector<do
bias = MLPPUtilities::biasInitialization(); bias = MLPPUtilities::biasInitialization();
} }
std::vector<double> MLPPExpReg::modelSetTest(std::vector<std::vector<double>> X) { std::vector<real_t> MLPPExpReg::modelSetTest(std::vector<std::vector<real_t>> X) {
return Evaluate(X); return Evaluate(X);
} }
double MLPPExpReg::modelTest(std::vector<double> x) { real_t MLPPExpReg::modelTest(std::vector<real_t> x) {
return Evaluate(x); return Evaluate(x);
} }
void MLPPExpReg::gradientDescent(double learning_rate, int max_epoch, bool UI) { void MLPPExpReg::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
forwardPass(); forwardPass();
while (true) { while (true) {
cost_prev = Cost(y_hat, outputSet); cost_prev = Cost(y_hat, outputSet);
std::vector<double> error = alg.subtraction(y_hat, outputSet); std::vector<real_t> error = alg.subtraction(y_hat, outputSet);
for (int i = 0; i < k; i++) { for (int i = 0; i < k; i++) {
// Calculating the weight gradient // Calculating the weight gradient
double sum = 0; real_t sum = 0;
for (int j = 0; j < n; j++) { for (int j = 0; j < n; j++) {
sum += error[j] * inputSet[j][i] * std::pow(weights[i], inputSet[j][i] - 1); sum += error[j] * inputSet[j][i] * std::pow(weights[i], inputSet[j][i] - 1);
} }
double w_gradient = sum / n; real_t w_gradient = sum / n;
// Calculating the initial gradient // Calculating the initial gradient
double sum2 = 0; real_t sum2 = 0;
for (int j = 0; j < n; j++) { for (int j = 0; j < n; j++) {
sum2 += error[j] * std::pow(weights[i], inputSet[j][i]); sum2 += error[j] * std::pow(weights[i], inputSet[j][i]);
} }
double i_gradient = sum2 / n; real_t i_gradient = sum2 / n;
// Weight/initial updation // Weight/initial updation
weights[i] -= learning_rate * w_gradient; weights[i] -= learning_rate * w_gradient;
@ -66,11 +66,11 @@ void MLPPExpReg::gradientDescent(double learning_rate, int max_epoch, bool UI) {
weights = regularization.regWeights(weights, lambda, alpha, reg); weights = regularization.regWeights(weights, lambda, alpha, reg);
// Calculating the bias gradient // Calculating the bias gradient
double sum = 0; real_t sum = 0;
for (int j = 0; j < n; j++) { for (int j = 0; j < n; j++) {
sum += (y_hat[j] - outputSet[j]); sum += (y_hat[j] - outputSet[j]);
} }
double b_gradient = sum / n; real_t b_gradient = sum / n;
// bias updation // bias updation
bias -= learning_rate * b_gradient; bias -= learning_rate * b_gradient;
@ -88,9 +88,9 @@ void MLPPExpReg::gradientDescent(double learning_rate, int max_epoch, bool UI) {
} }
} }
void MLPPExpReg::SGD(double learning_rate, int max_epoch, bool UI) { void MLPPExpReg::SGD(real_t learning_rate, int max_epoch, bool UI) {
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
while (true) { while (true) {
@ -99,14 +99,14 @@ void MLPPExpReg::SGD(double learning_rate, int max_epoch, bool UI) {
std::uniform_int_distribution<int> distribution(0, int(n - 1)); std::uniform_int_distribution<int> distribution(0, int(n - 1));
int outputIndex = distribution(generator); int outputIndex = distribution(generator);
double y_hat = Evaluate(inputSet[outputIndex]); real_t y_hat = Evaluate(inputSet[outputIndex]);
cost_prev = Cost({ y_hat }, { outputSet[outputIndex] }); cost_prev = Cost({ y_hat }, { outputSet[outputIndex] });
for (int i = 0; i < k; i++) { for (int i = 0; i < k; i++) {
// Calculating the weight gradients // Calculating the weight gradients
double w_gradient = (y_hat - outputSet[outputIndex]) * inputSet[outputIndex][i] * std::pow(weights[i], inputSet[outputIndex][i] - 1); real_t w_gradient = (y_hat - outputSet[outputIndex]) * inputSet[outputIndex][i] * std::pow(weights[i], inputSet[outputIndex][i] - 1);
double i_gradient = (y_hat - outputSet[outputIndex]) * std::pow(weights[i], inputSet[outputIndex][i]); real_t i_gradient = (y_hat - outputSet[outputIndex]) * std::pow(weights[i], inputSet[outputIndex][i]);
// Weight/initial updation // Weight/initial updation
weights[i] -= learning_rate * w_gradient; weights[i] -= learning_rate * w_gradient;
@ -115,7 +115,7 @@ void MLPPExpReg::SGD(double learning_rate, int max_epoch, bool UI) {
weights = regularization.regWeights(weights, lambda, alpha, reg); weights = regularization.regWeights(weights, lambda, alpha, reg);
// Calculating the bias gradients // Calculating the bias gradients
double b_gradient = (y_hat - outputSet[outputIndex]); real_t b_gradient = (y_hat - outputSet[outputIndex]);
// Bias updation // Bias updation
bias -= learning_rate * b_gradient; bias -= learning_rate * b_gradient;
@ -134,10 +134,10 @@ void MLPPExpReg::SGD(double learning_rate, int max_epoch, bool UI) {
forwardPass(); forwardPass();
} }
void MLPPExpReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI) { void MLPPExpReg::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
// Creating the mini-batches // Creating the mini-batches
@ -146,25 +146,25 @@ void MLPPExpReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size,
while (true) { while (true) {
for (int i = 0; i < n_mini_batch; i++) { for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = Evaluate(inputMiniBatches[i]); std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]); cost_prev = Cost(y_hat, outputMiniBatches[i]);
std::vector<double> error = alg.subtraction(y_hat, outputMiniBatches[i]); std::vector<real_t> error = alg.subtraction(y_hat, outputMiniBatches[i]);
for (int j = 0; j < k; j++) { for (int j = 0; j < k; j++) {
// Calculating the weight gradient // Calculating the weight gradient
double sum = 0; real_t sum = 0;
for (int k = 0; k < outputMiniBatches[i].size(); k++) { for (int k = 0; k < outputMiniBatches[i].size(); k++) {
sum += error[k] * inputMiniBatches[i][k][j] * std::pow(weights[j], inputMiniBatches[i][k][j] - 1); sum += error[k] * inputMiniBatches[i][k][j] * std::pow(weights[j], inputMiniBatches[i][k][j] - 1);
} }
double w_gradient = sum / outputMiniBatches[i].size(); real_t w_gradient = sum / outputMiniBatches[i].size();
// Calculating the initial gradient // Calculating the initial gradient
double sum2 = 0; real_t sum2 = 0;
for (int k = 0; k < outputMiniBatches[i].size(); k++) { for (int k = 0; k < outputMiniBatches[i].size(); k++) {
sum2 += error[k] * std::pow(weights[j], inputMiniBatches[i][k][j]); sum2 += error[k] * std::pow(weights[j], inputMiniBatches[i][k][j]);
} }
double i_gradient = sum2 / outputMiniBatches[i].size(); real_t i_gradient = sum2 / outputMiniBatches[i].size();
// Weight/initial updation // Weight/initial updation
weights[j] -= learning_rate * w_gradient; weights[j] -= learning_rate * w_gradient;
@ -173,11 +173,11 @@ void MLPPExpReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size,
weights = regularization.regWeights(weights, lambda, alpha, reg); weights = regularization.regWeights(weights, lambda, alpha, reg);
// Calculating the bias gradient // Calculating the bias gradient
double sum = 0; real_t sum = 0;
for (int j = 0; j < outputMiniBatches[i].size(); j++) { for (int j = 0; j < outputMiniBatches[i].size(); j++) {
sum += (y_hat[j] - outputMiniBatches[i][j]); sum += (y_hat[j] - outputMiniBatches[i][j]);
} }
double b_gradient = sum / outputMiniBatches[i].size(); real_t b_gradient = sum / outputMiniBatches[i].size();
y_hat = Evaluate(inputMiniBatches[i]); y_hat = Evaluate(inputMiniBatches[i]);
if (UI) { if (UI) {
@ -193,7 +193,7 @@ void MLPPExpReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size,
forwardPass(); forwardPass();
} }
double MLPPExpReg::score() { real_t MLPPExpReg::score() {
MLPPUtilities util; MLPPUtilities util;
return util.performance(y_hat, outputSet); return util.performance(y_hat, outputSet);
} }
@ -203,14 +203,14 @@ void MLPPExpReg::save(std::string fileName) {
util.saveParameters(fileName, weights, initial, bias); util.saveParameters(fileName, weights, initial, bias);
} }
double MLPPExpReg::Cost(std::vector<double> y_hat, std::vector<double> y) { real_t MLPPExpReg::Cost(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPReg regularization; MLPPReg regularization;
class MLPPCost cost; class MLPPCost cost;
return cost.MSE(y_hat, y) + regularization.regTerm(weights, lambda, alpha, reg); return cost.MSE(y_hat, y) + regularization.regTerm(weights, lambda, alpha, reg);
} }
std::vector<double> MLPPExpReg::Evaluate(std::vector<std::vector<double>> X) { std::vector<real_t> MLPPExpReg::Evaluate(std::vector<std::vector<real_t>> X) {
std::vector<double> y_hat; std::vector<real_t> y_hat;
y_hat.resize(X.size()); y_hat.resize(X.size());
for (int i = 0; i < X.size(); i++) { for (int i = 0; i < X.size(); i++) {
y_hat[i] = 0; y_hat[i] = 0;
@ -222,8 +222,8 @@ std::vector<double> MLPPExpReg::Evaluate(std::vector<std::vector<double>> X) {
return y_hat; return y_hat;
} }
double MLPPExpReg::Evaluate(std::vector<double> x) { real_t MLPPExpReg::Evaluate(std::vector<real_t> x) {
double y_hat = 0; real_t y_hat = 0;
for (int i = 0; i < x.size(); i++) { for (int i = 0; i < x.size(); i++) {
y_hat += initial[i] * std::pow(weights[i], x[i]); y_hat += initial[i] * std::pow(weights[i], x[i]);
} }

38
mlpp/exp_reg/exp_reg.h
View File

@ -8,42 +8,44 @@
// Created by Marc Melikyan on 10/2/20. // Created by Marc Melikyan on 10/2/20.
// //
#include "core/math/math_defs.h"
#include <string> #include <string>
#include <vector> #include <vector>
class MLPPExpReg { class MLPPExpReg {
public: public:
MLPPExpReg(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, std::string reg = "None", double lambda = 0.5, double alpha = 0.5); MLPPExpReg(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
std::vector<double> modelSetTest(std::vector<std::vector<double>> X); std::vector<real_t> modelSetTest(std::vector<std::vector<real_t>> X);
double modelTest(std::vector<double> x); real_t modelTest(std::vector<real_t> x);
void gradientDescent(double learning_rate, int max_epoch, bool UI = 1); void gradientDescent(real_t learning_rate, int max_epoch, bool UI = 1);
void SGD(double learning_rate, int max_epoch, bool UI = 1); void SGD(real_t learning_rate, int max_epoch, bool UI = 1);
void MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI = 1); void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
double score(); real_t score();
void save(std::string fileName); void save(std::string fileName);
private: private:
double Cost(std::vector<double> y_hat, std::vector<double> y); real_t Cost(std::vector<real_t> y_hat, std::vector<real_t> y);
std::vector<double> Evaluate(std::vector<std::vector<double>> X); std::vector<real_t> Evaluate(std::vector<std::vector<real_t>> X);
double Evaluate(std::vector<double> x); real_t Evaluate(std::vector<real_t> x);
void forwardPass(); void forwardPass();
std::vector<std::vector<double>> inputSet; std::vector<std::vector<real_t>> inputSet;
std::vector<double> outputSet; std::vector<real_t> outputSet;
std::vector<double> y_hat; std::vector<real_t> y_hat;
std::vector<double> weights; std::vector<real_t> weights;
std::vector<double> initial; std::vector<real_t> initial;
double bias; real_t bias;
int n; int n;
int k; int k;
// Regularization Params // Regularization Params
std::string reg; std::string reg;
double lambda; real_t lambda;
double alpha; /* This is the controlling param for Elastic Net*/ real_t alpha; /* This is the controlling param for Elastic Net*/
}; };

60
mlpp/gan/gan.cpp
View File

@ -15,7 +15,7 @@
#include <iostream> #include <iostream>
MLPPGAN::MLPPGAN(double k, std::vector<std::vector<double>> outputSet) : MLPPGAN::MLPPGAN(real_t k, std::vector<std::vector<real_t>> outputSet) :
outputSet(outputSet), n(outputSet.size()), k(k) { outputSet(outputSet), n(outputSet.size()), k(k) {
} }
@ -23,15 +23,15 @@ MLPPGAN::~MLPPGAN() {
delete outputLayer; delete outputLayer;
} }
std::vector<std::vector<double>> MLPPGAN::generateExample(int n) { std::vector<std::vector<real_t>> MLPPGAN::generateExample(int n) {
MLPPLinAlg alg; MLPPLinAlg alg;
return modelSetTestGenerator(alg.gaussianNoise(n, k)); return modelSetTestGenerator(alg.gaussianNoise(n, k));
} }
void MLPPGAN::gradientDescent(double learning_rate, int max_epoch, bool UI) { void MLPPGAN::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
class MLPPCost cost; class MLPPCost cost;
MLPPLinAlg alg; MLPPLinAlg alg;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
forwardPass(); forwardPass();
@ -40,13 +40,13 @@ void MLPPGAN::gradientDescent(double learning_rate, int max_epoch, bool UI) {
// Training of the discriminator. // Training of the discriminator.
std::vector<std::vector<double>> generatorInputSet = alg.gaussianNoise(n, k); std::vector<std::vector<real_t>> generatorInputSet = alg.gaussianNoise(n, k);
std::vector<std::vector<double>> discriminatorInputSet = modelSetTestGenerator(generatorInputSet); std::vector<std::vector<real_t>> discriminatorInputSet = modelSetTestGenerator(generatorInputSet);
discriminatorInputSet.insert(discriminatorInputSet.end(), outputSet.begin(), outputSet.end()); // Fake + real inputs. discriminatorInputSet.insert(discriminatorInputSet.end(), outputSet.begin(), outputSet.end()); // Fake + real inputs.
std::vector<double> y_hat = modelSetTestDiscriminator(discriminatorInputSet); std::vector<real_t> y_hat = modelSetTestDiscriminator(discriminatorInputSet);
std::vector<double> outputSet = alg.zerovec(n); std::vector<real_t> outputSet = alg.zerovec(n);
std::vector<double> outputSetReal = alg.onevec(n); std::vector<real_t> outputSetReal = alg.onevec(n);
outputSet.insert(outputSet.end(), outputSetReal.begin(), outputSetReal.end()); // Fake + real output scores. outputSet.insert(outputSet.end(), outputSetReal.begin(), outputSetReal.end()); // Fake + real output scores.
auto [cumulativeDiscriminatorHiddenLayerWGrad, outputDiscriminatorWGrad] = computeDiscriminatorGradients(y_hat, outputSet); auto [cumulativeDiscriminatorHiddenLayerWGrad, outputDiscriminatorWGrad] = computeDiscriminatorGradients(y_hat, outputSet);
@ -60,7 +60,7 @@ void MLPPGAN::gradientDescent(double learning_rate, int max_epoch, bool UI) {
y_hat = modelSetTestDiscriminator(discriminatorInputSet); y_hat = modelSetTestDiscriminator(discriminatorInputSet);
outputSet = alg.onevec(n); outputSet = alg.onevec(n);
std::vector<std::vector<std::vector<double>>> cumulativeGeneratorHiddenLayerWGrad = computeGeneratorGradients(y_hat, outputSet); std::vector<std::vector<std::vector<real_t>>> cumulativeGeneratorHiddenLayerWGrad = computeGeneratorGradients(y_hat, outputSet);
cumulativeGeneratorHiddenLayerWGrad = alg.scalarMultiply(learning_rate / n, cumulativeGeneratorHiddenLayerWGrad); cumulativeGeneratorHiddenLayerWGrad = alg.scalarMultiply(learning_rate / n, cumulativeGeneratorHiddenLayerWGrad);
updateGeneratorParameters(cumulativeGeneratorHiddenLayerWGrad, learning_rate); updateGeneratorParameters(cumulativeGeneratorHiddenLayerWGrad, learning_rate);
@ -76,7 +76,7 @@ void MLPPGAN::gradientDescent(double learning_rate, int max_epoch, bool UI) {
} }
} }
double MLPPGAN::score() { real_t MLPPGAN::score() {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPUtilities util; MLPPUtilities util;
forwardPass(); forwardPass();
@ -96,7 +96,7 @@ void MLPPGAN::save(std::string fileName) {
} }
} }
void MLPPGAN::addLayer(int n_hidden, std::string activation, std::string weightInit, std::string reg, double lambda, double alpha) { void MLPPGAN::addLayer(int n_hidden, std::string activation, std::string weightInit, std::string reg, real_t lambda, real_t alpha) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (network.empty()) { if (network.empty()) {
network.push_back(MLPPHiddenLayer(n_hidden, activation, alg.gaussianNoise(n, k), weightInit, reg, lambda, alpha)); network.push_back(MLPPHiddenLayer(n_hidden, activation, alg.gaussianNoise(n, k), weightInit, reg, lambda, alpha));
@ -107,7 +107,7 @@ void MLPPGAN::addLayer(int n_hidden, std::string activation, std::string weightI
} }
} }
void MLPPGAN::addOutputLayer(std::string weightInit, std::string reg, double lambda, double alpha) { void MLPPGAN::addOutputLayer(std::string weightInit, std::string reg, real_t lambda, real_t alpha) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (!network.empty()) { if (!network.empty()) {
outputLayer = new MLPPOutputLayer(network[network.size() - 1].n_hidden, "Sigmoid", "LogLoss", network[network.size() - 1].a, weightInit, reg, lambda, alpha); outputLayer = new MLPPOutputLayer(network[network.size() - 1].n_hidden, "Sigmoid", "LogLoss", network[network.size() - 1].a, weightInit, reg, lambda, alpha);
@ -116,7 +116,7 @@ void MLPPGAN::addOutputLayer(std::string weightInit, std::string reg, double lam
} }
} }
std::vector<std::vector<double>> MLPPGAN::modelSetTestGenerator(std::vector<std::vector<double>> X) { std::vector<std::vector<real_t>> MLPPGAN::modelSetTestGenerator(std::vector<std::vector<real_t>> X) {
if (!network.empty()) { if (!network.empty()) {
network[0].input = X; network[0].input = X;
network[0].forwardPass(); network[0].forwardPass();
@ -129,7 +129,7 @@ std::vector<std::vector<double>> MLPPGAN::modelSetTestGenerator(std::vector<std:
return network[network.size() / 2].a; return network[network.size() / 2].a;
} }
std::vector<double> MLPPGAN::modelSetTestDiscriminator(std::vector<std::vector<double>> X) { std::vector<real_t> MLPPGAN::modelSetTestDiscriminator(std::vector<std::vector<real_t>> X) {
if (!network.empty()) { if (!network.empty()) {
for (int i = network.size() / 2 + 1; i < network.size(); i++) { for (int i = network.size() / 2 + 1; i < network.size(); i++) {
if (i == network.size() / 2 + 1) { if (i == network.size() / 2 + 1) {
@ -145,10 +145,10 @@ std::vector<double> MLPPGAN::modelSetTestDiscriminator(std::vector<std::vector<d
return outputLayer->a; return outputLayer->a;
} }
double MLPPGAN::Cost(std::vector<double> y_hat, std::vector<double> y) { real_t MLPPGAN::Cost(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPReg regularization; MLPPReg regularization;
class MLPPCost cost; class MLPPCost cost;
double totalRegTerm = 0; real_t totalRegTerm = 0;
auto cost_function = outputLayer->cost_map[outputLayer->cost]; auto cost_function = outputLayer->cost_map[outputLayer->cost];
if (!network.empty()) { if (!network.empty()) {
@ -177,7 +177,7 @@ void MLPPGAN::forwardPass() {
y_hat = outputLayer->a; y_hat = outputLayer->a;
} }
void MLPPGAN::updateDiscriminatorParameters(std::vector<std::vector<std::vector<double>>> hiddenLayerUpdations, std::vector<double> outputLayerUpdation, double learning_rate) { void MLPPGAN::updateDiscriminatorParameters(std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations, std::vector<real_t> outputLayerUpdation, real_t learning_rate) {
MLPPLinAlg alg; MLPPLinAlg alg;
outputLayer->weights = alg.subtraction(outputLayer->weights, outputLayerUpdation); outputLayer->weights = alg.subtraction(outputLayer->weights, outputLayerUpdation);
@ -194,7 +194,7 @@ void MLPPGAN::updateDiscriminatorParameters(std::vector<std::vector<std::vector<
} }
} }
void MLPPGAN::updateGeneratorParameters(std::vector<std::vector<std::vector<double>>> hiddenLayerUpdations, double learning_rate) { void MLPPGAN::updateGeneratorParameters(std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations, real_t learning_rate) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (!network.empty()) { if (!network.empty()) {
@ -207,25 +207,25 @@ void MLPPGAN::updateGeneratorParameters(std::vector<std::vector<std::vector<doub
} }
} }
std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<double>> MLPPGAN::computeDiscriminatorGradients(std::vector<double> y_hat, std::vector<double> outputSet) { std::tuple<std::vector<std::vector<std::vector<real_t>>>, std::vector<real_t>> MLPPGAN::computeDiscriminatorGradients(std::vector<real_t> y_hat, std::vector<real_t> outputSet) {
class MLPPCost cost; class MLPPCost cost;
MLPPActivation avn; MLPPActivation avn;
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
std::vector<std::vector<std::vector<double>>> cumulativeHiddenLayerWGrad; // Tensor containing ALL hidden grads. std::vector<std::vector<std::vector<real_t>>> cumulativeHiddenLayerWGrad; // Tensor containing ALL hidden grads.
auto costDeriv = outputLayer->costDeriv_map[outputLayer->cost]; auto costDeriv = outputLayer->costDeriv_map[outputLayer->cost];
auto outputAvn = outputLayer->activation_map[outputLayer->activation]; auto outputAvn = outputLayer->activation_map[outputLayer->activation];
outputLayer->delta = alg.hadamard_product((cost.*costDeriv)(y_hat, outputSet), (avn.*outputAvn)(outputLayer->z, 1)); outputLayer->delta = alg.hadamard_product((cost.*costDeriv)(y_hat, outputSet), (avn.*outputAvn)(outputLayer->z, 1));
std::vector<double> outputWGrad = alg.mat_vec_mult(alg.transpose(outputLayer->input), outputLayer->delta); std::vector<real_t> outputWGrad = alg.mat_vec_mult(alg.transpose(outputLayer->input), outputLayer->delta);
outputWGrad = alg.addition(outputWGrad, regularization.regDerivTerm(outputLayer->weights, outputLayer->lambda, outputLayer->alpha, outputLayer->reg)); outputWGrad = alg.addition(outputWGrad, regularization.regDerivTerm(outputLayer->weights, outputLayer->lambda, outputLayer->alpha, outputLayer->reg));
if (!network.empty()) { if (!network.empty()) {
auto hiddenLayerAvn = network[network.size() - 1].activation_map[network[network.size() - 1].activation]; auto hiddenLayerAvn = network[network.size() - 1].activation_map[network[network.size() - 1].activation];
network[network.size() - 1].delta = alg.hadamard_product(alg.outerProduct(outputLayer->delta, outputLayer->weights), (avn.*hiddenLayerAvn)(network[network.size() - 1].z, 1)); network[network.size() - 1].delta = alg.hadamard_product(alg.outerProduct(outputLayer->delta, outputLayer->weights), (avn.*hiddenLayerAvn)(network[network.size() - 1].z, 1));
std::vector<std::vector<double>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[network.size() - 1].input), network[network.size() - 1].delta); std::vector<std::vector<real_t>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[network.size() - 1].input), network[network.size() - 1].delta);
cumulativeHiddenLayerWGrad.push_back(alg.addition(hiddenLayerWGrad, regularization.regDerivTerm(network[network.size() - 1].weights, network[network.size() - 1].lambda, network[network.size() - 1].alpha, network[network.size() - 1].reg))); // Adding to our cumulative hidden layer grads. Maintain reg terms as well. cumulativeHiddenLayerWGrad.push_back(alg.addition(hiddenLayerWGrad, regularization.regDerivTerm(network[network.size() - 1].weights, network[network.size() - 1].lambda, network[network.size() - 1].alpha, network[network.size() - 1].reg))); // Adding to our cumulative hidden layer grads. Maintain reg terms as well.
@ -235,7 +235,7 @@ std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<double>> M
for (int i = network.size() - 2; i > network.size() / 2; i--) { for (int i = network.size() - 2; i > network.size() / 2; i--) {
auto hiddenLayerAvn = network[i].activation_map[network[i].activation]; auto hiddenLayerAvn = network[i].activation_map[network[i].activation];
network[i].delta = alg.hadamard_product(alg.matmult(network[i + 1].delta, alg.transpose(network[i + 1].weights)), (avn.*hiddenLayerAvn)(network[i].z, 1)); network[i].delta = alg.hadamard_product(alg.matmult(network[i + 1].delta, alg.transpose(network[i + 1].weights)), (avn.*hiddenLayerAvn)(network[i].z, 1));
std::vector<std::vector<double>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[i].input), network[i].delta); std::vector<std::vector<real_t>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[i].input), network[i].delta);
cumulativeHiddenLayerWGrad.push_back(alg.addition(hiddenLayerWGrad, regularization.regDerivTerm(network[i].weights, network[i].lambda, network[i].alpha, network[i].reg))); // Adding to our cumulative hidden layer grads. Maintain reg terms as well. cumulativeHiddenLayerWGrad.push_back(alg.addition(hiddenLayerWGrad, regularization.regDerivTerm(network[i].weights, network[i].lambda, network[i].alpha, network[i].reg))); // Adding to our cumulative hidden layer grads. Maintain reg terms as well.
} }
@ -243,36 +243,36 @@ std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<double>> M
return { cumulativeHiddenLayerWGrad, outputWGrad }; return { cumulativeHiddenLayerWGrad, outputWGrad };
} }
std::vector<std::vector<std::vector<double>>> MLPPGAN::computeGeneratorGradients(std::vector<double> y_hat, std::vector<double> outputSet) { std::vector<std::vector<std::vector<real_t>>> MLPPGAN::computeGeneratorGradients(std::vector<real_t> y_hat, std::vector<real_t> outputSet) {
class MLPPCost cost; class MLPPCost cost;
MLPPActivation avn; MLPPActivation avn;
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
std::vector<std::vector<std::vector<double>>> cumulativeHiddenLayerWGrad; // Tensor containing ALL hidden grads. std::vector<std::vector<std::vector<real_t>>> cumulativeHiddenLayerWGrad; // Tensor containing ALL hidden grads.
auto costDeriv = outputLayer->costDeriv_map[outputLayer->cost]; auto costDeriv = outputLayer->costDeriv_map[outputLayer->cost];
auto outputAvn = outputLayer->activation_map[outputLayer->activation]; auto outputAvn = outputLayer->activation_map[outputLayer->activation];
outputLayer->delta = alg.hadamard_product((cost.*costDeriv)(y_hat, outputSet), (avn.*outputAvn)(outputLayer->z, 1)); outputLayer->delta = alg.hadamard_product((cost.*costDeriv)(y_hat, outputSet), (avn.*outputAvn)(outputLayer->z, 1));
std::vector<double> outputWGrad = alg.mat_vec_mult(alg.transpose(outputLayer->input), outputLayer->delta); std::vector<real_t> outputWGrad = alg.mat_vec_mult(alg.transpose(outputLayer->input), outputLayer->delta);
outputWGrad = alg.addition(outputWGrad, regularization.regDerivTerm(outputLayer->weights, outputLayer->lambda, outputLayer->alpha, outputLayer->reg)); outputWGrad = alg.addition(outputWGrad, regularization.regDerivTerm(outputLayer->weights, outputLayer->lambda, outputLayer->alpha, outputLayer->reg));
if (!network.empty()) { if (!network.empty()) {
auto hiddenLayerAvn = network[network.size() - 1].activation_map[network[network.size() - 1].activation]; auto hiddenLayerAvn = network[network.size() - 1].activation_map[network[network.size() - 1].activation];
network[network.size() - 1].delta = alg.hadamard_product(alg.outerProduct(outputLayer->delta, outputLayer->weights), (avn.*hiddenLayerAvn)(network[network.size() - 1].z, 1)); network[network.size() - 1].delta = alg.hadamard_product(alg.outerProduct(outputLayer->delta, outputLayer->weights), (avn.*hiddenLayerAvn)(network[network.size() - 1].z, 1));
std::vector<std::vector<double>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[network.size() - 1].input), network[network.size() - 1].delta); std::vector<std::vector<real_t>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[network.size() - 1].input), network[network.size() - 1].delta);
cumulativeHiddenLayerWGrad.push_back(alg.addition(hiddenLayerWGrad, regularization.regDerivTerm(network[network.size() - 1].weights, network[network.size() - 1].lambda, network[network.size() - 1].alpha, network[network.size() - 1].reg))); // Adding to our cumulative hidden layer grads. Maintain reg terms as well. cumulativeHiddenLayerWGrad.push_back(alg.addition(hiddenLayerWGrad, regularization.regDerivTerm(network[network.size() - 1].weights, network[network.size() - 1].lambda, network[network.size() - 1].alpha, network[network.size() - 1].reg))); // Adding to our cumulative hidden layer grads. Maintain reg terms as well.
for (int i = network.size() - 2; i >= 0; i--) { for (int i = network.size() - 2; i >= 0; i--) {
auto hiddenLayerAvn = network[i].activation_map[network[i].activation]; auto hiddenLayerAvn = network[i].activation_map[network[i].activation];
network[i].delta = alg.hadamard_product(alg.matmult(network[i + 1].delta, alg.transpose(network[i + 1].weights)), (avn.*hiddenLayerAvn)(network[i].z, 1)); network[i].delta = alg.hadamard_product(alg.matmult(network[i + 1].delta, alg.transpose(network[i + 1].weights)), (avn.*hiddenLayerAvn)(network[i].z, 1));
std::vector<std::vector<double>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[i].input), network[i].delta); std::vector<std::vector<real_t>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[i].input), network[i].delta);
cumulativeHiddenLayerWGrad.push_back(alg.addition(hiddenLayerWGrad, regularization.regDerivTerm(network[i].weights, network[i].lambda, network[i].alpha, network[i].reg))); // Adding to our cumulative hidden layer grads. Maintain reg terms as well. cumulativeHiddenLayerWGrad.push_back(alg.addition(hiddenLayerWGrad, regularization.regDerivTerm(network[i].weights, network[i].lambda, network[i].alpha, network[i].reg))); // Adding to our cumulative hidden layer grads. Maintain reg terms as well.
} }
} }
return cumulativeHiddenLayerWGrad; return cumulativeHiddenLayerWGrad;
} }
void MLPPGAN::UI(int epoch, double cost_prev, std::vector<double> y_hat, std::vector<double> outputSet) { void MLPPGAN::UI(int epoch, real_t cost_prev, std::vector<real_t> y_hat, std::vector<real_t> outputSet) {
MLPPUtilities::CostInfo(epoch, cost_prev, Cost(y_hat, outputSet)); MLPPUtilities::CostInfo(epoch, cost_prev, Cost(y_hat, outputSet));
std::cout << "Layer " << network.size() + 1 << ": " << std::endl; std::cout << "Layer " << network.size() + 1 << ": " << std::endl;
MLPPUtilities::UI(outputLayer->weights, outputLayer->bias); MLPPUtilities::UI(outputLayer->weights, outputLayer->bias);

34
mlpp/gan/gan.h
View File

@ -8,6 +8,8 @@
// Created by Marc Melikyan on 11/4/20. // Created by Marc Melikyan on 11/4/20.
// //
#include "core/math/math_defs.h"
#include "../hidden_layer/hidden_layer.h" #include "../hidden_layer/hidden_layer.h"
#include "../output_layer/output_layer.h" #include "../output_layer/output_layer.h"
@ -19,32 +21,32 @@
class MLPPGAN { class MLPPGAN {
public: public:
MLPPGAN(double k, std::vector<std::vector<double>> outputSet); MLPPGAN(real_t k, std::vector<std::vector<real_t>> outputSet);
~MLPPGAN(); ~MLPPGAN();
std::vector<std::vector<double>> generateExample(int n); std::vector<std::vector<real_t>> generateExample(int n);
void gradientDescent(double learning_rate, int max_epoch, bool UI = 1); void gradientDescent(real_t learning_rate, int max_epoch, bool UI = 1);
double score(); real_t score();
void save(std::string fileName); void save(std::string fileName);
void addLayer(int n_hidden, std::string activation, std::string weightInit = "Default", std::string reg = "None", double lambda = 0.5, double alpha = 0.5); void addLayer(int n_hidden, std::string activation, std::string weightInit = "Default", std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
void addOutputLayer(std::string weightInit = "Default", std::string reg = "None", double lambda = 0.5, double alpha = 0.5); void addOutputLayer(std::string weightInit = "Default", std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
private: private:
std::vector<std::vector<double>> modelSetTestGenerator(std::vector<std::vector<double>> X); // Evaluator for the generator of the gan. std::vector<std::vector<real_t>> modelSetTestGenerator(std::vector<std::vector<real_t>> X); // Evaluator for the generator of the gan.
std::vector<double> modelSetTestDiscriminator(std::vector<std::vector<double>> X); // Evaluator for the discriminator of the gan. std::vector<real_t> modelSetTestDiscriminator(std::vector<std::vector<real_t>> X); // Evaluator for the discriminator of the gan.
double Cost(std::vector<double> y_hat, std::vector<double> y); real_t Cost(std::vector<real_t> y_hat, std::vector<real_t> y);
void forwardPass(); void forwardPass();
void updateDiscriminatorParameters(std::vector<std::vector<std::vector<double>>> hiddenLayerUpdations, std::vector<double> outputLayerUpdation, double learning_rate); void updateDiscriminatorParameters(std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations, std::vector<real_t> outputLayerUpdation, real_t learning_rate);
void updateGeneratorParameters(std::vector<std::vector<std::vector<double>>> hiddenLayerUpdations, double learning_rate); void updateGeneratorParameters(std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations, real_t learning_rate);
std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<double>> computeDiscriminatorGradients(std::vector<double> y_hat, std::vector<double> outputSet); std::tuple<std::vector<std::vector<std::vector<real_t>>>, std::vector<real_t>> computeDiscriminatorGradients(std::vector<real_t> y_hat, std::vector<real_t> outputSet);
std::vector<std::vector<std::vector<double>>> computeGeneratorGradients(std::vector<double> y_hat, std::vector<double> outputSet); std::vector<std::vector<std::vector<real_t>>> computeGeneratorGradients(std::vector<real_t> y_hat, std::vector<real_t> outputSet);
void UI(int epoch, double cost_prev, std::vector<double> y_hat, std::vector<double> outputSet); void UI(int epoch, real_t cost_prev, std::vector<real_t> y_hat, std::vector<real_t> outputSet);
std::vector<std::vector<double>> outputSet; std::vector<std::vector<real_t>> outputSet;
std::vector<double> y_hat; std::vector<real_t> y_hat;
std::vector<MLPPHiddenLayer> network; std::vector<MLPPHiddenLayer> network;
MLPPOutputLayer *outputLayer; MLPPOutputLayer *outputLayer;

10
mlpp/gauss_markov_checker/gauss_markov_checker.cpp
View File

@ -9,7 +9,7 @@
#include <iostream> #include <iostream>
void MLPPGaussMarkovChecker::checkGMConditions(std::vector<double> eps) { void MLPPGaussMarkovChecker::checkGMConditions(std::vector<real_t> eps) {
bool condition1 = arithmeticMean(eps); bool condition1 = arithmeticMean(eps);
bool condition2 = homoscedasticity(eps); bool condition2 = homoscedasticity(eps);
bool condition3 = exogeneity(eps); bool condition3 = exogeneity(eps);
@ -21,7 +21,7 @@ void MLPPGaussMarkovChecker::checkGMConditions(std::vector<double> eps) {
} }
} }
bool MLPPGaussMarkovChecker::arithmeticMean(std::vector<double> eps) { bool MLPPGaussMarkovChecker::arithmeticMean(std::vector<real_t> eps) {
MLPPStat stat; MLPPStat stat;
if (stat.mean(eps) == 0) { if (stat.mean(eps) == 0) {
return 1; return 1;
@ -30,9 +30,9 @@ bool MLPPGaussMarkovChecker::arithmeticMean(std::vector<double> eps) {
} }
} }
bool MLPPGaussMarkovChecker::homoscedasticity(std::vector<double> eps) { bool MLPPGaussMarkovChecker::homoscedasticity(std::vector<real_t> eps) {
MLPPStat stat; MLPPStat stat;
double currentVar = (eps[0] - stat.mean(eps)) * (eps[0] - stat.mean(eps)) / eps.size(); real_t currentVar = (eps[0] - stat.mean(eps)) * (eps[0] - stat.mean(eps)) / eps.size();
for (int i = 0; i < eps.size(); i++) { for (int i = 0; i < eps.size(); i++) {
if (currentVar != (eps[i] - stat.mean(eps)) * (eps[i] - stat.mean(eps)) / eps.size()) { if (currentVar != (eps[i] - stat.mean(eps)) * (eps[i] - stat.mean(eps)) / eps.size()) {
return 0; return 0;
@ -41,7 +41,7 @@ bool MLPPGaussMarkovChecker::homoscedasticity(std::vector<double> eps) {
return 1; return 1;
} }
bool MLPPGaussMarkovChecker::exogeneity(std::vector<double> eps) { bool MLPPGaussMarkovChecker::exogeneity(std::vector<real_t> eps) {
MLPPStat stat; MLPPStat stat;
for (int i = 0; i < eps.size(); i++) { for (int i = 0; i < eps.size(); i++) {
for (int j = 0; j < eps.size(); j++) { for (int j = 0; j < eps.size(); j++) {

10
mlpp/gauss_markov_checker/gauss_markov_checker.h
View File

@ -8,18 +8,20 @@
// Created by Marc Melikyan on 11/13/20. // Created by Marc Melikyan on 11/13/20.
// //
#include "core/math/math_defs.h"
#include <string> #include <string>
#include <vector> #include <vector>
class MLPPGaussMarkovChecker { class MLPPGaussMarkovChecker {
public: public:
void checkGMConditions(std::vector<double> eps); void checkGMConditions(std::vector<real_t> eps);
// Independent, 3 Gauss-Markov Conditions // Independent, 3 Gauss-Markov Conditions
bool arithmeticMean(std::vector<double> eps); // 1) Arithmetic Mean of 0. bool arithmeticMean(std::vector<real_t> eps); // 1) Arithmetic Mean of 0.
bool homoscedasticity(std::vector<double> eps); // 2) Homoscedasticity bool homoscedasticity(std::vector<real_t> eps); // 2) Homoscedasticity
bool exogeneity(std::vector<double> eps); // 3) Cov of any 2 non-equal eps values = 0. bool exogeneity(std::vector<real_t> eps); // 3) Cov of any 2 non-equal eps values = 0.
private: private:
}; };

28
mlpp/gaussian_nb/gaussian_nb.cpp
View File

@ -14,35 +14,35 @@
#include <random> #include <random>
MLPPGaussianNB::MLPPGaussianNB(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, int class_num) : MLPPGaussianNB::MLPPGaussianNB(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, int class_num) :
inputSet(inputSet), outputSet(outputSet), class_num(class_num) { inputSet(inputSet), outputSet(outputSet), class_num(class_num) {
y_hat.resize(outputSet.size()); y_hat.resize(outputSet.size());
Evaluate(); Evaluate();
MLPPLinAlg alg; MLPPLinAlg alg;
} }
std::vector<double> MLPPGaussianNB::modelSetTest(std::vector<std::vector<double>> X) { std::vector<real_t> MLPPGaussianNB::modelSetTest(std::vector<std::vector<real_t>> X) {
std::vector<double> y_hat; std::vector<real_t> y_hat;
for (int i = 0; i < X.size(); i++) { for (int i = 0; i < X.size(); i++) {
y_hat.push_back(modelTest(X[i])); y_hat.push_back(modelTest(X[i]));
} }
return y_hat; return y_hat;
} }
double MLPPGaussianNB::modelTest(std::vector<double> x) { real_t MLPPGaussianNB::modelTest(std::vector<real_t> x) {
MLPPStat stat; MLPPStat stat;
MLPPLinAlg alg; MLPPLinAlg alg;
double score[class_num]; real_t score[class_num];
double y_hat_i = 1; real_t y_hat_i = 1;
for (int i = class_num - 1; i >= 0; i--) { for (int i = class_num - 1; i >= 0; i--) {
y_hat_i += std::log(priors[i] * (1 / sqrt(2 * M_PI * sigma[i] * sigma[i])) * exp(-(x[i] * mu[i]) * (x[i] * mu[i]) / (2 * sigma[i] * sigma[i]))); y_hat_i += std::log(priors[i] * (1 / sqrt(2 * M_PI * sigma[i] * sigma[i])) * exp(-(x[i] * mu[i]) * (x[i] * mu[i]) / (2 * sigma[i] * sigma[i])));
score[i] = exp(y_hat_i); score[i] = exp(y_hat_i);
} }
return std::distance(score, std::max_element(score, score + sizeof(score) / sizeof(double))); return std::distance(score, std::max_element(score, score + sizeof(score) / sizeof(real_t)));
} }
double MLPPGaussianNB::score() { real_t MLPPGaussianNB::score() {
MLPPUtilities util; MLPPUtilities util;
return util.performance(y_hat, outputSet); return util.performance(y_hat, outputSet);
} }
@ -55,7 +55,7 @@ void MLPPGaussianNB::Evaluate() {
mu.resize(class_num); mu.resize(class_num);
sigma.resize(class_num); sigma.resize(class_num);
for (int i = class_num - 1; i >= 0; i--) { for (int i = class_num - 1; i >= 0; i--) {
std::vector<double> set; std::vector<real_t> set;
for (int j = 0; j < inputSet.size(); j++) { for (int j = 0; j < inputSet.size(); j++) {
for (int k = 0; k < inputSet[j].size(); k++) { for (int k = 0; k < inputSet[j].size(); k++) {
if (outputSet[j] == i) { if (outputSet[j] == i) {
@ -72,11 +72,11 @@ void MLPPGaussianNB::Evaluate() {
for (int i = 0; i < outputSet.size(); i++) { for (int i = 0; i < outputSet.size(); i++) {
priors[int(outputSet[i])]++; priors[int(outputSet[i])]++;
} }
priors = alg.scalarMultiply(double(1) / double(outputSet.size()), priors); priors = alg.scalarMultiply(real_t(1) / real_t(outputSet.size()), priors);
for (int i = 0; i < outputSet.size(); i++) { for (int i = 0; i < outputSet.size(); i++) {
double score[class_num]; real_t score[class_num];
double y_hat_i = 1; real_t y_hat_i = 1;
for (int j = class_num - 1; j >= 0; j--) { for (int j = class_num - 1; j >= 0; j--) {
for (int k = 0; k < inputSet[i].size(); k++) { for (int k = 0; k < inputSet[i].size(); k++) {
y_hat_i += std::log(priors[j] * (1 / sqrt(2 * M_PI * sigma[j] * sigma[j])) * exp(-(inputSet[i][k] * mu[j]) * (inputSet[i][k] * mu[j]) / (2 * sigma[j] * sigma[j]))); y_hat_i += std::log(priors[j] * (1 / sqrt(2 * M_PI * sigma[j] * sigma[j])) * exp(-(inputSet[i][k] * mu[j]) * (inputSet[i][k] * mu[j]) / (2 * sigma[j] * sigma[j])));
@ -84,7 +84,7 @@ void MLPPGaussianNB::Evaluate() {
score[j] = exp(y_hat_i); score[j] = exp(y_hat_i);
std::cout << score[j] << std::endl; std::cout << score[j] << std::endl;
} }
y_hat[i] = std::distance(score, std::max_element(score, score + sizeof(score) / sizeof(double))); y_hat[i] = std::distance(score, std::max_element(score, score + sizeof(score) / sizeof(real_t)));
std::cout << std::distance(score, std::max_element(score, score + sizeof(score) / sizeof(double))) << std::endl; std::cout << std::distance(score, std::max_element(score, score + sizeof(score) / sizeof(real_t))) << std::endl;
} }
} }

22
mlpp/gaussian_nb/gaussian_nb.h
View File

@ -8,29 +8,31 @@
// Created by Marc Melikyan on 1/17/21. // Created by Marc Melikyan on 1/17/21.
// //
#include "core/math/math_defs.h"
#include <vector> #include <vector>
class MLPPGaussianNB { class MLPPGaussianNB {
public: public:
MLPPGaussianNB(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, int class_num); MLPPGaussianNB(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, int class_num);
std::vector<double> modelSetTest(std::vector<std::vector<double>> X); std::vector<real_t> modelSetTest(std::vector<std::vector<real_t>> X);
double modelTest(std::vector<double> x); real_t modelTest(std::vector<real_t> x);
double score(); real_t score();
private: private:
void Evaluate(); void Evaluate();
int class_num; int class_num;
std::vector<double> priors; std::vector<real_t> priors;
std::vector<double> mu; std::vector<real_t> mu;
std::vector<double> sigma; std::vector<real_t> sigma;
std::vector<std::vector<double>> inputSet; std::vector<std::vector<real_t>> inputSet;
std::vector<double> outputSet; std::vector<real_t> outputSet;
std::vector<double> y_hat; std::vector<real_t> y_hat;
}; };
#endif /* GaussianNB_hpp */ #endif /* GaussianNB_hpp */

4
mlpp/hidden_layer/hidden_layer.cpp
View File

@ -13,7 +13,7 @@
#include <random> #include <random>
MLPPHiddenLayer::MLPPHiddenLayer(int n_hidden, std::string activation, std::vector<std::vector<double>> input, std::string weightInit, std::string reg, double lambda, double alpha) : MLPPHiddenLayer::MLPPHiddenLayer(int n_hidden, std::string activation, std::vector<std::vector<real_t>> input, std::string weightInit, std::string reg, real_t lambda, real_t alpha) :
n_hidden(n_hidden), activation(activation), input(input), weightInit(weightInit), reg(reg), lambda(lambda), alpha(alpha) { n_hidden(n_hidden), activation(activation), input(input), weightInit(weightInit), reg(reg), lambda(lambda), alpha(alpha) {
weights = MLPPUtilities::weightInitialization(input[0].size(), n_hidden, weightInit); weights = MLPPUtilities::weightInitialization(input[0].size(), n_hidden, weightInit);
bias = MLPPUtilities::biasInitialization(n_hidden); bias = MLPPUtilities::biasInitialization(n_hidden);
@ -104,7 +104,7 @@ void MLPPHiddenLayer::forwardPass() {
a = (avn.*activation_map[activation])(z, 0); a = (avn.*activation_map[activation])(z, 0);
} }
void MLPPHiddenLayer::Test(std::vector<double> x) { void MLPPHiddenLayer::Test(std::vector<real_t> x) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPActivation avn; MLPPActivation avn;
z_test = alg.addition(alg.mat_vec_mult(alg.transpose(weights), x), bias); z_test = alg.addition(alg.mat_vec_mult(alg.transpose(weights), x), bias);

30
mlpp/hidden_layer/hidden_layer.h
View File

@ -8,6 +8,8 @@
// Created by Marc Melikyan on 11/4/20. // Created by Marc Melikyan on 11/4/20.
// //
#include "core/math/math_defs.h"
#include "../activation/activation.h" #include "../activation/activation.h"
#include <map> #include <map>
@ -17,36 +19,36 @@
class MLPPHiddenLayer { class MLPPHiddenLayer {
public: public:
MLPPHiddenLayer(int n_hidden, std::string activation, std::vector<std::vector<double>> input, std::string weightInit, std::string reg, double lambda, double alpha); MLPPHiddenLayer(int n_hidden, std::string activation, std::vector<std::vector<real_t>> input, std::string weightInit, std::string reg, real_t lambda, real_t alpha);
int n_hidden; int n_hidden;
std::string activation; std::string activation;
std::vector<std::vector<double>> input; std::vector<std::vector<real_t>> input;
std::vector<std::vector<double>> weights; std::vector<std::vector<real_t>> weights;
std::vector<double> bias; std::vector<real_t> bias;
std::vector<std::vector<double>> z; std::vector<std::vector<real_t>> z;
std::vector<std::vector<double>> a; std::vector<std::vector<real_t>> a;
std::map<std::string, std::vector<std::vector<double>> (MLPPActivation::*)(std::vector<std::vector<double>>, bool)> activation_map; std::map<std::string, std::vector<std::vector<real_t>> (MLPPActivation::*)(std::vector<std::vector<real_t>>, bool)> activation_map;
std::map<std::string, std::vector<double> (MLPPActivation::*)(std::vector<double>, bool)> activationTest_map; std::map<std::string, std::vector<real_t> (MLPPActivation::*)(std::vector<real_t>, bool)> activationTest_map;
std::vector<double> z_test; std::vector<real_t> z_test;
std::vector<double> a_test; std::vector<real_t> a_test;
std::vector<std::vector<double>> delta; std::vector<std::vector<real_t>> delta;
// Regularization Params // Regularization Params
std::string reg; std::string reg;
double lambda; /* Regularization Parameter */ real_t lambda; /* Regularization Parameter */
double alpha; /* This is the controlling param for Elastic Net*/ real_t alpha; /* This is the controlling param for Elastic Net*/
std::string weightInit; std::string weightInit;
void forwardPass(); void forwardPass();
void Test(std::vector<double> x); void Test(std::vector<real_t> x);
}; };

6
mlpp/hypothesis_testing/hypothesis_testing.cpp
View File

@ -8,9 +8,9 @@
std::tuple<bool, double> MLPPHypothesisTesting::chiSquareTest(std::vector<double> observed, std::vector<double> expected) { std::tuple<bool, real_t> MLPPHypothesisTesting::chiSquareTest(std::vector<real_t> observed, std::vector<real_t> expected) {
double df = observed.size() - 1; // These are our degrees of freedom real_t df = observed.size() - 1; // These are our degrees of freedom
double sum = 0; real_t sum = 0;
for (int i = 0; i < observed.size(); i++) { for (int i = 0; i < observed.size(); i++) {
sum += (observed[i] - expected[i]) * (observed[i] - expected[i]) / expected[i]; sum += (observed[i] - expected[i]) * (observed[i] - expected[i]) / expected[i];
} }

4
mlpp/hypothesis_testing/hypothesis_testing.h
View File

@ -8,13 +8,15 @@
// Created by Marc Melikyan on 3/10/21. // Created by Marc Melikyan on 3/10/21.
// //
#include "core/math/math_defs.h"
#include <tuple> #include <tuple>
#include <vector> #include <vector>
class MLPPHypothesisTesting { class MLPPHypothesisTesting {
public: public:
std::tuple<bool, double> chiSquareTest(std::vector<double> observed, std::vector<double> expected); std::tuple<bool, real_t> chiSquareTest(std::vector<real_t> observed, std::vector<real_t> expected);
private: private:
}; };

48
mlpp/kmeans/kmeans.cpp
View File

@ -13,7 +13,7 @@
#include <random> #include <random>
MLPPKMeans::MLPPKMeans(std::vector<std::vector<double>> inputSet, int k, std::string init_type) : MLPPKMeans::MLPPKMeans(std::vector<std::vector<real_t>> inputSet, int k, std::string init_type) :
inputSet(inputSet), k(k), init_type(init_type) { inputSet(inputSet), k(k), init_type(init_type) {
if (init_type == "KMeans++") { if (init_type == "KMeans++") {
kmeansppInitialization(k); kmeansppInitialization(k);
@ -22,11 +22,11 @@ MLPPKMeans::MLPPKMeans(std::vector<std::vector<double>> inputSet, int k, std::st
} }
} }
std::vector<std::vector<double>> MLPPKMeans::modelSetTest(std::vector<std::vector<double>> X) { std::vector<std::vector<real_t>> MLPPKMeans::modelSetTest(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg; MLPPLinAlg alg;
std::vector<std::vector<double>> closestCentroids; std::vector<std::vector<real_t>> closestCentroids;
for (int i = 0; i < inputSet.size(); i++) { for (int i = 0; i < inputSet.size(); i++) {
std::vector<double> closestCentroid = mu[0]; std::vector<real_t> closestCentroid = mu[0];
for (int j = 0; j < r[0].size(); j++) { for (int j = 0; j < r[0].size(); j++) {
bool isCentroidCloser = alg.euclideanDistance(X[i], mu[j]) < alg.euclideanDistance(X[i], closestCentroid); bool isCentroidCloser = alg.euclideanDistance(X[i], mu[j]) < alg.euclideanDistance(X[i], closestCentroid);
if (isCentroidCloser) { if (isCentroidCloser) {
@ -38,9 +38,9 @@ std::vector<std::vector<double>> MLPPKMeans::modelSetTest(std::vector<std::vecto
return closestCentroids; return closestCentroids;
} }
std::vector<double> MLPPKMeans::modelTest(std::vector<double> x) { std::vector<real_t> MLPPKMeans::modelTest(std::vector<real_t> x) {
MLPPLinAlg alg; MLPPLinAlg alg;
std::vector<double> closestCentroid = mu[0]; std::vector<real_t> closestCentroid = mu[0];
for (int j = 0; j < mu.size(); j++) { for (int j = 0; j < mu.size(); j++) {
if (alg.euclideanDistance(x, mu[j]) < alg.euclideanDistance(x, closestCentroid)) { if (alg.euclideanDistance(x, mu[j]) < alg.euclideanDistance(x, closestCentroid)) {
closestCentroid = mu[j]; closestCentroid = mu[j];
@ -50,7 +50,7 @@ std::vector<double> MLPPKMeans::modelTest(std::vector<double> x) {
} }
void MLPPKMeans::train(int epoch_num, bool UI) { void MLPPKMeans::train(int epoch_num, bool UI) {
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
Evaluate(); Evaluate();
@ -80,17 +80,17 @@ void MLPPKMeans::train(int epoch_num, bool UI) {
} }
} }
double MLPPKMeans::score() { real_t MLPPKMeans::score() {
return Cost(); return Cost();
} }
std::vector<double> MLPPKMeans::silhouette_scores() { std::vector<real_t> MLPPKMeans::silhouette_scores() {
MLPPLinAlg alg; MLPPLinAlg alg;
std::vector<std::vector<double>> closestCentroids = modelSetTest(inputSet); std::vector<std::vector<real_t>> closestCentroids = modelSetTest(inputSet);
std::vector<double> silhouette_scores; std::vector<real_t> silhouette_scores;
for (int i = 0; i < inputSet.size(); i++) { for (int i = 0; i < inputSet.size(); i++) {
// COMPUTING a[i] // COMPUTING a[i]
double a = 0; real_t a = 0;
for (int j = 0; j < inputSet.size(); j++) { for (int j = 0; j < inputSet.size(); j++) {
if (i != j && r[i] == r[j]) { if (i != j && r[i] == r[j]) {
a += alg.euclideanDistance(inputSet[i], inputSet[j]); a += alg.euclideanDistance(inputSet[i], inputSet[j]);
@ -100,15 +100,15 @@ std::vector<double> MLPPKMeans::silhouette_scores() {
a /= closestCentroids[i].size() - 1; a /= closestCentroids[i].size() - 1;
// COMPUTING b[i] // COMPUTING b[i]
double b = INT_MAX; real_t b = INT_MAX;
for (int j = 0; j < mu.size(); j++) { for (int j = 0; j < mu.size(); j++) {
if (closestCentroids[i] != mu[j]) { if (closestCentroids[i] != mu[j]) {
double sum = 0; real_t sum = 0;
for (int k = 0; k < inputSet.size(); k++) { for (int k = 0; k < inputSet.size(); k++) {
sum += alg.euclideanDistance(inputSet[i], inputSet[k]); sum += alg.euclideanDistance(inputSet[i], inputSet[k]);
} }
// NORMALIZE b[i] // NORMALIZE b[i]
double k_clusterSize = 0; real_t k_clusterSize = 0;
for (int k = 0; k < closestCentroids.size(); k++) { for (int k = 0; k < closestCentroids.size(); k++) {
if (closestCentroids[k] == mu[j]) { if (closestCentroids[k] == mu[j]) {
k_clusterSize++; k_clusterSize++;
@ -144,7 +144,7 @@ void MLPPKMeans::Evaluate() {
} }
for (int i = 0; i < r.size(); i++) { for (int i = 0; i < r.size(); i++) {
std::vector<double> closestCentroid = mu[0]; std::vector<real_t> closestCentroid = mu[0];
for (int j = 0; j < r[0].size(); j++) { for (int j = 0; j < r[0].size(); j++) {
bool isCentroidCloser = alg.euclideanDistance(inputSet[i], mu[j]) < alg.euclideanDistance(inputSet[i], closestCentroid); bool isCentroidCloser = alg.euclideanDistance(inputSet[i], mu[j]) < alg.euclideanDistance(inputSet[i], closestCentroid);
if (isCentroidCloser) { if (isCentroidCloser) {
@ -165,21 +165,21 @@ void MLPPKMeans::Evaluate() {
void MLPPKMeans::computeMu() { void MLPPKMeans::computeMu() {
MLPPLinAlg alg; MLPPLinAlg alg;
for (int i = 0; i < mu.size(); i++) { for (int i = 0; i < mu.size(); i++) {
std::vector<double> num; std::vector<real_t> num;
num.resize(r.size()); num.resize(r.size());
for (int i = 0; i < num.size(); i++) { for (int i = 0; i < num.size(); i++) {
num[i] = 0; num[i] = 0;
} }
double den = 0; real_t den = 0;
for (int j = 0; j < r.size(); j++) { for (int j = 0; j < r.size(); j++) {
num = alg.addition(num, alg.scalarMultiply(r[j][i], inputSet[j])); num = alg.addition(num, alg.scalarMultiply(r[j][i], inputSet[j]));
} }
for (int j = 0; j < r.size(); j++) { for (int j = 0; j < r.size(); j++) {
den += r[j][i]; den += r[j][i];
} }
mu[i] = alg.scalarMultiply(double(1) / double(den), num); mu[i] = alg.scalarMultiply(real_t(1) / real_t(den), num);
} }
} }
@ -204,12 +204,12 @@ void MLPPKMeans::kmeansppInitialization(int k) {
mu.push_back(inputSet[distribution(generator)]); mu.push_back(inputSet[distribution(generator)]);
for (int i = 0; i < k - 1; i++) { for (int i = 0; i < k - 1; i++) {
std::vector<double> farthestCentroid; std::vector<real_t> farthestCentroid;
for (int j = 0; j < inputSet.size(); j++) { for (int j = 0; j < inputSet.size(); j++) {
double max_dist = 0; real_t max_dist = 0;
/* SUM ALL THE SQUARED DISTANCES, CHOOSE THE ONE THAT'S FARTHEST /* SUM ALL THE SQUARED DISTANCES, CHOOSE THE ONE THAT'S FARTHEST
AS TO SPREAD OUT THE CLUSTER CENTROIDS. */ AS TO SPREAD OUT THE CLUSTER CENTROIDS. */
double sum = 0; real_t sum = 0;
for (int k = 0; k < mu.size(); k++) { for (int k = 0; k < mu.size(); k++) {
sum += alg.euclideanDistance(inputSet[j], mu[k]); sum += alg.euclideanDistance(inputSet[j], mu[k]);
} }
@ -222,9 +222,9 @@ void MLPPKMeans::kmeansppInitialization(int k) {
} }
} }
double MLPPKMeans::Cost() { real_t MLPPKMeans::Cost() {
MLPPLinAlg alg; MLPPLinAlg alg;
double sum = 0; real_t sum = 0;
for (int i = 0; i < r.size(); i++) { for (int i = 0; i < r.size(); i++) {
for (int j = 0; j < r[0].size(); j++) { for (int j = 0; j < r[0].size(); j++) {
sum += r[i][j] * alg.norm_sq(alg.subtraction(inputSet[i], mu[j])); sum += r[i][j] * alg.norm_sq(alg.subtraction(inputSet[i], mu[j]));

24
mlpp/kmeans/kmeans.h
View File

@ -8,18 +8,20 @@
// Created by Marc Melikyan on 10/2/20. // Created by Marc Melikyan on 10/2/20.
// //
#include "core/math/math_defs.h"
#include <string> #include <string>
#include <vector> #include <vector>
class MLPPKMeans { class MLPPKMeans {
public: public:
MLPPKMeans(std::vector<std::vector<double>> inputSet, int k, std::string init_type = "Default"); MLPPKMeans(std::vector<std::vector<real_t>> inputSet, int k, std::string init_type = "Default");
std::vector<std::vector<double>> modelSetTest(std::vector<std::vector<double>> X); std::vector<std::vector<real_t>> modelSetTest(std::vector<std::vector<real_t>> X);
std::vector<double> modelTest(std::vector<double> x); std::vector<real_t> modelTest(std::vector<real_t> x);
void train(int epoch_num, bool UI = 1); void train(int epoch_num, bool UI = 1);
double score(); real_t score();
std::vector<double> silhouette_scores(); std::vector<real_t> silhouette_scores();
private: private:
void Evaluate(); void Evaluate();
@ -27,15 +29,15 @@ private:
void centroidInitialization(int k); void centroidInitialization(int k);
void kmeansppInitialization(int k); void kmeansppInitialization(int k);
double Cost(); real_t Cost();
std::vector<std::vector<double>> inputSet; std::vector<std::vector<real_t>> inputSet;
std::vector<std::vector<double>> mu; std::vector<std::vector<real_t>> mu;
std::vector<std::vector<double>> r; std::vector<std::vector<real_t>> r;
double euclideanDistance(std::vector<double> A, std::vector<double> B); real_t euclideanDistance(std::vector<real_t> A, std::vector<real_t> B);
double accuracy_threshold; real_t accuracy_threshold;
int k; int k;
std::string init_type; std::string init_type;

18
mlpp/knn/knn.cpp
View File

@ -13,28 +13,28 @@
#include <map> #include <map>
MLPPKNN::MLPPKNN(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, int k) : MLPPKNN::MLPPKNN(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, int k) :
inputSet(inputSet), outputSet(outputSet), k(k) { inputSet(inputSet), outputSet(outputSet), k(k) {
} }
std::vector<double> MLPPKNN::modelSetTest(std::vector<std::vector<double>> X) { std::vector<real_t> MLPPKNN::modelSetTest(std::vector<std::vector<real_t>> X) {
std::vector<double> y_hat; std::vector<real_t> y_hat;
for (int i = 0; i < X.size(); i++) { for (int i = 0; i < X.size(); i++) {
y_hat.push_back(modelTest(X[i])); y_hat.push_back(modelTest(X[i]));
} }
return y_hat; return y_hat;
} }
int MLPPKNN::modelTest(std::vector<double> x) { int MLPPKNN::modelTest(std::vector<real_t> x) {
return determineClass(nearestNeighbors(x)); return determineClass(nearestNeighbors(x));
} }
double MLPPKNN::score() { real_t MLPPKNN::score() {
MLPPUtilities util; MLPPUtilities util;
return util.performance(modelSetTest(inputSet), outputSet); return util.performance(modelSetTest(inputSet), outputSet);
} }
int MLPPKNN::determineClass(std::vector<double> knn) { int MLPPKNN::determineClass(std::vector<real_t> knn) {
std::map<int, int> class_nums; std::map<int, int> class_nums;
for (int i = 0; i < outputSet.size(); i++) { for (int i = 0; i < outputSet.size(); i++) {
class_nums[outputSet[i]] = 0; class_nums[outputSet[i]] = 0;
@ -62,12 +62,12 @@ int MLPPKNN::determineClass(std::vector<double> knn) {
return final_class; return final_class;
} }
std::vector<double> MLPPKNN::nearestNeighbors(std::vector<double> x) { std::vector<real_t> MLPPKNN::nearestNeighbors(std::vector<real_t> x) {
MLPPLinAlg alg; MLPPLinAlg alg;
// The nearest neighbors // The nearest neighbors
std::vector<double> knn; std::vector<real_t> knn;
std::vector<std::vector<double>> inputUseSet = inputSet; std::vector<std::vector<real_t>> inputUseSet = inputSet;
//Perfom this loop unless and until all k nearest neighbors are found, appended, and returned //Perfom this loop unless and until all k nearest neighbors are found, appended, and returned
for (int i = 0; i < k; i++) { for (int i = 0; i < k; i++) {
int neighbor = 0; int neighbor = 0;

18
mlpp/knn/knn.h
View File

@ -8,24 +8,26 @@
// Created by Marc Melikyan on 10/2/20. // Created by Marc Melikyan on 10/2/20.
// //
#include "core/math/math_defs.h"
#include <vector> #include <vector>
class MLPPKNN { class MLPPKNN {
public: public:
MLPPKNN(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, int k); MLPPKNN(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, int k);
std::vector<double> modelSetTest(std::vector<std::vector<double>> X); std::vector<real_t> modelSetTest(std::vector<std::vector<real_t>> X);
int modelTest(std::vector<double> x); int modelTest(std::vector<real_t> x);
double score(); real_t score();
private: private:
// Private Model Functions // Private Model Functions
std::vector<double> nearestNeighbors(std::vector<double> x); std::vector<real_t> nearestNeighbors(std::vector<real_t> x);
int determineClass(std::vector<double> knn); int determineClass(std::vector<real_t> knn);
// Model Inputs and Parameters // Model Inputs and Parameters
std::vector<std::vector<double>> inputSet; std::vector<std::vector<real_t>> inputSet;
std::vector<double> outputSet; std::vector<real_t> outputSet;
int k; int k;
}; };

442
mlpp/lin_alg/lin_alg.cpp
View File

File diff suppressed because it is too large Load Diff

236
mlpp/lin_alg/lin_alg.h
View File

@ -8,6 +8,8 @@
// Created by Marc Melikyan on 1/8/21. // Created by Marc Melikyan on 1/8/21.
// //
#include "core/math/math_defs.h"
#include <tuple> #include <tuple>
#include <vector> #include <vector>
@ -15,246 +17,246 @@ class MLPPLinAlg {
public: public:
// MATRIX FUNCTIONS // MATRIX FUNCTIONS
std::vector<std::vector<double>> gramMatrix(std::vector<std::vector<double>> A); std::vector<std::vector<real_t>> gramMatrix(std::vector<std::vector<real_t>> A);
bool linearIndependenceChecker(std::vector<std::vector<double>> A); bool linearIndependenceChecker(std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> gaussianNoise(int n, int m); std::vector<std::vector<real_t>> gaussianNoise(int n, int m);
std::vector<std::vector<double>> addition(std::vector<std::vector<double>> A, std::vector<std::vector<double>> B); std::vector<std::vector<real_t>> addition(std::vector<std::vector<real_t>> A, std::vector<std::vector<real_t>> B);
std::vector<std::vector<double>> subtraction(std::vector<std::vector<double>> A, std::vector<std::vector<double>> B); std::vector<std::vector<real_t>> subtraction(std::vector<std::vector<real_t>> A, std::vector<std::vector<real_t>> B);
std::vector<std::vector<double>> matmult(std::vector<std::vector<double>> A, std::vector<std::vector<double>> B); std::vector<std::vector<real_t>> matmult(std::vector<std::vector<real_t>> A, std::vector<std::vector<real_t>> B);
std::vector<std::vector<double>> hadamard_product(std::vector<std::vector<double>> A, std::vector<std::vector<double>> B); std::vector<std::vector<real_t>> hadamard_product(std::vector<std::vector<real_t>> A, std::vector<std::vector<real_t>> B);
std::vector<std::vector<double>> kronecker_product(std::vector<std::vector<double>> A, std::vector<std::vector<double>> B); std::vector<std::vector<real_t>> kronecker_product(std::vector<std::vector<real_t>> A, std::vector<std::vector<real_t>> B);
std::vector<std::vector<double>> elementWiseDivision(std::vector<std::vector<double>> A, std::vector<std::vector<double>> B); std::vector<std::vector<real_t>> elementWiseDivision(std::vector<std::vector<real_t>> A, std::vector<std::vector<real_t>> B);
std::vector<std::vector<double>> transpose(std::vector<std::vector<double>> A); std::vector<std::vector<real_t>> transpose(std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> scalarMultiply(double scalar, std::vector<std::vector<double>> A); std::vector<std::vector<real_t>> scalarMultiply(real_t scalar, std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> scalarAdd(double scalar, std::vector<std::vector<double>> A); std::vector<std::vector<real_t>> scalarAdd(real_t scalar, std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> log(std::vector<std::vector<double>> A); std::vector<std::vector<real_t>> log(std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> log10(std::vector<std::vector<double>> A); std::vector<std::vector<real_t>> log10(std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> exp(std::vector<std::vector<double>> A); std::vector<std::vector<real_t>> exp(std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> erf(std::vector<std::vector<double>> A); std::vector<std::vector<real_t>> erf(std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> exponentiate(std::vector<std::vector<double>> A, double p); std::vector<std::vector<real_t>> exponentiate(std::vector<std::vector<real_t>> A, real_t p);
std::vector<std::vector<double>> sqrt(std::vector<std::vector<double>> A); std::vector<std::vector<real_t>> sqrt(std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> cbrt(std::vector<std::vector<double>> A); std::vector<std::vector<real_t>> cbrt(std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> matrixPower(std::vector<std::vector<double>> A, int n); std::vector<std::vector<real_t>> matrixPower(std::vector<std::vector<real_t>> A, int n);
std::vector<std::vector<double>> abs(std::vector<std::vector<double>> A); std::vector<std::vector<real_t>> abs(std::vector<std::vector<real_t>> A);
double det(std::vector<std::vector<double>> A, int d); real_t det(std::vector<std::vector<real_t>> A, int d);
double trace(std::vector<std::vector<double>> A); real_t trace(std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> cofactor(std::vector<std::vector<double>> A, int n, int i, int j); std::vector<std::vector<real_t>> cofactor(std::vector<std::vector<real_t>> A, int n, int i, int j);
std::vector<std::vector<double>> adjoint(std::vector<std::vector<double>> A); std::vector<std::vector<real_t>> adjoint(std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> inverse(std::vector<std::vector<double>> A); std::vector<std::vector<real_t>> inverse(std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> pinverse(std::vector<std::vector<double>> A); std::vector<std::vector<real_t>> pinverse(std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> zeromat(int n, int m); std::vector<std::vector<real_t>> zeromat(int n, int m);
std::vector<std::vector<double>> onemat(int n, int m); std::vector<std::vector<real_t>> onemat(int n, int m);
std::vector<std::vector<double>> full(int n, int m, int k); std::vector<std::vector<real_t>> full(int n, int m, int k);
std::vector<std::vector<double>> sin(std::vector<std::vector<double>> A); std::vector<std::vector<real_t>> sin(std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> cos(std::vector<std::vector<double>> A); std::vector<std::vector<real_t>> cos(std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> rotate(std::vector<std::vector<double>> A, double theta, int axis = -1); std::vector<std::vector<real_t>> rotate(std::vector<std::vector<real_t>> A, real_t theta, int axis = -1);
std::vector<std::vector<double>> max(std::vector<std::vector<double>> A, std::vector<std::vector<double>> B); std::vector<std::vector<real_t>> max(std::vector<std::vector<real_t>> A, std::vector<std::vector<real_t>> B);
double max(std::vector<std::vector<double>> A); real_t max(std::vector<std::vector<real_t>> A);
double min(std::vector<std::vector<double>> A); real_t min(std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> round(std::vector<std::vector<double>> A); std::vector<std::vector<real_t>> round(std::vector<std::vector<real_t>> A);
double norm_2(std::vector<std::vector<double>> A); real_t norm_2(std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> identity(double d); std::vector<std::vector<real_t>> identity(real_t d);
std::vector<std::vector<double>> cov(std::vector<std::vector<double>> A); std::vector<std::vector<real_t>> cov(std::vector<std::vector<real_t>> A);
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> eig(std::vector<std::vector<double>> A); std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> eig(std::vector<std::vector<real_t>> A);
struct EigenResult { struct EigenResult {
std::vector<std::vector<double>> eigen_vectors; std::vector<std::vector<real_t>> eigen_vectors;
std::vector<std::vector<double>> eigen_values; std::vector<std::vector<real_t>> eigen_values;
}; };
EigenResult eigen(std::vector<std::vector<double>> A); EigenResult eigen(std::vector<std::vector<real_t>> A);
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>, std::vector<std::vector<double>>> SVD(std::vector<std::vector<double>> A); std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> SVD(std::vector<std::vector<real_t>> A);
struct SDVResult { struct SDVResult {
std::vector<std::vector<double>> U; std::vector<std::vector<real_t>> U;
std::vector<std::vector<double>> S; std::vector<std::vector<real_t>> S;
std::vector<std::vector<double>> Vt; std::vector<std::vector<real_t>> Vt;
}; };
SDVResult svd(std::vector<std::vector<double>> A); SDVResult svd(std::vector<std::vector<real_t>> A);
std::vector<double> vectorProjection(std::vector<double> a, std::vector<double> b); std::vector<real_t> vectorProjection(std::vector<real_t> a, std::vector<real_t> b);
std::vector<std::vector<double>> gramSchmidtProcess(std::vector<std::vector<double>> A); std::vector<std::vector<real_t>> gramSchmidtProcess(std::vector<std::vector<real_t>> A);
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> QRD(std::vector<std::vector<double>> A); std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> QRD(std::vector<std::vector<real_t>> A);
struct QRDResult { struct QRDResult {
std::vector<std::vector<double>> Q; std::vector<std::vector<real_t>> Q;
std::vector<std::vector<double>> R; std::vector<std::vector<real_t>> R;
}; };
QRDResult qrd(std::vector<std::vector<double>> A); QRDResult qrd(std::vector<std::vector<real_t>> A);
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> chol(std::vector<std::vector<double>> A); std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> chol(std::vector<std::vector<real_t>> A);
struct CholeskyResult { struct CholeskyResult {
std::vector<std::vector<double>> L; std::vector<std::vector<real_t>> L;
std::vector<std::vector<double>> Lt; std::vector<std::vector<real_t>> Lt;
}; };
CholeskyResult cholesky(std::vector<std::vector<double>> A); CholeskyResult cholesky(std::vector<std::vector<real_t>> A);
double sum_elements(std::vector<std::vector<double>> A); real_t sum_elements(std::vector<std::vector<real_t>> A);
std::vector<double> flatten(std::vector<std::vector<double>> A); std::vector<real_t> flatten(std::vector<std::vector<real_t>> A);
std::vector<double> solve(std::vector<std::vector<double>> A, std::vector<double> b); std::vector<real_t> solve(std::vector<std::vector<real_t>> A, std::vector<real_t> b);
bool positiveDefiniteChecker(std::vector<std::vector<double>> A); bool positiveDefiniteChecker(std::vector<std::vector<real_t>> A);
bool negativeDefiniteChecker(std::vector<std::vector<double>> A); bool negativeDefiniteChecker(std::vector<std::vector<real_t>> A);
bool zeroEigenvalue(std::vector<std::vector<double>> A); bool zeroEigenvalue(std::vector<std::vector<real_t>> A);
void printMatrix(std::vector<std::vector<double>> A); void printMatrix(std::vector<std::vector<real_t>> A);
// VECTOR FUNCTIONS // VECTOR FUNCTIONS
std::vector<std::vector<double>> outerProduct(std::vector<double> a, std::vector<double> b); // This multiplies a, bT std::vector<std::vector<real_t>> outerProduct(std::vector<real_t> a, std::vector<real_t> b); // This multiplies a, bT
std::vector<double> hadamard_product(std::vector<double> a, std::vector<double> b); std::vector<real_t> hadamard_product(std::vector<real_t> a, std::vector<real_t> b);
std::vector<double> elementWiseDivision(std::vector<double> a, std::vector<double> b); std::vector<real_t> elementWiseDivision(std::vector<real_t> a, std::vector<real_t> b);
std::vector<double> scalarMultiply(double scalar, std::vector<double> a); std::vector<real_t> scalarMultiply(real_t scalar, std::vector<real_t> a);
std::vector<double> scalarAdd(double scalar, std::vector<double> a); std::vector<real_t> scalarAdd(real_t scalar, std::vector<real_t> a);
std::vector<double> addition(std::vector<double> a, std::vector<double> b); std::vector<real_t> addition(std::vector<real_t> a, std::vector<real_t> b);
std::vector<double> subtraction(std::vector<double> a, std::vector<double> b); std::vector<real_t> subtraction(std::vector<real_t> a, std::vector<real_t> b);
std::vector<double> subtractMatrixRows(std::vector<double> a, std::vector<std::vector<double>> B); std::vector<real_t> subtractMatrixRows(std::vector<real_t> a, std::vector<std::vector<real_t>> B);
std::vector<double> log(std::vector<double> a); std::vector<real_t> log(std::vector<real_t> a);
std::vector<double> log10(std::vector<double> a); std::vector<real_t> log10(std::vector<real_t> a);
std::vector<double> exp(std::vector<double> a); std::vector<real_t> exp(std::vector<real_t> a);
std::vector<double> erf(std::vector<double> a); std::vector<real_t> erf(std::vector<real_t> a);
std::vector<double> exponentiate(std::vector<double> a, double p); std::vector<real_t> exponentiate(std::vector<real_t> a, real_t p);
std::vector<double> sqrt(std::vector<double> a); std::vector<real_t> sqrt(std::vector<real_t> a);
std::vector<double> cbrt(std::vector<double> a); std::vector<real_t> cbrt(std::vector<real_t> a);
double dot(std::vector<double> a, std::vector<double> b); real_t dot(std::vector<real_t> a, std::vector<real_t> b);
std::vector<double> cross(std::vector<double> a, std::vector<double> b); std::vector<real_t> cross(std::vector<real_t> a, std::vector<real_t> b);
std::vector<double> abs(std::vector<double> a); std::vector<real_t> abs(std::vector<real_t> a);
std::vector<double> zerovec(int n); std::vector<real_t> zerovec(int n);
std::vector<double> onevec(int n); std::vector<real_t> onevec(int n);
std::vector<std::vector<double>> diag(std::vector<double> a); std::vector<std::vector<real_t>> diag(std::vector<real_t> a);
std::vector<double> full(int n, int k); std::vector<real_t> full(int n, int k);
std::vector<double> sin(std::vector<double> a); std::vector<real_t> sin(std::vector<real_t> a);
std::vector<double> cos(std::vector<double> a); std::vector<real_t> cos(std::vector<real_t> a);
std::vector<double> max(std::vector<double> a, std::vector<double> b); std::vector<real_t> max(std::vector<real_t> a, std::vector<real_t> b);
double max(std::vector<double> a); real_t max(std::vector<real_t> a);
double min(std::vector<double> a); real_t min(std::vector<real_t> a);
std::vector<double> round(std::vector<double> a); std::vector<real_t> round(std::vector<real_t> a);
double euclideanDistance(std::vector<double> a, std::vector<double> b); real_t euclideanDistance(std::vector<real_t> a, std::vector<real_t> b);
double norm_2(std::vector<double> a); real_t norm_2(std::vector<real_t> a);
double norm_sq(std::vector<double> a); real_t norm_sq(std::vector<real_t> a);
double sum_elements(std::vector<double> a); real_t sum_elements(std::vector<real_t> a);
double cosineSimilarity(std::vector<double> a, std::vector<double> b); real_t cosineSimilarity(std::vector<real_t> a, std::vector<real_t> b);
void printVector(std::vector<double> a); void printVector(std::vector<real_t> a);
// MATRIX-VECTOR FUNCTIONS // MATRIX-VECTOR FUNCTIONS
std::vector<std::vector<double>> mat_vec_add(std::vector<std::vector<double>> A, std::vector<double> b); std::vector<std::vector<real_t>> mat_vec_add(std::vector<std::vector<real_t>> A, std::vector<real_t> b);
std::vector<double> mat_vec_mult(std::vector<std::vector<double>> A, std::vector<double> b); std::vector<real_t> mat_vec_mult(std::vector<std::vector<real_t>> A, std::vector<real_t> b);
// TENSOR FUNCTIONS // TENSOR FUNCTIONS
std::vector<std::vector<std::vector<double>>> addition(std::vector<std::vector<std::vector<double>>> A, std::vector<std::vector<std::vector<double>>> B); std::vector<std::vector<std::vector<real_t>>> addition(std::vector<std::vector<std::vector<real_t>>> A, std::vector<std::vector<std::vector<real_t>>> B);
std::vector<std::vector<std::vector<double>>> elementWiseDivision(std::vector<std::vector<std::vector<double>>> A, std::vector<std::vector<std::vector<double>>> B); std::vector<std::vector<std::vector<real_t>>> elementWiseDivision(std::vector<std::vector<std::vector<real_t>>> A, std::vector<std::vector<std::vector<real_t>>> B);
std::vector<std::vector<std::vector<double>>> sqrt(std::vector<std::vector<std::vector<double>>> A); std::vector<std::vector<std::vector<real_t>>> sqrt(std::vector<std::vector<std::vector<real_t>>> A);
std::vector<std::vector<std::vector<double>>> exponentiate(std::vector<std::vector<std::vector<double>>> A, double p); std::vector<std::vector<std::vector<real_t>>> exponentiate(std::vector<std::vector<std::vector<real_t>>> A, real_t p);
std::vector<std::vector<double>> tensor_vec_mult(std::vector<std::vector<std::vector<double>>> A, std::vector<double> b); std::vector<std::vector<real_t>> tensor_vec_mult(std::vector<std::vector<std::vector<real_t>>> A, std::vector<real_t> b);
std::vector<double> flatten(std::vector<std::vector<std::vector<double>>> A); std::vector<real_t> flatten(std::vector<std::vector<std::vector<real_t>>> A);
void printTensor(std::vector<std::vector<std::vector<double>>> A); void printTensor(std::vector<std::vector<std::vector<real_t>>> A);
std::vector<std::vector<std::vector<double>>> scalarMultiply(double scalar, std::vector<std::vector<std::vector<double>>> A); std::vector<std::vector<std::vector<real_t>>> scalarMultiply(real_t scalar, std::vector<std::vector<std::vector<real_t>>> A);
std::vector<std::vector<std::vector<double>>> scalarAdd(double scalar, std::vector<std::vector<std::vector<double>>> A); std::vector<std::vector<std::vector<real_t>>> scalarAdd(real_t scalar, std::vector<std::vector<std::vector<real_t>>> A);
std::vector<std::vector<std::vector<double>>> resize(std::vector<std::vector<std::vector<double>>> A, std::vector<std::vector<std::vector<double>>> B); std::vector<std::vector<std::vector<real_t>>> resize(std::vector<std::vector<std::vector<real_t>>> A, std::vector<std::vector<std::vector<real_t>>> B);
std::vector<std::vector<std::vector<double>>> hadamard_product(std::vector<std::vector<std::vector<double>>> A, std::vector<std::vector<std::vector<double>>> B); std::vector<std::vector<std::vector<real_t>>> hadamard_product(std::vector<std::vector<std::vector<real_t>>> A, std::vector<std::vector<std::vector<real_t>>> B);
std::vector<std::vector<std::vector<double>>> max(std::vector<std::vector<std::vector<double>>> A, std::vector<std::vector<std::vector<double>>> B); std::vector<std::vector<std::vector<real_t>>> max(std::vector<std::vector<std::vector<real_t>>> A, std::vector<std::vector<std::vector<real_t>>> B);
std::vector<std::vector<std::vector<double>>> abs(std::vector<std::vector<std::vector<double>>> A); std::vector<std::vector<std::vector<real_t>>> abs(std::vector<std::vector<std::vector<real_t>>> A);
double norm_2(std::vector<std::vector<std::vector<double>>> A); real_t norm_2(std::vector<std::vector<std::vector<real_t>>> A);
std::vector<std::vector<std::vector<double>>> vector_wise_tensor_product(std::vector<std::vector<std::vector<double>>> A, std::vector<std::vector<double>> B); std::vector<std::vector<std::vector<real_t>>> vector_wise_tensor_product(std::vector<std::vector<std::vector<real_t>>> A, std::vector<std::vector<real_t>> B);
private: private:
}; };

84
mlpp/lin_alg/mlpp_matrix.h
View File

@ -1,6 +1,8 @@
#ifndef MLPP_MATRIX_H #ifndef MLPP_MATRIX_H
#define MLPP_MATRIX_H #define MLPP_MATRIX_H
#include "core/math/math_defs.h"
#include "core/containers/pool_vector.h" #include "core/containers/pool_vector.h"
#include "core/containers/sort_array.h" #include "core/containers/sort_array.h"
#include "core/containers/vector.h" #include "core/containers/vector.h"
@ -19,15 +21,15 @@ class MLPPMatrix : public Reference {
GDCLASS(MLPPMatrix, Reference); GDCLASS(MLPPMatrix, Reference);
public: public:
double *ptr() { real_t *ptr() {
return _data; return _data;
} }
const double *ptr() const { const real_t *ptr() const {
return _data; return _data;
} }
_FORCE_INLINE_ void add_row(const Vector<double> &p_row) { _FORCE_INLINE_ void add_row(const Vector<real_t> &p_row) {
if (_size.x == 0) { if (_size.x == 0) {
_size.x = p_row.size(); _size.x = p_row.size();
} }
@ -38,10 +40,10 @@ public:
++_size.y; ++_size.y;
_data = (double *)memrealloc(_data, data_size() * sizeof(double)); _data = (real_t *)memrealloc(_data, data_size() * sizeof(real_t));
CRASH_COND_MSG(!_data, "Out of memory"); CRASH_COND_MSG(!_data, "Out of memory");
const double *row_arr = p_row.ptr(); const real_t *row_arr = p_row.ptr();
for (int i = 0; i < p_row.size(); ++i) { for (int i = 0; i < p_row.size(); ++i) {
_data[ci + i] = row_arr[i]; _data[ci + i] = row_arr[i];
@ -59,7 +61,7 @@ public:
++_size.y; ++_size.y;
_data = (double *)memrealloc(_data, data_size() * sizeof(double)); _data = (real_t *)memrealloc(_data, data_size() * sizeof(real_t));
CRASH_COND_MSG(!_data, "Out of memory"); CRASH_COND_MSG(!_data, "Out of memory");
PoolRealArray::Read rread = p_row.read(); PoolRealArray::Read rread = p_row.read();
@ -70,7 +72,7 @@ public:
} }
} }
void remove_row(double p_index) { void remove_row(real_t p_index) {
ERR_FAIL_INDEX(p_index, _size.y); ERR_FAIL_INDEX(p_index, _size.y);
--_size.y; --_size.y;
@ -87,7 +89,7 @@ public:
_data[i] = _data[i + _size.x]; _data[i] = _data[i + _size.x];
} }
_data = (double *)memrealloc(_data, data_size() * sizeof(double)); _data = (real_t *)memrealloc(_data, data_size() * sizeof(real_t));
CRASH_COND_MSG(!_data, "Out of memory"); CRASH_COND_MSG(!_data, "Out of memory");
} }
@ -113,7 +115,7 @@ public:
_data[i] = _data[ds + i]; _data[i] = _data[ds + i];
} }
_data = (double *)memrealloc(_data, data_size() * sizeof(double)); _data = (real_t *)memrealloc(_data, data_size() * sizeof(real_t));
CRASH_COND_MSG(!_data, "Out of memory"); CRASH_COND_MSG(!_data, "Out of memory");
} }
@ -154,26 +156,26 @@ public:
return; return;
} }
_data = (double *)memrealloc(_data, ds * sizeof(double)); _data = (real_t *)memrealloc(_data, ds * sizeof(real_t));
CRASH_COND_MSG(!_data, "Out of memory"); CRASH_COND_MSG(!_data, "Out of memory");
} }
_FORCE_INLINE_ const double &operator[](int p_index) const { _FORCE_INLINE_ const real_t &operator[](int p_index) const {
CRASH_BAD_INDEX(p_index, data_size()); CRASH_BAD_INDEX(p_index, data_size());
return _data[p_index]; return _data[p_index];
} }
_FORCE_INLINE_ double &operator[](int p_index) { _FORCE_INLINE_ real_t &operator[](int p_index) {
CRASH_BAD_INDEX(p_index, data_size()); CRASH_BAD_INDEX(p_index, data_size());
return _data[p_index]; return _data[p_index];
} }
_FORCE_INLINE_ double get_element(int p_index_x, int p_index_y) const { _FORCE_INLINE_ real_t get_element(int p_index_x, int p_index_y) const {
ERR_FAIL_INDEX_V(p_index_x, _size.x, 0); ERR_FAIL_INDEX_V(p_index_x, _size.x, 0);
ERR_FAIL_INDEX_V(p_index_y, _size.y, 0); ERR_FAIL_INDEX_V(p_index_y, _size.y, 0);
return _data[p_index_x * p_index_y]; return _data[p_index_x * p_index_y];
} }
_FORCE_INLINE_ double get_element(int p_index_x, int p_index_y) { _FORCE_INLINE_ real_t get_element(int p_index_x, int p_index_y) {
ERR_FAIL_INDEX_V(p_index_x, _size.x, 0); ERR_FAIL_INDEX_V(p_index_x, _size.x, 0);
ERR_FAIL_INDEX_V(p_index_y, _size.y, 0); ERR_FAIL_INDEX_V(p_index_y, _size.y, 0);
@ -187,7 +189,7 @@ public:
return static_cast<real_t>(_data[p_index_x * p_index_y]); return static_cast<real_t>(_data[p_index_x * p_index_y]);
} }
_FORCE_INLINE_ void set_element(int p_index_x, int p_index_y, double p_val) { _FORCE_INLINE_ void set_element(int p_index_x, int p_index_y, real_t p_val) {
ERR_FAIL_INDEX(p_index_x, _size.x); ERR_FAIL_INDEX(p_index_x, _size.x);
ERR_FAIL_INDEX(p_index_y, _size.y); ERR_FAIL_INDEX(p_index_y, _size.y);
@ -201,13 +203,13 @@ public:
_data[p_index_x * p_index_y] = p_val; _data[p_index_x * p_index_y] = p_val;
} }
_FORCE_INLINE_ void set_row_vector(int p_index_y, const Vector<double> &p_row) { _FORCE_INLINE_ void set_row_vector(int p_index_y, const Vector<real_t> &p_row) {
ERR_FAIL_COND(p_row.size() != _size.x); ERR_FAIL_COND(p_row.size() != _size.x);
ERR_FAIL_INDEX(p_index_y, _size.y); ERR_FAIL_INDEX(p_index_y, _size.y);
int ind_start = p_index_y * _size.x; int ind_start = p_index_y * _size.x;
const double *row_ptr = p_row.ptr(); const real_t *row_ptr = p_row.ptr();
for (int i = 0; i < _size.x; ++i) { for (int i = 0; i < _size.x; ++i) {
_data[ind_start + i] = row_ptr[i]; _data[ind_start + i] = row_ptr[i];
@ -228,18 +230,18 @@ public:
} }
} }
void fill(double p_val) { void fill(real_t p_val) {
int ds = data_size(); int ds = data_size();
for (int i = 0; i < ds; i++) { for (int i = 0; i < ds; i++) {
_data[i] = p_val; _data[i] = p_val;
} }
} }
Vector<double> to_flat_vector() const { Vector<real_t> to_flat_vector() const {
Vector<double> ret; Vector<real_t> ret;
ret.resize(data_size()); ret.resize(data_size());
double *w = ret.ptrw(); real_t *w = ret.ptrw();
memcpy(w, _data, sizeof(double) * data_size()); memcpy(w, _data, sizeof(real_t) * data_size());
return ret; return ret;
} }
@ -259,9 +261,9 @@ public:
Vector<uint8_t> to_flat_byte_array() const { Vector<uint8_t> to_flat_byte_array() const {
Vector<uint8_t> ret; Vector<uint8_t> ret;
ret.resize(data_size() * sizeof(double)); ret.resize(data_size() * sizeof(real_t));
uint8_t *w = ret.ptrw(); uint8_t *w = ret.ptrw();
memcpy(w, _data, sizeof(double) * data_size()); memcpy(w, _data, sizeof(real_t) * data_size());
return ret; return ret;
} }
@ -307,7 +309,7 @@ public:
int start_index = i * _size.x; int start_index = i * _size.x;
const double *from_ptr = r->ptr(); const real_t *from_ptr = r->ptr();
for (int j = 0; j < _size.x; j++) { for (int j = 0; j < _size.x; j++) {
_data[start_index + j] = from_ptr[j]; _data[start_index + j] = from_ptr[j];
} }
@ -342,14 +344,14 @@ public:
int start_index = i * _size.x; int start_index = i * _size.x;
const double *from_ptr = r->ptr(); const real_t *from_ptr = r->ptr();
for (int j = 0; j < _size.x; j++) { for (int j = 0; j < _size.x; j++) {
_data[start_index + j] = from_ptr[j]; _data[start_index + j] = from_ptr[j];
} }
} }
} }
_FORCE_INLINE_ void set_from_vectors(const Vector<Vector<double>> &p_from) { _FORCE_INLINE_ void set_from_vectors(const Vector<Vector<real_t>> &p_from) {
if (p_from.size() == 0) { if (p_from.size() == 0) {
reset(); reset();
return; return;
@ -363,13 +365,13 @@ public:
} }
for (int i = 0; i < p_from.size(); ++i) { for (int i = 0; i < p_from.size(); ++i) {
const Vector<double> &r = p_from[i]; const Vector<real_t> &r = p_from[i];
ERR_CONTINUE(r.size() != _size.x); ERR_CONTINUE(r.size() != _size.x);
int start_index = i * _size.x; int start_index = i * _size.x;
const double *from_ptr = r.ptr(); const real_t *from_ptr = r.ptr();
for (int j = 0; j < _size.x; j++) { for (int j = 0; j < _size.x; j++) {
_data[start_index + j] = from_ptr[j]; _data[start_index + j] = from_ptr[j];
} }
@ -420,7 +422,7 @@ public:
} }
} }
MLPPMatrix(const Vector<Vector<double>> &p_from) { MLPPMatrix(const Vector<Vector<real_t>> &p_from) {
_data = NULL; _data = NULL;
set_from_vectors(p_from); set_from_vectors(p_from);
@ -439,15 +441,15 @@ public:
} }
// TODO: These are temporary // TODO: These are temporary
std::vector<double> to_flat_std_vector() const { std::vector<real_t> to_flat_std_vector() const {
std::vector<double> ret; std::vector<real_t> ret;
ret.resize(data_size()); ret.resize(data_size());
double *w = &ret[0]; real_t *w = &ret[0];
memcpy(w, _data, sizeof(double) * data_size()); memcpy(w, _data, sizeof(real_t) * data_size());
return ret; return ret;
} }
_FORCE_INLINE_ void set_from_std_vectors(const std::vector<std::vector<double>> &p_from) { _FORCE_INLINE_ void set_from_std_vectors(const std::vector<std::vector<real_t>> &p_from) {
if (p_from.size() == 0) { if (p_from.size() == 0) {
reset(); reset();
return; return;
@ -461,33 +463,33 @@ public:
} }
for (uint32_t i = 0; i < p_from.size(); ++i) { for (uint32_t i = 0; i < p_from.size(); ++i) {
const std::vector<double> &r = p_from[i]; const std::vector<real_t> &r = p_from[i];
ERR_CONTINUE(r.size() != static_cast<uint32_t>(_size.x)); ERR_CONTINUE(r.size() != static_cast<uint32_t>(_size.x));
int start_index = i * _size.x; int start_index = i * _size.x;
const double *from_ptr = &r[0]; const real_t *from_ptr = &r[0];
for (int j = 0; j < _size.x; j++) { for (int j = 0; j < _size.x; j++) {
_data[start_index + j] = from_ptr[j]; _data[start_index + j] = from_ptr[j];
} }
} }
} }
_FORCE_INLINE_ void set_row_std_vector(int p_index_y, const std::vector<double> &p_row) { _FORCE_INLINE_ void set_row_std_vector(int p_index_y, const std::vector<real_t> &p_row) {
ERR_FAIL_COND(p_row.size() != static_cast<uint32_t>(_size.x)); ERR_FAIL_COND(p_row.size() != static_cast<uint32_t>(_size.x));
ERR_FAIL_INDEX(p_index_y, _size.y); ERR_FAIL_INDEX(p_index_y, _size.y);
int ind_start = p_index_y * _size.x; int ind_start = p_index_y * _size.x;
const double *row_ptr = &p_row[0]; const real_t *row_ptr = &p_row[0];
for (int i = 0; i < _size.x; ++i) { for (int i = 0; i < _size.x; ++i) {
_data[ind_start + i] = row_ptr[i]; _data[ind_start + i] = row_ptr[i];
} }
} }
MLPPMatrix(const std::vector<std::vector<double>> &p_from) { MLPPMatrix(const std::vector<std::vector<real_t>> &p_from) {
_data = NULL; _data = NULL;
set_from_std_vectors(p_from); set_from_std_vectors(p_from);
@ -498,7 +500,7 @@ protected:
protected: protected:
Size2i _size; Size2i _size;
double *_data; real_t *_data;
}; };
#endif #endif

74
mlpp/lin_alg/mlpp_vector.h
View File

@ -1,6 +1,8 @@
#ifndef MLPP_VECTOR_H #ifndef MLPP_VECTOR_H
#define MLPP_VECTOR_H #define MLPP_VECTOR_H
#include "core/math/math_defs.h"
#include "core/containers/pool_vector.h" #include "core/containers/pool_vector.h"
#include "core/containers/sort_array.h" #include "core/containers/sort_array.h"
#include "core/containers/vector.h" #include "core/containers/vector.h"
@ -16,24 +18,24 @@ class MLPPVector : public Reference {
GDCLASS(MLPPVector, Reference); GDCLASS(MLPPVector, Reference);
public: public:
double *ptr() { real_t *ptr() {
return _data; return _data;
} }
const double *ptr() const { const real_t *ptr() const {
return _data; return _data;
} }
_FORCE_INLINE_ void push_back(double p_elem) { _FORCE_INLINE_ void push_back(real_t p_elem) {
++_size; ++_size;
_data = (double *)memrealloc(_data, _size * sizeof(double)); _data = (real_t *)memrealloc(_data, _size * sizeof(real_t));
CRASH_COND_MSG(!_data, "Out of memory"); CRASH_COND_MSG(!_data, "Out of memory");
_data[_size - 1] = p_elem; _data[_size - 1] = p_elem;
} }
void remove(double p_index) { void remove(real_t p_index) {
ERR_FAIL_INDEX(p_index, _size); ERR_FAIL_INDEX(p_index, _size);
--_size; --_size;
@ -42,7 +44,7 @@ public:
_data[i] = _data[i + 1]; _data[i] = _data[i + 1];
} }
_data = (double *)memrealloc(_data, _size * sizeof(double)); _data = (real_t *)memrealloc(_data, _size * sizeof(real_t));
CRASH_COND_MSG(!_data, "Out of memory"); CRASH_COND_MSG(!_data, "Out of memory");
} }
@ -56,18 +58,18 @@ public:
_data[p_index] = _data[_size]; _data[p_index] = _data[_size];
} }
_data = (double *)memrealloc(_data, _size * sizeof(double)); _data = (real_t *)memrealloc(_data, _size * sizeof(real_t));
CRASH_COND_MSG(!_data, "Out of memory"); CRASH_COND_MSG(!_data, "Out of memory");
} }
void erase(const double &p_val) { void erase(const real_t &p_val) {
int idx = find(p_val); int idx = find(p_val);
if (idx >= 0) { if (idx >= 0) {
remove(idx); remove(idx);
} }
} }
int erase_multiple_unordered(const double &p_val) { int erase_multiple_unordered(const real_t &p_val) {
int from = 0; int from = 0;
int count = 0; int count = 0;
while (true) { while (true) {
@ -111,24 +113,24 @@ public:
return; return;
} }
_data = (double *)memrealloc(_data, _size * sizeof(double)); _data = (real_t *)memrealloc(_data, _size * sizeof(real_t));
CRASH_COND_MSG(!_data, "Out of memory"); CRASH_COND_MSG(!_data, "Out of memory");
} }
_FORCE_INLINE_ const double &operator[](int p_index) const { _FORCE_INLINE_ const real_t &operator[](int p_index) const {
CRASH_BAD_INDEX(p_index, _size); CRASH_BAD_INDEX(p_index, _size);
return _data[p_index]; return _data[p_index];
} }
_FORCE_INLINE_ double &operator[](int p_index) { _FORCE_INLINE_ real_t &operator[](int p_index) {
CRASH_BAD_INDEX(p_index, _size); CRASH_BAD_INDEX(p_index, _size);
return _data[p_index]; return _data[p_index];
} }
_FORCE_INLINE_ double get_element(int p_index) const { _FORCE_INLINE_ real_t get_element(int p_index) const {
ERR_FAIL_INDEX_V(p_index, _size, 0); ERR_FAIL_INDEX_V(p_index, _size, 0);
return _data[p_index]; return _data[p_index];
} }
_FORCE_INLINE_ double get_element(int p_index) { _FORCE_INLINE_ real_t get_element(int p_index) {
ERR_FAIL_INDEX_V(p_index, _size, 0); ERR_FAIL_INDEX_V(p_index, _size, 0);
return _data[p_index]; return _data[p_index];
} }
@ -138,7 +140,7 @@ public:
return static_cast<real_t>(_data[p_index]); return static_cast<real_t>(_data[p_index]);
} }
_FORCE_INLINE_ void set_element(int p_index, double p_val) { _FORCE_INLINE_ void set_element(int p_index, real_t p_val) {
ERR_FAIL_INDEX(p_index, _size); ERR_FAIL_INDEX(p_index, _size);
_data[p_index] = p_val; _data[p_index] = p_val;
} }
@ -148,13 +150,13 @@ public:
_data[p_index] = p_val; _data[p_index] = p_val;
} }
void fill(double p_val) { void fill(real_t p_val) {
for (int i = 0; i < _size; i++) { for (int i = 0; i < _size; i++) {
_data[i] = p_val; _data[i] = p_val;
} }
} }
void insert(int p_pos, double p_val) { void insert(int p_pos, real_t p_val) {
ERR_FAIL_INDEX(p_pos, _size + 1); ERR_FAIL_INDEX(p_pos, _size + 1);
if (p_pos == _size) { if (p_pos == _size) {
push_back(p_val); push_back(p_val);
@ -167,7 +169,7 @@ public:
} }
} }
int find(const double &p_val, int p_from = 0) const { int find(const real_t &p_val, int p_from = 0) const {
for (int i = p_from; i < _size; i++) { for (int i = p_from; i < _size; i++) {
if (_data[i] == p_val) { if (_data[i] == p_val) {
return i; return i;
@ -183,15 +185,15 @@ public:
return; return;
} }
SortArray<double, C> sorter; SortArray<real_t, C> sorter;
sorter.sort(_data, len); sorter.sort(_data, len);
} }
void sort() { void sort() {
sort_custom<_DefaultComparator<double>>(); sort_custom<_DefaultComparator<real_t>>();
} }
void ordered_insert(double p_val) { void ordered_insert(real_t p_val) {
int i; int i;
for (i = 0; i < _size; i++) { for (i = 0; i < _size; i++) {
if (p_val < _data[i]) { if (p_val < _data[i]) {
@ -201,11 +203,11 @@ public:
insert(i, p_val); insert(i, p_val);
} }
Vector<double> to_vector() const { Vector<real_t> to_vector() const {
Vector<double> ret; Vector<real_t> ret;
ret.resize(size()); ret.resize(size());
double *w = ret.ptrw(); real_t *w = ret.ptrw();
memcpy(w, _data, sizeof(double) * _size); memcpy(w, _data, sizeof(real_t) * _size);
return ret; return ret;
} }
@ -225,9 +227,9 @@ public:
Vector<uint8_t> to_byte_array() const { Vector<uint8_t> to_byte_array() const {
Vector<uint8_t> ret; Vector<uint8_t> ret;
ret.resize(_size * sizeof(double)); ret.resize(_size * sizeof(real_t));
uint8_t *w = ret.ptrw(); uint8_t *w = ret.ptrw();
memcpy(w, _data, sizeof(double) * _size); memcpy(w, _data, sizeof(real_t) * _size);
return ret; return ret;
} }
@ -255,7 +257,7 @@ public:
} }
} }
_FORCE_INLINE_ void set_from_vector(const Vector<double> &p_from) { _FORCE_INLINE_ void set_from_vector(const Vector<real_t> &p_from) {
resize(p_from.size()); resize(p_from.size());
for (int i = 0; i < _size; i++) { for (int i = 0; i < _size; i++) {
_data[i] = p_from[i]; _data[i] = p_from[i];
@ -286,7 +288,7 @@ public:
} }
} }
MLPPVector(const Vector<double> &p_from) { MLPPVector(const Vector<real_t> &p_from) {
_size = 0; _size = 0;
_data = NULL; _data = NULL;
@ -314,22 +316,22 @@ public:
} }
// TODO: These are temporary // TODO: These are temporary
std::vector<double> to_std_vector() const { std::vector<real_t> to_std_vector() const {
std::vector<double> ret; std::vector<real_t> ret;
ret.resize(size()); ret.resize(size());
double *w = &ret[0]; real_t *w = &ret[0];
memcpy(w, _data, sizeof(double) * _size); memcpy(w, _data, sizeof(real_t) * _size);
return ret; return ret;
} }
_FORCE_INLINE_ void set_from_std_vector(const std::vector<double> &p_from) { _FORCE_INLINE_ void set_from_std_vector(const std::vector<real_t> &p_from) {
resize(p_from.size()); resize(p_from.size());
for (int i = 0; i < _size; i++) { for (int i = 0; i < _size; i++) {
_data[i] = p_from[i]; _data[i] = p_from[i];
} }
} }
MLPPVector(const std::vector<double> &p_from) { MLPPVector(const std::vector<real_t> &p_from) {
_size = 0; _size = 0;
_data = NULL; _data = NULL;
@ -344,7 +346,7 @@ protected:
protected: protected:
int _size; int _size;
double *_data; real_t *_data;
}; };
#endif #endif

182
mlpp/lin_reg/lin_reg.cpp
View File

@ -15,7 +15,7 @@
#include <iostream> #include <iostream>
#include <random> #include <random>
MLPPLinReg::MLPPLinReg(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, std::string reg, double lambda, double alpha) : MLPPLinReg::MLPPLinReg(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, std::string reg, real_t lambda, real_t alpha) :
inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) { inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) {
y_hat.resize(n); y_hat.resize(n);
@ -23,28 +23,28 @@ MLPPLinReg::MLPPLinReg(std::vector<std::vector<double>> inputSet, std::vector<do
bias = MLPPUtilities::biasInitialization(); bias = MLPPUtilities::biasInitialization();
} }
std::vector<double> MLPPLinReg::modelSetTest(std::vector<std::vector<double>> X) { std::vector<real_t> MLPPLinReg::modelSetTest(std::vector<std::vector<real_t>> X) {
return Evaluate(X); return Evaluate(X);
} }
double MLPPLinReg::modelTest(std::vector<double> x) { real_t MLPPLinReg::modelTest(std::vector<real_t> x) {
return Evaluate(x); return Evaluate(x);
} }
void MLPPLinReg::NewtonRaphson(double learning_rate, int max_epoch, bool UI) { void MLPPLinReg::NewtonRaphson(real_t learning_rate, int max_epoch, bool UI) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
forwardPass(); forwardPass();
while (true) { while (true) {
cost_prev = Cost(y_hat, outputSet); cost_prev = Cost(y_hat, outputSet);
std::vector<double> error = alg.subtraction(y_hat, outputSet); std::vector<real_t> error = alg.subtraction(y_hat, outputSet);
// Calculating the weight gradients (2nd derivative) // Calculating the weight gradients (2nd derivative)
std::vector<double> first_derivative = alg.mat_vec_mult(alg.transpose(inputSet), error); std::vector<real_t> first_derivative = alg.mat_vec_mult(alg.transpose(inputSet), error);
std::vector<std::vector<double>> second_derivative = alg.matmult(alg.transpose(inputSet), inputSet); std::vector<std::vector<real_t>> second_derivative = alg.matmult(alg.transpose(inputSet), inputSet);
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(alg.inverse(second_derivative)), first_derivative))); weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(alg.inverse(second_derivative)), first_derivative)));
weights = regularization.regWeights(weights, lambda, alpha, reg); weights = regularization.regWeights(weights, lambda, alpha, reg);
@ -63,17 +63,17 @@ void MLPPLinReg::NewtonRaphson(double learning_rate, int max_epoch, bool UI) {
} }
} }
void MLPPLinReg::gradientDescent(double learning_rate, int max_epoch, bool UI) { void MLPPLinReg::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
forwardPass(); forwardPass();
while (true) { while (true) {
cost_prev = Cost(y_hat, outputSet); cost_prev = Cost(y_hat, outputSet);
std::vector<double> error = alg.subtraction(y_hat, outputSet); std::vector<real_t> error = alg.subtraction(y_hat, outputSet);
// Calculating the weight gradients // Calculating the weight gradients
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputSet), error))); weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputSet), error)));
@ -94,10 +94,10 @@ void MLPPLinReg::gradientDescent(double learning_rate, int max_epoch, bool UI) {
} }
} }
void MLPPLinReg::SGD(double learning_rate, int max_epoch, bool UI) { void MLPPLinReg::SGD(real_t learning_rate, int max_epoch, bool UI) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
while (true) { while (true) {
@ -106,10 +106,10 @@ void MLPPLinReg::SGD(double learning_rate, int max_epoch, bool UI) {
std::uniform_int_distribution<int> distribution(0, int(n - 1)); std::uniform_int_distribution<int> distribution(0, int(n - 1));
int outputIndex = distribution(generator); int outputIndex = distribution(generator);
double y_hat = Evaluate(inputSet[outputIndex]); real_t y_hat = Evaluate(inputSet[outputIndex]);
cost_prev = Cost({ y_hat }, { outputSet[outputIndex] }); cost_prev = Cost({ y_hat }, { outputSet[outputIndex] });
double error = y_hat - outputSet[outputIndex]; real_t error = y_hat - outputSet[outputIndex];
// Weight updation // Weight updation
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate * error, inputSet[outputIndex])); weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate * error, inputSet[outputIndex]));
@ -133,10 +133,10 @@ void MLPPLinReg::SGD(double learning_rate, int max_epoch, bool UI) {
forwardPass(); forwardPass();
} }
void MLPPLinReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI) { void MLPPLinReg::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
// Creating the mini-batches // Creating the mini-batches
@ -145,10 +145,10 @@ void MLPPLinReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size,
while (true) { while (true) {
for (int i = 0; i < n_mini_batch; i++) { for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = Evaluate(inputMiniBatches[i]); std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]); cost_prev = Cost(y_hat, outputMiniBatches[i]);
std::vector<double> error = alg.subtraction(y_hat, outputMiniBatches[i]); std::vector<real_t> error = alg.subtraction(y_hat, outputMiniBatches[i]);
// Calculating the weight gradients // Calculating the weight gradients
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error))); weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error)));
@ -171,10 +171,10 @@ void MLPPLinReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size,
forwardPass(); forwardPass();
} }
void MLPPLinReg::Momentum(double learning_rate, int max_epoch, int mini_batch_size, double gamma, bool UI) { void MLPPLinReg::Momentum(real_t learning_rate, int max_epoch, int mini_batch_size, real_t gamma, bool UI) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
// Creating the mini-batches // Creating the mini-batches
@ -182,18 +182,18 @@ void MLPPLinReg::Momentum(double learning_rate, int max_epoch, int mini_batch_si
auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch); auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch);
// Initializing necessary components for Momentum. // Initializing necessary components for Momentum.
std::vector<double> v = alg.zerovec(weights.size()); std::vector<real_t> v = alg.zerovec(weights.size());
while (true) { while (true) {
for (int i = 0; i < n_mini_batch; i++) { for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = Evaluate(inputMiniBatches[i]); std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]); cost_prev = Cost(y_hat, outputMiniBatches[i]);
std::vector<double> error = alg.subtraction(y_hat, outputMiniBatches[i]); std::vector<real_t> error = alg.subtraction(y_hat, outputMiniBatches[i]);
// Calculating the weight gradients // Calculating the weight gradients
std::vector<double> gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error)); std::vector<real_t> gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error));
std::vector<double> RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg); std::vector<real_t> RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg);
std::vector<double> weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final std::vector<real_t> weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final
v = alg.addition(alg.scalarMultiply(gamma, v), alg.scalarMultiply(learning_rate, weight_grad)); v = alg.addition(alg.scalarMultiply(gamma, v), alg.scalarMultiply(learning_rate, weight_grad));
@ -216,10 +216,10 @@ void MLPPLinReg::Momentum(double learning_rate, int max_epoch, int mini_batch_si
forwardPass(); forwardPass();
} }
void MLPPLinReg::NAG(double learning_rate, int max_epoch, int mini_batch_size, double gamma, bool UI) { void MLPPLinReg::NAG(real_t learning_rate, int max_epoch, int mini_batch_size, real_t gamma, bool UI) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
// Creating the mini-batches // Creating the mini-batches
@ -227,20 +227,20 @@ void MLPPLinReg::NAG(double learning_rate, int max_epoch, int mini_batch_size, d
auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch); auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch);
// Initializing necessary components for Momentum. // Initializing necessary components for Momentum.
std::vector<double> v = alg.zerovec(weights.size()); std::vector<real_t> v = alg.zerovec(weights.size());
while (true) { while (true) {
for (int i = 0; i < n_mini_batch; i++) { for (int i = 0; i < n_mini_batch; i++) {
weights = alg.subtraction(weights, alg.scalarMultiply(gamma, v)); // "Aposterori" calculation weights = alg.subtraction(weights, alg.scalarMultiply(gamma, v)); // "Aposterori" calculation
std::vector<double> y_hat = Evaluate(inputMiniBatches[i]); std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]); cost_prev = Cost(y_hat, outputMiniBatches[i]);
std::vector<double> error = alg.subtraction(y_hat, outputMiniBatches[i]); std::vector<real_t> error = alg.subtraction(y_hat, outputMiniBatches[i]);
// Calculating the weight gradients // Calculating the weight gradients
std::vector<double> gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error)); std::vector<real_t> gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error));
std::vector<double> RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg); std::vector<real_t> RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg);
std::vector<double> weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final std::vector<real_t> weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final
v = alg.addition(alg.scalarMultiply(gamma, v), alg.scalarMultiply(learning_rate, weight_grad)); v = alg.addition(alg.scalarMultiply(gamma, v), alg.scalarMultiply(learning_rate, weight_grad));
@ -263,10 +263,10 @@ void MLPPLinReg::NAG(double learning_rate, int max_epoch, int mini_batch_size, d
forwardPass(); forwardPass();
} }
void MLPPLinReg::Adagrad(double learning_rate, int max_epoch, int mini_batch_size, double e, bool UI) { void MLPPLinReg::Adagrad(real_t learning_rate, int max_epoch, int mini_batch_size, real_t e, bool UI) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
// Creating the mini-batches // Creating the mini-batches
@ -274,18 +274,18 @@ void MLPPLinReg::Adagrad(double learning_rate, int max_epoch, int mini_batch_siz
auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch); auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch);
// Initializing necessary components for Adagrad. // Initializing necessary components for Adagrad.
std::vector<double> v = alg.zerovec(weights.size()); std::vector<real_t> v = alg.zerovec(weights.size());
while (true) { while (true) {
for (int i = 0; i < n_mini_batch; i++) { for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = Evaluate(inputMiniBatches[i]); std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]); cost_prev = Cost(y_hat, outputMiniBatches[i]);
std::vector<double> error = alg.subtraction(y_hat, outputMiniBatches[i]); std::vector<real_t> error = alg.subtraction(y_hat, outputMiniBatches[i]);
// Calculating the weight gradients // Calculating the weight gradients
std::vector<double> gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error)); std::vector<real_t> gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error));
std::vector<double> RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg); std::vector<real_t> RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg);
std::vector<double> weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final std::vector<real_t> weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final
v = alg.hadamard_product(weight_grad, weight_grad); v = alg.hadamard_product(weight_grad, weight_grad);
@ -308,11 +308,11 @@ void MLPPLinReg::Adagrad(double learning_rate, int max_epoch, int mini_batch_siz
forwardPass(); forwardPass();
} }
void MLPPLinReg::Adadelta(double learning_rate, int max_epoch, int mini_batch_size, double b1, double e, bool UI) { void MLPPLinReg::Adadelta(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t e, bool UI) {
// Adagrad upgrade. Momentum is applied. // Adagrad upgrade. Momentum is applied.
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
// Creating the mini-batches // Creating the mini-batches
@ -320,18 +320,18 @@ void MLPPLinReg::Adadelta(double learning_rate, int max_epoch, int mini_batch_si
auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch); auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch);
// Initializing necessary components for Adagrad. // Initializing necessary components for Adagrad.
std::vector<double> v = alg.zerovec(weights.size()); std::vector<real_t> v = alg.zerovec(weights.size());
while (true) { while (true) {
for (int i = 0; i < n_mini_batch; i++) { for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = Evaluate(inputMiniBatches[i]); std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]); cost_prev = Cost(y_hat, outputMiniBatches[i]);
std::vector<double> error = alg.subtraction(y_hat, outputMiniBatches[i]); std::vector<real_t> error = alg.subtraction(y_hat, outputMiniBatches[i]);
// Calculating the weight gradients // Calculating the weight gradients
std::vector<double> gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error)); std::vector<real_t> gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error));
std::vector<double> RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg); std::vector<real_t> RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg);
std::vector<double> weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final std::vector<real_t> weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final
v = alg.addition(alg.scalarMultiply(b1, v), alg.scalarMultiply(1 - b1, alg.hadamard_product(weight_grad, weight_grad))); v = alg.addition(alg.scalarMultiply(b1, v), alg.scalarMultiply(1 - b1, alg.hadamard_product(weight_grad, weight_grad)));
@ -354,10 +354,10 @@ void MLPPLinReg::Adadelta(double learning_rate, int max_epoch, int mini_batch_si
forwardPass(); forwardPass();
} }
void MLPPLinReg::Adam(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI) { void MLPPLinReg::Adam(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t b2, real_t e, bool UI) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
// Creating the mini-batches // Creating the mini-batches
@ -365,26 +365,26 @@ void MLPPLinReg::Adam(double learning_rate, int max_epoch, int mini_batch_size,
auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch); auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch);
// Initializing necessary components for Adam. // Initializing necessary components for Adam.
std::vector<double> m = alg.zerovec(weights.size()); std::vector<real_t> m = alg.zerovec(weights.size());
std::vector<double> v = alg.zerovec(weights.size()); std::vector<real_t> v = alg.zerovec(weights.size());
while (true) { while (true) {
for (int i = 0; i < n_mini_batch; i++) { for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = Evaluate(inputMiniBatches[i]); std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]); cost_prev = Cost(y_hat, outputMiniBatches[i]);
std::vector<double> error = alg.subtraction(y_hat, outputMiniBatches[i]); std::vector<real_t> error = alg.subtraction(y_hat, outputMiniBatches[i]);
// Calculating the weight gradients // Calculating the weight gradients
std::vector<double> gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error)); std::vector<real_t> gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error));
std::vector<double> RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg); std::vector<real_t> RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg);
std::vector<double> weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final std::vector<real_t> weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final
m = alg.addition(alg.scalarMultiply(b1, m), alg.scalarMultiply(1 - b1, weight_grad)); m = alg.addition(alg.scalarMultiply(b1, m), alg.scalarMultiply(1 - b1, weight_grad));
v = alg.addition(alg.scalarMultiply(b2, v), alg.scalarMultiply(1 - b2, alg.exponentiate(weight_grad, 2))); v = alg.addition(alg.scalarMultiply(b2, v), alg.scalarMultiply(1 - b2, alg.exponentiate(weight_grad, 2)));
std::vector<double> m_hat = alg.scalarMultiply(1 / (1 - pow(b1, epoch)), m); std::vector<real_t> m_hat = alg.scalarMultiply(1 / (1 - pow(b1, epoch)), m);
std::vector<double> v_hat = alg.scalarMultiply(1 / (1 - pow(b2, epoch)), v); std::vector<real_t> v_hat = alg.scalarMultiply(1 / (1 - pow(b2, epoch)), v);
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate, alg.elementWiseDivision(m_hat, alg.scalarAdd(e, alg.sqrt(v_hat))))); weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate, alg.elementWiseDivision(m_hat, alg.scalarAdd(e, alg.sqrt(v_hat)))));
@ -405,35 +405,35 @@ void MLPPLinReg::Adam(double learning_rate, int max_epoch, int mini_batch_size,
forwardPass(); forwardPass();
} }
void MLPPLinReg::Adamax(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI) { void MLPPLinReg::Adamax(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t b2, real_t e, bool UI) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
// Creating the mini-batches // Creating the mini-batches
int n_mini_batch = n / mini_batch_size; int n_mini_batch = n / mini_batch_size;
auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch); auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch);
std::vector<double> m = alg.zerovec(weights.size()); std::vector<real_t> m = alg.zerovec(weights.size());
std::vector<double> u = alg.zerovec(weights.size()); std::vector<real_t> u = alg.zerovec(weights.size());
while (true) { while (true) {
for (int i = 0; i < n_mini_batch; i++) { for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = Evaluate(inputMiniBatches[i]); std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]); cost_prev = Cost(y_hat, outputMiniBatches[i]);
std::vector<double> error = alg.subtraction(y_hat, outputMiniBatches[i]); std::vector<real_t> error = alg.subtraction(y_hat, outputMiniBatches[i]);
// Calculating the weight gradients // Calculating the weight gradients
std::vector<double> gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error)); std::vector<real_t> gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error));
std::vector<double> RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg); std::vector<real_t> RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg);
std::vector<double> weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final std::vector<real_t> weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final
m = alg.addition(alg.scalarMultiply(b1, m), alg.scalarMultiply(1 - b1, weight_grad)); m = alg.addition(alg.scalarMultiply(b1, m), alg.scalarMultiply(1 - b1, weight_grad));
u = alg.max(alg.scalarMultiply(b2, u), alg.abs(weight_grad)); u = alg.max(alg.scalarMultiply(b2, u), alg.abs(weight_grad));
std::vector<double> m_hat = alg.scalarMultiply(1 / (1 - pow(b1, epoch)), m); std::vector<real_t> m_hat = alg.scalarMultiply(1 / (1 - pow(b1, epoch)), m);
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate, alg.elementWiseDivision(m_hat, u))); weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate, alg.elementWiseDivision(m_hat, u)));
@ -454,10 +454,10 @@ void MLPPLinReg::Adamax(double learning_rate, int max_epoch, int mini_batch_size
forwardPass(); forwardPass();
} }
void MLPPLinReg::Nadam(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI) { void MLPPLinReg::Nadam(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t b2, real_t e, bool UI) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
// Creating the mini-batches // Creating the mini-batches
@ -465,27 +465,27 @@ void MLPPLinReg::Nadam(double learning_rate, int max_epoch, int mini_batch_size,
auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch); auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch);
// Initializing necessary components for Adam. // Initializing necessary components for Adam.
std::vector<double> m = alg.zerovec(weights.size()); std::vector<real_t> m = alg.zerovec(weights.size());
std::vector<double> v = alg.zerovec(weights.size()); std::vector<real_t> v = alg.zerovec(weights.size());
std::vector<double> m_final = alg.zerovec(weights.size()); std::vector<real_t> m_final = alg.zerovec(weights.size());
while (true) { while (true) {
for (int i = 0; i < n_mini_batch; i++) { for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = Evaluate(inputMiniBatches[i]); std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]); cost_prev = Cost(y_hat, outputMiniBatches[i]);
std::vector<double> error = alg.subtraction(y_hat, outputMiniBatches[i]); std::vector<real_t> error = alg.subtraction(y_hat, outputMiniBatches[i]);
// Calculating the weight gradients // Calculating the weight gradients
std::vector<double> gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error)); std::vector<real_t> gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error));
std::vector<double> RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg); std::vector<real_t> RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg);
std::vector<double> weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final std::vector<real_t> weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final
m = alg.addition(alg.scalarMultiply(b1, m), alg.scalarMultiply(1 - b1, weight_grad)); m = alg.addition(alg.scalarMultiply(b1, m), alg.scalarMultiply(1 - b1, weight_grad));
v = alg.addition(alg.scalarMultiply(b2, v), alg.scalarMultiply(1 - b2, alg.exponentiate(weight_grad, 2))); v = alg.addition(alg.scalarMultiply(b2, v), alg.scalarMultiply(1 - b2, alg.exponentiate(weight_grad, 2)));
m_final = alg.addition(alg.scalarMultiply(b1, m), alg.scalarMultiply((1 - b1) / (1 - pow(b1, epoch)), weight_grad)); m_final = alg.addition(alg.scalarMultiply(b1, m), alg.scalarMultiply((1 - b1) / (1 - pow(b1, epoch)), weight_grad));
std::vector<double> m_hat = alg.scalarMultiply(1 / (1 - pow(b1, epoch)), m); std::vector<real_t> m_hat = alg.scalarMultiply(1 / (1 - pow(b1, epoch)), m);
std::vector<double> v_hat = alg.scalarMultiply(1 / (1 - pow(b2, epoch)), v); std::vector<real_t> v_hat = alg.scalarMultiply(1 / (1 - pow(b2, epoch)), v);
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate, alg.elementWiseDivision(m_final, alg.scalarAdd(e, alg.sqrt(v_hat))))); weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate, alg.elementWiseDivision(m_final, alg.scalarAdd(e, alg.sqrt(v_hat)))));
@ -509,8 +509,8 @@ void MLPPLinReg::Nadam(double learning_rate, int max_epoch, int mini_batch_size,
void MLPPLinReg::normalEquation() { void MLPPLinReg::normalEquation() {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPStat stat; MLPPStat stat;
std::vector<double> x_means; std::vector<real_t> x_means;
std::vector<std::vector<double>> inputSetT = alg.transpose(inputSet); std::vector<std::vector<real_t>> inputSetT = alg.transpose(inputSet);
x_means.resize(inputSetT.size()); x_means.resize(inputSetT.size());
for (int i = 0; i < inputSetT.size(); i++) { for (int i = 0; i < inputSetT.size(); i++) {
@ -518,7 +518,7 @@ void MLPPLinReg::normalEquation() {
} }
//try { //try {
std::vector<double> temp; std::vector<real_t> temp;
temp.resize(k); temp.resize(k);
temp = alg.mat_vec_mult(alg.inverse(alg.matmult(alg.transpose(inputSet), inputSet)), alg.mat_vec_mult(alg.transpose(inputSet), outputSet)); temp = alg.mat_vec_mult(alg.inverse(alg.matmult(alg.transpose(inputSet), inputSet)), alg.mat_vec_mult(alg.transpose(inputSet), outputSet));
if (std::isnan(temp[0])) { if (std::isnan(temp[0])) {
@ -542,7 +542,7 @@ void MLPPLinReg::normalEquation() {
//} //}
} }
double MLPPLinReg::score() { real_t MLPPLinReg::score() {
MLPPUtilities util; MLPPUtilities util;
return util.performance(y_hat, outputSet); return util.performance(y_hat, outputSet);
} }
@ -552,18 +552,18 @@ void MLPPLinReg::save(std::string fileName) {
util.saveParameters(fileName, weights, bias); util.saveParameters(fileName, weights, bias);
} }
double MLPPLinReg::Cost(std::vector<double> y_hat, std::vector<double> y) { real_t MLPPLinReg::Cost(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPReg regularization; MLPPReg regularization;
class MLPPCost cost; class MLPPCost cost;
return cost.MSE(y_hat, y) + regularization.regTerm(weights, lambda, alpha, reg); return cost.MSE(y_hat, y) + regularization.regTerm(weights, lambda, alpha, reg);
} }
std::vector<double> MLPPLinReg::Evaluate(std::vector<std::vector<double>> X) { std::vector<real_t> MLPPLinReg::Evaluate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.scalarAdd(bias, alg.mat_vec_mult(X, weights)); return alg.scalarAdd(bias, alg.mat_vec_mult(X, weights));
} }
double MLPPLinReg::Evaluate(std::vector<double> x) { real_t MLPPLinReg::Evaluate(std::vector<real_t> x) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.dot(weights, x) + bias; return alg.dot(weights, x) + bias;
} }

48
mlpp/lin_reg/lin_reg.h
View File

@ -8,43 +8,45 @@
// Created by Marc Melikyan on 10/2/20. // Created by Marc Melikyan on 10/2/20.
// //
#include "core/math/math_defs.h"
#include <string> #include <string>
#include <vector> #include <vector>
class MLPPLinReg { class MLPPLinReg {
public: public:
MLPPLinReg(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, std::string reg = "None", double lambda = 0.5, double alpha = 0.5); MLPPLinReg(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
std::vector<double> modelSetTest(std::vector<std::vector<double>> X); std::vector<real_t> modelSetTest(std::vector<std::vector<real_t>> X);
double modelTest(std::vector<double> x); real_t modelTest(std::vector<real_t> x);
void NewtonRaphson(double learning_rate, int max_epoch, bool UI); void NewtonRaphson(real_t learning_rate, int max_epoch, bool UI);
void gradientDescent(double learning_rate, int max_epoch, bool UI = 1); void gradientDescent(real_t learning_rate, int max_epoch, bool UI = 1);
void SGD(double learning_rate, int max_epoch, bool UI = 1); void SGD(real_t learning_rate, int max_epoch, bool UI = 1);
void Momentum(double learning_rate, int max_epoch, int mini_batch_size, double gamma, bool UI = 1); void Momentum(real_t learning_rate, int max_epoch, int mini_batch_size, real_t gamma, bool UI = 1);
void NAG(double learning_rate, int max_epoch, int mini_batch_size, double gamma, bool UI = 1); void NAG(real_t learning_rate, int max_epoch, int mini_batch_size, real_t gamma, bool UI = 1);
void Adagrad(double learning_rate, int max_epoch, int mini_batch_size, double e, bool UI = 1); void Adagrad(real_t learning_rate, int max_epoch, int mini_batch_size, real_t e, bool UI = 1);
void Adadelta(double learning_rate, int max_epoch, int mini_batch_size, double b1, double e, bool UI = 1); void Adadelta(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t e, bool UI = 1);
void Adam(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI = 1); void Adam(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t b2, real_t e, bool UI = 1);
void Adamax(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI = 1); void Adamax(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t b2, real_t e, bool UI = 1);
void Nadam(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI = 1); void Nadam(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t b2, real_t e, bool UI = 1);
void MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI = 1); void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
void normalEquation(); void normalEquation();
double score(); real_t score();
void save(std::string fileName); void save(std::string fileName);
private: private:
double Cost(std::vector<double> y_hat, std::vector<double> y); real_t Cost(std::vector<real_t> y_hat, std::vector<real_t> y);
std::vector<double> Evaluate(std::vector<std::vector<double>> X); std::vector<real_t> Evaluate(std::vector<std::vector<real_t>> X);
double Evaluate(std::vector<double> x); real_t Evaluate(std::vector<real_t> x);
void forwardPass(); void forwardPass();
std::vector<std::vector<double>> inputSet; std::vector<std::vector<real_t>> inputSet;
std::vector<double> outputSet; std::vector<real_t> outputSet;
std::vector<double> y_hat; std::vector<real_t> y_hat;
std::vector<double> weights; std::vector<real_t> weights;
double bias; real_t bias;
int n; int n;
int k; int k;

42
mlpp/log_reg/log_reg.cpp
View File

@ -15,32 +15,32 @@
#include <random> #include <random>
MLPPLogReg::MLPPLogReg(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, std::string reg, double lambda, double alpha) : MLPPLogReg::MLPPLogReg(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, std::string reg, real_t lambda, real_t alpha) :
inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) { inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) {
y_hat.resize(n); y_hat.resize(n);
weights = MLPPUtilities::weightInitialization(k); weights = MLPPUtilities::weightInitialization(k);
bias = MLPPUtilities::biasInitialization(); bias = MLPPUtilities::biasInitialization();
} }
std::vector<double> MLPPLogReg::modelSetTest(std::vector<std::vector<double>> X) { std::vector<real_t> MLPPLogReg::modelSetTest(std::vector<std::vector<real_t>> X) {
return Evaluate(X); return Evaluate(X);
} }
double MLPPLogReg::modelTest(std::vector<double> x) { real_t MLPPLogReg::modelTest(std::vector<real_t> x) {
return Evaluate(x); return Evaluate(x);
} }
void MLPPLogReg::gradientDescent(double learning_rate, int max_epoch, bool UI) { void MLPPLogReg::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
forwardPass(); forwardPass();
while (true) { while (true) {
cost_prev = Cost(y_hat, outputSet); cost_prev = Cost(y_hat, outputSet);
std::vector<double> error = alg.subtraction(y_hat, outputSet); std::vector<real_t> error = alg.subtraction(y_hat, outputSet);
// Calculating the weight gradients // Calculating the weight gradients
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputSet), error))); weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputSet), error)));
@ -62,17 +62,17 @@ void MLPPLogReg::gradientDescent(double learning_rate, int max_epoch, bool UI) {
} }
} }
void MLPPLogReg::MLE(double learning_rate, int max_epoch, bool UI) { void MLPPLogReg::MLE(real_t learning_rate, int max_epoch, bool UI) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
forwardPass(); forwardPass();
while (true) { while (true) {
cost_prev = Cost(y_hat, outputSet); cost_prev = Cost(y_hat, outputSet);
std::vector<double> error = alg.subtraction(outputSet, y_hat); std::vector<real_t> error = alg.subtraction(outputSet, y_hat);
// Calculating the weight gradients // Calculating the weight gradients
weights = alg.addition(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputSet), error))); weights = alg.addition(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputSet), error)));
@ -93,10 +93,10 @@ void MLPPLogReg::MLE(double learning_rate, int max_epoch, bool UI) {
} }
} }
void MLPPLogReg::SGD(double learning_rate, int max_epoch, bool UI) { void MLPPLogReg::SGD(real_t learning_rate, int max_epoch, bool UI) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
while (true) { while (true) {
@ -105,10 +105,10 @@ void MLPPLogReg::SGD(double learning_rate, int max_epoch, bool UI) {
std::uniform_int_distribution<int> distribution(0, int(n - 1)); std::uniform_int_distribution<int> distribution(0, int(n - 1));
int outputIndex = distribution(generator); int outputIndex = distribution(generator);
double y_hat = Evaluate(inputSet[outputIndex]); real_t y_hat = Evaluate(inputSet[outputIndex]);
cost_prev = Cost({ y_hat }, { outputSet[outputIndex] }); cost_prev = Cost({ y_hat }, { outputSet[outputIndex] });
double error = y_hat - outputSet[outputIndex]; real_t error = y_hat - outputSet[outputIndex];
// Weight updation // Weight updation
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate * error, inputSet[outputIndex])); weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate * error, inputSet[outputIndex]));
@ -132,10 +132,10 @@ void MLPPLogReg::SGD(double learning_rate, int max_epoch, bool UI) {
forwardPass(); forwardPass();
} }
void MLPPLogReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI) { void MLPPLogReg::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
// Creating the mini-batches // Creating the mini-batches
@ -144,10 +144,10 @@ void MLPPLogReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size,
while (true) { while (true) {
for (int i = 0; i < n_mini_batch; i++) { for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = Evaluate(inputMiniBatches[i]); std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]); cost_prev = Cost(y_hat, outputMiniBatches[i]);
std::vector<double> error = alg.subtraction(y_hat, outputMiniBatches[i]); std::vector<real_t> error = alg.subtraction(y_hat, outputMiniBatches[i]);
// Calculating the weight gradients // Calculating the weight gradients
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error))); weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error)));
@ -170,7 +170,7 @@ void MLPPLogReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size,
forwardPass(); forwardPass();
} }
double MLPPLogReg::score() { real_t MLPPLogReg::score() {
MLPPUtilities util; MLPPUtilities util;
return util.performance(y_hat, outputSet); return util.performance(y_hat, outputSet);
} }
@ -180,19 +180,19 @@ void MLPPLogReg::save(std::string fileName) {
util.saveParameters(fileName, weights, bias); util.saveParameters(fileName, weights, bias);
} }
double MLPPLogReg::Cost(std::vector<double> y_hat, std::vector<double> y) { real_t MLPPLogReg::Cost(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPReg regularization; MLPPReg regularization;
class MLPPCost cost; class MLPPCost cost;
return cost.LogLoss(y_hat, y) + regularization.regTerm(weights, lambda, alpha, reg); return cost.LogLoss(y_hat, y) + regularization.regTerm(weights, lambda, alpha, reg);
} }
std::vector<double> MLPPLogReg::Evaluate(std::vector<std::vector<double>> X) { std::vector<real_t> MLPPLogReg::Evaluate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPActivation avn; MLPPActivation avn;
return avn.sigmoid(alg.scalarAdd(bias, alg.mat_vec_mult(X, weights))); return avn.sigmoid(alg.scalarAdd(bias, alg.mat_vec_mult(X, weights)));
} }
double MLPPLogReg::Evaluate(std::vector<double> x) { real_t MLPPLogReg::Evaluate(std::vector<real_t> x) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPActivation avn; MLPPActivation avn;
return avn.sigmoid(alg.dot(weights, x) + bias); return avn.sigmoid(alg.dot(weights, x) + bias);

40
mlpp/log_reg/log_reg.h
View File

@ -8,6 +8,8 @@
// Created by Marc Melikyan on 10/2/20. // Created by Marc Melikyan on 10/2/20.
// //
#include "core/math/math_defs.h"
#include <string> #include <string>
#include <vector> #include <vector>
@ -15,37 +17,37 @@
class MLPPLogReg { class MLPPLogReg {
public: public:
MLPPLogReg(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, std::string reg = "None", double lambda = 0.5, double alpha = 0.5); MLPPLogReg(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
std::vector<double> modelSetTest(std::vector<std::vector<double>> X); std::vector<real_t> modelSetTest(std::vector<std::vector<real_t>> X);
double modelTest(std::vector<double> x); real_t modelTest(std::vector<real_t> x);
void gradientDescent(double learning_rate, int max_epoch, bool UI = 1); void gradientDescent(real_t learning_rate, int max_epoch, bool UI = 1);
void MLE(double learning_rate, int max_epoch, bool UI = 1); void MLE(real_t learning_rate, int max_epoch, bool UI = 1);
void SGD(double learning_rate, int max_epoch, bool UI = 1); void SGD(real_t learning_rate, int max_epoch, bool UI = 1);
void MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI = 1); void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
double score(); real_t score();
void save(std::string fileName); void save(std::string fileName);
private: private:
double Cost(std::vector<double> y_hat, std::vector<double> y); real_t Cost(std::vector<real_t> y_hat, std::vector<real_t> y);
std::vector<double> Evaluate(std::vector<std::vector<double>> X); std::vector<real_t> Evaluate(std::vector<std::vector<real_t>> X);
double Evaluate(std::vector<double> x); real_t Evaluate(std::vector<real_t> x);
void forwardPass(); void forwardPass();
std::vector<std::vector<double>> inputSet; std::vector<std::vector<real_t>> inputSet;
std::vector<double> outputSet; std::vector<real_t> outputSet;
std::vector<double> y_hat; std::vector<real_t> y_hat;
std::vector<double> weights; std::vector<real_t> weights;
double bias; real_t bias;
int n; int n;
int k; int k;
double learning_rate; real_t learning_rate;
// Regularization Params // Regularization Params
std::string reg; std::string reg;
double lambda; /* Regularization Parameter */ real_t lambda; /* Regularization Parameter */
double alpha; /* This is the controlling param for Elastic Net*/ real_t alpha; /* This is the controlling param for Elastic Net*/
}; };

26
mlpp/mann/mann.cpp
View File

@ -14,7 +14,7 @@
#include <iostream> #include <iostream>
MLPPMANN::MLPPMANN(std::vector<std::vector<double>> inputSet, std::vector<std::vector<double>> outputSet) : MLPPMANN::MLPPMANN(std::vector<std::vector<real_t>> inputSet, std::vector<std::vector<real_t>> outputSet) :
inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), n_output(outputSet[0].size()) { inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), n_output(outputSet[0].size()) {
} }
@ -22,7 +22,7 @@ MLPPMANN::~MLPPMANN() {
delete outputLayer; delete outputLayer;
} }
std::vector<std::vector<double>> MLPPMANN::modelSetTest(std::vector<std::vector<double>> X) { std::vector<std::vector<real_t>> MLPPMANN::modelSetTest(std::vector<std::vector<real_t>> X) {
if (!network.empty()) { if (!network.empty()) {
network[0].input = X; network[0].input = X;
network[0].forwardPass(); network[0].forwardPass();
@ -39,7 +39,7 @@ std::vector<std::vector<double>> MLPPMANN::modelSetTest(std::vector<std::vector<
return outputLayer->a; return outputLayer->a;
} }
std::vector<double> MLPPMANN::modelTest(std::vector<double> x) { std::vector<real_t> MLPPMANN::modelTest(std::vector<real_t> x) {
if (!network.empty()) { if (!network.empty()) {
network[0].Test(x); network[0].Test(x);
for (int i = 1; i < network.size(); i++) { for (int i = 1; i < network.size(); i++) {
@ -52,13 +52,13 @@ std::vector<double> MLPPMANN::modelTest(std::vector<double> x) {
return outputLayer->a_test; return outputLayer->a_test;
} }
void MLPPMANN::gradientDescent(double learning_rate, int max_epoch, bool UI) { void MLPPMANN::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
class MLPPCost cost; class MLPPCost cost;
MLPPActivation avn; MLPPActivation avn;
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
forwardPass(); forwardPass();
@ -73,7 +73,7 @@ void MLPPMANN::gradientDescent(double learning_rate, int max_epoch, bool UI) {
outputLayer->delta = alg.hadamard_product((cost.*costDeriv)(y_hat, outputSet), (avn.*outputAvn)(outputLayer->z, 1)); outputLayer->delta = alg.hadamard_product((cost.*costDeriv)(y_hat, outputSet), (avn.*outputAvn)(outputLayer->z, 1));
} }
std::vector<std::vector<double>> outputWGrad = alg.matmult(alg.transpose(outputLayer->input), outputLayer->delta); std::vector<std::vector<real_t>> outputWGrad = alg.matmult(alg.transpose(outputLayer->input), outputLayer->delta);
outputLayer->weights = alg.subtraction(outputLayer->weights, alg.scalarMultiply(learning_rate / n, outputWGrad)); outputLayer->weights = alg.subtraction(outputLayer->weights, alg.scalarMultiply(learning_rate / n, outputWGrad));
outputLayer->weights = regularization.regWeights(outputLayer->weights, outputLayer->lambda, outputLayer->alpha, outputLayer->reg); outputLayer->weights = regularization.regWeights(outputLayer->weights, outputLayer->lambda, outputLayer->alpha, outputLayer->reg);
@ -82,7 +82,7 @@ void MLPPMANN::gradientDescent(double learning_rate, int max_epoch, bool UI) {
if (!network.empty()) { if (!network.empty()) {
auto hiddenLayerAvn = network[network.size() - 1].activation_map[network[network.size() - 1].activation]; auto hiddenLayerAvn = network[network.size() - 1].activation_map[network[network.size() - 1].activation];
network[network.size() - 1].delta = alg.hadamard_product(alg.matmult(outputLayer->delta, alg.transpose(outputLayer->weights)), (avn.*hiddenLayerAvn)(network[network.size() - 1].z, 1)); network[network.size() - 1].delta = alg.hadamard_product(alg.matmult(outputLayer->delta, alg.transpose(outputLayer->weights)), (avn.*hiddenLayerAvn)(network[network.size() - 1].z, 1));
std::vector<std::vector<double>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[network.size() - 1].input), network[network.size() - 1].delta); std::vector<std::vector<real_t>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[network.size() - 1].input), network[network.size() - 1].delta);
network[network.size() - 1].weights = alg.subtraction(network[network.size() - 1].weights, alg.scalarMultiply(learning_rate / n, hiddenLayerWGrad)); network[network.size() - 1].weights = alg.subtraction(network[network.size() - 1].weights, alg.scalarMultiply(learning_rate / n, hiddenLayerWGrad));
network[network.size() - 1].weights = regularization.regWeights(network[network.size() - 1].weights, network[network.size() - 1].lambda, network[network.size() - 1].alpha, network[network.size() - 1].reg); network[network.size() - 1].weights = regularization.regWeights(network[network.size() - 1].weights, network[network.size() - 1].lambda, network[network.size() - 1].alpha, network[network.size() - 1].reg);
@ -91,7 +91,7 @@ void MLPPMANN::gradientDescent(double learning_rate, int max_epoch, bool UI) {
for (int i = network.size() - 2; i >= 0; i--) { for (int i = network.size() - 2; i >= 0; i--) {
auto hiddenLayerAvn = network[i].activation_map[network[i].activation]; auto hiddenLayerAvn = network[i].activation_map[network[i].activation];
network[i].delta = alg.hadamard_product(alg.matmult(network[i + 1].delta, network[i + 1].weights), (avn.*hiddenLayerAvn)(network[i].z, 1)); network[i].delta = alg.hadamard_product(alg.matmult(network[i + 1].delta, network[i + 1].weights), (avn.*hiddenLayerAvn)(network[i].z, 1));
std::vector<std::vector<double>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[i].input), network[i].delta); std::vector<std::vector<real_t>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[i].input), network[i].delta);
network[i].weights = alg.subtraction(network[i].weights, alg.scalarMultiply(learning_rate / n, hiddenLayerWGrad)); network[i].weights = alg.subtraction(network[i].weights, alg.scalarMultiply(learning_rate / n, hiddenLayerWGrad));
network[i].weights = regularization.regWeights(network[i].weights, network[i].lambda, network[i].alpha, network[i].reg); network[i].weights = regularization.regWeights(network[i].weights, network[i].lambda, network[i].alpha, network[i].reg);
network[i].bias = alg.subtractMatrixRows(network[i].bias, alg.scalarMultiply(learning_rate / n, network[i].delta)); network[i].bias = alg.subtractMatrixRows(network[i].bias, alg.scalarMultiply(learning_rate / n, network[i].delta));
@ -120,7 +120,7 @@ void MLPPMANN::gradientDescent(double learning_rate, int max_epoch, bool UI) {
} }
} }
double MLPPMANN::score() { real_t MLPPMANN::score() {
MLPPUtilities util; MLPPUtilities util;
forwardPass(); forwardPass();
return util.performance(y_hat, outputSet); return util.performance(y_hat, outputSet);
@ -139,7 +139,7 @@ void MLPPMANN::save(std::string fileName) {
} }
} }
void MLPPMANN::addLayer(int n_hidden, std::string activation, std::string weightInit, std::string reg, double lambda, double alpha) { void MLPPMANN::addLayer(int n_hidden, std::string activation, std::string weightInit, std::string reg, real_t lambda, real_t alpha) {
if (network.empty()) { if (network.empty()) {
network.push_back(MLPPHiddenLayer(n_hidden, activation, inputSet, weightInit, reg, lambda, alpha)); network.push_back(MLPPHiddenLayer(n_hidden, activation, inputSet, weightInit, reg, lambda, alpha));
network[0].forwardPass(); network[0].forwardPass();
@ -149,7 +149,7 @@ void MLPPMANN::addLayer(int n_hidden, std::string activation, std::string weight
} }
} }
void MLPPMANN::addOutputLayer(std::string activation, std::string loss, std::string weightInit, std::string reg, double lambda, double alpha) { void MLPPMANN::addOutputLayer(std::string activation, std::string loss, std::string weightInit, std::string reg, real_t lambda, real_t alpha) {
if (!network.empty()) { if (!network.empty()) {
outputLayer = new MLPPMultiOutputLayer(n_output, network[0].n_hidden, activation, loss, network[network.size() - 1].a, weightInit, reg, lambda, alpha); outputLayer = new MLPPMultiOutputLayer(n_output, network[0].n_hidden, activation, loss, network[network.size() - 1].a, weightInit, reg, lambda, alpha);
} else { } else {
@ -157,10 +157,10 @@ void MLPPMANN::addOutputLayer(std::string activation, std::string loss, std::str
} }
} }
double MLPPMANN::Cost(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) { real_t MLPPMANN::Cost(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
MLPPReg regularization; MLPPReg regularization;
class MLPPCost cost; class MLPPCost cost;
double totalRegTerm = 0; real_t totalRegTerm = 0;
auto cost_function = outputLayer->cost_map[outputLayer->cost]; auto cost_function = outputLayer->cost_map[outputLayer->cost];
if (!network.empty()) { if (!network.empty()) {

24
mlpp/mann/mann.h
View File

@ -8,6 +8,8 @@
// Created by Marc Melikyan on 11/4/20. // Created by Marc Melikyan on 11/4/20.
// //
#include "core/math/math_defs.h"
#include "../hidden_layer/hidden_layer.h" #include "../hidden_layer/hidden_layer.h"
#include "../multi_output_layer/multi_output_layer.h" #include "../multi_output_layer/multi_output_layer.h"
@ -18,24 +20,24 @@
class MLPPMANN { class MLPPMANN {
public: public:
MLPPMANN(std::vector<std::vector<double>> inputSet, std::vector<std::vector<double>> outputSet); MLPPMANN(std::vector<std::vector<real_t>> inputSet, std::vector<std::vector<real_t>> outputSet);
~MLPPMANN(); ~MLPPMANN();
std::vector<std::vector<double>> modelSetTest(std::vector<std::vector<double>> X); std::vector<std::vector<real_t>> modelSetTest(std::vector<std::vector<real_t>> X);
std::vector<double> modelTest(std::vector<double> x); std::vector<real_t> modelTest(std::vector<real_t> x);
void gradientDescent(double learning_rate, int max_epoch, bool UI = 1); void gradientDescent(real_t learning_rate, int max_epoch, bool UI = 1);
double score(); real_t score();
void save(std::string fileName); void save(std::string fileName);
void addLayer(int n_hidden, std::string activation, std::string weightInit = "Default", std::string reg = "None", double lambda = 0.5, double alpha = 0.5); void addLayer(int n_hidden, std::string activation, std::string weightInit = "Default", std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
void addOutputLayer(std::string activation, std::string loss, std::string weightInit = "Default", std::string reg = "None", double lambda = 0.5, double alpha = 0.5); void addOutputLayer(std::string activation, std::string loss, std::string weightInit = "Default", std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
private: private:
double Cost(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y); real_t Cost(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
void forwardPass(); void forwardPass();
std::vector<std::vector<double>> inputSet; std::vector<std::vector<real_t>> inputSet;
std::vector<std::vector<double>> outputSet; std::vector<std::vector<real_t>> outputSet;
std::vector<std::vector<double>> y_hat; std::vector<std::vector<real_t>> y_hat;
std::vector<MLPPHiddenLayer> network; std::vector<MLPPHiddenLayer> network;
MLPPMultiOutputLayer *outputLayer; MLPPMultiOutputLayer *outputLayer;

82
mlpp/mlp/mlp.cpp
View File

@ -16,7 +16,7 @@
#include <random> #include <random>
MLPPMLP::MLPPMLP(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, int n_hidden, std::string reg, double lambda, double alpha) : MLPPMLP::MLPPMLP(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, int n_hidden, std::string reg, real_t lambda, real_t alpha) :
inputSet(inputSet), outputSet(outputSet), n_hidden(n_hidden), n(inputSet.size()), k(inputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) { inputSet(inputSet), outputSet(outputSet), n_hidden(n_hidden), n(inputSet.size()), k(inputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) {
MLPPActivation avn; MLPPActivation avn;
y_hat.resize(n); y_hat.resize(n);
@ -27,19 +27,19 @@ MLPPMLP::MLPPMLP(std::vector<std::vector<double>> inputSet, std::vector<double>
bias2 = MLPPUtilities::biasInitialization(); bias2 = MLPPUtilities::biasInitialization();
} }
std::vector<double> MLPPMLP::modelSetTest(std::vector<std::vector<double>> X) { std::vector<real_t> MLPPMLP::modelSetTest(std::vector<std::vector<real_t>> X) {
return Evaluate(X); return Evaluate(X);
} }
double MLPPMLP::modelTest(std::vector<double> x) { real_t MLPPMLP::modelTest(std::vector<real_t> x) {
return Evaluate(x); return Evaluate(x);
} }
void MLPPMLP::gradientDescent(double learning_rate, int max_epoch, bool UI) { void MLPPMLP::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
MLPPActivation avn; MLPPActivation avn;
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
forwardPass(); forwardPass();
@ -47,11 +47,11 @@ void MLPPMLP::gradientDescent(double learning_rate, int max_epoch, bool UI) {
cost_prev = Cost(y_hat, outputSet); cost_prev = Cost(y_hat, outputSet);
// Calculating the errors // Calculating the errors
std::vector<double> error = alg.subtraction(y_hat, outputSet); std::vector<real_t> error = alg.subtraction(y_hat, outputSet);
// Calculating the weight/bias gradients for layer 2 // Calculating the weight/bias gradients for layer 2
std::vector<double> D2_1 = alg.mat_vec_mult(alg.transpose(a2), error); std::vector<real_t> D2_1 = alg.mat_vec_mult(alg.transpose(a2), error);
// weights and bias updation for layer 2 // weights and bias updation for layer 2
weights2 = alg.subtraction(weights2, alg.scalarMultiply(learning_rate / n, D2_1)); weights2 = alg.subtraction(weights2, alg.scalarMultiply(learning_rate / n, D2_1));
@ -61,14 +61,14 @@ void MLPPMLP::gradientDescent(double learning_rate, int max_epoch, bool UI) {
// Calculating the weight/bias for layer 1 // Calculating the weight/bias for layer 1
std::vector<std::vector<double>> D1_1; std::vector<std::vector<real_t>> D1_1;
D1_1.resize(n); D1_1.resize(n);
D1_1 = alg.outerProduct(error, weights2); D1_1 = alg.outerProduct(error, weights2);
std::vector<std::vector<double>> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1)); std::vector<std::vector<real_t>> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1));
std::vector<std::vector<double>> D1_3 = alg.matmult(alg.transpose(inputSet), D1_2); std::vector<std::vector<real_t>> D1_3 = alg.matmult(alg.transpose(inputSet), D1_2);
// weight an bias updation for layer 1 // weight an bias updation for layer 1
weights1 = alg.subtraction(weights1, alg.scalarMultiply(learning_rate / n, D1_3)); weights1 = alg.subtraction(weights1, alg.scalarMultiply(learning_rate / n, D1_3));
@ -94,11 +94,11 @@ void MLPPMLP::gradientDescent(double learning_rate, int max_epoch, bool UI) {
} }
} }
void MLPPMLP::SGD(double learning_rate, int max_epoch, bool UI) { void MLPPMLP::SGD(real_t learning_rate, int max_epoch, bool UI) {
MLPPActivation avn; MLPPActivation avn;
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
while (true) { while (true) {
@ -107,13 +107,13 @@ void MLPPMLP::SGD(double learning_rate, int max_epoch, bool UI) {
std::uniform_int_distribution<int> distribution(0, int(n - 1)); std::uniform_int_distribution<int> distribution(0, int(n - 1));
int outputIndex = distribution(generator); int outputIndex = distribution(generator);
double y_hat = Evaluate(inputSet[outputIndex]); real_t y_hat = Evaluate(inputSet[outputIndex]);
auto [z2, a2] = propagate(inputSet[outputIndex]); auto [z2, a2] = propagate(inputSet[outputIndex]);
cost_prev = Cost({ y_hat }, { outputSet[outputIndex] }); cost_prev = Cost({ y_hat }, { outputSet[outputIndex] });
double error = y_hat - outputSet[outputIndex]; real_t error = y_hat - outputSet[outputIndex];
// Weight updation for layer 2 // Weight updation for layer 2
std::vector<double> D2_1 = alg.scalarMultiply(error, a2); std::vector<real_t> D2_1 = alg.scalarMultiply(error, a2);
weights2 = alg.subtraction(weights2, alg.scalarMultiply(learning_rate, D2_1)); weights2 = alg.subtraction(weights2, alg.scalarMultiply(learning_rate, D2_1));
weights2 = regularization.regWeights(weights2, lambda, alpha, reg); weights2 = regularization.regWeights(weights2, lambda, alpha, reg);
@ -121,9 +121,9 @@ void MLPPMLP::SGD(double learning_rate, int max_epoch, bool UI) {
bias2 -= learning_rate * error; bias2 -= learning_rate * error;
// Weight updation for layer 1 // Weight updation for layer 1
std::vector<double> D1_1 = alg.scalarMultiply(error, weights2); std::vector<real_t> D1_1 = alg.scalarMultiply(error, weights2);
std::vector<double> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1)); std::vector<real_t> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1));
std::vector<std::vector<double>> D1_3 = alg.outerProduct(inputSet[outputIndex], D1_2); std::vector<std::vector<real_t>> D1_3 = alg.outerProduct(inputSet[outputIndex], D1_2);
weights1 = alg.subtraction(weights1, alg.scalarMultiply(learning_rate, D1_3)); weights1 = alg.subtraction(weights1, alg.scalarMultiply(learning_rate, D1_3));
weights1 = regularization.regWeights(weights1, lambda, alpha, reg); weights1 = regularization.regWeights(weights1, lambda, alpha, reg);
@ -148,11 +148,11 @@ void MLPPMLP::SGD(double learning_rate, int max_epoch, bool UI) {
forwardPass(); forwardPass();
} }
void MLPPMLP::MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI) { void MLPPMLP::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI) {
MLPPActivation avn; MLPPActivation avn;
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
// Creating the mini-batches // Creating the mini-batches
@ -161,34 +161,34 @@ void MLPPMLP::MBGD(double learning_rate, int max_epoch, int mini_batch_size, boo
while (true) { while (true) {
for (int i = 0; i < n_mini_batch; i++) { for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = Evaluate(inputMiniBatches[i]); std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]);
auto [z2, a2] = propagate(inputMiniBatches[i]); auto [z2, a2] = propagate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]); cost_prev = Cost(y_hat, outputMiniBatches[i]);
// Calculating the errors // Calculating the errors
std::vector<double> error = alg.subtraction(y_hat, outputMiniBatches[i]); std::vector<real_t> error = alg.subtraction(y_hat, outputMiniBatches[i]);
// Calculating the weight/bias gradients for layer 2 // Calculating the weight/bias gradients for layer 2
std::vector<double> D2_1 = alg.mat_vec_mult(alg.transpose(a2), error); std::vector<real_t> D2_1 = alg.mat_vec_mult(alg.transpose(a2), error);
// weights and bias updation for layser 2 // weights and bias updation for layser 2
weights2 = alg.subtraction(weights2, alg.scalarMultiply(learning_rate / outputMiniBatches[i].size(), D2_1)); weights2 = alg.subtraction(weights2, alg.scalarMultiply(learning_rate / outputMiniBatches[i].size(), D2_1));
weights2 = regularization.regWeights(weights2, lambda, alpha, reg); weights2 = regularization.regWeights(weights2, lambda, alpha, reg);
// Calculating the bias gradients for layer 2 // Calculating the bias gradients for layer 2
double b_gradient = alg.sum_elements(error); real_t b_gradient = alg.sum_elements(error);
// Bias Updation for layer 2 // Bias Updation for layer 2
bias2 -= learning_rate * alg.sum_elements(error) / outputMiniBatches[i].size(); bias2 -= learning_rate * alg.sum_elements(error) / outputMiniBatches[i].size();
//Calculating the weight/bias for layer 1 //Calculating the weight/bias for layer 1
std::vector<std::vector<double>> D1_1 = alg.outerProduct(error, weights2); std::vector<std::vector<real_t>> D1_1 = alg.outerProduct(error, weights2);
std::vector<std::vector<double>> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1)); std::vector<std::vector<real_t>> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1));
std::vector<std::vector<double>> D1_3 = alg.matmult(alg.transpose(inputMiniBatches[i]), D1_2); std::vector<std::vector<real_t>> D1_3 = alg.matmult(alg.transpose(inputMiniBatches[i]), D1_2);
// weight an bias updation for layer 1 // weight an bias updation for layer 1
weights1 = alg.subtraction(weights1, alg.scalarMultiply(learning_rate / outputMiniBatches[i].size(), D1_3)); weights1 = alg.subtraction(weights1, alg.scalarMultiply(learning_rate / outputMiniBatches[i].size(), D1_3));
@ -214,7 +214,7 @@ void MLPPMLP::MBGD(double learning_rate, int max_epoch, int mini_batch_size, boo
forwardPass(); forwardPass();
} }
double MLPPMLP::score() { real_t MLPPMLP::score() {
MLPPUtilities util; MLPPUtilities util;
return util.performance(y_hat, outputSet); return util.performance(y_hat, outputSet);
} }
@ -225,41 +225,41 @@ void MLPPMLP::save(std::string fileName) {
util.saveParameters(fileName, weights2, bias2, 1, 2); util.saveParameters(fileName, weights2, bias2, 1, 2);
} }
double MLPPMLP::Cost(std::vector<double> y_hat, std::vector<double> y) { real_t MLPPMLP::Cost(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPReg regularization; MLPPReg regularization;
class MLPPCost cost; class MLPPCost cost;
return cost.LogLoss(y_hat, y) + regularization.regTerm(weights2, lambda, alpha, reg) + regularization.regTerm(weights1, lambda, alpha, reg); return cost.LogLoss(y_hat, y) + regularization.regTerm(weights2, lambda, alpha, reg) + regularization.regTerm(weights1, lambda, alpha, reg);
} }
std::vector<double> MLPPMLP::Evaluate(std::vector<std::vector<double>> X) { std::vector<real_t> MLPPMLP::Evaluate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPActivation avn; MLPPActivation avn;
std::vector<std::vector<double>> z2 = alg.mat_vec_add(alg.matmult(X, weights1), bias1); std::vector<std::vector<real_t>> z2 = alg.mat_vec_add(alg.matmult(X, weights1), bias1);
std::vector<std::vector<double>> a2 = avn.sigmoid(z2); std::vector<std::vector<real_t>> a2 = avn.sigmoid(z2);
return avn.sigmoid(alg.scalarAdd(bias2, alg.mat_vec_mult(a2, weights2))); return avn.sigmoid(alg.scalarAdd(bias2, alg.mat_vec_mult(a2, weights2)));
} }
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> MLPPMLP::propagate(std::vector<std::vector<double>> X) { std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> MLPPMLP::propagate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPActivation avn; MLPPActivation avn;
std::vector<std::vector<double>> z2 = alg.mat_vec_add(alg.matmult(X, weights1), bias1); std::vector<std::vector<real_t>> z2 = alg.mat_vec_add(alg.matmult(X, weights1), bias1);
std::vector<std::vector<double>> a2 = avn.sigmoid(z2); std::vector<std::vector<real_t>> a2 = avn.sigmoid(z2);
return { z2, a2 }; return { z2, a2 };
} }
double MLPPMLP::Evaluate(std::vector<double> x) { real_t MLPPMLP::Evaluate(std::vector<real_t> x) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPActivation avn; MLPPActivation avn;
std::vector<double> z2 = alg.addition(alg.mat_vec_mult(alg.transpose(weights1), x), bias1); std::vector<real_t> z2 = alg.addition(alg.mat_vec_mult(alg.transpose(weights1), x), bias1);
std::vector<double> a2 = avn.sigmoid(z2); std::vector<real_t> a2 = avn.sigmoid(z2);
return avn.sigmoid(alg.dot(weights2, a2) + bias2); return avn.sigmoid(alg.dot(weights2, a2) + bias2);
} }
std::tuple<std::vector<double>, std::vector<double>> MLPPMLP::propagate(std::vector<double> x) { std::tuple<std::vector<real_t>, std::vector<real_t>> MLPPMLP::propagate(std::vector<real_t> x) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPActivation avn; MLPPActivation avn;
std::vector<double> z2 = alg.addition(alg.mat_vec_mult(alg.transpose(weights1), x), bias1); std::vector<real_t> z2 = alg.addition(alg.mat_vec_mult(alg.transpose(weights1), x), bias1);
std::vector<double> a2 = avn.sigmoid(z2); std::vector<real_t> a2 = avn.sigmoid(z2);
return { z2, a2 }; return { z2, a2 };
} }

48
mlpp/mlp/mlp.h
View File

@ -8,6 +8,8 @@
// Created by Marc Melikyan on 11/4/20. // Created by Marc Melikyan on 11/4/20.
// //
#include "core/math/math_defs.h"
#include <map> #include <map>
#include <string> #include <string>
#include <vector> #include <vector>
@ -16,36 +18,36 @@
class MLPPMLP { class MLPPMLP {
public: public:
MLPPMLP(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, int n_hidden, std::string reg = "None", double lambda = 0.5, double alpha = 0.5); MLPPMLP(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, int n_hidden, std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
std::vector<double> modelSetTest(std::vector<std::vector<double>> X); std::vector<real_t> modelSetTest(std::vector<std::vector<real_t>> X);
double modelTest(std::vector<double> x); real_t modelTest(std::vector<real_t> x);
void gradientDescent(double learning_rate, int max_epoch, bool UI = 1); void gradientDescent(real_t learning_rate, int max_epoch, bool UI = 1);
void SGD(double learning_rate, int max_epoch, bool UI = 1); void SGD(real_t learning_rate, int max_epoch, bool UI = 1);
void MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI = 1); void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
double score(); real_t score();
void save(std::string fileName); void save(std::string fileName);
private: private:
double Cost(std::vector<double> y_hat, std::vector<double> y); real_t Cost(std::vector<real_t> y_hat, std::vector<real_t> y);
std::vector<double> Evaluate(std::vector<std::vector<double>> X); std::vector<real_t> Evaluate(std::vector<std::vector<real_t>> X);
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> propagate(std::vector<std::vector<double>> X); std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> propagate(std::vector<std::vector<real_t>> X);
double Evaluate(std::vector<double> x); real_t Evaluate(std::vector<real_t> x);
std::tuple<std::vector<double>, std::vector<double>> propagate(std::vector<double> x); std::tuple<std::vector<real_t>, std::vector<real_t>> propagate(std::vector<real_t> x);
void forwardPass(); void forwardPass();
std::vector<std::vector<double>> inputSet; std::vector<std::vector<real_t>> inputSet;
std::vector<double> outputSet; std::vector<real_t> outputSet;
std::vector<double> y_hat; std::vector<real_t> y_hat;
std::vector<std::vector<double>> weights1; std::vector<std::vector<real_t>> weights1;
std::vector<double> weights2; std::vector<real_t> weights2;
std::vector<double> bias1; std::vector<real_t> bias1;
double bias2; real_t bias2;
std::vector<std::vector<double>> z2; std::vector<std::vector<real_t>> z2;
std::vector<std::vector<double>> a2; std::vector<std::vector<real_t>> a2;
int n; int n;
int k; int k;
@ -53,8 +55,8 @@ private:
// Regularization Params // Regularization Params
std::string reg; std::string reg;
double lambda; /* Regularization Parameter */ real_t lambda; /* Regularization Parameter */
double alpha; /* This is the controlling param for Elastic Net*/ real_t alpha; /* This is the controlling param for Elastic Net*/
}; };

4
mlpp/multi_output_layer/multi_output_layer.cpp
View File

@ -12,7 +12,7 @@
#include <random> #include <random>
MLPPMultiOutputLayer::MLPPMultiOutputLayer(int n_output, int n_hidden, std::string activation, std::string cost, std::vector<std::vector<double>> input, std::string weightInit, std::string reg, double lambda, double alpha) : MLPPMultiOutputLayer::MLPPMultiOutputLayer(int n_output, int n_hidden, std::string activation, std::string cost, std::vector<std::vector<real_t>> input, std::string weightInit, std::string reg, real_t lambda, real_t alpha) :
n_output(n_output), n_hidden(n_hidden), activation(activation), cost(cost), input(input), weightInit(weightInit), reg(reg), lambda(lambda), alpha(alpha) { n_output(n_output), n_hidden(n_hidden), activation(activation), cost(cost), input(input), weightInit(weightInit), reg(reg), lambda(lambda), alpha(alpha) {
weights = MLPPUtilities::weightInitialization(n_hidden, n_output, weightInit); weights = MLPPUtilities::weightInitialization(n_hidden, n_output, weightInit);
bias = MLPPUtilities::biasInitialization(n_output); bias = MLPPUtilities::biasInitialization(n_output);
@ -123,7 +123,7 @@ void MLPPMultiOutputLayer::forwardPass() {
a = (avn.*activation_map[activation])(z, 0); a = (avn.*activation_map[activation])(z, 0);
} }
void MLPPMultiOutputLayer::Test(std::vector<double> x) { void MLPPMultiOutputLayer::Test(std::vector<real_t> x) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPActivation avn; MLPPActivation avn;
z_test = alg.addition(alg.mat_vec_mult(alg.transpose(weights), x), bias); z_test = alg.addition(alg.mat_vec_mult(alg.transpose(weights), x), bias);

34
mlpp/multi_output_layer/multi_output_layer.h
View File

@ -8,6 +8,8 @@
// Created by Marc Melikyan on 11/4/20. // Created by Marc Melikyan on 11/4/20.
// //
#include "core/math/math_defs.h"
#include "../activation/activation.h" #include "../activation/activation.h"
#include "../cost/cost.h" #include "../cost/cost.h"
@ -18,40 +20,40 @@
class MLPPMultiOutputLayer { class MLPPMultiOutputLayer {
public: public:
MLPPMultiOutputLayer(int n_output, int n_hidden, std::string activation, std::string cost, std::vector<std::vector<double>> input, std::string weightInit, std::string reg, double lambda, double alpha); MLPPMultiOutputLayer(int n_output, int n_hidden, std::string activation, std::string cost, std::vector<std::vector<real_t>> input, std::string weightInit, std::string reg, real_t lambda, real_t alpha);
int n_output; int n_output;
int n_hidden; int n_hidden;
std::string activation; std::string activation;
std::string cost; std::string cost;
std::vector<std::vector<double>> input; std::vector<std::vector<real_t>> input;
std::vector<std::vector<double>> weights; std::vector<std::vector<real_t>> weights;
std::vector<double> bias; std::vector<real_t> bias;
std::vector<std::vector<double>> z; std::vector<std::vector<real_t>> z;
std::vector<std::vector<double>> a; std::vector<std::vector<real_t>> a;
std::map<std::string, std::vector<std::vector<double>> (MLPPActivation::*)(std::vector<std::vector<double>>, bool)> activation_map; std::map<std::string, std::vector<std::vector<real_t>> (MLPPActivation::*)(std::vector<std::vector<real_t>>, bool)> activation_map;
std::map<std::string, std::vector<double> (MLPPActivation::*)(std::vector<double>, bool)> activationTest_map; std::map<std::string, std::vector<real_t> (MLPPActivation::*)(std::vector<real_t>, bool)> activationTest_map;
std::map<std::string, double (MLPPCost::*)(std::vector<std::vector<double>>, std::vector<std::vector<double>>)> cost_map; std::map<std::string, real_t (MLPPCost::*)(std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>)> cost_map;
std::map<std::string, std::vector<std::vector<double>> (MLPPCost::*)(std::vector<std::vector<double>>, std::vector<std::vector<double>>)> costDeriv_map; std::map<std::string, std::vector<std::vector<real_t>> (MLPPCost::*)(std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>)> costDeriv_map;
std::vector<double> z_test; std::vector<real_t> z_test;
std::vector<double> a_test; std::vector<real_t> a_test;
std::vector<std::vector<double>> delta; std::vector<std::vector<real_t>> delta;
// Regularization Params // Regularization Params
std::string reg; std::string reg;
double lambda; /* Regularization Parameter */ real_t lambda; /* Regularization Parameter */
double alpha; /* This is the controlling param for Elastic Net*/ real_t alpha; /* This is the controlling param for Elastic Net*/
std::string weightInit; std::string weightInit;
void forwardPass(); void forwardPass();
void Test(std::vector<double> x); void Test(std::vector<real_t> x);
}; };

20
mlpp/multinomial_nb/multinomial_nb.cpp
View File

@ -13,22 +13,22 @@
#include <random> #include <random>
MLPPMultinomialNB::MLPPMultinomialNB(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, int class_num) : MLPPMultinomialNB::MLPPMultinomialNB(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, int class_num) :
inputSet(inputSet), outputSet(outputSet), class_num(class_num) { inputSet(inputSet), outputSet(outputSet), class_num(class_num) {
y_hat.resize(outputSet.size()); y_hat.resize(outputSet.size());
Evaluate(); Evaluate();
} }
std::vector<double> MLPPMultinomialNB::modelSetTest(std::vector<std::vector<double>> X) { std::vector<real_t> MLPPMultinomialNB::modelSetTest(std::vector<std::vector<real_t>> X) {
std::vector<double> y_hat; std::vector<real_t> y_hat;
for (int i = 0; i < X.size(); i++) { for (int i = 0; i < X.size(); i++) {
y_hat.push_back(modelTest(X[i])); y_hat.push_back(modelTest(X[i]));
} }
return y_hat; return y_hat;
} }
double MLPPMultinomialNB::modelTest(std::vector<double> x) { real_t MLPPMultinomialNB::modelTest(std::vector<real_t> x) {
double score[class_num]; real_t score[class_num];
computeTheta(); computeTheta();
for (int j = 0; j < x.size(); j++) { for (int j = 0; j < x.size(); j++) {
@ -45,10 +45,10 @@ double MLPPMultinomialNB::modelTest(std::vector<double> x) {
score[i] += std::log(priors[i]); score[i] += std::log(priors[i]);
} }
return std::distance(score, std::max_element(score, score + sizeof(score) / sizeof(double))); return std::distance(score, std::max_element(score, score + sizeof(score) / sizeof(real_t)));
} }
double MLPPMultinomialNB::score() { real_t MLPPMultinomialNB::score() {
MLPPUtilities util; MLPPUtilities util;
return util.performance(y_hat, outputSet); return util.performance(y_hat, outputSet);
} }
@ -81,14 +81,14 @@ void MLPPMultinomialNB::Evaluate() {
MLPPLinAlg alg; MLPPLinAlg alg;
for (int i = 0; i < outputSet.size(); i++) { for (int i = 0; i < outputSet.size(); i++) {
// Pr(B | A) * Pr(A) // Pr(B | A) * Pr(A)
double score[class_num]; real_t score[class_num];
// Easy computation of priors, i.e. Pr(C_k) // Easy computation of priors, i.e. Pr(C_k)
priors.resize(class_num); priors.resize(class_num);
for (int i = 0; i < outputSet.size(); i++) { for (int i = 0; i < outputSet.size(); i++) {
priors[int(outputSet[i])]++; priors[int(outputSet[i])]++;
} }
priors = alg.scalarMultiply(double(1) / double(outputSet.size()), priors); priors = alg.scalarMultiply(real_t(1) / real_t(outputSet.size()), priors);
// Evaluating Theta... // Evaluating Theta...
computeTheta(); computeTheta();
@ -113,6 +113,6 @@ void MLPPMultinomialNB::Evaluate() {
} }
// Assigning the traning example's y_hat to a class // Assigning the traning example's y_hat to a class
y_hat[i] = std::distance(score, std::max_element(score, score + sizeof(score) / sizeof(double))); y_hat[i] = std::distance(score, std::max_element(score, score + sizeof(score) / sizeof(real_t)));
} }
} }

22
mlpp/multinomial_nb/multinomial_nb.h
View File

@ -8,32 +8,34 @@
// Created by Marc Melikyan on 1/17/21. // Created by Marc Melikyan on 1/17/21.
// //
#include "core/math/math_defs.h"
#include <map> #include <map>
#include <vector> #include <vector>
class MLPPMultinomialNB { class MLPPMultinomialNB {
public: public:
MLPPMultinomialNB(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, int class_num); MLPPMultinomialNB(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, int class_num);
std::vector<double> modelSetTest(std::vector<std::vector<double>> X); std::vector<real_t> modelSetTest(std::vector<std::vector<real_t>> X);
double modelTest(std::vector<double> x); real_t modelTest(std::vector<real_t> x);
double score(); real_t score();
private: private:
void computeTheta(); void computeTheta();
void Evaluate(); void Evaluate();
// Model Params // Model Params
std::vector<double> priors; std::vector<real_t> priors;
std::vector<std::map<double, int>> theta; std::vector<std::map<real_t, int>> theta;
std::vector<double> vocab; std::vector<real_t> vocab;
int class_num; int class_num;
// Datasets // Datasets
std::vector<std::vector<double>> inputSet; std::vector<std::vector<real_t>> inputSet;
std::vector<double> outputSet; std::vector<real_t> outputSet;
std::vector<double> y_hat; std::vector<real_t> y_hat;
}; };
#endif /* MultinomialNB_hpp */ #endif /* MultinomialNB_hpp */

140
mlpp/numerical_analysis/numerical_analysis.cpp
View File

@ -14,127 +14,127 @@
double MLPPNumericalAnalysis::numDiff(double (*function)(double), double x) { real_t MLPPNumericalAnalysis::numDiff(real_t (*function)(real_t), real_t x) {
double eps = 1e-10; real_t eps = 1e-10;
return (function(x + eps) - function(x)) / eps; // This is just the formal def. of the derivative. return (function(x + eps) - function(x)) / eps; // This is just the formal def. of the derivative.
} }
double MLPPNumericalAnalysis::numDiff_2(double (*function)(double), double x) { real_t MLPPNumericalAnalysis::numDiff_2(real_t (*function)(real_t), real_t x) {
double eps = 1e-5; real_t eps = 1e-5;
return (function(x + 2 * eps) - 2 * function(x + eps) + function(x)) / (eps * eps); return (function(x + 2 * eps) - 2 * function(x + eps) + function(x)) / (eps * eps);
} }
double MLPPNumericalAnalysis::numDiff_3(double (*function)(double), double x) { real_t MLPPNumericalAnalysis::numDiff_3(real_t (*function)(real_t), real_t x) {
double eps = 1e-5; real_t eps = 1e-5;
double t1 = function(x + 3 * eps) - 2 * function(x + 2 * eps) + function(x + eps); real_t t1 = function(x + 3 * eps) - 2 * function(x + 2 * eps) + function(x + eps);
double t2 = function(x + 2 * eps) - 2 * function(x + eps) + function(x); real_t t2 = function(x + 2 * eps) - 2 * function(x + eps) + function(x);
return (t1 - t2) / (eps * eps * eps); return (t1 - t2) / (eps * eps * eps);
} }
double MLPPNumericalAnalysis::constantApproximation(double (*function)(double), double c) { real_t MLPPNumericalAnalysis::constantApproximation(real_t (*function)(real_t), real_t c) {
return function(c); return function(c);
} }
double MLPPNumericalAnalysis::linearApproximation(double (*function)(double), double c, double x) { real_t MLPPNumericalAnalysis::linearApproximation(real_t (*function)(real_t), real_t c, real_t x) {
return constantApproximation(function, c) + numDiff(function, c) * (x - c); return constantApproximation(function, c) + numDiff(function, c) * (x - c);
} }
double MLPPNumericalAnalysis::quadraticApproximation(double (*function)(double), double c, double x) { real_t MLPPNumericalAnalysis::quadraticApproximation(real_t (*function)(real_t), real_t c, real_t x) {
return linearApproximation(function, c, x) + 0.5 * numDiff_2(function, c) * (x - c) * (x - c); return linearApproximation(function, c, x) + 0.5 * numDiff_2(function, c) * (x - c) * (x - c);
} }
double MLPPNumericalAnalysis::cubicApproximation(double (*function)(double), double c, double x) { real_t MLPPNumericalAnalysis::cubicApproximation(real_t (*function)(real_t), real_t c, real_t x) {
return quadraticApproximation(function, c, x) + (1 / 6) * numDiff_3(function, c) * (x - c) * (x - c) * (x - c); return quadraticApproximation(function, c, x) + (1 / 6) * numDiff_3(function, c) * (x - c) * (x - c) * (x - c);
} }
double MLPPNumericalAnalysis::numDiff(double (*function)(std::vector<double>), std::vector<double> x, int axis) { real_t MLPPNumericalAnalysis::numDiff(real_t (*function)(std::vector<real_t>), std::vector<real_t> x, int axis) {
// For multivariable function analysis. // For multivariable function analysis.
// This will be used for calculating Jacobian vectors. // This will be used for calculating Jacobian vectors.
// Diffrentiate with respect to indicated axis. (0, 1, 2 ...) // Diffrentiate with respect to indicated axis. (0, 1, 2 ...)
double eps = 1e-10; real_t eps = 1e-10;
std::vector<double> x_eps = x; std::vector<real_t> x_eps = x;
x_eps[axis] += eps; x_eps[axis] += eps;
return (function(x_eps) - function(x)) / eps; return (function(x_eps) - function(x)) / eps;
} }
double MLPPNumericalAnalysis::numDiff_2(double (*function)(std::vector<double>), std::vector<double> x, int axis1, int axis2) { real_t MLPPNumericalAnalysis::numDiff_2(real_t (*function)(std::vector<real_t>), std::vector<real_t> x, int axis1, int axis2) {
//For Hessians. //For Hessians.
double eps = 1e-5; real_t eps = 1e-5;
std::vector<double> x_pp = x; std::vector<real_t> x_pp = x;
x_pp[axis1] += eps; x_pp[axis1] += eps;
x_pp[axis2] += eps; x_pp[axis2] += eps;
std::vector<double> x_np = x; std::vector<real_t> x_np = x;
x_np[axis2] += eps; x_np[axis2] += eps;
std::vector<double> x_pn = x; std::vector<real_t> x_pn = x;
x_pn[axis1] += eps; x_pn[axis1] += eps;
return (function(x_pp) - function(x_np) - function(x_pn) + function(x)) / (eps * eps); return (function(x_pp) - function(x_np) - function(x_pn) + function(x)) / (eps * eps);
} }
double MLPPNumericalAnalysis::numDiff_3(double (*function)(std::vector<double>), std::vector<double> x, int axis1, int axis2, int axis3) { real_t MLPPNumericalAnalysis::numDiff_3(real_t (*function)(std::vector<real_t>), std::vector<real_t> x, int axis1, int axis2, int axis3) {
// For third order derivative tensors. // For third order derivative tensors.
// NOTE: Approximations do not appear to be accurate for sinusodial functions... // NOTE: Approximations do not appear to be accurate for sinusodial functions...
// Should revisit this later. // Should revisit this later.
double eps = INT_MAX; real_t eps = INT_MAX;
std::vector<double> x_ppp = x; std::vector<real_t> x_ppp = x;
x_ppp[axis1] += eps; x_ppp[axis1] += eps;
x_ppp[axis2] += eps; x_ppp[axis2] += eps;
x_ppp[axis3] += eps; x_ppp[axis3] += eps;
std::vector<double> x_npp = x; std::vector<real_t> x_npp = x;
x_npp[axis2] += eps; x_npp[axis2] += eps;
x_npp[axis3] += eps; x_npp[axis3] += eps;
std::vector<double> x_pnp = x; std::vector<real_t> x_pnp = x;
x_pnp[axis1] += eps; x_pnp[axis1] += eps;
x_pnp[axis3] += eps; x_pnp[axis3] += eps;
std::vector<double> x_nnp = x; std::vector<real_t> x_nnp = x;
x_nnp[axis3] += eps; x_nnp[axis3] += eps;
std::vector<double> x_ppn = x; std::vector<real_t> x_ppn = x;
x_ppn[axis1] += eps; x_ppn[axis1] += eps;
x_ppn[axis2] += eps; x_ppn[axis2] += eps;
std::vector<double> x_npn = x; std::vector<real_t> x_npn = x;
x_npn[axis2] += eps; x_npn[axis2] += eps;
std::vector<double> x_pnn = x; std::vector<real_t> x_pnn = x;
x_pnn[axis1] += eps; x_pnn[axis1] += eps;
double thirdAxis = function(x_ppp) - function(x_npp) - function(x_pnp) + function(x_nnp); real_t thirdAxis = function(x_ppp) - function(x_npp) - function(x_pnp) + function(x_nnp);
double noThirdAxis = function(x_ppn) - function(x_npn) - function(x_pnn) + function(x); real_t noThirdAxis = function(x_ppn) - function(x_npn) - function(x_pnn) + function(x);
return (thirdAxis - noThirdAxis) / (eps * eps * eps); return (thirdAxis - noThirdAxis) / (eps * eps * eps);
} }
double MLPPNumericalAnalysis::newtonRaphsonMethod(double (*function)(double), double x_0, double epoch_num) { real_t MLPPNumericalAnalysis::newtonRaphsonMethod(real_t (*function)(real_t), real_t x_0, real_t epoch_num) {
double x = x_0; real_t x = x_0;
for (int i = 0; i < epoch_num; i++) { for (int i = 0; i < epoch_num; i++) {
x -= function(x) / numDiff(function, x); x -= function(x) / numDiff(function, x);
} }
return x; return x;
} }
double MLPPNumericalAnalysis::halleyMethod(double (*function)(double), double x_0, double epoch_num) { real_t MLPPNumericalAnalysis::halleyMethod(real_t (*function)(real_t), real_t x_0, real_t epoch_num) {
double x = x_0; real_t x = x_0;
for (int i = 0; i < epoch_num; i++) { for (int i = 0; i < epoch_num; i++) {
x -= ((2 * function(x) * numDiff(function, x)) / (2 * numDiff(function, x) * numDiff(function, x) - function(x) * numDiff_2(function, x))); x -= ((2 * function(x) * numDiff(function, x)) / (2 * numDiff(function, x) * numDiff(function, x) - function(x) * numDiff_2(function, x)));
} }
return x; return x;
} }
double MLPPNumericalAnalysis::invQuadraticInterpolation(double (*function)(double), std::vector<double> x_0, double epoch_num) { real_t MLPPNumericalAnalysis::invQuadraticInterpolation(real_t (*function)(real_t), std::vector<real_t> x_0, real_t epoch_num) {
double x = 0; real_t x = 0;
std::vector<double> currentThree = x_0; std::vector<real_t> currentThree = x_0;
for (int i = 0; i < epoch_num; i++) { for (int i = 0; i < epoch_num; i++) {
double t1 = ((function(currentThree[1]) * function(currentThree[2])) / ((function(currentThree[0]) - function(currentThree[1])) * (function(currentThree[0]) - function(currentThree[2])))) * currentThree[0]; real_t t1 = ((function(currentThree[1]) * function(currentThree[2])) / ((function(currentThree[0]) - function(currentThree[1])) * (function(currentThree[0]) - function(currentThree[2])))) * currentThree[0];
double t2 = ((function(currentThree[0]) * function(currentThree[2])) / ((function(currentThree[1]) - function(currentThree[0])) * (function(currentThree[1]) - function(currentThree[2])))) * currentThree[1]; real_t t2 = ((function(currentThree[0]) * function(currentThree[2])) / ((function(currentThree[1]) - function(currentThree[0])) * (function(currentThree[1]) - function(currentThree[2])))) * currentThree[1];
double t3 = ((function(currentThree[0]) * function(currentThree[1])) / ((function(currentThree[2]) - function(currentThree[0])) * (function(currentThree[2]) - function(currentThree[1])))) * currentThree[2]; real_t t3 = ((function(currentThree[0]) * function(currentThree[1])) / ((function(currentThree[2]) - function(currentThree[0])) * (function(currentThree[2]) - function(currentThree[1])))) * currentThree[2];
x = t1 + t2 + t3; x = t1 + t2 + t3;
currentThree.erase(currentThree.begin()); currentThree.erase(currentThree.begin());
@ -143,10 +143,10 @@ double MLPPNumericalAnalysis::invQuadraticInterpolation(double (*function)(doubl
return x; return x;
} }
double MLPPNumericalAnalysis::eulerianMethod(double (*derivative)(double), std::vector<double> q_0, double p, double h) { real_t MLPPNumericalAnalysis::eulerianMethod(real_t (*derivative)(real_t), std::vector<real_t> q_0, real_t p, real_t h) {
double max_epoch = (p - q_0[0]) / h; real_t max_epoch = (p - q_0[0]) / h;
double x = q_0[0]; real_t x = q_0[0];
double y = q_0[1]; real_t y = q_0[1];
for (int i = 0; i < max_epoch; i++) { for (int i = 0; i < max_epoch; i++) {
y = y + h * derivative(x); y = y + h * derivative(x);
x += h; x += h;
@ -154,10 +154,10 @@ double MLPPNumericalAnalysis::eulerianMethod(double (*derivative)(double), std::
return y; return y;
} }
double MLPPNumericalAnalysis::eulerianMethod(double (*derivative)(std::vector<double>), std::vector<double> q_0, double p, double h) { real_t MLPPNumericalAnalysis::eulerianMethod(real_t (*derivative)(std::vector<real_t>), std::vector<real_t> q_0, real_t p, real_t h) {
double max_epoch = (p - q_0[0]) / h; real_t max_epoch = (p - q_0[0]) / h;
double x = q_0[0]; real_t x = q_0[0];
double y = q_0[1]; real_t y = q_0[1];
for (int i = 0; i < max_epoch; i++) { for (int i = 0; i < max_epoch; i++) {
y = y + h * derivative({ x, y }); y = y + h * derivative({ x, y });
x += h; x += h;
@ -165,7 +165,7 @@ double MLPPNumericalAnalysis::eulerianMethod(double (*derivative)(std::vector<do
return y; return y;
} }
double MLPPNumericalAnalysis::growthMethod(double C, double k, double t) { real_t MLPPNumericalAnalysis::growthMethod(real_t C, real_t k, real_t t) {
/* /*
dP/dt = kP dP/dt = kP
dP/P = kdt dP/P = kdt
@ -177,20 +177,20 @@ double MLPPNumericalAnalysis::growthMethod(double C, double k, double t) {
P = C * e^(kt) P = C * e^(kt)
*/ */
// auto growthFunction = [&C, &k](double t) { return C * exp(k * t); }; // auto growthFunction = [&C, &k](real_t t) { return C * exp(k * t); };
return C * std::exp(k * t); return C * std::exp(k * t);
} }
std::vector<double> MLPPNumericalAnalysis::jacobian(double (*function)(std::vector<double>), std::vector<double> x) { std::vector<real_t> MLPPNumericalAnalysis::jacobian(real_t (*function)(std::vector<real_t>), std::vector<real_t> x) {
std::vector<double> jacobian; std::vector<real_t> jacobian;
jacobian.resize(x.size()); jacobian.resize(x.size());
for (int i = 0; i < jacobian.size(); i++) { for (int i = 0; i < jacobian.size(); i++) {
jacobian[i] = numDiff(function, x, i); // Derivative w.r.t axis i evaluated at x. For all x_i. jacobian[i] = numDiff(function, x, i); // Derivative w.r.t axis i evaluated at x. For all x_i.
} }
return jacobian; return jacobian;
} }
std::vector<std::vector<double>> MLPPNumericalAnalysis::hessian(double (*function)(std::vector<double>), std::vector<double> x) { std::vector<std::vector<real_t>> MLPPNumericalAnalysis::hessian(real_t (*function)(std::vector<real_t>), std::vector<real_t> x) {
std::vector<std::vector<double>> hessian; std::vector<std::vector<real_t>> hessian;
hessian.resize(x.size()); hessian.resize(x.size());
for (int i = 0; i < hessian.size(); i++) { for (int i = 0; i < hessian.size(); i++) {
hessian[i].resize(x.size()); hessian[i].resize(x.size());
@ -203,8 +203,8 @@ std::vector<std::vector<double>> MLPPNumericalAnalysis::hessian(double (*functio
return hessian; return hessian;
} }
std::vector<std::vector<std::vector<double>>> MLPPNumericalAnalysis::thirdOrderTensor(double (*function)(std::vector<double>), std::vector<double> x) { std::vector<std::vector<std::vector<real_t>>> MLPPNumericalAnalysis::thirdOrderTensor(real_t (*function)(std::vector<real_t>), std::vector<real_t> x) {
std::vector<std::vector<std::vector<double>>> tensor; std::vector<std::vector<std::vector<real_t>>> tensor;
tensor.resize(x.size()); tensor.resize(x.size());
for (int i = 0; i < tensor.size(); i++) { for (int i = 0; i < tensor.size(); i++) {
tensor[i].resize(x.size()); tensor[i].resize(x.size());
@ -221,21 +221,21 @@ std::vector<std::vector<std::vector<double>>> MLPPNumericalAnalysis::thirdOrderT
return tensor; return tensor;
} }
double MLPPNumericalAnalysis::constantApproximation(double (*function)(std::vector<double>), std::vector<double> c) { real_t MLPPNumericalAnalysis::constantApproximation(real_t (*function)(std::vector<real_t>), std::vector<real_t> c) {
return function(c); return function(c);
} }
double MLPPNumericalAnalysis::linearApproximation(double (*function)(std::vector<double>), std::vector<double> c, std::vector<double> x) { real_t MLPPNumericalAnalysis::linearApproximation(real_t (*function)(std::vector<real_t>), std::vector<real_t> c, std::vector<real_t> x) {
MLPPLinAlg alg; MLPPLinAlg alg;
return constantApproximation(function, c) + alg.matmult(alg.transpose({ jacobian(function, c) }), { alg.subtraction(x, c) })[0][0]; return constantApproximation(function, c) + alg.matmult(alg.transpose({ jacobian(function, c) }), { alg.subtraction(x, c) })[0][0];
} }
double MLPPNumericalAnalysis::quadraticApproximation(double (*function)(std::vector<double>), std::vector<double> c, std::vector<double> x) { real_t MLPPNumericalAnalysis::quadraticApproximation(real_t (*function)(std::vector<real_t>), std::vector<real_t> c, std::vector<real_t> x) {
MLPPLinAlg alg; MLPPLinAlg alg;
return linearApproximation(function, c, x) + 0.5 * alg.matmult({ (alg.subtraction(x, c)) }, alg.matmult(hessian(function, c), alg.transpose({ alg.subtraction(x, c) })))[0][0]; return linearApproximation(function, c, x) + 0.5 * alg.matmult({ (alg.subtraction(x, c)) }, alg.matmult(hessian(function, c), alg.transpose({ alg.subtraction(x, c) })))[0][0];
} }
double MLPPNumericalAnalysis::cubicApproximation(double (*function)(std::vector<double>), std::vector<double> c, std::vector<double> x) { real_t MLPPNumericalAnalysis::cubicApproximation(real_t (*function)(std::vector<real_t>), std::vector<real_t> c, std::vector<real_t> x) {
/* /*
Not completely sure as the literature seldom discusses the third order taylor approximation, Not completely sure as the literature seldom discusses the third order taylor approximation,
in particular for multivariate cases, but ostensibly, the matrix/tensor/vector multiplies in particular for multivariate cases, but ostensibly, the matrix/tensor/vector multiplies
@ -246,32 +246,32 @@ double MLPPNumericalAnalysis::cubicApproximation(double (*function)(std::vector<
Result is a scalar. Result is a scalar.
*/ */
MLPPLinAlg alg; MLPPLinAlg alg;
std::vector<std::vector<double>> resultMat = alg.tensor_vec_mult(thirdOrderTensor(function, c), alg.subtraction(x, c)); std::vector<std::vector<real_t>> resultMat = alg.tensor_vec_mult(thirdOrderTensor(function, c), alg.subtraction(x, c));
double resultScalar = alg.matmult({ (alg.subtraction(x, c)) }, alg.matmult(resultMat, alg.transpose({ alg.subtraction(x, c) })))[0][0]; real_t resultScalar = alg.matmult({ (alg.subtraction(x, c)) }, alg.matmult(resultMat, alg.transpose({ alg.subtraction(x, c) })))[0][0];
return quadraticApproximation(function, c, x) + (1 / 6) * resultScalar; return quadraticApproximation(function, c, x) + (1 / 6) * resultScalar;
} }
double MLPPNumericalAnalysis::laplacian(double (*function)(std::vector<double>), std::vector<double> x) { real_t MLPPNumericalAnalysis::laplacian(real_t (*function)(std::vector<real_t>), std::vector<real_t> x) {
MLPPLinAlg alg; MLPPLinAlg alg;
std::vector<std::vector<double>> hessian_matrix = hessian(function, x); std::vector<std::vector<real_t>> hessian_matrix = hessian(function, x);
double laplacian = 0; real_t laplacian = 0;
for (int i = 0; i < hessian_matrix.size(); i++) { for (int i = 0; i < hessian_matrix.size(); i++) {
laplacian += hessian_matrix[i][i]; // homogenous 2nd derivs w.r.t i, then i laplacian += hessian_matrix[i][i]; // homogenous 2nd derivs w.r.t i, then i
} }
return laplacian; return laplacian;
} }
std::string MLPPNumericalAnalysis::secondPartialDerivativeTest(double (*function)(std::vector<double>), std::vector<double> x) { std::string MLPPNumericalAnalysis::secondPartialDerivativeTest(real_t (*function)(std::vector<real_t>), std::vector<real_t> x) {
MLPPLinAlg alg; MLPPLinAlg alg;
std::vector<std::vector<double>> hessianMatrix = hessian(function, x); std::vector<std::vector<real_t>> hessianMatrix = hessian(function, x);
/* /*
The reason we do this is because the 2nd partial derivative test is less conclusive for functions of variables greater than The reason we do this is because the 2nd partial derivative test is less conclusive for functions of variables greater than
2, and the calculations specific to the bivariate case are less computationally intensive. 2, and the calculations specific to the bivariate case are less computationally intensive.
*/ */
if (x.size() == 2) { if (x.size() == 2) {
double det = alg.det(hessianMatrix, hessianMatrix.size()); real_t det = alg.det(hessianMatrix, hessianMatrix.size());
double secondDerivative = numDiff_2(function, x, 0, 0); real_t secondDerivative = numDiff_2(function, x, 0, 0);
if (secondDerivative > 0 && det > 0) { if (secondDerivative > 0 && det > 0) {
return "min"; return "min";
} else if (secondDerivative < 0 && det > 0) { } else if (secondDerivative < 0 && det > 0) {

52
mlpp/numerical_analysis/numerical_analysis.h
View File

@ -7,6 +7,8 @@
// //
// //
#include "core/math/math_defs.h"
#include <string> #include <string>
#include <vector> #include <vector>
@ -17,40 +19,40 @@ public:
as an analytical method for calculating derivatives will most likely be used in as an analytical method for calculating derivatives will most likely be used in
the future. the future.
*/ */
double numDiff(double (*function)(double), double x); real_t numDiff(real_t (*function)(real_t), real_t x);
double numDiff_2(double (*function)(double), double x); real_t numDiff_2(real_t (*function)(real_t), real_t x);
double numDiff_3(double (*function)(double), double x); real_t numDiff_3(real_t (*function)(real_t), real_t x);
double constantApproximation(double (*function)(double), double c); real_t constantApproximation(real_t (*function)(real_t), real_t c);
double linearApproximation(double (*function)(double), double c, double x); real_t linearApproximation(real_t (*function)(real_t), real_t c, real_t x);
double quadraticApproximation(double (*function)(double), double c, double x); real_t quadraticApproximation(real_t (*function)(real_t), real_t c, real_t x);
double cubicApproximation(double (*function)(double), double c, double x); real_t cubicApproximation(real_t (*function)(real_t), real_t c, real_t x);
double numDiff(double (*function)(std::vector<double>), std::vector<double> x, int axis); real_t numDiff(real_t (*function)(std::vector<real_t>), std::vector<real_t> x, int axis);
double numDiff_2(double (*function)(std::vector<double>), std::vector<double> x, int axis1, int axis2); real_t numDiff_2(real_t (*function)(std::vector<real_t>), std::vector<real_t> x, int axis1, int axis2);
double numDiff_3(double (*function)(std::vector<double>), std::vector<double> x, int axis1, int axis2, int axis3); real_t numDiff_3(real_t (*function)(std::vector<real_t>), std::vector<real_t> x, int axis1, int axis2, int axis3);
double newtonRaphsonMethod(double (*function)(double), double x_0, double epoch_num); real_t newtonRaphsonMethod(real_t (*function)(real_t), real_t x_0, real_t epoch_num);
double halleyMethod(double (*function)(double), double x_0, double epoch_num); real_t halleyMethod(real_t (*function)(real_t), real_t x_0, real_t epoch_num);
double invQuadraticInterpolation(double (*function)(double), std::vector<double> x_0, double epoch_num); real_t invQuadraticInterpolation(real_t (*function)(real_t), std::vector<real_t> x_0, real_t epoch_num);
double eulerianMethod(double (*derivative)(double), std::vector<double> q_0, double p, double h); // Euler's method for solving diffrential equations. real_t eulerianMethod(real_t (*derivative)(real_t), std::vector<real_t> q_0, real_t p, real_t h); // Euler's method for solving diffrential equations.
double eulerianMethod(double (*derivative)(std::vector<double>), std::vector<double> q_0, double p, double h); // Euler's method for solving diffrential equations. real_t eulerianMethod(real_t (*derivative)(std::vector<real_t>), std::vector<real_t> q_0, real_t p, real_t h); // Euler's method for solving diffrential equations.
double growthMethod(double C, double k, double t); // General growth-based diffrential equations can be solved by seperation of variables. real_t growthMethod(real_t C, real_t k, real_t t); // General growth-based diffrential equations can be solved by seperation of variables.
std::vector<double> jacobian(double (*function)(std::vector<double>), std::vector<double> x); // Indeed, for functions with scalar outputs the Jacobians will be vectors. std::vector<real_t> jacobian(real_t (*function)(std::vector<real_t>), std::vector<real_t> x); // Indeed, for functions with scalar outputs the Jacobians will be vectors.
std::vector<std::vector<double>> hessian(double (*function)(std::vector<double>), std::vector<double> x); std::vector<std::vector<real_t>> hessian(real_t (*function)(std::vector<real_t>), std::vector<real_t> x);
std::vector<std::vector<std::vector<double>>> thirdOrderTensor(double (*function)(std::vector<double>), std::vector<double> x); std::vector<std::vector<std::vector<real_t>>> thirdOrderTensor(real_t (*function)(std::vector<real_t>), std::vector<real_t> x);
double constantApproximation(double (*function)(std::vector<double>), std::vector<double> c); real_t constantApproximation(real_t (*function)(std::vector<real_t>), std::vector<real_t> c);
double linearApproximation(double (*function)(std::vector<double>), std::vector<double> c, std::vector<double> x); real_t linearApproximation(real_t (*function)(std::vector<real_t>), std::vector<real_t> c, std::vector<real_t> x);
double quadraticApproximation(double (*function)(std::vector<double>), std::vector<double> c, std::vector<double> x); real_t quadraticApproximation(real_t (*function)(std::vector<real_t>), std::vector<real_t> c, std::vector<real_t> x);
double cubicApproximation(double (*function)(std::vector<double>), std::vector<double> c, std::vector<double> x); real_t cubicApproximation(real_t (*function)(std::vector<real_t>), std::vector<real_t> c, std::vector<real_t> x);
double laplacian(double (*function)(std::vector<double>), std::vector<double> x); // laplacian real_t laplacian(real_t (*function)(std::vector<real_t>), std::vector<real_t> x); // laplacian
std::string secondPartialDerivativeTest(double (*function)(std::vector<double>), std::vector<double> x); std::string secondPartialDerivativeTest(real_t (*function)(std::vector<real_t>), std::vector<real_t> x);
}; };

12
mlpp/outlier_finder/outlier_finder.cpp
View File

@ -13,13 +13,13 @@ MLPPOutlierFinder::MLPPOutlierFinder(int threshold) :
threshold(threshold) { threshold(threshold) {
} }
std::vector<std::vector<double>> MLPPOutlierFinder::modelSetTest(std::vector<std::vector<double>> inputSet) { std::vector<std::vector<real_t>> MLPPOutlierFinder::modelSetTest(std::vector<std::vector<real_t>> inputSet) {
MLPPStat stat; MLPPStat stat;
std::vector<std::vector<double>> outliers; std::vector<std::vector<real_t>> outliers;
outliers.resize(inputSet.size()); outliers.resize(inputSet.size());
for (int i = 0; i < inputSet.size(); i++) { for (int i = 0; i < inputSet.size(); i++) {
for (int j = 0; j < inputSet[i].size(); j++) { for (int j = 0; j < inputSet[i].size(); j++) {
double z = (inputSet[i][j] - stat.mean(inputSet[i])) / stat.standardDeviation(inputSet[i]); real_t z = (inputSet[i][j] - stat.mean(inputSet[i])) / stat.standardDeviation(inputSet[i]);
if (abs(z) > threshold) { if (abs(z) > threshold) {
outliers[i].push_back(inputSet[i][j]); outliers[i].push_back(inputSet[i][j]);
} }
@ -28,11 +28,11 @@ std::vector<std::vector<double>> MLPPOutlierFinder::modelSetTest(std::vector<std
return outliers; return outliers;
} }
std::vector<double> MLPPOutlierFinder::modelTest(std::vector<double> inputSet) { std::vector<real_t> MLPPOutlierFinder::modelTest(std::vector<real_t> inputSet) {
MLPPStat stat; MLPPStat stat;
std::vector<double> outliers; std::vector<real_t> outliers;
for (int i = 0; i < inputSet.size(); i++) { for (int i = 0; i < inputSet.size(); i++) {
double z = (inputSet[i] - stat.mean(inputSet)) / stat.standardDeviation(inputSet); real_t z = (inputSet[i] - stat.mean(inputSet)) / stat.standardDeviation(inputSet);
if (abs(z) > threshold) { if (abs(z) > threshold) {
outliers.push_back(inputSet[i]); outliers.push_back(inputSet[i]);
} }

6
mlpp/outlier_finder/outlier_finder.h
View File

@ -8,6 +8,8 @@
// Created by Marc Melikyan on 11/13/20. // Created by Marc Melikyan on 11/13/20.
// //
#include "core/math/math_defs.h"
#include <vector> #include <vector>
@ -16,8 +18,8 @@ public:
// Cnstr // Cnstr
MLPPOutlierFinder(int threshold); MLPPOutlierFinder(int threshold);
std::vector<std::vector<double>> modelSetTest(std::vector<std::vector<double>> inputSet); std::vector<std::vector<real_t>> modelSetTest(std::vector<std::vector<real_t>> inputSet);
std::vector<double> modelTest(std::vector<double> inputSet); std::vector<real_t> modelTest(std::vector<real_t> inputSet);
// Variables required // Variables required
int threshold; int threshold;

4
mlpp/output_layer/output_layer.cpp
View File

@ -12,7 +12,7 @@
#include <random> #include <random>
MLPPOutputLayer::MLPPOutputLayer(int n_hidden, std::string activation, std::string cost, std::vector<std::vector<double>> input, std::string weightInit, std::string reg, double lambda, double alpha) : MLPPOutputLayer::MLPPOutputLayer(int n_hidden, std::string activation, std::string cost, std::vector<std::vector<real_t>> input, std::string weightInit, std::string reg, real_t lambda, real_t alpha) :
n_hidden(n_hidden), activation(activation), cost(cost), input(input), weightInit(weightInit), reg(reg), lambda(lambda), alpha(alpha) { n_hidden(n_hidden), activation(activation), cost(cost), input(input), weightInit(weightInit), reg(reg), lambda(lambda), alpha(alpha) {
weights = MLPPUtilities::weightInitialization(n_hidden, weightInit); weights = MLPPUtilities::weightInitialization(n_hidden, weightInit);
bias = MLPPUtilities::biasInitialization(); bias = MLPPUtilities::biasInitialization();
@ -120,7 +120,7 @@ void MLPPOutputLayer::forwardPass() {
a = (avn.*activation_map[activation])(z, 0); a = (avn.*activation_map[activation])(z, 0);
} }
void MLPPOutputLayer::Test(std::vector<double> x) { void MLPPOutputLayer::Test(std::vector<real_t> x) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPActivation avn; MLPPActivation avn;
z_test = alg.dot(weights, x) + bias; z_test = alg.dot(weights, x) + bias;

34
mlpp/output_layer/output_layer.h
View File

@ -8,6 +8,8 @@
// Created by Marc Melikyan on 11/4/20. // Created by Marc Melikyan on 11/4/20.
// //
#include "core/math/math_defs.h"
#include "../activation/activation.h" #include "../activation/activation.h"
#include "../cost/cost.h" #include "../cost/cost.h"
@ -18,39 +20,39 @@
class MLPPOutputLayer { class MLPPOutputLayer {
public: public:
MLPPOutputLayer(int n_hidden, std::string activation, std::string cost, std::vector<std::vector<double>> input, std::string weightInit, std::string reg, double lambda, double alpha); MLPPOutputLayer(int n_hidden, std::string activation, std::string cost, std::vector<std::vector<real_t>> input, std::string weightInit, std::string reg, real_t lambda, real_t alpha);
int n_hidden; int n_hidden;
std::string activation; std::string activation;
std::string cost; std::string cost;
std::vector<std::vector<double>> input; std::vector<std::vector<real_t>> input;
std::vector<double> weights; std::vector<real_t> weights;
double bias; real_t bias;
std::vector<double> z; std::vector<real_t> z;
std::vector<double> a; std::vector<real_t> a;
std::map<std::string, std::vector<double> (MLPPActivation::*)(std::vector<double>, bool)> activation_map; std::map<std::string, std::vector<real_t> (MLPPActivation::*)(std::vector<real_t>, bool)> activation_map;
std::map<std::string, double (MLPPActivation::*)(double, bool)> activationTest_map; std::map<std::string, real_t (MLPPActivation::*)(real_t, bool)> activationTest_map;
std::map<std::string, double (MLPPCost::*)(std::vector<double>, std::vector<double>)> cost_map; std::map<std::string, real_t (MLPPCost::*)(std::vector<real_t>, std::vector<real_t>)> cost_map;
std::map<std::string, std::vector<double> (MLPPCost::*)(std::vector<double>, std::vector<double>)> costDeriv_map; std::map<std::string, std::vector<real_t> (MLPPCost::*)(std::vector<real_t>, std::vector<real_t>)> costDeriv_map;
double z_test; real_t z_test;
double a_test; real_t a_test;
std::vector<double> delta; std::vector<real_t> delta;
// Regularization Params // Regularization Params
std::string reg; std::string reg;
double lambda; /* Regularization Parameter */ real_t lambda; /* Regularization Parameter */
double alpha; /* This is the controlling param for Elastic Net*/ real_t alpha; /* This is the controlling param for Elastic Net*/
std::string weightInit; std::string weightInit;
void forwardPass(); void forwardPass();
void Test(std::vector<double> x); void Test(std::vector<real_t> x);
}; };

10
mlpp/pca/pca.cpp
View File

@ -13,11 +13,11 @@
MLPPPCA::MLPPPCA(std::vector<std::vector<double>> inputSet, int k) : MLPPPCA::MLPPPCA(std::vector<std::vector<real_t>> inputSet, int k) :
inputSet(inputSet), k(k) { inputSet(inputSet), k(k) {
} }
std::vector<std::vector<double>> MLPPPCA::principalComponents() { std::vector<std::vector<real_t>> MLPPPCA::principalComponents() {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPData data; MLPPData data;
@ -33,10 +33,10 @@ std::vector<std::vector<double>> MLPPPCA::principalComponents() {
return Z; return Z;
} }
// Simply tells us the percentage of variance maintained. // Simply tells us the percentage of variance maintained.
double MLPPPCA::score() { real_t MLPPPCA::score() {
MLPPLinAlg alg; MLPPLinAlg alg;
std::vector<std::vector<double>> X_approx = alg.matmult(U_reduce, Z); std::vector<std::vector<real_t>> X_approx = alg.matmult(U_reduce, Z);
double num, den = 0; real_t num, den = 0;
for (int i = 0; i < X_normalized.size(); i++) { for (int i = 0; i < X_normalized.size(); i++) {
num += alg.norm_sq(alg.subtraction(X_normalized[i], X_approx[i])); num += alg.norm_sq(alg.subtraction(X_normalized[i], X_approx[i]));
} }

16
mlpp/pca/pca.h
View File

@ -8,20 +8,22 @@
// Created by Marc Melikyan on 10/2/20. // Created by Marc Melikyan on 10/2/20.
// //
#include "core/math/math_defs.h"
#include <vector> #include <vector>
class MLPPPCA { class MLPPPCA {
public: public:
MLPPPCA(std::vector<std::vector<double>> inputSet, int k); MLPPPCA(std::vector<std::vector<real_t>> inputSet, int k);
std::vector<std::vector<double>> principalComponents(); std::vector<std::vector<real_t>> principalComponents();
double score(); real_t score();
private: private:
std::vector<std::vector<double>> inputSet; std::vector<std::vector<real_t>> inputSet;
std::vector<std::vector<double>> X_normalized; std::vector<std::vector<real_t>> X_normalized;
std::vector<std::vector<double>> U_reduce; std::vector<std::vector<real_t>> U_reduce;
std::vector<std::vector<double>> Z; std::vector<std::vector<real_t>> Z;
int k; int k;
}; };

56
mlpp/probit_reg/probit_reg.cpp
View File

@ -15,33 +15,33 @@
#include <random> #include <random>
MLPPProbitReg::MLPPProbitReg(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, std::string reg, double lambda, double alpha) : MLPPProbitReg::MLPPProbitReg(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, std::string reg, real_t lambda, real_t alpha) :
inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) { inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) {
y_hat.resize(n); y_hat.resize(n);
weights = MLPPUtilities::weightInitialization(k); weights = MLPPUtilities::weightInitialization(k);
bias = MLPPUtilities::biasInitialization(); bias = MLPPUtilities::biasInitialization();
} }
std::vector<double> MLPPProbitReg::modelSetTest(std::vector<std::vector<double>> X) { std::vector<real_t> MLPPProbitReg::modelSetTest(std::vector<std::vector<real_t>> X) {
return Evaluate(X); return Evaluate(X);
} }
double MLPPProbitReg::modelTest(std::vector<double> x) { real_t MLPPProbitReg::modelTest(std::vector<real_t> x) {
return Evaluate(x); return Evaluate(x);
} }
void MLPPProbitReg::gradientDescent(double learning_rate, int max_epoch, bool UI) { void MLPPProbitReg::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
MLPPActivation avn; MLPPActivation avn;
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
forwardPass(); forwardPass();
while (true) { while (true) {
cost_prev = Cost(y_hat, outputSet); cost_prev = Cost(y_hat, outputSet);
std::vector<double> error = alg.subtraction(y_hat, outputSet); std::vector<real_t> error = alg.subtraction(y_hat, outputSet);
// Calculating the weight gradients // Calculating the weight gradients
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputSet), alg.hadamard_product(error, avn.gaussianCDF(z, 1))))); weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputSet), alg.hadamard_product(error, avn.gaussianCDF(z, 1)))));
@ -63,18 +63,18 @@ void MLPPProbitReg::gradientDescent(double learning_rate, int max_epoch, bool UI
} }
} }
void MLPPProbitReg::MLE(double learning_rate, int max_epoch, bool UI) { void MLPPProbitReg::MLE(real_t learning_rate, int max_epoch, bool UI) {
MLPPActivation avn; MLPPActivation avn;
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
forwardPass(); forwardPass();
while (true) { while (true) {
cost_prev = Cost(y_hat, outputSet); cost_prev = Cost(y_hat, outputSet);
std::vector<double> error = alg.subtraction(outputSet, y_hat); std::vector<real_t> error = alg.subtraction(outputSet, y_hat);
// Calculating the weight gradients // Calculating the weight gradients
weights = alg.addition(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputSet), alg.hadamard_product(error, avn.gaussianCDF(z, 1))))); weights = alg.addition(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputSet), alg.hadamard_product(error, avn.gaussianCDF(z, 1)))));
@ -96,12 +96,12 @@ void MLPPProbitReg::MLE(double learning_rate, int max_epoch, bool UI) {
} }
} }
void MLPPProbitReg::SGD(double learning_rate, int max_epoch, bool UI) { void MLPPProbitReg::SGD(real_t learning_rate, int max_epoch, bool UI) {
// NOTE: ∂y_hat/∂z is sparse // NOTE: ∂y_hat/∂z is sparse
MLPPActivation avn; MLPPActivation avn;
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
while (true) { while (true) {
@ -110,11 +110,11 @@ void MLPPProbitReg::SGD(double learning_rate, int max_epoch, bool UI) {
std::uniform_int_distribution<int> distribution(0, int(n - 1)); std::uniform_int_distribution<int> distribution(0, int(n - 1));
int outputIndex = distribution(generator); int outputIndex = distribution(generator);
double y_hat = Evaluate(inputSet[outputIndex]); real_t y_hat = Evaluate(inputSet[outputIndex]);
double z = propagate(inputSet[outputIndex]); real_t z = propagate(inputSet[outputIndex]);
cost_prev = Cost({ y_hat }, { outputSet[outputIndex] }); cost_prev = Cost({ y_hat }, { outputSet[outputIndex] });
double error = y_hat - outputSet[outputIndex]; real_t error = y_hat - outputSet[outputIndex];
// Weight Updation // Weight Updation
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate * error * ((1 / sqrt(2 * M_PI)) * exp(-z * z / 2)), inputSet[outputIndex])); weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate * error * ((1 / sqrt(2 * M_PI)) * exp(-z * z / 2)), inputSet[outputIndex]));
@ -138,11 +138,11 @@ void MLPPProbitReg::SGD(double learning_rate, int max_epoch, bool UI) {
forwardPass(); forwardPass();
} }
void MLPPProbitReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI) { void MLPPProbitReg::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI) {
MLPPActivation avn; MLPPActivation avn;
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
// Creating the mini-batches // Creating the mini-batches
@ -151,8 +151,8 @@ void MLPPProbitReg::MBGD(double learning_rate, int max_epoch, int mini_batch_siz
// Creating the mini-batches // Creating the mini-batches
for (int i = 0; i < n_mini_batch; i++) { for (int i = 0; i < n_mini_batch; i++) {
std::vector<std::vector<double>> currentInputSet; std::vector<std::vector<real_t>> currentInputSet;
std::vector<double> currentOutputSet; std::vector<real_t> currentOutputSet;
for (int j = 0; j < n / n_mini_batch; j++) { for (int j = 0; j < n / n_mini_batch; j++) {
currentInputSet.push_back(inputSet[n / n_mini_batch * i + j]); currentInputSet.push_back(inputSet[n / n_mini_batch * i + j]);
currentOutputSet.push_back(outputSet[n / n_mini_batch * i + j]); currentOutputSet.push_back(outputSet[n / n_mini_batch * i + j]);
@ -161,7 +161,7 @@ void MLPPProbitReg::MBGD(double learning_rate, int max_epoch, int mini_batch_siz
outputMiniBatches.push_back(currentOutputSet); outputMiniBatches.push_back(currentOutputSet);
} }
if (double(n) / double(n_mini_batch) - int(n / n_mini_batch) != 0) { if (real_t(n) / real_t(n_mini_batch) - int(n / n_mini_batch) != 0) {
for (int i = 0; i < n - n / n_mini_batch * n_mini_batch; i++) { for (int i = 0; i < n - n / n_mini_batch * n_mini_batch; i++) {
inputMiniBatches[n_mini_batch - 1].push_back(inputSet[n / n_mini_batch * n_mini_batch + i]); inputMiniBatches[n_mini_batch - 1].push_back(inputSet[n / n_mini_batch * n_mini_batch + i]);
outputMiniBatches[n_mini_batch - 1].push_back(outputSet[n / n_mini_batch * n_mini_batch + i]); outputMiniBatches[n_mini_batch - 1].push_back(outputSet[n / n_mini_batch * n_mini_batch + i]);
@ -170,11 +170,11 @@ void MLPPProbitReg::MBGD(double learning_rate, int max_epoch, int mini_batch_siz
while (true) { while (true) {
for (int i = 0; i < n_mini_batch; i++) { for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = Evaluate(inputMiniBatches[i]); std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]);
std::vector<double> z = propagate(inputMiniBatches[i]); std::vector<real_t> z = propagate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]); cost_prev = Cost(y_hat, outputMiniBatches[i]);
std::vector<double> error = alg.subtraction(y_hat, outputMiniBatches[i]); std::vector<real_t> error = alg.subtraction(y_hat, outputMiniBatches[i]);
// Calculating the weight gradients // Calculating the weight gradients
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / outputMiniBatches.size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), alg.hadamard_product(error, avn.gaussianCDF(z, 1))))); weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / outputMiniBatches.size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), alg.hadamard_product(error, avn.gaussianCDF(z, 1)))));
@ -197,7 +197,7 @@ void MLPPProbitReg::MBGD(double learning_rate, int max_epoch, int mini_batch_siz
forwardPass(); forwardPass();
} }
double MLPPProbitReg::score() { real_t MLPPProbitReg::score() {
MLPPUtilities util; MLPPUtilities util;
return util.performance(y_hat, outputSet); return util.performance(y_hat, outputSet);
} }
@ -207,30 +207,30 @@ void MLPPProbitReg::save(std::string fileName) {
util.saveParameters(fileName, weights, bias); util.saveParameters(fileName, weights, bias);
} }
double MLPPProbitReg::Cost(std::vector<double> y_hat, std::vector<double> y) { real_t MLPPProbitReg::Cost(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPReg regularization; MLPPReg regularization;
class MLPPCost cost; class MLPPCost cost;
return cost.MSE(y_hat, y) + regularization.regTerm(weights, lambda, alpha, reg); return cost.MSE(y_hat, y) + regularization.regTerm(weights, lambda, alpha, reg);
} }
std::vector<double> MLPPProbitReg::Evaluate(std::vector<std::vector<double>> X) { std::vector<real_t> MLPPProbitReg::Evaluate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPActivation avn; MLPPActivation avn;
return avn.gaussianCDF(alg.scalarAdd(bias, alg.mat_vec_mult(X, weights))); return avn.gaussianCDF(alg.scalarAdd(bias, alg.mat_vec_mult(X, weights)));
} }
std::vector<double> MLPPProbitReg::propagate(std::vector<std::vector<double>> X) { std::vector<real_t> MLPPProbitReg::propagate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.scalarAdd(bias, alg.mat_vec_mult(X, weights)); return alg.scalarAdd(bias, alg.mat_vec_mult(X, weights));
} }
double MLPPProbitReg::Evaluate(std::vector<double> x) { real_t MLPPProbitReg::Evaluate(std::vector<real_t> x) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPActivation avn; MLPPActivation avn;
return avn.gaussianCDF(alg.dot(weights, x) + bias); return avn.gaussianCDF(alg.dot(weights, x) + bias);
} }
double MLPPProbitReg::propagate(std::vector<double> x) { real_t MLPPProbitReg::propagate(std::vector<real_t> x) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.dot(weights, x) + bias; return alg.dot(weights, x) + bias;
} }

44
mlpp/probit_reg/probit_reg.h
View File

@ -8,6 +8,8 @@
// Created by Marc Melikyan on 10/2/20. // Created by Marc Melikyan on 10/2/20.
// //
#include "core/math/math_defs.h"
#include <string> #include <string>
#include <vector> #include <vector>
@ -15,39 +17,39 @@
class MLPPProbitReg { class MLPPProbitReg {
public: public:
MLPPProbitReg(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, std::string reg = "None", double lambda = 0.5, double alpha = 0.5); MLPPProbitReg(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
std::vector<double> modelSetTest(std::vector<std::vector<double>> X); std::vector<real_t> modelSetTest(std::vector<std::vector<real_t>> X);
double modelTest(std::vector<double> x); real_t modelTest(std::vector<real_t> x);
void gradientDescent(double learning_rate, int max_epoch = 0, bool UI = 1); void gradientDescent(real_t learning_rate, int max_epoch = 0, bool UI = 1);
void MLE(double learning_rate, int max_epoch = 0, bool UI = 1); void MLE(real_t learning_rate, int max_epoch = 0, bool UI = 1);
void SGD(double learning_rate, int max_epoch = 0, bool UI = 1); void SGD(real_t learning_rate, int max_epoch = 0, bool UI = 1);
void MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI = 1); void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
double score(); real_t score();
void save(std::string fileName); void save(std::string fileName);
private: private:
double Cost(std::vector<double> y_hat, std::vector<double> y); real_t Cost(std::vector<real_t> y_hat, std::vector<real_t> y);
std::vector<double> Evaluate(std::vector<std::vector<double>> X); std::vector<real_t> Evaluate(std::vector<std::vector<real_t>> X);
std::vector<double> propagate(std::vector<std::vector<double>> X); std::vector<real_t> propagate(std::vector<std::vector<real_t>> X);
double Evaluate(std::vector<double> x); real_t Evaluate(std::vector<real_t> x);
double propagate(std::vector<double> x); real_t propagate(std::vector<real_t> x);
void forwardPass(); void forwardPass();
std::vector<std::vector<double>> inputSet; std::vector<std::vector<real_t>> inputSet;
std::vector<double> outputSet; std::vector<real_t> outputSet;
std::vector<double> z; std::vector<real_t> z;
std::vector<double> y_hat; std::vector<real_t> y_hat;
std::vector<double> weights; std::vector<real_t> weights;
double bias; real_t bias;
int n; int n;
int k; int k;
// Regularization Params // Regularization Params
std::string reg; std::string reg;
double lambda; real_t lambda;
double alpha; /* This is the controlling param for Elastic Net*/ real_t alpha; /* This is the controlling param for Elastic Net*/
}; };

32
mlpp/regularization/reg.cpp
View File

@ -12,21 +12,21 @@
double MLPPReg::regTerm(std::vector<double> weights, double lambda, double alpha, std::string reg) { real_t MLPPReg::regTerm(std::vector<real_t> weights, real_t lambda, real_t alpha, std::string reg) {
if (reg == "Ridge") { if (reg == "Ridge") {
double reg = 0; real_t reg = 0;
for (int i = 0; i < weights.size(); i++) { for (int i = 0; i < weights.size(); i++) {
reg += weights[i] * weights[i]; reg += weights[i] * weights[i];
} }
return reg * lambda / 2; return reg * lambda / 2;
} else if (reg == "Lasso") { } else if (reg == "Lasso") {
double reg = 0; real_t reg = 0;
for (int i = 0; i < weights.size(); i++) { for (int i = 0; i < weights.size(); i++) {
reg += abs(weights[i]); reg += abs(weights[i]);
} }
return reg * lambda; return reg * lambda;
} else if (reg == "ElasticNet") { } else if (reg == "ElasticNet") {
double reg = 0; real_t reg = 0;
for (int i = 0; i < weights.size(); i++) { for (int i = 0; i < weights.size(); i++) {
reg += alpha * abs(weights[i]); // Lasso Reg reg += alpha * abs(weights[i]); // Lasso Reg
reg += ((1 - alpha) / 2) * weights[i] * weights[i]; // Ridge Reg reg += ((1 - alpha) / 2) * weights[i] * weights[i]; // Ridge Reg
@ -36,9 +36,9 @@ double MLPPReg::regTerm(std::vector<double> weights, double lambda, double alpha
return 0; return 0;
} }
double MLPPReg::regTerm(std::vector<std::vector<double>> weights, double lambda, double alpha, std::string reg) { real_t MLPPReg::regTerm(std::vector<std::vector<real_t>> weights, real_t lambda, real_t alpha, std::string reg) {
if (reg == "Ridge") { if (reg == "Ridge") {
double reg = 0; real_t reg = 0;
for (int i = 0; i < weights.size(); i++) { for (int i = 0; i < weights.size(); i++) {
for (int j = 0; j < weights[i].size(); j++) { for (int j = 0; j < weights[i].size(); j++) {
reg += weights[i][j] * weights[i][j]; reg += weights[i][j] * weights[i][j];
@ -46,7 +46,7 @@ double MLPPReg::regTerm(std::vector<std::vector<double>> weights, double lambda,
} }
return reg * lambda / 2; return reg * lambda / 2;
} else if (reg == "Lasso") { } else if (reg == "Lasso") {
double reg = 0; real_t reg = 0;
for (int i = 0; i < weights.size(); i++) { for (int i = 0; i < weights.size(); i++) {
for (int j = 0; j < weights[i].size(); j++) { for (int j = 0; j < weights[i].size(); j++) {
reg += abs(weights[i][j]); reg += abs(weights[i][j]);
@ -54,7 +54,7 @@ double MLPPReg::regTerm(std::vector<std::vector<double>> weights, double lambda,
} }
return reg * lambda; return reg * lambda;
} else if (reg == "ElasticNet") { } else if (reg == "ElasticNet") {
double reg = 0; real_t reg = 0;
for (int i = 0; i < weights.size(); i++) { for (int i = 0; i < weights.size(); i++) {
for (int j = 0; j < weights[i].size(); j++) { for (int j = 0; j < weights[i].size(); j++) {
reg += alpha * abs(weights[i][j]); // Lasso Reg reg += alpha * abs(weights[i][j]); // Lasso Reg
@ -66,7 +66,7 @@ double MLPPReg::regTerm(std::vector<std::vector<double>> weights, double lambda,
return 0; return 0;
} }
std::vector<double> MLPPReg::regWeights(std::vector<double> weights, double lambda, double alpha, std::string reg) { std::vector<real_t> MLPPReg::regWeights(std::vector<real_t> weights, real_t lambda, real_t alpha, std::string reg) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (reg == "WeightClipping") { if (reg == "WeightClipping") {
return regDerivTerm(weights, lambda, alpha, reg); return regDerivTerm(weights, lambda, alpha, reg);
@ -78,7 +78,7 @@ std::vector<double> MLPPReg::regWeights(std::vector<double> weights, double lamb
// return weights; // return weights;
} }
std::vector<std::vector<double>> MLPPReg::regWeights(std::vector<std::vector<double>> weights, double lambda, double alpha, std::string reg) { std::vector<std::vector<real_t>> MLPPReg::regWeights(std::vector<std::vector<real_t>> weights, real_t lambda, real_t alpha, std::string reg) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (reg == "WeightClipping") { if (reg == "WeightClipping") {
return regDerivTerm(weights, lambda, alpha, reg); return regDerivTerm(weights, lambda, alpha, reg);
@ -92,8 +92,8 @@ std::vector<std::vector<double>> MLPPReg::regWeights(std::vector<std::vector<dou
// return weights; // return weights;
} }
std::vector<double> MLPPReg::regDerivTerm(std::vector<double> weights, double lambda, double alpha, std::string reg) { std::vector<real_t> MLPPReg::regDerivTerm(std::vector<real_t> weights, real_t lambda, real_t alpha, std::string reg) {
std::vector<double> regDeriv; std::vector<real_t> regDeriv;
regDeriv.resize(weights.size()); regDeriv.resize(weights.size());
for (int i = 0; i < regDeriv.size(); i++) { for (int i = 0; i < regDeriv.size(); i++) {
@ -102,8 +102,8 @@ std::vector<double> MLPPReg::regDerivTerm(std::vector<double> weights, double la
return regDeriv; return regDeriv;
} }
std::vector<std::vector<double>> MLPPReg::regDerivTerm(std::vector<std::vector<double>> weights, double lambda, double alpha, std::string reg) { std::vector<std::vector<real_t>> MLPPReg::regDerivTerm(std::vector<std::vector<real_t>> weights, real_t lambda, real_t alpha, std::string reg) {
std::vector<std::vector<double>> regDeriv; std::vector<std::vector<real_t>> regDeriv;
regDeriv.resize(weights.size()); regDeriv.resize(weights.size());
for (int i = 0; i < regDeriv.size(); i++) { for (int i = 0; i < regDeriv.size(); i++) {
regDeriv[i].resize(weights[0].size()); regDeriv[i].resize(weights[0].size());
@ -117,7 +117,7 @@ std::vector<std::vector<double>> MLPPReg::regDerivTerm(std::vector<std::vector<d
return regDeriv; return regDeriv;
} }
double MLPPReg::regDerivTerm(std::vector<double> weights, double lambda, double alpha, std::string reg, int j) { real_t MLPPReg::regDerivTerm(std::vector<real_t> weights, real_t lambda, real_t alpha, std::string reg, int j) {
MLPPActivation act; MLPPActivation act;
if (reg == "Ridge") { if (reg == "Ridge") {
return lambda * weights[j]; return lambda * weights[j];
@ -140,7 +140,7 @@ double MLPPReg::regDerivTerm(std::vector<double> weights, double lambda, double
} }
} }
double MLPPReg::regDerivTerm(std::vector<std::vector<double>> weights, double lambda, double alpha, std::string reg, int i, int j) { real_t MLPPReg::regDerivTerm(std::vector<std::vector<real_t>> weights, real_t lambda, real_t alpha, std::string reg, int i, int j) {
MLPPActivation act; MLPPActivation act;
if (reg == "Ridge") { if (reg == "Ridge") {
return lambda * weights[i][j]; return lambda * weights[i][j];

18
mlpp/regularization/reg.h
View File

@ -9,24 +9,26 @@
// Created by Marc Melikyan on 1/16/21. // Created by Marc Melikyan on 1/16/21.
// //
#include "core/math/math_defs.h"
#include <vector> #include <vector>
#include <string> #include <string>
class MLPPReg { class MLPPReg {
public: public:
double regTerm(std::vector<double> weights, double lambda, double alpha, std::string reg); real_t regTerm(std::vector<real_t> weights, real_t lambda, real_t alpha, std::string reg);
double regTerm(std::vector<std::vector<double>> weights, double lambda, double alpha, std::string reg); real_t regTerm(std::vector<std::vector<real_t>> weights, real_t lambda, real_t alpha, std::string reg);
std::vector<double> regWeights(std::vector<double> weights, double lambda, double alpha, std::string reg); std::vector<real_t> regWeights(std::vector<real_t> weights, real_t lambda, real_t alpha, std::string reg);
std::vector<std::vector<double>> regWeights(std::vector<std::vector<double>> weights, double lambda, double alpha, std::string reg); std::vector<std::vector<real_t>> regWeights(std::vector<std::vector<real_t>> weights, real_t lambda, real_t alpha, std::string reg);
std::vector<double> regDerivTerm(std::vector<double> weights, double lambda, double alpha, std::string reg); std::vector<real_t> regDerivTerm(std::vector<real_t> weights, real_t lambda, real_t alpha, std::string reg);
std::vector<std::vector<double>> regDerivTerm(std::vector<std::vector<double>>, double lambda, double alpha, std::string reg); std::vector<std::vector<real_t>> regDerivTerm(std::vector<std::vector<real_t>>, real_t lambda, real_t alpha, std::string reg);
private: private:
double regDerivTerm(std::vector<double> weights, double lambda, double alpha, std::string reg, int j); real_t regDerivTerm(std::vector<real_t> weights, real_t lambda, real_t alpha, std::string reg, int j);
double regDerivTerm(std::vector<std::vector<double>> weights, double lambda, double alpha, std::string reg, int i, int j); real_t regDerivTerm(std::vector<std::vector<real_t>> weights, real_t lambda, real_t alpha, std::string reg, int i, int j);
}; };

88
mlpp/softmax_net/softmax_net.cpp
View File

@ -16,7 +16,7 @@
#include <random> #include <random>
MLPPSoftmaxNet::MLPPSoftmaxNet(std::vector<std::vector<double>> inputSet, std::vector<std::vector<double>> outputSet, int n_hidden, std::string reg, double lambda, double alpha) : MLPPSoftmaxNet::MLPPSoftmaxNet(std::vector<std::vector<real_t>> inputSet, std::vector<std::vector<real_t>> outputSet, int n_hidden, std::string reg, real_t lambda, real_t alpha) :
inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), n_hidden(n_hidden), n_class(outputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) { inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), n_hidden(n_hidden), n_class(outputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) {
y_hat.resize(n); y_hat.resize(n);
@ -26,19 +26,19 @@ MLPPSoftmaxNet::MLPPSoftmaxNet(std::vector<std::vector<double>> inputSet, std::v
bias2 = MLPPUtilities::biasInitialization(n_class); bias2 = MLPPUtilities::biasInitialization(n_class);
} }
std::vector<double> MLPPSoftmaxNet::modelTest(std::vector<double> x) { std::vector<real_t> MLPPSoftmaxNet::modelTest(std::vector<real_t> x) {
return Evaluate(x); return Evaluate(x);
} }
std::vector<std::vector<double>> MLPPSoftmaxNet::modelSetTest(std::vector<std::vector<double>> X) { std::vector<std::vector<real_t>> MLPPSoftmaxNet::modelSetTest(std::vector<std::vector<real_t>> X) {
return Evaluate(X); return Evaluate(X);
} }
void MLPPSoftmaxNet::gradientDescent(double learning_rate, int max_epoch, bool UI) { void MLPPSoftmaxNet::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
MLPPActivation avn; MLPPActivation avn;
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
forwardPass(); forwardPass();
@ -46,11 +46,11 @@ void MLPPSoftmaxNet::gradientDescent(double learning_rate, int max_epoch, bool U
cost_prev = Cost(y_hat, outputSet); cost_prev = Cost(y_hat, outputSet);
// Calculating the errors // Calculating the errors
std::vector<std::vector<double>> error = alg.subtraction(y_hat, outputSet); std::vector<std::vector<real_t>> error = alg.subtraction(y_hat, outputSet);
// Calculating the weight/bias gradients for layer 2 // Calculating the weight/bias gradients for layer 2
std::vector<std::vector<double>> D2_1 = alg.matmult(alg.transpose(a2), error); std::vector<std::vector<real_t>> D2_1 = alg.matmult(alg.transpose(a2), error);
// weights and bias updation for layer 2 // weights and bias updation for layer 2
weights2 = alg.subtraction(weights2, alg.scalarMultiply(learning_rate, D2_1)); weights2 = alg.subtraction(weights2, alg.scalarMultiply(learning_rate, D2_1));
@ -60,11 +60,11 @@ void MLPPSoftmaxNet::gradientDescent(double learning_rate, int max_epoch, bool U
//Calculating the weight/bias for layer 1 //Calculating the weight/bias for layer 1
std::vector<std::vector<double>> D1_1 = alg.matmult(error, alg.transpose(weights2)); std::vector<std::vector<real_t>> D1_1 = alg.matmult(error, alg.transpose(weights2));
std::vector<std::vector<double>> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1)); std::vector<std::vector<real_t>> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1));
std::vector<std::vector<double>> D1_3 = alg.matmult(alg.transpose(inputSet), D1_2); std::vector<std::vector<real_t>> D1_3 = alg.matmult(alg.transpose(inputSet), D1_2);
// weight an bias updation for layer 1 // weight an bias updation for layer 1
weights1 = alg.subtraction(weights1, alg.scalarMultiply(learning_rate, D1_3)); weights1 = alg.subtraction(weights1, alg.scalarMultiply(learning_rate, D1_3));
@ -90,11 +90,11 @@ void MLPPSoftmaxNet::gradientDescent(double learning_rate, int max_epoch, bool U
} }
} }
void MLPPSoftmaxNet::SGD(double learning_rate, int max_epoch, bool UI) { void MLPPSoftmaxNet::SGD(real_t learning_rate, int max_epoch, bool UI) {
MLPPActivation avn; MLPPActivation avn;
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
while (true) { while (true) {
@ -103,13 +103,13 @@ void MLPPSoftmaxNet::SGD(double learning_rate, int max_epoch, bool UI) {
std::uniform_int_distribution<int> distribution(0, int(n - 1)); std::uniform_int_distribution<int> distribution(0, int(n - 1));
int outputIndex = distribution(generator); int outputIndex = distribution(generator);
std::vector<double> y_hat = Evaluate(inputSet[outputIndex]); std::vector<real_t> y_hat = Evaluate(inputSet[outputIndex]);
auto [z2, a2] = propagate(inputSet[outputIndex]); auto [z2, a2] = propagate(inputSet[outputIndex]);
cost_prev = Cost({ y_hat }, { outputSet[outputIndex] }); cost_prev = Cost({ y_hat }, { outputSet[outputIndex] });
std::vector<double> error = alg.subtraction(y_hat, outputSet[outputIndex]); std::vector<real_t> error = alg.subtraction(y_hat, outputSet[outputIndex]);
// Weight updation for layer 2 // Weight updation for layer 2
std::vector<std::vector<double>> D2_1 = alg.outerProduct(error, a2); std::vector<std::vector<real_t>> D2_1 = alg.outerProduct(error, a2);
weights2 = alg.subtraction(weights2, alg.scalarMultiply(learning_rate, alg.transpose(D2_1))); weights2 = alg.subtraction(weights2, alg.scalarMultiply(learning_rate, alg.transpose(D2_1)));
weights2 = regularization.regWeights(weights2, lambda, alpha, reg); weights2 = regularization.regWeights(weights2, lambda, alpha, reg);
@ -117,9 +117,9 @@ void MLPPSoftmaxNet::SGD(double learning_rate, int max_epoch, bool UI) {
bias2 = alg.subtraction(bias2, alg.scalarMultiply(learning_rate, error)); bias2 = alg.subtraction(bias2, alg.scalarMultiply(learning_rate, error));
// Weight updation for layer 1 // Weight updation for layer 1
std::vector<double> D1_1 = alg.mat_vec_mult(weights2, error); std::vector<real_t> D1_1 = alg.mat_vec_mult(weights2, error);
std::vector<double> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1)); std::vector<real_t> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1));
std::vector<std::vector<double>> D1_3 = alg.outerProduct(inputSet[outputIndex], D1_2); std::vector<std::vector<real_t>> D1_3 = alg.outerProduct(inputSet[outputIndex], D1_2);
weights1 = alg.subtraction(weights1, alg.scalarMultiply(learning_rate, D1_3)); weights1 = alg.subtraction(weights1, alg.scalarMultiply(learning_rate, D1_3));
weights1 = regularization.regWeights(weights1, lambda, alpha, reg); weights1 = regularization.regWeights(weights1, lambda, alpha, reg);
@ -144,11 +144,11 @@ void MLPPSoftmaxNet::SGD(double learning_rate, int max_epoch, bool UI) {
forwardPass(); forwardPass();
} }
void MLPPSoftmaxNet::MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI) { void MLPPSoftmaxNet::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI) {
MLPPActivation avn; MLPPActivation avn;
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
// Creating the mini-batches // Creating the mini-batches
@ -157,8 +157,8 @@ void MLPPSoftmaxNet::MBGD(double learning_rate, int max_epoch, int mini_batch_si
// Creating the mini-batches // Creating the mini-batches
for (int i = 0; i < n_mini_batch; i++) { for (int i = 0; i < n_mini_batch; i++) {
std::vector<std::vector<double>> currentInputSet; std::vector<std::vector<real_t>> currentInputSet;
std::vector<std::vector<double>> currentOutputSet; std::vector<std::vector<real_t>> currentOutputSet;
for (int j = 0; j < n / n_mini_batch; j++) { for (int j = 0; j < n / n_mini_batch; j++) {
currentInputSet.push_back(inputSet[n / n_mini_batch * i + j]); currentInputSet.push_back(inputSet[n / n_mini_batch * i + j]);
currentOutputSet.push_back(outputSet[n / n_mini_batch * i + j]); currentOutputSet.push_back(outputSet[n / n_mini_batch * i + j]);
@ -167,7 +167,7 @@ void MLPPSoftmaxNet::MBGD(double learning_rate, int max_epoch, int mini_batch_si
outputMiniBatches.push_back(currentOutputSet); outputMiniBatches.push_back(currentOutputSet);
} }
if (double(n) / double(n_mini_batch) - int(n / n_mini_batch) != 0) { if (real_t(n) / real_t(n_mini_batch) - int(n / n_mini_batch) != 0) {
for (int i = 0; i < n - n / n_mini_batch * n_mini_batch; i++) { for (int i = 0; i < n - n / n_mini_batch * n_mini_batch; i++) {
inputMiniBatches[n_mini_batch - 1].push_back(inputSet[n / n_mini_batch * n_mini_batch + i]); inputMiniBatches[n_mini_batch - 1].push_back(inputSet[n / n_mini_batch * n_mini_batch + i]);
outputMiniBatches[n_mini_batch - 1].push_back(outputSet[n / n_mini_batch * n_mini_batch + i]); outputMiniBatches[n_mini_batch - 1].push_back(outputSet[n / n_mini_batch * n_mini_batch + i]);
@ -176,16 +176,16 @@ void MLPPSoftmaxNet::MBGD(double learning_rate, int max_epoch, int mini_batch_si
while (true) { while (true) {
for (int i = 0; i < n_mini_batch; i++) { for (int i = 0; i < n_mini_batch; i++) {
std::vector<std::vector<double>> y_hat = Evaluate(inputMiniBatches[i]); std::vector<std::vector<real_t>> y_hat = Evaluate(inputMiniBatches[i]);
auto [z2, a2] = propagate(inputMiniBatches[i]); auto [z2, a2] = propagate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]); cost_prev = Cost(y_hat, outputMiniBatches[i]);
// Calculating the errors // Calculating the errors
std::vector<std::vector<double>> error = alg.subtraction(y_hat, outputMiniBatches[i]); std::vector<std::vector<real_t>> error = alg.subtraction(y_hat, outputMiniBatches[i]);
// Calculating the weight/bias gradients for layer 2 // Calculating the weight/bias gradients for layer 2
std::vector<std::vector<double>> D2_1 = alg.matmult(alg.transpose(a2), error); std::vector<std::vector<real_t>> D2_1 = alg.matmult(alg.transpose(a2), error);
// weights and bias updation for layser 2 // weights and bias updation for layser 2
weights2 = alg.subtraction(weights2, alg.scalarMultiply(learning_rate, D2_1)); weights2 = alg.subtraction(weights2, alg.scalarMultiply(learning_rate, D2_1));
@ -196,11 +196,11 @@ void MLPPSoftmaxNet::MBGD(double learning_rate, int max_epoch, int mini_batch_si
//Calculating the weight/bias for layer 1 //Calculating the weight/bias for layer 1
std::vector<std::vector<double>> D1_1 = alg.matmult(error, alg.transpose(weights2)); std::vector<std::vector<real_t>> D1_1 = alg.matmult(error, alg.transpose(weights2));
std::vector<std::vector<double>> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1)); std::vector<std::vector<real_t>> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1));
std::vector<std::vector<double>> D1_3 = alg.matmult(alg.transpose(inputMiniBatches[i]), D1_2); std::vector<std::vector<real_t>> D1_3 = alg.matmult(alg.transpose(inputMiniBatches[i]), D1_2);
// weight an bias updation for layer 1 // weight an bias updation for layer 1
weights1 = alg.subtraction(weights1, alg.scalarMultiply(learning_rate, D1_3)); weights1 = alg.subtraction(weights1, alg.scalarMultiply(learning_rate, D1_3));
@ -226,7 +226,7 @@ void MLPPSoftmaxNet::MBGD(double learning_rate, int max_epoch, int mini_batch_si
forwardPass(); forwardPass();
} }
double MLPPSoftmaxNet::score() { real_t MLPPSoftmaxNet::score() {
MLPPUtilities util; MLPPUtilities util;
return util.performance(y_hat, outputSet); return util.performance(y_hat, outputSet);
} }
@ -239,46 +239,46 @@ void MLPPSoftmaxNet::save(std::string fileName) {
MLPPLinAlg alg; MLPPLinAlg alg;
} }
std::vector<std::vector<double>> MLPPSoftmaxNet::getEmbeddings() { std::vector<std::vector<real_t>> MLPPSoftmaxNet::getEmbeddings() {
return weights1; return weights1;
} }
double MLPPSoftmaxNet::Cost(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) { real_t MLPPSoftmaxNet::Cost(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
MLPPReg regularization; MLPPReg regularization;
MLPPData data; MLPPData data;
class MLPPCost cost; class MLPPCost cost;
return cost.CrossEntropy(y_hat, y) + regularization.regTerm(weights1, lambda, alpha, reg) + regularization.regTerm(weights2, lambda, alpha, reg); return cost.CrossEntropy(y_hat, y) + regularization.regTerm(weights1, lambda, alpha, reg) + regularization.regTerm(weights2, lambda, alpha, reg);
} }
std::vector<std::vector<double>> MLPPSoftmaxNet::Evaluate(std::vector<std::vector<double>> X) { std::vector<std::vector<real_t>> MLPPSoftmaxNet::Evaluate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPActivation avn; MLPPActivation avn;
std::vector<std::vector<double>> z2 = alg.mat_vec_add(alg.matmult(X, weights1), bias1); std::vector<std::vector<real_t>> z2 = alg.mat_vec_add(alg.matmult(X, weights1), bias1);
std::vector<std::vector<double>> a2 = avn.sigmoid(z2); std::vector<std::vector<real_t>> a2 = avn.sigmoid(z2);
return avn.adjSoftmax(alg.mat_vec_add(alg.matmult(a2, weights2), bias2)); return avn.adjSoftmax(alg.mat_vec_add(alg.matmult(a2, weights2), bias2));
} }
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> MLPPSoftmaxNet::propagate(std::vector<std::vector<double>> X) { std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> MLPPSoftmaxNet::propagate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPActivation avn; MLPPActivation avn;
std::vector<std::vector<double>> z2 = alg.mat_vec_add(alg.matmult(X, weights1), bias1); std::vector<std::vector<real_t>> z2 = alg.mat_vec_add(alg.matmult(X, weights1), bias1);
std::vector<std::vector<double>> a2 = avn.sigmoid(z2); std::vector<std::vector<real_t>> a2 = avn.sigmoid(z2);
return { z2, a2 }; return { z2, a2 };
} }
std::vector<double> MLPPSoftmaxNet::Evaluate(std::vector<double> x) { std::vector<real_t> MLPPSoftmaxNet::Evaluate(std::vector<real_t> x) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPActivation avn; MLPPActivation avn;
std::vector<double> z2 = alg.addition(alg.mat_vec_mult(alg.transpose(weights1), x), bias1); std::vector<real_t> z2 = alg.addition(alg.mat_vec_mult(alg.transpose(weights1), x), bias1);
std::vector<double> a2 = avn.sigmoid(z2); std::vector<real_t> a2 = avn.sigmoid(z2);
return avn.adjSoftmax(alg.addition(alg.mat_vec_mult(alg.transpose(weights2), a2), bias2)); return avn.adjSoftmax(alg.addition(alg.mat_vec_mult(alg.transpose(weights2), a2), bias2));
} }
std::tuple<std::vector<double>, std::vector<double>> MLPPSoftmaxNet::propagate(std::vector<double> x) { std::tuple<std::vector<real_t>, std::vector<real_t>> MLPPSoftmaxNet::propagate(std::vector<real_t> x) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPActivation avn; MLPPActivation avn;
std::vector<double> z2 = alg.addition(alg.mat_vec_mult(alg.transpose(weights1), x), bias1); std::vector<real_t> z2 = alg.addition(alg.mat_vec_mult(alg.transpose(weights1), x), bias1);
std::vector<double> a2 = avn.sigmoid(z2); std::vector<real_t> a2 = avn.sigmoid(z2);
return { z2, a2 }; return { z2, a2 };
} }

50
mlpp/softmax_net/softmax_net.h
View File

@ -8,6 +8,8 @@
// Created by Marc Melikyan on 10/2/20. // Created by Marc Melikyan on 10/2/20.
// //
#include "core/math/math_defs.h"
#include <string> #include <string>
#include <vector> #include <vector>
@ -15,37 +17,37 @@
class MLPPSoftmaxNet { class MLPPSoftmaxNet {
public: public:
MLPPSoftmaxNet(std::vector<std::vector<double>> inputSet, std::vector<std::vector<double>> outputSet, int n_hidden, std::string reg = "None", double lambda = 0.5, double alpha = 0.5); MLPPSoftmaxNet(std::vector<std::vector<real_t>> inputSet, std::vector<std::vector<real_t>> outputSet, int n_hidden, std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
std::vector<double> modelTest(std::vector<double> x); std::vector<real_t> modelTest(std::vector<real_t> x);
std::vector<std::vector<double>> modelSetTest(std::vector<std::vector<double>> X); std::vector<std::vector<real_t>> modelSetTest(std::vector<std::vector<real_t>> X);
void gradientDescent(double learning_rate, int max_epoch, bool UI = 1); void gradientDescent(real_t learning_rate, int max_epoch, bool UI = 1);
void SGD(double learning_rate, int max_epoch, bool UI = 1); void SGD(real_t learning_rate, int max_epoch, bool UI = 1);
void MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI = 1); void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
double score(); real_t score();
void save(std::string fileName); void save(std::string fileName);
std::vector<std::vector<double>> getEmbeddings(); // This class is used (mostly) for word2Vec. This function returns our embeddings. std::vector<std::vector<real_t>> getEmbeddings(); // This class is used (mostly) for word2Vec. This function returns our embeddings.
private: private:
double Cost(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y); real_t Cost(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
std::vector<std::vector<double>> Evaluate(std::vector<std::vector<double>> X); std::vector<std::vector<real_t>> Evaluate(std::vector<std::vector<real_t>> X);
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> propagate(std::vector<std::vector<double>> X); std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> propagate(std::vector<std::vector<real_t>> X);
std::vector<double> Evaluate(std::vector<double> x); std::vector<real_t> Evaluate(std::vector<real_t> x);
std::tuple<std::vector<double>, std::vector<double>> propagate(std::vector<double> x); std::tuple<std::vector<real_t>, std::vector<real_t>> propagate(std::vector<real_t> x);
void forwardPass(); void forwardPass();
std::vector<std::vector<double>> inputSet; std::vector<std::vector<real_t>> inputSet;
std::vector<std::vector<double>> outputSet; std::vector<std::vector<real_t>> outputSet;
std::vector<std::vector<double>> y_hat; std::vector<std::vector<real_t>> y_hat;
std::vector<std::vector<double>> weights1; std::vector<std::vector<real_t>> weights1;
std::vector<std::vector<double>> weights2; std::vector<std::vector<real_t>> weights2;
std::vector<double> bias1; std::vector<real_t> bias1;
std::vector<double> bias2; std::vector<real_t> bias2;
std::vector<std::vector<double>> z2; std::vector<std::vector<real_t>> z2;
std::vector<std::vector<double>> a2; std::vector<std::vector<real_t>> a2;
int n; int n;
int k; int k;
@ -54,8 +56,8 @@ private:
// Regularization Params // Regularization Params
std::string reg; std::string reg;
double lambda; real_t lambda;
double alpha; /* This is the controlling param for Elastic Net*/ real_t alpha; /* This is the controlling param for Elastic Net*/
}; };

46
mlpp/softmax_reg/softmax_reg.cpp
View File

@ -15,41 +15,41 @@
#include <random> #include <random>
MLPPSoftmaxReg::MLPPSoftmaxReg(std::vector<std::vector<double>> inputSet, std::vector<std::vector<double>> outputSet, std::string reg, double lambda, double alpha) : MLPPSoftmaxReg::MLPPSoftmaxReg(std::vector<std::vector<real_t>> inputSet, std::vector<std::vector<real_t>> outputSet, std::string reg, real_t lambda, real_t alpha) :
inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), n_class(outputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) { inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), n_class(outputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) {
y_hat.resize(n); y_hat.resize(n);
weights = MLPPUtilities::weightInitialization(k, n_class); weights = MLPPUtilities::weightInitialization(k, n_class);
bias = MLPPUtilities::biasInitialization(n_class); bias = MLPPUtilities::biasInitialization(n_class);
} }
std::vector<double> MLPPSoftmaxReg::modelTest(std::vector<double> x) { std::vector<real_t> MLPPSoftmaxReg::modelTest(std::vector<real_t> x) {
return Evaluate(x); return Evaluate(x);
} }
std::vector<std::vector<double>> MLPPSoftmaxReg::modelSetTest(std::vector<std::vector<double>> X) { std::vector<std::vector<real_t>> MLPPSoftmaxReg::modelSetTest(std::vector<std::vector<real_t>> X) {
return Evaluate(X); return Evaluate(X);
} }
void MLPPSoftmaxReg::gradientDescent(double learning_rate, int max_epoch, bool UI) { void MLPPSoftmaxReg::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
forwardPass(); forwardPass();
while (true) { while (true) {
cost_prev = Cost(y_hat, outputSet); cost_prev = Cost(y_hat, outputSet);
std::vector<std::vector<double>> error = alg.subtraction(y_hat, outputSet); std::vector<std::vector<real_t>> error = alg.subtraction(y_hat, outputSet);
//Calculating the weight gradients //Calculating the weight gradients
std::vector<std::vector<double>> w_gradient = alg.matmult(alg.transpose(inputSet), error); std::vector<std::vector<real_t>> w_gradient = alg.matmult(alg.transpose(inputSet), error);
//Weight updation //Weight updation
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate, w_gradient)); weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate, w_gradient));
weights = regularization.regWeights(weights, lambda, alpha, reg); weights = regularization.regWeights(weights, lambda, alpha, reg);
// Calculating the bias gradients // Calculating the bias gradients
//double b_gradient = alg.sum_elements(error); //real_t b_gradient = alg.sum_elements(error);
// Bias Updation // Bias Updation
bias = alg.subtractMatrixRows(bias, alg.scalarMultiply(learning_rate, error)); bias = alg.subtractMatrixRows(bias, alg.scalarMultiply(learning_rate, error));
@ -69,30 +69,30 @@ void MLPPSoftmaxReg::gradientDescent(double learning_rate, int max_epoch, bool U
} }
} }
void MLPPSoftmaxReg::SGD(double learning_rate, int max_epoch, bool UI) { void MLPPSoftmaxReg::SGD(real_t learning_rate, int max_epoch, bool UI) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
while (true) { while (true) {
std::random_device rd; std::random_device rd;
std::default_random_engine generator(rd()); std::default_random_engine generator(rd());
std::uniform_int_distribution<int> distribution(0, int(n - 1)); std::uniform_int_distribution<int> distribution(0, int(n - 1));
double outputIndex = distribution(generator); real_t outputIndex = distribution(generator);
std::vector<double> y_hat = Evaluate(inputSet[outputIndex]); std::vector<real_t> y_hat = Evaluate(inputSet[outputIndex]);
cost_prev = Cost({ y_hat }, { outputSet[outputIndex] }); cost_prev = Cost({ y_hat }, { outputSet[outputIndex] });
// Calculating the weight gradients // Calculating the weight gradients
std::vector<std::vector<double>> w_gradient = alg.outerProduct(inputSet[outputIndex], alg.subtraction(y_hat, outputSet[outputIndex])); std::vector<std::vector<real_t>> w_gradient = alg.outerProduct(inputSet[outputIndex], alg.subtraction(y_hat, outputSet[outputIndex]));
// Weight Updation // Weight Updation
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate, w_gradient)); weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate, w_gradient));
weights = regularization.regWeights(weights, lambda, alpha, reg); weights = regularization.regWeights(weights, lambda, alpha, reg);
// Calculating the bias gradients // Calculating the bias gradients
std::vector<double> b_gradient = alg.subtraction(y_hat, outputSet[outputIndex]); std::vector<real_t> b_gradient = alg.subtraction(y_hat, outputSet[outputIndex]);
// Bias updation // Bias updation
bias = alg.subtraction(bias, alg.scalarMultiply(learning_rate, b_gradient)); bias = alg.subtraction(bias, alg.scalarMultiply(learning_rate, b_gradient));
@ -112,10 +112,10 @@ void MLPPSoftmaxReg::SGD(double learning_rate, int max_epoch, bool UI) {
forwardPass(); forwardPass();
} }
void MLPPSoftmaxReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI) { void MLPPSoftmaxReg::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
// Creating the mini-batches // Creating the mini-batches
@ -124,13 +124,13 @@ void MLPPSoftmaxReg::MBGD(double learning_rate, int max_epoch, int mini_batch_si
while (true) { while (true) {
for (int i = 0; i < n_mini_batch; i++) { for (int i = 0; i < n_mini_batch; i++) {
std::vector<std::vector<double>> y_hat = Evaluate(inputMiniBatches[i]); std::vector<std::vector<real_t>> y_hat = Evaluate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]); cost_prev = Cost(y_hat, outputMiniBatches[i]);
std::vector<std::vector<double>> error = alg.subtraction(y_hat, outputMiniBatches[i]); std::vector<std::vector<real_t>> error = alg.subtraction(y_hat, outputMiniBatches[i]);
// Calculating the weight gradients // Calculating the weight gradients
std::vector<std::vector<double>> w_gradient = alg.matmult(alg.transpose(inputMiniBatches[i]), error); std::vector<std::vector<real_t>> w_gradient = alg.matmult(alg.transpose(inputMiniBatches[i]), error);
//Weight updation //Weight updation
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate, w_gradient)); weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate, w_gradient));
@ -153,7 +153,7 @@ void MLPPSoftmaxReg::MBGD(double learning_rate, int max_epoch, int mini_batch_si
forwardPass(); forwardPass();
} }
double MLPPSoftmaxReg::score() { real_t MLPPSoftmaxReg::score() {
MLPPUtilities util; MLPPUtilities util;
return util.performance(y_hat, outputSet); return util.performance(y_hat, outputSet);
} }
@ -163,19 +163,19 @@ void MLPPSoftmaxReg::save(std::string fileName) {
util.saveParameters(fileName, weights, bias); util.saveParameters(fileName, weights, bias);
} }
double MLPPSoftmaxReg::Cost(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) { real_t MLPPSoftmaxReg::Cost(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
MLPPReg regularization; MLPPReg regularization;
class MLPPCost cost; class MLPPCost cost;
return cost.CrossEntropy(y_hat, y) + regularization.regTerm(weights, lambda, alpha, reg); return cost.CrossEntropy(y_hat, y) + regularization.regTerm(weights, lambda, alpha, reg);
} }
std::vector<double> MLPPSoftmaxReg::Evaluate(std::vector<double> x) { std::vector<real_t> MLPPSoftmaxReg::Evaluate(std::vector<real_t> x) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPActivation avn; MLPPActivation avn;
return avn.softmax(alg.addition(bias, alg.mat_vec_mult(alg.transpose(weights), x))); return avn.softmax(alg.addition(bias, alg.mat_vec_mult(alg.transpose(weights), x)));
} }
std::vector<std::vector<double>> MLPPSoftmaxReg::Evaluate(std::vector<std::vector<double>> X) { std::vector<std::vector<real_t>> MLPPSoftmaxReg::Evaluate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPActivation avn; MLPPActivation avn;

36
mlpp/softmax_reg/softmax_reg.h
View File

@ -8,6 +8,8 @@
// Created by Marc Melikyan on 10/2/20. // Created by Marc Melikyan on 10/2/20.
// //
#include "core/math/math_defs.h"
#include <string> #include <string>
#include <vector> #include <vector>
@ -15,27 +17,27 @@
class MLPPSoftmaxReg { class MLPPSoftmaxReg {
public: public:
MLPPSoftmaxReg(std::vector<std::vector<double>> inputSet, std::vector<std::vector<double>> outputSet, std::string reg = "None", double lambda = 0.5, double alpha = 0.5); MLPPSoftmaxReg(std::vector<std::vector<real_t>> inputSet, std::vector<std::vector<real_t>> outputSet, std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
std::vector<double> modelTest(std::vector<double> x); std::vector<real_t> modelTest(std::vector<real_t> x);
std::vector<std::vector<double>> modelSetTest(std::vector<std::vector<double>> X); std::vector<std::vector<real_t>> modelSetTest(std::vector<std::vector<real_t>> X);
void gradientDescent(double learning_rate, int max_epoch, bool UI = 1); void gradientDescent(real_t learning_rate, int max_epoch, bool UI = 1);
void SGD(double learning_rate, int max_epoch, bool UI = 1); void SGD(real_t learning_rate, int max_epoch, bool UI = 1);
void MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI = 1); void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
double score(); real_t score();
void save(std::string fileName); void save(std::string fileName);
private: private:
double Cost(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y); real_t Cost(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
std::vector<std::vector<double>> Evaluate(std::vector<std::vector<double>> X); std::vector<std::vector<real_t>> Evaluate(std::vector<std::vector<real_t>> X);
std::vector<double> Evaluate(std::vector<double> x); std::vector<real_t> Evaluate(std::vector<real_t> x);
void forwardPass(); void forwardPass();
std::vector<std::vector<double>> inputSet; std::vector<std::vector<real_t>> inputSet;
std::vector<std::vector<double>> outputSet; std::vector<std::vector<real_t>> outputSet;
std::vector<std::vector<double>> y_hat; std::vector<std::vector<real_t>> y_hat;
std::vector<std::vector<double>> weights; std::vector<std::vector<real_t>> weights;
std::vector<double> bias; std::vector<real_t> bias;
int n; int n;
int k; int k;
@ -43,8 +45,8 @@ private:
// Regularization Params // Regularization Params
std::string reg; std::string reg;
double lambda; real_t lambda;
double alpha; /* This is the controlling param for Elastic Net*/ real_t alpha; /* This is the controlling param for Elastic Net*/
}; };

94
mlpp/stat/stat.cpp
View File

@ -15,24 +15,24 @@
#include <iostream> #include <iostream>
double MLPPStat::b0Estimation(const std::vector<double> &x, const std::vector<double> &y) { real_t MLPPStat::b0Estimation(const std::vector<real_t> &x, const std::vector<real_t> &y) {
return mean(y) - b1Estimation(x, y) * mean(x); return mean(y) - b1Estimation(x, y) * mean(x);
} }
double MLPPStat::b1Estimation(const std::vector<double> &x, const std::vector<double> &y) { real_t MLPPStat::b1Estimation(const std::vector<real_t> &x, const std::vector<real_t> &y) {
return covariance(x, y) / variance(x); return covariance(x, y) / variance(x);
} }
double MLPPStat::mean(const std::vector<double> &x) { real_t MLPPStat::mean(const std::vector<real_t> &x) {
double sum = 0; real_t sum = 0;
for (int i = 0; i < x.size(); i++) { for (int i = 0; i < x.size(); i++) {
sum += x[i]; sum += x[i];
} }
return sum / x.size(); return sum / x.size();
} }
double MLPPStat::median(std::vector<double> x) { real_t MLPPStat::median(std::vector<real_t> x) {
double center = double(x.size()) / double(2); real_t center = real_t(x.size()) / real_t(2);
sort(x.begin(), x.end()); sort(x.begin(), x.end());
if (x.size() % 2 == 0) { if (x.size() % 2 == 0) {
return mean({ x[center - 1], x[center] }); return mean({ x[center - 1], x[center] });
@ -41,18 +41,18 @@ double MLPPStat::median(std::vector<double> x) {
} }
} }
std::vector<double> MLPPStat::mode(const std::vector<double> &x) { std::vector<real_t> MLPPStat::mode(const std::vector<real_t> &x) {
MLPPData data; MLPPData data;
std::vector<double> x_set = data.vecToSet(x); std::vector<real_t> x_set = data.vecToSet(x);
std::map<double, int> element_num; std::map<real_t, int> element_num;
for (int i = 0; i < x_set.size(); i++) { for (int i = 0; i < x_set.size(); i++) {
element_num[x[i]] = 0; element_num[x[i]] = 0;
} }
for (int i = 0; i < x.size(); i++) { for (int i = 0; i < x.size(); i++) {
element_num[x[i]]++; element_num[x[i]]++;
} }
std::vector<double> modes; std::vector<real_t> modes;
double max_num = element_num[x_set[0]]; real_t max_num = element_num[x_set[0]];
for (int i = 0; i < x_set.size(); i++) { for (int i = 0; i < x_set.size(); i++) {
if (element_num[x_set[i]] > max_num) { if (element_num[x_set[i]] > max_num) {
max_num = element_num[x_set[i]]; max_num = element_num[x_set[i]];
@ -65,59 +65,59 @@ std::vector<double> MLPPStat::mode(const std::vector<double> &x) {
return modes; return modes;
} }
double MLPPStat::range(const std::vector<double> &x) { real_t MLPPStat::range(const std::vector<real_t> &x) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.max(x) - alg.min(x); return alg.max(x) - alg.min(x);
} }
double MLPPStat::midrange(const std::vector<double> &x) { real_t MLPPStat::midrange(const std::vector<real_t> &x) {
return range(x) / 2; return range(x) / 2;
} }
double MLPPStat::absAvgDeviation(const std::vector<double> &x) { real_t MLPPStat::absAvgDeviation(const std::vector<real_t> &x) {
double sum = 0; real_t sum = 0;
for (int i = 0; i < x.size(); i++) { for (int i = 0; i < x.size(); i++) {
sum += std::abs(x[i] - mean(x)); sum += std::abs(x[i] - mean(x));
} }
return sum / x.size(); return sum / x.size();
} }
double MLPPStat::standardDeviation(const std::vector<double> &x) { real_t MLPPStat::standardDeviation(const std::vector<real_t> &x) {
return std::sqrt(variance(x)); return std::sqrt(variance(x));
} }
double MLPPStat::variance(const std::vector<double> &x) { real_t MLPPStat::variance(const std::vector<real_t> &x) {
double sum = 0; real_t sum = 0;
for (int i = 0; i < x.size(); i++) { for (int i = 0; i < x.size(); i++) {
sum += (x[i] - mean(x)) * (x[i] - mean(x)); sum += (x[i] - mean(x)) * (x[i] - mean(x));
} }
return sum / (x.size() - 1); return sum / (x.size() - 1);
} }
double MLPPStat::covariance(const std::vector<double> &x, const std::vector<double> &y) { real_t MLPPStat::covariance(const std::vector<real_t> &x, const std::vector<real_t> &y) {
double sum = 0; real_t sum = 0;
for (int i = 0; i < x.size(); i++) { for (int i = 0; i < x.size(); i++) {
sum += (x[i] - mean(x)) * (y[i] - mean(y)); sum += (x[i] - mean(x)) * (y[i] - mean(y));
} }
return sum / (x.size() - 1); return sum / (x.size() - 1);
} }
double MLPPStat::correlation(const std::vector<double> &x, const std::vector<double> &y) { real_t MLPPStat::correlation(const std::vector<real_t> &x, const std::vector<real_t> &y) {
return covariance(x, y) / (standardDeviation(x) * standardDeviation(y)); return covariance(x, y) / (standardDeviation(x) * standardDeviation(y));
} }
double MLPPStat::R2(const std::vector<double> &x, const std::vector<double> &y) { real_t MLPPStat::R2(const std::vector<real_t> &x, const std::vector<real_t> &y) {
return correlation(x, y) * correlation(x, y); return correlation(x, y) * correlation(x, y);
} }
double MLPPStat::chebyshevIneq(const double k) { real_t MLPPStat::chebyshevIneq(const real_t k) {
// X may or may not belong to a Gaussian Distribution // X may or may not belong to a Gaussian Distribution
return 1 - 1 / (k * k); return 1 - 1 / (k * k);
} }
double MLPPStat::weightedMean(const std::vector<double> &x, const std::vector<double> &weights) { real_t MLPPStat::weightedMean(const std::vector<real_t> &x, const std::vector<real_t> &weights) {
double sum = 0; real_t sum = 0;
double weights_sum = 0; real_t weights_sum = 0;
for (int i = 0; i < x.size(); i++) { for (int i = 0; i < x.size(); i++) {
sum += x[i] * weights[i]; sum += x[i] * weights[i];
weights_sum += weights[i]; weights_sum += weights[i];
@ -125,41 +125,41 @@ double MLPPStat::weightedMean(const std::vector<double> &x, const std::vector<do
return sum / weights_sum; return sum / weights_sum;
} }
double MLPPStat::geometricMean(const std::vector<double> &x) { real_t MLPPStat::geometricMean(const std::vector<real_t> &x) {
double product = 1; real_t product = 1;
for (int i = 0; i < x.size(); i++) { for (int i = 0; i < x.size(); i++) {
product *= x[i]; product *= x[i];
} }
return std::pow(product, 1.0 / x.size()); return std::pow(product, 1.0 / x.size());
} }
double MLPPStat::harmonicMean(const std::vector<double> &x) { real_t MLPPStat::harmonicMean(const std::vector<real_t> &x) {
double sum = 0; real_t sum = 0;
for (int i = 0; i < x.size(); i++) { for (int i = 0; i < x.size(); i++) {
sum += 1 / x[i]; sum += 1 / x[i];
} }
return x.size() / sum; return x.size() / sum;
} }
double MLPPStat::RMS(const std::vector<double> &x) { real_t MLPPStat::RMS(const std::vector<real_t> &x) {
double sum = 0; real_t sum = 0;
for (int i = 0; i < x.size(); i++) { for (int i = 0; i < x.size(); i++) {
sum += x[i] * x[i]; sum += x[i] * x[i];
} }
return sqrt(sum / x.size()); return sqrt(sum / x.size());
} }
double MLPPStat::powerMean(const std::vector<double> &x, const double p) { real_t MLPPStat::powerMean(const std::vector<real_t> &x, const real_t p) {
double sum = 0; real_t sum = 0;
for (int i = 0; i < x.size(); i++) { for (int i = 0; i < x.size(); i++) {
sum += std::pow(x[i], p); sum += std::pow(x[i], p);
} }
return std::pow(sum / x.size(), 1 / p); return std::pow(sum / x.size(), 1 / p);
} }
double MLPPStat::lehmerMean(const std::vector<double> &x, const double p) { real_t MLPPStat::lehmerMean(const std::vector<real_t> &x, const real_t p) {
double num = 0; real_t num = 0;
double den = 0; real_t den = 0;
for (int i = 0; i < x.size(); i++) { for (int i = 0; i < x.size(); i++) {
num += std::pow(x[i], p); num += std::pow(x[i], p);
den += std::pow(x[i], p - 1); den += std::pow(x[i], p - 1);
@ -167,9 +167,9 @@ double MLPPStat::lehmerMean(const std::vector<double> &x, const double p) {
return num / den; return num / den;
} }
double MLPPStat::weightedLehmerMean(const std::vector<double> &x, const std::vector<double> &weights, const double p) { real_t MLPPStat::weightedLehmerMean(const std::vector<real_t> &x, const std::vector<real_t> &weights, const real_t p) {
double num = 0; real_t num = 0;
double den = 0; real_t den = 0;
for (int i = 0; i < x.size(); i++) { for (int i = 0; i < x.size(); i++) {
num += weights[i] * std::pow(x[i], p); num += weights[i] * std::pow(x[i], p);
den += weights[i] * std::pow(x[i], p - 1); den += weights[i] * std::pow(x[i], p - 1);
@ -177,38 +177,38 @@ double MLPPStat::weightedLehmerMean(const std::vector<double> &x, const std::vec
return num / den; return num / den;
} }
double MLPPStat::heronianMean(const double A, const double B) { real_t MLPPStat::heronianMean(const real_t A, const real_t B) {
return (A + sqrt(A * B) + B) / 3; return (A + sqrt(A * B) + B) / 3;
} }
double MLPPStat::contraHarmonicMean(const std::vector<double> &x) { real_t MLPPStat::contraHarmonicMean(const std::vector<real_t> &x) {
return lehmerMean(x, 2); return lehmerMean(x, 2);
} }
double MLPPStat::heinzMean(const double A, const double B, const double x) { real_t MLPPStat::heinzMean(const real_t A, const real_t B, const real_t x) {
return (std::pow(A, x) * std::pow(B, 1 - x) + std::pow(A, 1 - x) * std::pow(B, x)) / 2; return (std::pow(A, x) * std::pow(B, 1 - x) + std::pow(A, 1 - x) * std::pow(B, x)) / 2;
} }
double MLPPStat::neumanSandorMean(const double a, const double b) { real_t MLPPStat::neumanSandorMean(const real_t a, const real_t b) {
MLPPActivation avn; MLPPActivation avn;
return (a - b) / 2 * avn.arsinh((a - b) / (a + b)); return (a - b) / 2 * avn.arsinh((a - b) / (a + b));
} }
double MLPPStat::stolarskyMean(const double x, const double y, const double p) { real_t MLPPStat::stolarskyMean(const real_t x, const real_t y, const real_t p) {
if (x == y) { if (x == y) {
return x; return x;
} }
return std::pow((std::pow(x, p) - std::pow(y, p)) / (p * (x - y)), 1 / (p - 1)); return std::pow((std::pow(x, p) - std::pow(y, p)) / (p * (x - y)), 1 / (p - 1));
} }
double MLPPStat::identricMean(const double x, const double y) { real_t MLPPStat::identricMean(const real_t x, const real_t y) {
if (x == y) { if (x == y) {
return x; return x;
} }
return (1 / M_E) * std::pow(std::pow(x, x) / std::pow(y, y), 1 / (x - y)); return (1 / M_E) * std::pow(std::pow(x, x) / std::pow(y, y), 1 / (x - y));
} }
double MLPPStat::logMean(const double x, const double y) { real_t MLPPStat::logMean(const real_t x, const real_t y) {
if (x == y) { if (x == y) {
return x; return x;
} }

58
mlpp/stat/stat.h
View File

@ -8,44 +8,46 @@
// Created by Marc Melikyan on 9/29/20. // Created by Marc Melikyan on 9/29/20.
// //
#include "core/math/math_defs.h"
#include <vector> #include <vector>
class MLPPStat { class MLPPStat {
public: public:
// These functions are for univariate lin reg module- not for users. // These functions are for univariate lin reg module- not for users.
double b0Estimation(const std::vector<double> &x, const std::vector<double> &y); real_t b0Estimation(const std::vector<real_t> &x, const std::vector<real_t> &y);
double b1Estimation(const std::vector<double> &x, const std::vector<double> &y); real_t b1Estimation(const std::vector<real_t> &x, const std::vector<real_t> &y);
// Statistical Functions // Statistical Functions
double mean(const std::vector<double> &x); real_t mean(const std::vector<real_t> &x);
double median(std::vector<double> x); real_t median(std::vector<real_t> x);
std::vector<double> mode(const std::vector<double> &x); std::vector<real_t> mode(const std::vector<real_t> &x);
double range(const std::vector<double> &x); real_t range(const std::vector<real_t> &x);
double midrange(const std::vector<double> &x); real_t midrange(const std::vector<real_t> &x);
double absAvgDeviation(const std::vector<double> &x); real_t absAvgDeviation(const std::vector<real_t> &x);
double standardDeviation(const std::vector<double> &x); real_t standardDeviation(const std::vector<real_t> &x);
double variance(const std::vector<double> &x); real_t variance(const std::vector<real_t> &x);
double covariance(const std::vector<double> &x, const std::vector<double> &y); real_t covariance(const std::vector<real_t> &x, const std::vector<real_t> &y);
double correlation(const std::vector<double> &x, const std::vector<double> &y); real_t correlation(const std::vector<real_t> &x, const std::vector<real_t> &y);
double R2(const std::vector<double> &x, const std::vector<double> &y); real_t R2(const std::vector<real_t> &x, const std::vector<real_t> &y);
double chebyshevIneq(const double k); real_t chebyshevIneq(const real_t k);
// Extras // Extras
double weightedMean(const std::vector<double> &x, const std::vector<double> &weights); real_t weightedMean(const std::vector<real_t> &x, const std::vector<real_t> &weights);
double geometricMean(const std::vector<double> &x); real_t geometricMean(const std::vector<real_t> &x);
double harmonicMean(const std::vector<double> &x); real_t harmonicMean(const std::vector<real_t> &x);
double RMS(const std::vector<double> &x); real_t RMS(const std::vector<real_t> &x);
double powerMean(const std::vector<double> &x, const double p); real_t powerMean(const std::vector<real_t> &x, const real_t p);
double lehmerMean(const std::vector<double> &x, const double p); real_t lehmerMean(const std::vector<real_t> &x, const real_t p);
double weightedLehmerMean(const std::vector<double> &x, const std::vector<double> &weights, const double p); real_t weightedLehmerMean(const std::vector<real_t> &x, const std::vector<real_t> &weights, const real_t p);
double contraHarmonicMean(const std::vector<double> &x); real_t contraHarmonicMean(const std::vector<real_t> &x);
double heronianMean(const double A, const double B); real_t heronianMean(const real_t A, const real_t B);
double heinzMean(const double A, const double B, const double x); real_t heinzMean(const real_t A, const real_t B, const real_t x);
double neumanSandorMean(const double a, const double b); real_t neumanSandorMean(const real_t a, const real_t b);
double stolarskyMean(const double x, const double y, const double p); real_t stolarskyMean(const real_t x, const real_t y, const real_t p);
double identricMean(const double x, const double y); real_t identricMean(const real_t x, const real_t y);
double logMean(const double x, const double y); real_t logMean(const real_t x, const real_t y);
}; };

40
mlpp/svc/svc.cpp
View File

@ -15,27 +15,27 @@
#include <random> #include <random>
MLPPSVC::MLPPSVC(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, double C) : MLPPSVC::MLPPSVC(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, real_t C) :
inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), C(C) { inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), C(C) {
y_hat.resize(n); y_hat.resize(n);
weights = MLPPUtilities::weightInitialization(k); weights = MLPPUtilities::weightInitialization(k);
bias = MLPPUtilities::biasInitialization(); bias = MLPPUtilities::biasInitialization();
} }
std::vector<double> MLPPSVC::modelSetTest(std::vector<std::vector<double>> X) { std::vector<real_t> MLPPSVC::modelSetTest(std::vector<std::vector<real_t>> X) {
return Evaluate(X); return Evaluate(X);
} }
double MLPPSVC::modelTest(std::vector<double> x) { real_t MLPPSVC::modelTest(std::vector<real_t> x) {
return Evaluate(x); return Evaluate(x);
} }
void MLPPSVC::gradientDescent(double learning_rate, int max_epoch, bool UI) { void MLPPSVC::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
class MLPPCost cost; class MLPPCost cost;
MLPPActivation avn; MLPPActivation avn;
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
forwardPass(); forwardPass();
@ -63,13 +63,13 @@ void MLPPSVC::gradientDescent(double learning_rate, int max_epoch, bool UI) {
} }
} }
void MLPPSVC::SGD(double learning_rate, int max_epoch, bool UI) { void MLPPSVC::SGD(real_t learning_rate, int max_epoch, bool UI) {
class MLPPCost cost; class MLPPCost cost;
MLPPActivation avn; MLPPActivation avn;
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
while (true) { while (true) {
@ -78,11 +78,11 @@ void MLPPSVC::SGD(double learning_rate, int max_epoch, bool UI) {
std::uniform_int_distribution<int> distribution(0, int(n - 1)); std::uniform_int_distribution<int> distribution(0, int(n - 1));
int outputIndex = distribution(generator); int outputIndex = distribution(generator);
double y_hat = Evaluate(inputSet[outputIndex]); real_t y_hat = Evaluate(inputSet[outputIndex]);
double z = propagate(inputSet[outputIndex]); real_t z = propagate(inputSet[outputIndex]);
cost_prev = Cost({ z }, { outputSet[outputIndex] }, weights, C); cost_prev = Cost({ z }, { outputSet[outputIndex] }, weights, C);
double costDeriv = cost.HingeLossDeriv(std::vector<double>({ z }), std::vector<double>({ outputSet[outputIndex] }), C)[0]; // Explicit conversion to avoid ambiguity with overloaded function. Error occured on Ubuntu. real_t costDeriv = cost.HingeLossDeriv(std::vector<real_t>({ z }), std::vector<real_t>({ outputSet[outputIndex] }), C)[0]; // Explicit conversion to avoid ambiguity with overloaded function. Error occured on Ubuntu.
// Weight Updation // Weight Updation
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate * costDeriv, inputSet[outputIndex])); weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate * costDeriv, inputSet[outputIndex]));
@ -106,12 +106,12 @@ void MLPPSVC::SGD(double learning_rate, int max_epoch, bool UI) {
forwardPass(); forwardPass();
} }
void MLPPSVC::MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI) { void MLPPSVC::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI) {
class MLPPCost cost; class MLPPCost cost;
MLPPActivation avn; MLPPActivation avn;
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
// Creating the mini-batches // Creating the mini-batches
@ -120,8 +120,8 @@ void MLPPSVC::MBGD(double learning_rate, int max_epoch, int mini_batch_size, boo
while (true) { while (true) {
for (int i = 0; i < n_mini_batch; i++) { for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = Evaluate(inputMiniBatches[i]); std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]);
std::vector<double> z = propagate(inputMiniBatches[i]); std::vector<real_t> z = propagate(inputMiniBatches[i]);
cost_prev = Cost(z, outputMiniBatches[i], weights, C); cost_prev = Cost(z, outputMiniBatches[i], weights, C);
// Calculating the weight gradients // Calculating the weight gradients
@ -148,7 +148,7 @@ void MLPPSVC::MBGD(double learning_rate, int max_epoch, int mini_batch_size, boo
forwardPass(); forwardPass();
} }
double MLPPSVC::score() { real_t MLPPSVC::score() {
MLPPUtilities util; MLPPUtilities util;
return util.performance(y_hat, outputSet); return util.performance(y_hat, outputSet);
} }
@ -158,30 +158,30 @@ void MLPPSVC::save(std::string fileName) {
util.saveParameters(fileName, weights, bias); util.saveParameters(fileName, weights, bias);
} }
double MLPPSVC::Cost(std::vector<double> z, std::vector<double> y, std::vector<double> weights, double C) { real_t MLPPSVC::Cost(std::vector<real_t> z, std::vector<real_t> y, std::vector<real_t> weights, real_t C) {
class MLPPCost cost; class MLPPCost cost;
return cost.HingeLoss(z, y, weights, C); return cost.HingeLoss(z, y, weights, C);
} }
std::vector<double> MLPPSVC::Evaluate(std::vector<std::vector<double>> X) { std::vector<real_t> MLPPSVC::Evaluate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPActivation avn; MLPPActivation avn;
return avn.sign(alg.scalarAdd(bias, alg.mat_vec_mult(X, weights))); return avn.sign(alg.scalarAdd(bias, alg.mat_vec_mult(X, weights)));
} }
std::vector<double> MLPPSVC::propagate(std::vector<std::vector<double>> X) { std::vector<real_t> MLPPSVC::propagate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPActivation avn; MLPPActivation avn;
return alg.scalarAdd(bias, alg.mat_vec_mult(X, weights)); return alg.scalarAdd(bias, alg.mat_vec_mult(X, weights));
} }
double MLPPSVC::Evaluate(std::vector<double> x) { real_t MLPPSVC::Evaluate(std::vector<real_t> x) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPActivation avn; MLPPActivation avn;
return avn.sign(alg.dot(weights, x) + bias); return avn.sign(alg.dot(weights, x) + bias);
} }
double MLPPSVC::propagate(std::vector<double> x) { real_t MLPPSVC::propagate(std::vector<real_t> x) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPActivation avn; MLPPActivation avn;
return alg.dot(weights, x) + bias; return alg.dot(weights, x) + bias;

42
mlpp/svc/svc.h
View File

@ -11,6 +11,8 @@
// https://towardsdatascience.com/svm-implementation-from-scratch-python-2db2fc52e5c2 // https://towardsdatascience.com/svm-implementation-from-scratch-python-2db2fc52e5c2
// Illustratd a practical definition of the Hinge Loss function and its gradient when optimizing with SGD. // Illustratd a practical definition of the Hinge Loss function and its gradient when optimizing with SGD.
#include "core/math/math_defs.h"
#include <string> #include <string>
#include <vector> #include <vector>
@ -18,37 +20,37 @@
class MLPPSVC { class MLPPSVC {
public: public:
MLPPSVC(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, double C); MLPPSVC(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, real_t C);
std::vector<double> modelSetTest(std::vector<std::vector<double>> X); std::vector<real_t> modelSetTest(std::vector<std::vector<real_t>> X);
double modelTest(std::vector<double> x); real_t modelTest(std::vector<real_t> x);
void gradientDescent(double learning_rate, int max_epoch, bool UI = 1); void gradientDescent(real_t learning_rate, int max_epoch, bool UI = 1);
void SGD(double learning_rate, int max_epoch, bool UI = 1); void SGD(real_t learning_rate, int max_epoch, bool UI = 1);
void MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI = 1); void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
double score(); real_t score();
void save(std::string fileName); void save(std::string fileName);
private: private:
double Cost(std::vector<double> y_hat, std::vector<double> y, std::vector<double> weights, double C); real_t Cost(std::vector<real_t> y_hat, std::vector<real_t> y, std::vector<real_t> weights, real_t C);
std::vector<double> Evaluate(std::vector<std::vector<double>> X); std::vector<real_t> Evaluate(std::vector<std::vector<real_t>> X);
std::vector<double> propagate(std::vector<std::vector<double>> X); std::vector<real_t> propagate(std::vector<std::vector<real_t>> X);
double Evaluate(std::vector<double> x); real_t Evaluate(std::vector<real_t> x);
double propagate(std::vector<double> x); real_t propagate(std::vector<real_t> x);
void forwardPass(); void forwardPass();
std::vector<std::vector<double>> inputSet; std::vector<std::vector<real_t>> inputSet;
std::vector<double> outputSet; std::vector<real_t> outputSet;
std::vector<double> z; std::vector<real_t> z;
std::vector<double> y_hat; std::vector<real_t> y_hat;
std::vector<double> weights; std::vector<real_t> weights;
double bias; real_t bias;
double C; real_t C;
int n; int n;
int k; int k;
// UI Portion // UI Portion
void UI(int epoch, double cost_prev); void UI(int epoch, real_t cost_prev);
}; };

42
mlpp/tanh_reg/tanh_reg.cpp
View File

@ -15,33 +15,33 @@
#include <random> #include <random>
MLPPTanhReg::MLPPTanhReg(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, std::string reg, double lambda, double alpha) : MLPPTanhReg::MLPPTanhReg(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, std::string reg, real_t lambda, real_t alpha) :
inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) { inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) {
y_hat.resize(n); y_hat.resize(n);
weights = MLPPUtilities::weightInitialization(k); weights = MLPPUtilities::weightInitialization(k);
bias = MLPPUtilities::biasInitialization(); bias = MLPPUtilities::biasInitialization();
} }
std::vector<double> MLPPTanhReg::modelSetTest(std::vector<std::vector<double>> X) { std::vector<real_t> MLPPTanhReg::modelSetTest(std::vector<std::vector<real_t>> X) {
return Evaluate(X); return Evaluate(X);
} }
double MLPPTanhReg::modelTest(std::vector<double> x) { real_t MLPPTanhReg::modelTest(std::vector<real_t> x) {
return Evaluate(x); return Evaluate(x);
} }
void MLPPTanhReg::gradientDescent(double learning_rate, int max_epoch, bool UI) { void MLPPTanhReg::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
MLPPActivation avn; MLPPActivation avn;
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
forwardPass(); forwardPass();
while (true) { while (true) {
cost_prev = Cost(y_hat, outputSet); cost_prev = Cost(y_hat, outputSet);
std::vector<double> error = alg.subtraction(y_hat, outputSet); std::vector<real_t> error = alg.subtraction(y_hat, outputSet);
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputSet), alg.hadamard_product(error, avn.tanh(z, 1))))); weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputSet), alg.hadamard_product(error, avn.tanh(z, 1)))));
weights = regularization.regWeights(weights, lambda, alpha, reg); weights = regularization.regWeights(weights, lambda, alpha, reg);
@ -64,10 +64,10 @@ void MLPPTanhReg::gradientDescent(double learning_rate, int max_epoch, bool UI)
} }
} }
void MLPPTanhReg::SGD(double learning_rate, int max_epoch, bool UI) { void MLPPTanhReg::SGD(real_t learning_rate, int max_epoch, bool UI) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
while (true) { while (true) {
@ -76,10 +76,10 @@ void MLPPTanhReg::SGD(double learning_rate, int max_epoch, bool UI) {
std::uniform_int_distribution<int> distribution(0, int(n - 1)); std::uniform_int_distribution<int> distribution(0, int(n - 1));
int outputIndex = distribution(generator); int outputIndex = distribution(generator);
double y_hat = Evaluate(inputSet[outputIndex]); real_t y_hat = Evaluate(inputSet[outputIndex]);
cost_prev = Cost({ y_hat }, { outputSet[outputIndex] }); cost_prev = Cost({ y_hat }, { outputSet[outputIndex] });
double error = y_hat - outputSet[outputIndex]; real_t error = y_hat - outputSet[outputIndex];
// Weight Updation // Weight Updation
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate * error * (1 - y_hat * y_hat), inputSet[outputIndex])); weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate * error * (1 - y_hat * y_hat), inputSet[outputIndex]));
@ -103,11 +103,11 @@ void MLPPTanhReg::SGD(double learning_rate, int max_epoch, bool UI) {
forwardPass(); forwardPass();
} }
void MLPPTanhReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI) { void MLPPTanhReg::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI) {
MLPPActivation avn; MLPPActivation avn;
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
// Creating the mini-batches // Creating the mini-batches
@ -116,11 +116,11 @@ void MLPPTanhReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size,
while (true) { while (true) {
for (int i = 0; i < n_mini_batch; i++) { for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = Evaluate(inputMiniBatches[i]); std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]);
std::vector<double> z = propagate(inputMiniBatches[i]); std::vector<real_t> z = propagate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]); cost_prev = Cost(y_hat, outputMiniBatches[i]);
std::vector<double> error = alg.subtraction(y_hat, outputMiniBatches[i]); std::vector<real_t> error = alg.subtraction(y_hat, outputMiniBatches[i]);
// Calculating the weight gradients // Calculating the weight gradients
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), alg.hadamard_product(error, avn.tanh(z, 1))))); weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), alg.hadamard_product(error, avn.tanh(z, 1)))));
@ -146,7 +146,7 @@ void MLPPTanhReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size,
forwardPass(); forwardPass();
} }
double MLPPTanhReg::score() { real_t MLPPTanhReg::score() {
MLPPUtilities util; MLPPUtilities util;
return util.performance(y_hat, outputSet); return util.performance(y_hat, outputSet);
} }
@ -156,30 +156,30 @@ void MLPPTanhReg::save(std::string fileName) {
util.saveParameters(fileName, weights, bias); util.saveParameters(fileName, weights, bias);
} }
double MLPPTanhReg::Cost(std::vector<double> y_hat, std::vector<double> y) { real_t MLPPTanhReg::Cost(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPReg regularization; MLPPReg regularization;
class MLPPCost cost; class MLPPCost cost;
return cost.MSE(y_hat, y) + regularization.regTerm(weights, lambda, alpha, reg); return cost.MSE(y_hat, y) + regularization.regTerm(weights, lambda, alpha, reg);
} }
std::vector<double> MLPPTanhReg::Evaluate(std::vector<std::vector<double>> X) { std::vector<real_t> MLPPTanhReg::Evaluate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPActivation avn; MLPPActivation avn;
return avn.tanh(alg.scalarAdd(bias, alg.mat_vec_mult(X, weights))); return avn.tanh(alg.scalarAdd(bias, alg.mat_vec_mult(X, weights)));
} }
std::vector<double> MLPPTanhReg::propagate(std::vector<std::vector<double>> X) { std::vector<real_t> MLPPTanhReg::propagate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.scalarAdd(bias, alg.mat_vec_mult(X, weights)); return alg.scalarAdd(bias, alg.mat_vec_mult(X, weights));
} }
double MLPPTanhReg::Evaluate(std::vector<double> x) { real_t MLPPTanhReg::Evaluate(std::vector<real_t> x) {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPActivation avn; MLPPActivation avn;
return avn.tanh(alg.dot(weights, x) + bias); return avn.tanh(alg.dot(weights, x) + bias);
} }
double MLPPTanhReg::propagate(std::vector<double> x) { real_t MLPPTanhReg::propagate(std::vector<real_t> x) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.dot(weights, x) + bias; return alg.dot(weights, x) + bias;
} }

44
mlpp/tanh_reg/tanh_reg.h
View File

@ -8,6 +8,8 @@
// Created by Marc Melikyan on 10/2/20. // Created by Marc Melikyan on 10/2/20.
// //
#include "core/math/math_defs.h"
#include <string> #include <string>
#include <vector> #include <vector>
@ -15,41 +17,41 @@
class MLPPTanhReg { class MLPPTanhReg {
public: public:
MLPPTanhReg(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, std::string reg = "None", double lambda = 0.5, double alpha = 0.5); MLPPTanhReg(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
std::vector<double> modelSetTest(std::vector<std::vector<double>> X); std::vector<real_t> modelSetTest(std::vector<std::vector<real_t>> X);
double modelTest(std::vector<double> x); real_t modelTest(std::vector<real_t> x);
void gradientDescent(double learning_rate, int max_epoch, bool UI = 1); void gradientDescent(real_t learning_rate, int max_epoch, bool UI = 1);
void SGD(double learning_rate, int max_epoch, bool UI = 1); void SGD(real_t learning_rate, int max_epoch, bool UI = 1);
void MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI = 1); void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
double score(); real_t score();
void save(std::string fileName); void save(std::string fileName);
private: private:
double Cost(std::vector<double> y_hat, std::vector<double> y); real_t Cost(std::vector<real_t> y_hat, std::vector<real_t> y);
std::vector<double> Evaluate(std::vector<std::vector<double>> X); std::vector<real_t> Evaluate(std::vector<std::vector<real_t>> X);
std::vector<double> propagate(std::vector<std::vector<double>> X); std::vector<real_t> propagate(std::vector<std::vector<real_t>> X);
double Evaluate(std::vector<double> x); real_t Evaluate(std::vector<real_t> x);
double propagate(std::vector<double> x); real_t propagate(std::vector<real_t> x);
void forwardPass(); void forwardPass();
std::vector<std::vector<double>> inputSet; std::vector<std::vector<real_t>> inputSet;
std::vector<double> outputSet; std::vector<real_t> outputSet;
std::vector<double> z; std::vector<real_t> z;
std::vector<double> y_hat; std::vector<real_t> y_hat;
std::vector<double> weights; std::vector<real_t> weights;
double bias; real_t bias;
int n; int n;
int k; int k;
// UI Portion // UI Portion
void UI(int epoch, double cost_prev); void UI(int epoch, real_t cost_prev);
// Regularization Params // Regularization Params
std::string reg; std::string reg;
double lambda; real_t lambda;
double alpha; /* This is the controlling param for Elastic Net*/ real_t alpha; /* This is the controlling param for Elastic Net*/
}; };

14
mlpp/transforms/transforms.cpp
View File

@ -14,11 +14,11 @@
// DCT ii. // DCT ii.
// https://www.mathworks.com/help/images/discrete-cosine-transform.html // https://www.mathworks.com/help/images/discrete-cosine-transform.html
std::vector<std::vector<double>> MLPPTransforms::discreteCosineTransform(std::vector<std::vector<double>> A) { std::vector<std::vector<real_t>> MLPPTransforms::discreteCosineTransform(std::vector<std::vector<real_t>> A) {
MLPPLinAlg alg; MLPPLinAlg alg;
A = alg.scalarAdd(-128, A); // Center around 0. A = alg.scalarAdd(-128, A); // Center around 0.
std::vector<std::vector<double>> B; std::vector<std::vector<real_t>> B;
B.resize(A.size()); B.resize(A.size());
for (int i = 0; i < B.size(); i++) { for (int i = 0; i < B.size(); i++) {
B[i].resize(A[i].size()); B[i].resize(A[i].size());
@ -28,18 +28,18 @@ std::vector<std::vector<double>> MLPPTransforms::discreteCosineTransform(std::ve
for (int i = 0; i < B.size(); i++) { for (int i = 0; i < B.size(); i++) {
for (int j = 0; j < B[i].size(); j++) { for (int j = 0; j < B[i].size(); j++) {
double sum = 0; real_t sum = 0;
double alphaI; real_t alphaI;
if (i == 0) { if (i == 0) {
alphaI = 1 / std::sqrt(M); alphaI = 1 / std::sqrt(M);
} else { } else {
alphaI = std::sqrt(double(2) / double(M)); alphaI = std::sqrt(real_t(2) / real_t(M));
} }
double alphaJ; real_t alphaJ;
if (j == 0) { if (j == 0) {
alphaJ = 1 / std::sqrt(M); alphaJ = 1 / std::sqrt(M);
} else { } else {
alphaJ = std::sqrt(double(2) / double(M)); alphaJ = std::sqrt(real_t(2) / real_t(M));
} }
for (int k = 0; k < B.size(); k++) { for (int k = 0; k < B.size(); k++) {

4
mlpp/transforms/transforms.h
View File

@ -7,13 +7,15 @@
// //
// //
#include "core/math/math_defs.h"
#include <string> #include <string>
#include <vector> #include <vector>
class MLPPTransforms { class MLPPTransforms {
public: public:
std::vector<std::vector<double>> discreteCosineTransform(std::vector<std::vector<double>> A); std::vector<std::vector<real_t>> discreteCosineTransform(std::vector<std::vector<real_t>> A);
}; };

6
mlpp/uni_lin_reg/uni_lin_reg.cpp
View File

@ -16,19 +16,19 @@
// ŷ = b0 + b1x1 // ŷ = b0 + b1x1
MLPPUniLinReg::MLPPUniLinReg(std::vector<double> x, std::vector<double> y) : MLPPUniLinReg::MLPPUniLinReg(std::vector<real_t> x, std::vector<real_t> y) :
inputSet(x), outputSet(y) { inputSet(x), outputSet(y) {
MLPPStat estimator; MLPPStat estimator;
b1 = estimator.b1Estimation(inputSet, outputSet); b1 = estimator.b1Estimation(inputSet, outputSet);
b0 = estimator.b0Estimation(inputSet, outputSet); b0 = estimator.b0Estimation(inputSet, outputSet);
} }
std::vector<double> MLPPUniLinReg::modelSetTest(std::vector<double> x) { std::vector<real_t> MLPPUniLinReg::modelSetTest(std::vector<real_t> x) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.scalarAdd(b0, alg.scalarMultiply(b1, x)); return alg.scalarAdd(b0, alg.scalarMultiply(b1, x));
} }
double MLPPUniLinReg::modelTest(double input) { real_t MLPPUniLinReg::modelTest(real_t input) {
return b0 + b1 * input; return b0 + b1 * input;
} }

16
mlpp/uni_lin_reg/uni_lin_reg.h
View File

@ -8,21 +8,23 @@
// Created by Marc Melikyan on 9/29/20. // Created by Marc Melikyan on 9/29/20.
// //
#include "core/math/math_defs.h"
#include <vector> #include <vector>
class MLPPUniLinReg { class MLPPUniLinReg {
public: public:
MLPPUniLinReg(std::vector<double> x, std::vector<double> y); MLPPUniLinReg(std::vector<real_t> x, std::vector<real_t> y);
std::vector<double> modelSetTest(std::vector<double> x); std::vector<real_t> modelSetTest(std::vector<real_t> x);
double modelTest(double x); real_t modelTest(real_t x);
private: private:
std::vector<double> inputSet; std::vector<real_t> inputSet;
std::vector<double> outputSet; std::vector<real_t> outputSet;
double b0; real_t b0;
double b1; real_t b1;
}; };

116
mlpp/utilities/utilities.cpp
View File

@ -12,81 +12,81 @@
std::vector<double> MLPPUtilities::weightInitialization(int n, std::string type) { std::vector<real_t> MLPPUtilities::weightInitialization(int n, std::string type) {
std::random_device rd; std::random_device rd;
std::default_random_engine generator(rd()); std::default_random_engine generator(rd());
std::vector<double> weights; std::vector<real_t> weights;
for (int i = 0; i < n; i++) { for (int i = 0; i < n; i++) {
if (type == "XavierNormal") { if (type == "XavierNormal") {
std::normal_distribution<double> distribution(0, sqrt(2 / (n + 1))); std::normal_distribution<real_t> distribution(0, sqrt(2 / (n + 1)));
weights.push_back(distribution(generator)); weights.push_back(distribution(generator));
} else if (type == "XavierUniform") { } else if (type == "XavierUniform") {
std::uniform_real_distribution<double> distribution(-sqrt(6 / (n + 1)), sqrt(6 / (n + 1))); std::uniform_real_distribution<real_t> distribution(-sqrt(6 / (n + 1)), sqrt(6 / (n + 1)));
weights.push_back(distribution(generator)); weights.push_back(distribution(generator));
} else if (type == "HeNormal") { } else if (type == "HeNormal") {
std::normal_distribution<double> distribution(0, sqrt(2 / n)); std::normal_distribution<real_t> distribution(0, sqrt(2 / n));
weights.push_back(distribution(generator)); weights.push_back(distribution(generator));
} else if (type == "HeUniform") { } else if (type == "HeUniform") {
std::uniform_real_distribution<double> distribution(-sqrt(6 / n), sqrt(6 / n)); std::uniform_real_distribution<real_t> distribution(-sqrt(6 / n), sqrt(6 / n));
weights.push_back(distribution(generator)); weights.push_back(distribution(generator));
} else if (type == "LeCunNormal") { } else if (type == "LeCunNormal") {
std::normal_distribution<double> distribution(0, sqrt(1 / n)); std::normal_distribution<real_t> distribution(0, sqrt(1 / n));
weights.push_back(distribution(generator)); weights.push_back(distribution(generator));
} else if (type == "LeCunUniform") { } else if (type == "LeCunUniform") {
std::uniform_real_distribution<double> distribution(-sqrt(3 / n), sqrt(3 / n)); std::uniform_real_distribution<real_t> distribution(-sqrt(3 / n), sqrt(3 / n));
weights.push_back(distribution(generator)); weights.push_back(distribution(generator));
} else if (type == "Uniform") { } else if (type == "Uniform") {
std::uniform_real_distribution<double> distribution(-1 / sqrt(n), 1 / sqrt(n)); std::uniform_real_distribution<real_t> distribution(-1 / sqrt(n), 1 / sqrt(n));
weights.push_back(distribution(generator)); weights.push_back(distribution(generator));
} else { } else {
std::uniform_real_distribution<double> distribution(0, 1); std::uniform_real_distribution<real_t> distribution(0, 1);
weights.push_back(distribution(generator)); weights.push_back(distribution(generator));
} }
} }
return weights; return weights;
} }
double MLPPUtilities::biasInitialization() { real_t MLPPUtilities::biasInitialization() {
std::random_device rd; std::random_device rd;
std::default_random_engine generator(rd()); std::default_random_engine generator(rd());
std::uniform_real_distribution<double> distribution(0, 1); std::uniform_real_distribution<real_t> distribution(0, 1);
return distribution(generator); return distribution(generator);
} }
std::vector<std::vector<double>> MLPPUtilities::weightInitialization(int n, int m, std::string type) { std::vector<std::vector<real_t>> MLPPUtilities::weightInitialization(int n, int m, std::string type) {
std::random_device rd; std::random_device rd;
std::default_random_engine generator(rd()); std::default_random_engine generator(rd());
std::vector<std::vector<double>> weights; std::vector<std::vector<real_t>> weights;
weights.resize(n); weights.resize(n);
for (int i = 0; i < n; i++) { for (int i = 0; i < n; i++) {
for (int j = 0; j < m; j++) { for (int j = 0; j < m; j++) {
if (type == "XavierNormal") { if (type == "XavierNormal") {
std::normal_distribution<double> distribution(0, sqrt(2 / (n + m))); std::normal_distribution<real_t> distribution(0, sqrt(2 / (n + m)));
weights[i].push_back(distribution(generator)); weights[i].push_back(distribution(generator));
} else if (type == "XavierUniform") { } else if (type == "XavierUniform") {
std::uniform_real_distribution<double> distribution(-sqrt(6 / (n + m)), sqrt(6 / (n + m))); std::uniform_real_distribution<real_t> distribution(-sqrt(6 / (n + m)), sqrt(6 / (n + m)));
weights[i].push_back(distribution(generator)); weights[i].push_back(distribution(generator));
} else if (type == "HeNormal") { } else if (type == "HeNormal") {
std::normal_distribution<double> distribution(0, sqrt(2 / n)); std::normal_distribution<real_t> distribution(0, sqrt(2 / n));
weights[i].push_back(distribution(generator)); weights[i].push_back(distribution(generator));
} else if (type == "HeUniform") { } else if (type == "HeUniform") {
std::uniform_real_distribution<double> distribution(-sqrt(6 / n), sqrt(6 / n)); std::uniform_real_distribution<real_t> distribution(-sqrt(6 / n), sqrt(6 / n));
weights[i].push_back(distribution(generator)); weights[i].push_back(distribution(generator));
} else if (type == "LeCunNormal") { } else if (type == "LeCunNormal") {
std::normal_distribution<double> distribution(0, sqrt(1 / n)); std::normal_distribution<real_t> distribution(0, sqrt(1 / n));
weights[i].push_back(distribution(generator)); weights[i].push_back(distribution(generator));
} else if (type == "LeCunUniform") { } else if (type == "LeCunUniform") {
std::uniform_real_distribution<double> distribution(-sqrt(3 / n), sqrt(3 / n)); std::uniform_real_distribution<real_t> distribution(-sqrt(3 / n), sqrt(3 / n));
weights[i].push_back(distribution(generator)); weights[i].push_back(distribution(generator));
} else if (type == "Uniform") { } else if (type == "Uniform") {
std::uniform_real_distribution<double> distribution(-1 / sqrt(n), 1 / sqrt(n)); std::uniform_real_distribution<real_t> distribution(-1 / sqrt(n), 1 / sqrt(n));
weights[i].push_back(distribution(generator)); weights[i].push_back(distribution(generator));
} else { } else {
std::uniform_real_distribution<double> distribution(0, 1); std::uniform_real_distribution<real_t> distribution(0, 1);
weights[i].push_back(distribution(generator)); weights[i].push_back(distribution(generator));
} }
} }
@ -94,11 +94,11 @@ std::vector<std::vector<double>> MLPPUtilities::weightInitialization(int n, int
return weights; return weights;
} }
std::vector<double> MLPPUtilities::biasInitialization(int n) { std::vector<real_t> MLPPUtilities::biasInitialization(int n) {
std::vector<double> bias; std::vector<real_t> bias;
std::random_device rd; std::random_device rd;
std::default_random_engine generator(rd()); std::default_random_engine generator(rd());
std::uniform_real_distribution<double> distribution(0, 1); std::uniform_real_distribution<real_t> distribution(0, 1);
for (int i = 0; i < n; i++) { for (int i = 0; i < n; i++) {
bias.push_back(distribution(generator)); bias.push_back(distribution(generator));
@ -106,8 +106,8 @@ std::vector<double> MLPPUtilities::biasInitialization(int n) {
return bias; return bias;
} }
double MLPPUtilities::performance(std::vector<double> y_hat, std::vector<double> outputSet) { real_t MLPPUtilities::performance(std::vector<real_t> y_hat, std::vector<real_t> outputSet) {
double correct = 0; real_t correct = 0;
for (int i = 0; i < y_hat.size(); i++) { for (int i = 0; i < y_hat.size(); i++) {
if (std::round(y_hat[i]) == outputSet[i]) { if (std::round(y_hat[i]) == outputSet[i]) {
correct++; correct++;
@ -116,8 +116,8 @@ double MLPPUtilities::performance(std::vector<double> y_hat, std::vector<double>
return correct / y_hat.size(); return correct / y_hat.size();
} }
double MLPPUtilities::performance(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) { real_t MLPPUtilities::performance(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
double correct = 0; real_t correct = 0;
for (int i = 0; i < y_hat.size(); i++) { for (int i = 0; i < y_hat.size(); i++) {
int sub_correct = 0; int sub_correct = 0;
for (int j = 0; j < y_hat[i].size(); j++) { for (int j = 0; j < y_hat[i].size(); j++) {
@ -132,7 +132,7 @@ double MLPPUtilities::performance(std::vector<std::vector<double>> y_hat, std::v
return correct / y_hat.size(); return correct / y_hat.size();
} }
void MLPPUtilities::saveParameters(std::string fileName, std::vector<double> weights, double bias, bool app, int layer) { void MLPPUtilities::saveParameters(std::string fileName, std::vector<real_t> weights, real_t bias, bool app, int layer) {
std::string layer_info = ""; std::string layer_info = "";
std::ofstream saveFile; std::ofstream saveFile;
@ -160,7 +160,7 @@ void MLPPUtilities::saveParameters(std::string fileName, std::vector<double> wei
saveFile.close(); saveFile.close();
} }
void MLPPUtilities::saveParameters(std::string fileName, std::vector<double> weights, std::vector<double> initial, double bias, bool app, int layer) { void MLPPUtilities::saveParameters(std::string fileName, std::vector<real_t> weights, std::vector<real_t> initial, real_t bias, bool app, int layer) {
std::string layer_info = ""; std::string layer_info = "";
std::ofstream saveFile; std::ofstream saveFile;
@ -194,7 +194,7 @@ void MLPPUtilities::saveParameters(std::string fileName, std::vector<double> wei
saveFile.close(); saveFile.close();
} }
void MLPPUtilities::saveParameters(std::string fileName, std::vector<std::vector<double>> weights, std::vector<double> bias, bool app, int layer) { void MLPPUtilities::saveParameters(std::string fileName, std::vector<std::vector<real_t>> weights, std::vector<real_t> bias, bool app, int layer) {
std::string layer_info = ""; std::string layer_info = "";
std::ofstream saveFile; std::ofstream saveFile;
@ -226,7 +226,7 @@ void MLPPUtilities::saveParameters(std::string fileName, std::vector<std::vector
saveFile.close(); saveFile.close();
} }
void MLPPUtilities::UI(std::vector<double> weights, double bias) { void MLPPUtilities::UI(std::vector<real_t> weights, real_t bias) {
std::cout << "Values of the weight(s):" << std::endl; std::cout << "Values of the weight(s):" << std::endl;
for (int i = 0; i < weights.size(); i++) { for (int i = 0; i < weights.size(); i++) {
std::cout << weights[i] << std::endl; std::cout << weights[i] << std::endl;
@ -235,7 +235,7 @@ void MLPPUtilities::UI(std::vector<double> weights, double bias) {
std::cout << bias << std::endl; std::cout << bias << std::endl;
} }
void MLPPUtilities::UI(std::vector<std::vector<double>> weights, std::vector<double> bias) { void MLPPUtilities::UI(std::vector<std::vector<real_t>> weights, std::vector<real_t> bias) {
std::cout << "Values of the weight(s):" << std::endl; std::cout << "Values of the weight(s):" << std::endl;
for (int i = 0; i < weights.size(); i++) { for (int i = 0; i < weights.size(); i++) {
for (int j = 0; j < weights[i].size(); j++) { for (int j = 0; j < weights[i].size(); j++) {
@ -248,7 +248,7 @@ void MLPPUtilities::UI(std::vector<std::vector<double>> weights, std::vector<dou
} }
} }
void MLPPUtilities::UI(std::vector<double> weights, std::vector<double> initial, double bias) { void MLPPUtilities::UI(std::vector<real_t> weights, std::vector<real_t> initial, real_t bias) {
std::cout << "Values of the weight(s):" << std::endl; std::cout << "Values of the weight(s):" << std::endl;
for (int i = 0; i < weights.size(); i++) { for (int i = 0; i < weights.size(); i++) {
std::cout << weights[i] << std::endl; std::cout << weights[i] << std::endl;
@ -261,7 +261,7 @@ void MLPPUtilities::UI(std::vector<double> weights, std::vector<double> initial,
std::cout << bias << std::endl; std::cout << bias << std::endl;
} }
void MLPPUtilities::CostInfo(int epoch, double cost_prev, double Cost) { void MLPPUtilities::CostInfo(int epoch, real_t cost_prev, real_t Cost) {
std::cout << "-----------------------------------" << std::endl; std::cout << "-----------------------------------" << std::endl;
std::cout << "This is epoch: " << epoch << std::endl; std::cout << "This is epoch: " << epoch << std::endl;
std::cout << "The cost function has been minimized by " << cost_prev - Cost << std::endl; std::cout << "The cost function has been minimized by " << cost_prev - Cost << std::endl;
@ -269,21 +269,21 @@ void MLPPUtilities::CostInfo(int epoch, double cost_prev, double Cost) {
std::cout << Cost << std::endl; std::cout << Cost << std::endl;
} }
std::vector<std::vector<std::vector<double>>> MLPPUtilities::createMiniBatches(std::vector<std::vector<double>> inputSet, int n_mini_batch) { std::vector<std::vector<std::vector<real_t>>> MLPPUtilities::createMiniBatches(std::vector<std::vector<real_t>> inputSet, int n_mini_batch) {
int n = inputSet.size(); int n = inputSet.size();
std::vector<std::vector<std::vector<double>>> inputMiniBatches; std::vector<std::vector<std::vector<real_t>>> inputMiniBatches;
// Creating the mini-batches // Creating the mini-batches
for (int i = 0; i < n_mini_batch; i++) { for (int i = 0; i < n_mini_batch; i++) {
std::vector<std::vector<double>> currentInputSet; std::vector<std::vector<real_t>> currentInputSet;
for (int j = 0; j < n / n_mini_batch; j++) { for (int j = 0; j < n / n_mini_batch; j++) {
currentInputSet.push_back(inputSet[n / n_mini_batch * i + j]); currentInputSet.push_back(inputSet[n / n_mini_batch * i + j]);
} }
inputMiniBatches.push_back(currentInputSet); inputMiniBatches.push_back(currentInputSet);
} }
if (double(n) / double(n_mini_batch) - int(n / n_mini_batch) != 0) { if (real_t(n) / real_t(n_mini_batch) - int(n / n_mini_batch) != 0) {
for (int i = 0; i < n - n / n_mini_batch * n_mini_batch; i++) { for (int i = 0; i < n - n / n_mini_batch * n_mini_batch; i++) {
inputMiniBatches[n_mini_batch - 1].push_back(inputSet[n / n_mini_batch * n_mini_batch + i]); inputMiniBatches[n_mini_batch - 1].push_back(inputSet[n / n_mini_batch * n_mini_batch + i]);
} }
@ -291,15 +291,15 @@ std::vector<std::vector<std::vector<double>>> MLPPUtilities::createMiniBatches(s
return inputMiniBatches; return inputMiniBatches;
} }
std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<std::vector<double>>> MLPPUtilities::createMiniBatches(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, int n_mini_batch) { std::tuple<std::vector<std::vector<std::vector<real_t>>>, std::vector<std::vector<real_t>>> MLPPUtilities::createMiniBatches(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, int n_mini_batch) {
int n = inputSet.size(); int n = inputSet.size();
std::vector<std::vector<std::vector<double>>> inputMiniBatches; std::vector<std::vector<std::vector<real_t>>> inputMiniBatches;
std::vector<std::vector<double>> outputMiniBatches; std::vector<std::vector<real_t>> outputMiniBatches;
for (int i = 0; i < n_mini_batch; i++) { for (int i = 0; i < n_mini_batch; i++) {
std::vector<std::vector<double>> currentInputSet; std::vector<std::vector<real_t>> currentInputSet;
std::vector<double> currentOutputSet; std::vector<real_t> currentOutputSet;
for (int j = 0; j < n / n_mini_batch; j++) { for (int j = 0; j < n / n_mini_batch; j++) {
currentInputSet.push_back(inputSet[n / n_mini_batch * i + j]); currentInputSet.push_back(inputSet[n / n_mini_batch * i + j]);
currentOutputSet.push_back(outputSet[n / n_mini_batch * i + j]); currentOutputSet.push_back(outputSet[n / n_mini_batch * i + j]);
@ -308,7 +308,7 @@ std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<std::vecto
outputMiniBatches.push_back(currentOutputSet); outputMiniBatches.push_back(currentOutputSet);
} }
if (double(n) / double(n_mini_batch) - int(n / n_mini_batch) != 0) { if (real_t(n) / real_t(n_mini_batch) - int(n / n_mini_batch) != 0) {
for (int i = 0; i < n - n / n_mini_batch * n_mini_batch; i++) { for (int i = 0; i < n - n / n_mini_batch * n_mini_batch; i++) {
inputMiniBatches[n_mini_batch - 1].push_back(inputSet[n / n_mini_batch * n_mini_batch + i]); inputMiniBatches[n_mini_batch - 1].push_back(inputSet[n / n_mini_batch * n_mini_batch + i]);
outputMiniBatches[n_mini_batch - 1].push_back(outputSet[n / n_mini_batch * n_mini_batch + i]); outputMiniBatches[n_mini_batch - 1].push_back(outputSet[n / n_mini_batch * n_mini_batch + i]);
@ -317,15 +317,15 @@ std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<std::vecto
return { inputMiniBatches, outputMiniBatches }; return { inputMiniBatches, outputMiniBatches };
} }
std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<std::vector<std::vector<double>>>> MLPPUtilities::createMiniBatches(std::vector<std::vector<double>> inputSet, std::vector<std::vector<double>> outputSet, int n_mini_batch) { std::tuple<std::vector<std::vector<std::vector<real_t>>>, std::vector<std::vector<std::vector<real_t>>>> MLPPUtilities::createMiniBatches(std::vector<std::vector<real_t>> inputSet, std::vector<std::vector<real_t>> outputSet, int n_mini_batch) {
int n = inputSet.size(); int n = inputSet.size();
std::vector<std::vector<std::vector<double>>> inputMiniBatches; std::vector<std::vector<std::vector<real_t>>> inputMiniBatches;
std::vector<std::vector<std::vector<double>>> outputMiniBatches; std::vector<std::vector<std::vector<real_t>>> outputMiniBatches;
for (int i = 0; i < n_mini_batch; i++) { for (int i = 0; i < n_mini_batch; i++) {
std::vector<std::vector<double>> currentInputSet; std::vector<std::vector<real_t>> currentInputSet;
std::vector<std::vector<double>> currentOutputSet; std::vector<std::vector<real_t>> currentOutputSet;
for (int j = 0; j < n / n_mini_batch; j++) { for (int j = 0; j < n / n_mini_batch; j++) {
currentInputSet.push_back(inputSet[n / n_mini_batch * i + j]); currentInputSet.push_back(inputSet[n / n_mini_batch * i + j]);
currentOutputSet.push_back(outputSet[n / n_mini_batch * i + j]); currentOutputSet.push_back(outputSet[n / n_mini_batch * i + j]);
@ -334,7 +334,7 @@ std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<std::vecto
outputMiniBatches.push_back(currentOutputSet); outputMiniBatches.push_back(currentOutputSet);
} }
if (double(n) / double(n_mini_batch) - int(n / n_mini_batch) != 0) { if (real_t(n) / real_t(n_mini_batch) - int(n / n_mini_batch) != 0) {
for (int i = 0; i < n - n / n_mini_batch * n_mini_batch; i++) { for (int i = 0; i < n - n / n_mini_batch * n_mini_batch; i++) {
inputMiniBatches[n_mini_batch - 1].push_back(inputSet[n / n_mini_batch * n_mini_batch + i]); inputMiniBatches[n_mini_batch - 1].push_back(inputSet[n / n_mini_batch * n_mini_batch + i]);
outputMiniBatches[n_mini_batch - 1].push_back(outputSet[n / n_mini_batch * n_mini_batch + i]); outputMiniBatches[n_mini_batch - 1].push_back(outputSet[n / n_mini_batch * n_mini_batch + i]);
@ -343,8 +343,8 @@ std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<std::vecto
return { inputMiniBatches, outputMiniBatches }; return { inputMiniBatches, outputMiniBatches };
} }
std::tuple<double, double, double, double> MLPPUtilities::TF_PN(std::vector<double> y_hat, std::vector<double> y) { std::tuple<real_t, real_t, real_t, real_t> MLPPUtilities::TF_PN(std::vector<real_t> y_hat, std::vector<real_t> y) {
double TP, FP, TN, FN = 0; real_t TP, FP, TN, FN = 0;
for (int i = 0; i < y_hat.size(); i++) { for (int i = 0; i < y_hat.size(); i++) {
if (y_hat[i] == y[i]) { if (y_hat[i] == y[i]) {
if (y_hat[i] == 1) { if (y_hat[i] == 1) {
@ -363,20 +363,20 @@ std::tuple<double, double, double, double> MLPPUtilities::TF_PN(std::vector<doub
return { TP, FP, TN, FN }; return { TP, FP, TN, FN };
} }
double MLPPUtilities::recall(std::vector<double> y_hat, std::vector<double> y) { real_t MLPPUtilities::recall(std::vector<real_t> y_hat, std::vector<real_t> y) {
auto [TP, FP, TN, FN] = TF_PN(y_hat, y); auto [TP, FP, TN, FN] = TF_PN(y_hat, y);
return TP / (TP + FN); return TP / (TP + FN);
} }
double MLPPUtilities::precision(std::vector<double> y_hat, std::vector<double> y) { real_t MLPPUtilities::precision(std::vector<real_t> y_hat, std::vector<real_t> y) {
auto [TP, FP, TN, FN] = TF_PN(y_hat, y); auto [TP, FP, TN, FN] = TF_PN(y_hat, y);
return TP / (TP + FP); return TP / (TP + FP);
} }
double MLPPUtilities::accuracy(std::vector<double> y_hat, std::vector<double> y) { real_t MLPPUtilities::accuracy(std::vector<real_t> y_hat, std::vector<real_t> y) {
auto [TP, FP, TN, FN] = TF_PN(y_hat, y); auto [TP, FP, TN, FN] = TF_PN(y_hat, y);
return (TP + TN) / (TP + FP + FN + TN); return (TP + TN) / (TP + FP + FN + TN);
} }
double MLPPUtilities::f1_score(std::vector<double> y_hat, std::vector<double> y) { real_t MLPPUtilities::f1_score(std::vector<real_t> y_hat, std::vector<real_t> y) {
return 2 * precision(y_hat, y) * recall(y_hat, y) / (precision(y_hat, y) + recall(y_hat, y)); return 2 * precision(y_hat, y) * recall(y_hat, y) / (precision(y_hat, y) + recall(y_hat, y));
} }

45
mlpp/utilities/utilities.h
View File

@ -8,6 +8,9 @@
// Created by Marc Melikyan on 1/16/21. // Created by Marc Melikyan on 1/16/21.
// //
#include "core/math/math_defs.h"
#include <string> #include <string>
#include <tuple> #include <tuple>
#include <vector> #include <vector>
@ -16,37 +19,37 @@
class MLPPUtilities { class MLPPUtilities {
public: public:
// Weight Init // Weight Init
static std::vector<double> weightInitialization(int n, std::string type = "Default"); static std::vector<real_t> weightInitialization(int n, std::string type = "Default");
static double biasInitialization(); static real_t biasInitialization();
static std::vector<std::vector<double>> weightInitialization(int n, int m, std::string type = "Default"); static std::vector<std::vector<real_t>> weightInitialization(int n, int m, std::string type = "Default");
static std::vector<double> biasInitialization(int n); static std::vector<real_t> biasInitialization(int n);
// Cost/Performance related Functions // Cost/Performance related Functions
double performance(std::vector<double> y_hat, std::vector<double> y); real_t performance(std::vector<real_t> y_hat, std::vector<real_t> y);
double performance(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y); real_t performance(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
// Parameter Saving Functions // Parameter Saving Functions
void saveParameters(std::string fileName, std::vector<double> weights, double bias, bool app = 0, int layer = -1); void saveParameters(std::string fileName, std::vector<real_t> weights, real_t bias, bool app = 0, int layer = -1);
void saveParameters(std::string fileName, std::vector<double> weights, std::vector<double> initial, double bias, bool app = 0, int layer = -1); void saveParameters(std::string fileName, std::vector<real_t> weights, std::vector<real_t> initial, real_t bias, bool app = 0, int layer = -1);
void saveParameters(std::string fileName, std::vector<std::vector<double>> weights, std::vector<double> bias, bool app = 0, int layer = -1); void saveParameters(std::string fileName, std::vector<std::vector<real_t>> weights, std::vector<real_t> bias, bool app = 0, int layer = -1);
// Gradient Descent related // Gradient Descent related
static void UI(std::vector<double> weights, double bias); static void UI(std::vector<real_t> weights, real_t bias);
static void UI(std::vector<double> weights, std::vector<double> initial, double bias); static void UI(std::vector<real_t> weights, std::vector<real_t> initial, real_t bias);
static void UI(std::vector<std::vector<double>>, std::vector<double> bias); static void UI(std::vector<std::vector<real_t>>, std::vector<real_t> bias);
static void CostInfo(int epoch, double cost_prev, double Cost); static void CostInfo(int epoch, real_t cost_prev, real_t Cost);
static std::vector<std::vector<std::vector<double>>> createMiniBatches(std::vector<std::vector<double>> inputSet, int n_mini_batch); static std::vector<std::vector<std::vector<real_t>>> createMiniBatches(std::vector<std::vector<real_t>> inputSet, int n_mini_batch);
static std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<std::vector<double>>> createMiniBatches(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, int n_mini_batch); static std::tuple<std::vector<std::vector<std::vector<real_t>>>, std::vector<std::vector<real_t>>> createMiniBatches(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, int n_mini_batch);
static std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<std::vector<std::vector<double>>>> createMiniBatches(std::vector<std::vector<double>> inputSet, std::vector<std::vector<double>> outputSet, int n_mini_batch); static std::tuple<std::vector<std::vector<std::vector<real_t>>>, std::vector<std::vector<std::vector<real_t>>>> createMiniBatches(std::vector<std::vector<real_t>> inputSet, std::vector<std::vector<real_t>> outputSet, int n_mini_batch);
// F1 score, Precision/Recall, TP, FP, TN, FN, etc. // F1 score, Precision/Recall, TP, FP, TN, FN, etc.
std::tuple<double, double, double, double> TF_PN(std::vector<double> y_hat, std::vector<double> y); //TF_PN = "True", "False", "Positive", "Negative" std::tuple<real_t, real_t, real_t, real_t> TF_PN(std::vector<real_t> y_hat, std::vector<real_t> y); //TF_PN = "True", "False", "Positive", "Negative"
double recall(std::vector<double> y_hat, std::vector<double> y); real_t recall(std::vector<real_t> y_hat, std::vector<real_t> y);
double precision(std::vector<double> y_hat, std::vector<double> y); real_t precision(std::vector<real_t> y_hat, std::vector<real_t> y);
double accuracy(std::vector<double> y_hat, std::vector<double> y); real_t accuracy(std::vector<real_t> y_hat, std::vector<real_t> y);
double f1_score(std::vector<double> y_hat, std::vector<double> y); real_t f1_score(std::vector<real_t> y_hat, std::vector<real_t> y);
private: private:
}; };

60
mlpp/wgan/wgan.cpp
View File

@ -15,7 +15,7 @@
#include <iostream> #include <iostream>
MLPPWGAN::MLPPWGAN(double k, std::vector<std::vector<double>> outputSet) : MLPPWGAN::MLPPWGAN(real_t k, std::vector<std::vector<real_t>> outputSet) :
outputSet(outputSet), n(outputSet.size()), k(k) { outputSet(outputSet), n(outputSet.size()), k(k) {
} }
@ -23,15 +23,15 @@ MLPPWGAN::~MLPPWGAN() {
delete outputLayer; delete outputLayer;
} }
std::vector<std::vector<double>> MLPPWGAN::generateExample(int n) { std::vector<std::vector<real_t>> MLPPWGAN::generateExample(int n) {
MLPPLinAlg alg; MLPPLinAlg alg;
return modelSetTestGenerator(alg.gaussianNoise(n, k)); return modelSetTestGenerator(alg.gaussianNoise(n, k));
} }
void MLPPWGAN::gradientDescent(double learning_rate, int max_epoch, bool UI) { void MLPPWGAN::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
class MLPPCost cost; class MLPPCost cost;
MLPPLinAlg alg; MLPPLinAlg alg;
double cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
forwardPass(); forwardPass();
@ -40,11 +40,11 @@ void MLPPWGAN::gradientDescent(double learning_rate, int max_epoch, bool UI) {
while (true) { while (true) {
cost_prev = Cost(y_hat, alg.onevec(n)); cost_prev = Cost(y_hat, alg.onevec(n));
std::vector<std::vector<double>> generatorInputSet; std::vector<std::vector<real_t>> generatorInputSet;
std::vector<std::vector<double>> discriminatorInputSet; std::vector<std::vector<real_t>> discriminatorInputSet;
std::vector<double> y_hat; std::vector<real_t> y_hat;
std::vector<double> outputSet; std::vector<real_t> outputSet;
// Training of the discriminator. // Training of the discriminator.
for (int i = 0; i < CRITIC_INTERATIONS; i++) { for (int i = 0; i < CRITIC_INTERATIONS; i++) {
@ -54,7 +54,7 @@ void MLPPWGAN::gradientDescent(double learning_rate, int max_epoch, bool UI) {
y_hat = modelSetTestDiscriminator(discriminatorInputSet); y_hat = modelSetTestDiscriminator(discriminatorInputSet);
outputSet = alg.scalarMultiply(-1, alg.onevec(n)); // WGAN changes y_i = 1 and y_i = 0 to y_i = 1 and y_i = -1 outputSet = alg.scalarMultiply(-1, alg.onevec(n)); // WGAN changes y_i = 1 and y_i = 0 to y_i = 1 and y_i = -1
std::vector<double> outputSetReal = alg.onevec(n); std::vector<real_t> outputSetReal = alg.onevec(n);
outputSet.insert(outputSet.end(), outputSetReal.begin(), outputSetReal.end()); // Fake + real output scores. outputSet.insert(outputSet.end(), outputSetReal.begin(), outputSetReal.end()); // Fake + real output scores.
auto [cumulativeDiscriminatorHiddenLayerWGrad, outputDiscriminatorWGrad] = computeDiscriminatorGradients(y_hat, outputSet); auto [cumulativeDiscriminatorHiddenLayerWGrad, outputDiscriminatorWGrad] = computeDiscriminatorGradients(y_hat, outputSet);
@ -69,7 +69,7 @@ void MLPPWGAN::gradientDescent(double learning_rate, int max_epoch, bool UI) {
y_hat = modelSetTestDiscriminator(discriminatorInputSet); y_hat = modelSetTestDiscriminator(discriminatorInputSet);
outputSet = alg.onevec(n); outputSet = alg.onevec(n);
std::vector<std::vector<std::vector<double>>> cumulativeGeneratorHiddenLayerWGrad = computeGeneratorGradients(y_hat, outputSet); std::vector<std::vector<std::vector<real_t>>> cumulativeGeneratorHiddenLayerWGrad = computeGeneratorGradients(y_hat, outputSet);
cumulativeGeneratorHiddenLayerWGrad = alg.scalarMultiply(learning_rate / n, cumulativeGeneratorHiddenLayerWGrad); cumulativeGeneratorHiddenLayerWGrad = alg.scalarMultiply(learning_rate / n, cumulativeGeneratorHiddenLayerWGrad);
updateGeneratorParameters(cumulativeGeneratorHiddenLayerWGrad, learning_rate); updateGeneratorParameters(cumulativeGeneratorHiddenLayerWGrad, learning_rate);
@ -85,7 +85,7 @@ void MLPPWGAN::gradientDescent(double learning_rate, int max_epoch, bool UI) {
} }
} }
double MLPPWGAN::score() { real_t MLPPWGAN::score() {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPUtilities util; MLPPUtilities util;
forwardPass(); forwardPass();
@ -105,7 +105,7 @@ void MLPPWGAN::save(std::string fileName) {
} }
} }
void MLPPWGAN::addLayer(int n_hidden, std::string activation, std::string weightInit, std::string reg, double lambda, double alpha) { void MLPPWGAN::addLayer(int n_hidden, std::string activation, std::string weightInit, std::string reg, real_t lambda, real_t alpha) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (network.empty()) { if (network.empty()) {
network.push_back(MLPPHiddenLayer(n_hidden, activation, alg.gaussianNoise(n, k), weightInit, reg, lambda, alpha)); network.push_back(MLPPHiddenLayer(n_hidden, activation, alg.gaussianNoise(n, k), weightInit, reg, lambda, alpha));
@ -116,7 +116,7 @@ void MLPPWGAN::addLayer(int n_hidden, std::string activation, std::string weight
} }
} }
void MLPPWGAN::addOutputLayer(std::string weightInit, std::string reg, double lambda, double alpha) { void MLPPWGAN::addOutputLayer(std::string weightInit, std::string reg, real_t lambda, real_t alpha) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (!network.empty()) { if (!network.empty()) {
outputLayer = new MLPPOutputLayer(network[network.size() - 1].n_hidden, "Linear", "WassersteinLoss", network[network.size() - 1].a, weightInit, "WeightClipping", -0.01, 0.01); outputLayer = new MLPPOutputLayer(network[network.size() - 1].n_hidden, "Linear", "WassersteinLoss", network[network.size() - 1].a, weightInit, "WeightClipping", -0.01, 0.01);
@ -125,7 +125,7 @@ void MLPPWGAN::addOutputLayer(std::string weightInit, std::string reg, double la
} }
} }
std::vector<std::vector<double>> MLPPWGAN::modelSetTestGenerator(std::vector<std::vector<double>> X) { std::vector<std::vector<real_t>> MLPPWGAN::modelSetTestGenerator(std::vector<std::vector<real_t>> X) {
if (!network.empty()) { if (!network.empty()) {
network[0].input = X; network[0].input = X;
network[0].forwardPass(); network[0].forwardPass();
@ -138,7 +138,7 @@ std::vector<std::vector<double>> MLPPWGAN::modelSetTestGenerator(std::vector<std
return network[network.size() / 2].a; return network[network.size() / 2].a;
} }
std::vector<double> MLPPWGAN::modelSetTestDiscriminator(std::vector<std::vector<double>> X) { std::vector<real_t> MLPPWGAN::modelSetTestDiscriminator(std::vector<std::vector<real_t>> X) {
if (!network.empty()) { if (!network.empty()) {
for (int i = network.size() / 2 + 1; i < network.size(); i++) { for (int i = network.size() / 2 + 1; i < network.size(); i++) {
if (i == network.size() / 2 + 1) { if (i == network.size() / 2 + 1) {
@ -154,10 +154,10 @@ std::vector<double> MLPPWGAN::modelSetTestDiscriminator(std::vector<std::vector<
return outputLayer->a; return outputLayer->a;
} }
double MLPPWGAN::Cost(std::vector<double> y_hat, std::vector<double> y) { real_t MLPPWGAN::Cost(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPReg regularization; MLPPReg regularization;
class MLPPCost cost; class MLPPCost cost;
double totalRegTerm = 0; real_t totalRegTerm = 0;
auto cost_function = outputLayer->cost_map[outputLayer->cost]; auto cost_function = outputLayer->cost_map[outputLayer->cost];
if (!network.empty()) { if (!network.empty()) {
@ -186,7 +186,7 @@ void MLPPWGAN::forwardPass() {
y_hat = outputLayer->a; y_hat = outputLayer->a;
} }
void MLPPWGAN::updateDiscriminatorParameters(std::vector<std::vector<std::vector<double>>> hiddenLayerUpdations, std::vector<double> outputLayerUpdation, double learning_rate) { void MLPPWGAN::updateDiscriminatorParameters(std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations, std::vector<real_t> outputLayerUpdation, real_t learning_rate) {
MLPPLinAlg alg; MLPPLinAlg alg;
outputLayer->weights = alg.subtraction(outputLayer->weights, outputLayerUpdation); outputLayer->weights = alg.subtraction(outputLayer->weights, outputLayerUpdation);
@ -203,7 +203,7 @@ void MLPPWGAN::updateDiscriminatorParameters(std::vector<std::vector<std::vector
} }
} }
void MLPPWGAN::updateGeneratorParameters(std::vector<std::vector<std::vector<double>>> hiddenLayerUpdations, double learning_rate) { void MLPPWGAN::updateGeneratorParameters(std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations, real_t learning_rate) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (!network.empty()) { if (!network.empty()) {
@ -216,25 +216,25 @@ void MLPPWGAN::updateGeneratorParameters(std::vector<std::vector<std::vector<dou
} }
} }
std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<double>> MLPPWGAN::computeDiscriminatorGradients(std::vector<double> y_hat, std::vector<double> outputSet) { std::tuple<std::vector<std::vector<std::vector<real_t>>>, std::vector<real_t>> MLPPWGAN::computeDiscriminatorGradients(std::vector<real_t> y_hat, std::vector<real_t> outputSet) {
class MLPPCost cost; class MLPPCost cost;
MLPPActivation avn; MLPPActivation avn;
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
std::vector<std::vector<std::vector<double>>> cumulativeHiddenLayerWGrad; // Tensor containing ALL hidden grads. std::vector<std::vector<std::vector<real_t>>> cumulativeHiddenLayerWGrad; // Tensor containing ALL hidden grads.
auto costDeriv = outputLayer->costDeriv_map[outputLayer->cost]; auto costDeriv = outputLayer->costDeriv_map[outputLayer->cost];
auto outputAvn = outputLayer->activation_map[outputLayer->activation]; auto outputAvn = outputLayer->activation_map[outputLayer->activation];
outputLayer->delta = alg.hadamard_product((cost.*costDeriv)(y_hat, outputSet), (avn.*outputAvn)(outputLayer->z, 1)); outputLayer->delta = alg.hadamard_product((cost.*costDeriv)(y_hat, outputSet), (avn.*outputAvn)(outputLayer->z, 1));
std::vector<double> outputWGrad = alg.mat_vec_mult(alg.transpose(outputLayer->input), outputLayer->delta); std::vector<real_t> outputWGrad = alg.mat_vec_mult(alg.transpose(outputLayer->input), outputLayer->delta);
outputWGrad = alg.addition(outputWGrad, regularization.regDerivTerm(outputLayer->weights, outputLayer->lambda, outputLayer->alpha, outputLayer->reg)); outputWGrad = alg.addition(outputWGrad, regularization.regDerivTerm(outputLayer->weights, outputLayer->lambda, outputLayer->alpha, outputLayer->reg));
if (!network.empty()) { if (!network.empty()) {
auto hiddenLayerAvn = network[network.size() - 1].activation_map[network[network.size() - 1].activation]; auto hiddenLayerAvn = network[network.size() - 1].activation_map[network[network.size() - 1].activation];
network[network.size() - 1].delta = alg.hadamard_product(alg.outerProduct(outputLayer->delta, outputLayer->weights), (avn.*hiddenLayerAvn)(network[network.size() - 1].z, 1)); network[network.size() - 1].delta = alg.hadamard_product(alg.outerProduct(outputLayer->delta, outputLayer->weights), (avn.*hiddenLayerAvn)(network[network.size() - 1].z, 1));
std::vector<std::vector<double>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[network.size() - 1].input), network[network.size() - 1].delta); std::vector<std::vector<real_t>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[network.size() - 1].input), network[network.size() - 1].delta);
cumulativeHiddenLayerWGrad.push_back(alg.addition(hiddenLayerWGrad, regularization.regDerivTerm(network[network.size() - 1].weights, network[network.size() - 1].lambda, network[network.size() - 1].alpha, network[network.size() - 1].reg))); // Adding to our cumulative hidden layer grads. Maintain reg terms as well. cumulativeHiddenLayerWGrad.push_back(alg.addition(hiddenLayerWGrad, regularization.regDerivTerm(network[network.size() - 1].weights, network[network.size() - 1].lambda, network[network.size() - 1].alpha, network[network.size() - 1].reg))); // Adding to our cumulative hidden layer grads. Maintain reg terms as well.
@ -244,7 +244,7 @@ std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<double>> M
for (int i = network.size() - 2; i > network.size() / 2; i--) { for (int i = network.size() - 2; i > network.size() / 2; i--) {
auto hiddenLayerAvn = network[i].activation_map[network[i].activation]; auto hiddenLayerAvn = network[i].activation_map[network[i].activation];
network[i].delta = alg.hadamard_product(alg.matmult(network[i + 1].delta, alg.transpose(network[i + 1].weights)), (avn.*hiddenLayerAvn)(network[i].z, 1)); network[i].delta = alg.hadamard_product(alg.matmult(network[i + 1].delta, alg.transpose(network[i + 1].weights)), (avn.*hiddenLayerAvn)(network[i].z, 1));
std::vector<std::vector<double>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[i].input), network[i].delta); std::vector<std::vector<real_t>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[i].input), network[i].delta);
cumulativeHiddenLayerWGrad.push_back(alg.addition(hiddenLayerWGrad, regularization.regDerivTerm(network[i].weights, network[i].lambda, network[i].alpha, network[i].reg))); // Adding to our cumulative hidden layer grads. Maintain reg terms as well. cumulativeHiddenLayerWGrad.push_back(alg.addition(hiddenLayerWGrad, regularization.regDerivTerm(network[i].weights, network[i].lambda, network[i].alpha, network[i].reg))); // Adding to our cumulative hidden layer grads. Maintain reg terms as well.
} }
@ -252,36 +252,36 @@ std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<double>> M
return { cumulativeHiddenLayerWGrad, outputWGrad }; return { cumulativeHiddenLayerWGrad, outputWGrad };
} }
std::vector<std::vector<std::vector<double>>> MLPPWGAN::computeGeneratorGradients(std::vector<double> y_hat, std::vector<double> outputSet) { std::vector<std::vector<std::vector<real_t>>> MLPPWGAN::computeGeneratorGradients(std::vector<real_t> y_hat, std::vector<real_t> outputSet) {
class MLPPCost cost; class MLPPCost cost;
MLPPActivation avn; MLPPActivation avn;
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPReg regularization; MLPPReg regularization;
std::vector<std::vector<std::vector<double>>> cumulativeHiddenLayerWGrad; // Tensor containing ALL hidden grads. std::vector<std::vector<std::vector<real_t>>> cumulativeHiddenLayerWGrad; // Tensor containing ALL hidden grads.
auto costDeriv = outputLayer->costDeriv_map[outputLayer->cost]; auto costDeriv = outputLayer->costDeriv_map[outputLayer->cost];
auto outputAvn = outputLayer->activation_map[outputLayer->activation]; auto outputAvn = outputLayer->activation_map[outputLayer->activation];
outputLayer->delta = alg.hadamard_product((cost.*costDeriv)(y_hat, outputSet), (avn.*outputAvn)(outputLayer->z, 1)); outputLayer->delta = alg.hadamard_product((cost.*costDeriv)(y_hat, outputSet), (avn.*outputAvn)(outputLayer->z, 1));
std::vector<double> outputWGrad = alg.mat_vec_mult(alg.transpose(outputLayer->input), outputLayer->delta); std::vector<real_t> outputWGrad = alg.mat_vec_mult(alg.transpose(outputLayer->input), outputLayer->delta);
outputWGrad = alg.addition(outputWGrad, regularization.regDerivTerm(outputLayer->weights, outputLayer->lambda, outputLayer->alpha, outputLayer->reg)); outputWGrad = alg.addition(outputWGrad, regularization.regDerivTerm(outputLayer->weights, outputLayer->lambda, outputLayer->alpha, outputLayer->reg));
if (!network.empty()) { if (!network.empty()) {
auto hiddenLayerAvn = network[network.size() - 1].activation_map[network[network.size() - 1].activation]; auto hiddenLayerAvn = network[network.size() - 1].activation_map[network[network.size() - 1].activation];
network[network.size() - 1].delta = alg.hadamard_product(alg.outerProduct(outputLayer->delta, outputLayer->weights), (avn.*hiddenLayerAvn)(network[network.size() - 1].z, 1)); network[network.size() - 1].delta = alg.hadamard_product(alg.outerProduct(outputLayer->delta, outputLayer->weights), (avn.*hiddenLayerAvn)(network[network.size() - 1].z, 1));
std::vector<std::vector<double>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[network.size() - 1].input), network[network.size() - 1].delta); std::vector<std::vector<real_t>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[network.size() - 1].input), network[network.size() - 1].delta);
cumulativeHiddenLayerWGrad.push_back(alg.addition(hiddenLayerWGrad, regularization.regDerivTerm(network[network.size() - 1].weights, network[network.size() - 1].lambda, network[network.size() - 1].alpha, network[network.size() - 1].reg))); // Adding to our cumulative hidden layer grads. Maintain reg terms as well. cumulativeHiddenLayerWGrad.push_back(alg.addition(hiddenLayerWGrad, regularization.regDerivTerm(network[network.size() - 1].weights, network[network.size() - 1].lambda, network[network.size() - 1].alpha, network[network.size() - 1].reg))); // Adding to our cumulative hidden layer grads. Maintain reg terms as well.
for (int i = network.size() - 2; i >= 0; i--) { for (int i = network.size() - 2; i >= 0; i--) {
auto hiddenLayerAvn = network[i].activation_map[network[i].activation]; auto hiddenLayerAvn = network[i].activation_map[network[i].activation];
network[i].delta = alg.hadamard_product(alg.matmult(network[i + 1].delta, alg.transpose(network[i + 1].weights)), (avn.*hiddenLayerAvn)(network[i].z, 1)); network[i].delta = alg.hadamard_product(alg.matmult(network[i + 1].delta, alg.transpose(network[i + 1].weights)), (avn.*hiddenLayerAvn)(network[i].z, 1));
std::vector<std::vector<double>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[i].input), network[i].delta); std::vector<std::vector<real_t>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[i].input), network[i].delta);
cumulativeHiddenLayerWGrad.push_back(alg.addition(hiddenLayerWGrad, regularization.regDerivTerm(network[i].weights, network[i].lambda, network[i].alpha, network[i].reg))); // Adding to our cumulative hidden layer grads. Maintain reg terms as well. cumulativeHiddenLayerWGrad.push_back(alg.addition(hiddenLayerWGrad, regularization.regDerivTerm(network[i].weights, network[i].lambda, network[i].alpha, network[i].reg))); // Adding to our cumulative hidden layer grads. Maintain reg terms as well.
} }
} }
return cumulativeHiddenLayerWGrad; return cumulativeHiddenLayerWGrad;
} }
void MLPPWGAN::UI(int epoch, double cost_prev, std::vector<double> y_hat, std::vector<double> outputSet) { void MLPPWGAN::UI(int epoch, real_t cost_prev, std::vector<real_t> y_hat, std::vector<real_t> outputSet) {
MLPPUtilities::CostInfo(epoch, cost_prev, Cost(y_hat, outputSet)); MLPPUtilities::CostInfo(epoch, cost_prev, Cost(y_hat, outputSet));
std::cout << "Layer " << network.size() + 1 << ": " << std::endl; std::cout << "Layer " << network.size() + 1 << ": " << std::endl;
MLPPUtilities::UI(outputLayer->weights, outputLayer->bias); MLPPUtilities::UI(outputLayer->weights, outputLayer->bias);

32
mlpp/wgan/wgan.h
View File

@ -19,32 +19,32 @@
class MLPPWGAN { class MLPPWGAN {
public: public:
MLPPWGAN(double k, std::vector<std::vector<double>> outputSet); MLPPWGAN(real_t k, std::vector<std::vector<real_t>> outputSet);
~MLPPWGAN(); ~MLPPWGAN();
std::vector<std::vector<double>> generateExample(int n); std::vector<std::vector<real_t>> generateExample(int n);
void gradientDescent(double learning_rate, int max_epoch, bool UI = 1); void gradientDescent(real_t learning_rate, int max_epoch, bool UI = 1);
double score(); real_t score();
void save(std::string fileName); void save(std::string fileName);
void addLayer(int n_hidden, std::string activation, std::string weightInit = "Default", std::string reg = "None", double lambda = 0.5, double alpha = 0.5); void addLayer(int n_hidden, std::string activation, std::string weightInit = "Default", std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
void addOutputLayer(std::string weightInit = "Default", std::string reg = "None", double lambda = 0.5, double alpha = 0.5); void addOutputLayer(std::string weightInit = "Default", std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
private: private:
std::vector<std::vector<double>> modelSetTestGenerator(std::vector<std::vector<double>> X); // Evaluator for the generator of the WGAN. std::vector<std::vector<real_t>> modelSetTestGenerator(std::vector<std::vector<real_t>> X); // Evaluator for the generator of the WGAN.
std::vector<double> modelSetTestDiscriminator(std::vector<std::vector<double>> X); // Evaluator for the discriminator of the WGAN. std::vector<real_t> modelSetTestDiscriminator(std::vector<std::vector<real_t>> X); // Evaluator for the discriminator of the WGAN.
double Cost(std::vector<double> y_hat, std::vector<double> y); real_t Cost(std::vector<real_t> y_hat, std::vector<real_t> y);
void forwardPass(); void forwardPass();
void updateDiscriminatorParameters(std::vector<std::vector<std::vector<double>>> hiddenLayerUpdations, std::vector<double> outputLayerUpdation, double learning_rate); void updateDiscriminatorParameters(std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations, std::vector<real_t> outputLayerUpdation, real_t learning_rate);
void updateGeneratorParameters(std::vector<std::vector<std::vector<double>>> hiddenLayerUpdations, double learning_rate); void updateGeneratorParameters(std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations, real_t learning_rate);
std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<double>> computeDiscriminatorGradients(std::vector<double> y_hat, std::vector<double> outputSet); std::tuple<std::vector<std::vector<std::vector<real_t>>>, std::vector<real_t>> computeDiscriminatorGradients(std::vector<real_t> y_hat, std::vector<real_t> outputSet);
std::vector<std::vector<std::vector<double>>> computeGeneratorGradients(std::vector<double> y_hat, std::vector<double> outputSet); std::vector<std::vector<std::vector<real_t>>> computeGeneratorGradients(std::vector<real_t> y_hat, std::vector<real_t> outputSet);
void UI(int epoch, double cost_prev, std::vector<double> y_hat, std::vector<double> outputSet); void UI(int epoch, real_t cost_prev, std::vector<real_t> y_hat, std::vector<real_t> outputSet);
std::vector<std::vector<double>> outputSet; std::vector<std::vector<real_t>> outputSet;
std::vector<double> y_hat; std::vector<real_t> y_hat;
std::vector<MLPPHiddenLayer> network; std::vector<MLPPHiddenLayer> network;
MLPPOutputLayer *outputLayer; MLPPOutputLayer *outputLayer;

165
test/mlpp_tests.cpp
View File

@ -45,11 +45,11 @@
#include "../mlpp/uni_lin_reg/uni_lin_reg.h" #include "../mlpp/uni_lin_reg/uni_lin_reg.h"
#include "../mlpp/wgan/wgan.h" #include "../mlpp/wgan/wgan.h"
Vector<double> dstd_vec_to_vec(const std::vector<double> &in) { Vector<real_t> dstd_vec_to_vec(const std::vector<real_t> &in) {
Vector<double> r; Vector<real_t> r;
r.resize(static_cast<int>(in.size())); r.resize(static_cast<int>(in.size()));
double *darr = r.ptrw(); real_t *darr = r.ptrw();
for (uint32_t i = 0; i < in.size(); ++i) { for (uint32_t i = 0; i < in.size(); ++i) {
darr[i] = in[i]; darr[i] = in[i];
@ -58,8 +58,8 @@ Vector<double> dstd_vec_to_vec(const std::vector<double> &in) {
return r; return r;
} }
Vector<Vector<double>> dstd_mat_to_mat(const std::vector<std::vector<double>> &in) { Vector<Vector<real_t>> dstd_mat_to_mat(const std::vector<std::vector<real_t>> &in) {
Vector<Vector<double>> r; Vector<Vector<real_t>> r;
for (uint32_t i = 0; i < in.size(); ++i) { for (uint32_t i = 0; i < in.size(); ++i) {
r.push_back(dstd_vec_to_vec(in[i])); r.push_back(dstd_vec_to_vec(in[i]));
@ -75,9 +75,9 @@ void MLPPTests::test_statistics() {
MLPPConvolutions conv; MLPPConvolutions conv;
// STATISTICS // STATISTICS
std::vector<double> x = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; std::vector<real_t> x = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
std::vector<double> y = { 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 }; std::vector<real_t> y = { 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 };
std::vector<double> w = { 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1 }; std::vector<real_t> w = { 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1 };
is_approx_equalsd(stat.mean(x), 5.5, "Arithmetic Mean"); is_approx_equalsd(stat.mean(x), 5.5, "Arithmetic Mean");
is_approx_equalsd(stat.mean(x), 5.5, "Median"); is_approx_equalsd(stat.mean(x), 5.5, "Median");
@ -117,19 +117,19 @@ void MLPPTests::test_statistics() {
void MLPPTests::test_linear_algebra() { void MLPPTests::test_linear_algebra() {
MLPPLinAlg alg; MLPPLinAlg alg;
std::vector<std::vector<double>> square = { { 1, 1 }, { -1, 1 }, { 1, -1 }, { -1, -1 } }; std::vector<std::vector<real_t>> square = { { 1, 1 }, { -1, 1 }, { 1, -1 }, { -1, -1 } };
std::vector<std::vector<double>> square_rot_res = { { 1.41421, 1.11022e-16 }, { -1.11022e-16, 1.41421 }, { 1.11022e-16, -1.41421 }, { -1.41421, -1.11022e-16 } }; std::vector<std::vector<real_t>> square_rot_res = { { 1.41421, 1.11022e-16 }, { -1.11022e-16, 1.41421 }, { 1.11022e-16, -1.41421 }, { -1.41421, -1.11022e-16 } };
is_approx_equals_dmat(dstd_mat_to_mat(alg.rotate(square, M_PI / 4)), dstd_mat_to_mat(square_rot_res), "alg.rotate(square, M_PI / 4)"); is_approx_equals_dmat(dstd_mat_to_mat(alg.rotate(square, M_PI / 4)), dstd_mat_to_mat(square_rot_res), "alg.rotate(square, M_PI / 4)");
std::vector<std::vector<double>> A = { std::vector<std::vector<real_t>> A = {
{ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 },
{ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 },
}; };
std::vector<double> a = { 4, 3, 1, 3 }; std::vector<real_t> a = { 4, 3, 1, 3 };
std::vector<double> b = { 3, 5, 6, 1 }; std::vector<real_t> b = { 3, 5, 6, 1 };
std::vector<std::vector<double>> mmtr_res = { std::vector<std::vector<real_t>> mmtr_res = {
{ 2, 4, 6, 8, 10, 12, 14, 16, 18, 20 }, { 2, 4, 6, 8, 10, 12, 14, 16, 18, 20 },
{ 4, 8, 12, 16, 20, 24, 28, 32, 36, 40 }, { 4, 8, 12, 16, 20, 24, 28, 32, 36, 40 },
{ 6, 12, 18, 24, 30, 36, 42, 48, 54, 60 }, { 6, 12, 18, 24, 30, 36, 42, 48, 54, 60 },
@ -146,14 +146,14 @@ void MLPPTests::test_linear_algebra() {
is_approx_equalsd(alg.dot(a, b), 36, "alg.dot(a, b)"); is_approx_equalsd(alg.dot(a, b), 36, "alg.dot(a, b)");
std::vector<std::vector<double>> had_prod_res = { std::vector<std::vector<real_t>> had_prod_res = {
{ 1, 4, 9, 16, 25, 36, 49, 64, 81, 100 }, { 1, 4, 9, 16, 25, 36, 49, 64, 81, 100 },
{ 1, 4, 9, 16, 25, 36, 49, 64, 81, 100 } { 1, 4, 9, 16, 25, 36, 49, 64, 81, 100 }
}; };
is_approx_equals_dmat(dstd_mat_to_mat(alg.hadamard_product(A, A)), dstd_mat_to_mat(had_prod_res), "alg.hadamard_product(A, A)"); is_approx_equals_dmat(dstd_mat_to_mat(alg.hadamard_product(A, A)), dstd_mat_to_mat(had_prod_res), "alg.hadamard_product(A, A)");
std::vector<std::vector<double>> id_10_res = { std::vector<std::vector<real_t>> id_10_res = {
{ 1, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
{ 0, 1, 0, 0, 0, 0, 0, 0, 0, 0 }, { 0, 1, 0, 0, 0, 0, 0, 0, 0, 0 },
{ 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 }, { 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 },
@ -177,7 +177,7 @@ void MLPPTests::test_univariate_linear_regression() {
MLPPUniLinReg model(ds->input, ds->output); MLPPUniLinReg model(ds->input, ds->output);
std::vector<double> slr_res = { std::vector<real_t> slr_res = {
24.1095, 28.4829, 29.8082, 26.0974, 27.2902, 61.0851, 30.4709, 25.0372, 25.5673, 35.9046, 24.1095, 28.4829, 29.8082, 26.0974, 27.2902, 61.0851, 30.4709, 25.0372, 25.5673, 35.9046,
54.4587, 18.8083, 23.4468, 18.5432, 19.2059, 21.1938, 23.0492, 18.8083, 25.4348, 35.9046, 54.4587, 18.8083, 23.4468, 18.5432, 19.2059, 21.1938, 23.0492, 18.8083, 25.4348, 35.9046,
37.76, 40.278, 63.8683, 68.5068, 40.4106, 46.772, 32.0612, 23.3143, 44.784, 44.519, 37.76, 40.278, 63.8683, 68.5068, 40.4106, 46.772, 32.0612, 23.3143, 44.784, 44.519,
@ -254,8 +254,8 @@ void MLPPTests::test_multivariate_linear_regression_score_sgd_adam(bool ui) {
const int TRIAL_NUM = 1000; const int TRIAL_NUM = 1000;
double scoreSGD = 0; real_t scoreSGD = 0;
double scoreADAM = 0; real_t scoreADAM = 0;
for (int i = 0; i < TRIAL_NUM; i++) { for (int i = 0; i < TRIAL_NUM; i++) {
MLPPLinReg modelf(alg.transpose(ds->input), ds->output); MLPPLinReg modelf(alg.transpose(ds->input), ds->output);
modelf.MBGD(0.001, 5, 1, ui); modelf.MBGD(0.001, 5, 1, ui);
@ -327,8 +327,8 @@ void MLPPTests::test_c_log_log_regression(bool ui) {
MLPPLinAlg alg; MLPPLinAlg alg;
// CLOGLOG REGRESSION // CLOGLOG REGRESSION
std::vector<std::vector<double>> inputSet = { { 1, 2, 3, 4, 5, 6, 7, 8 }, { 0, 0, 0, 0, 1, 1, 1, 1 } }; std::vector<std::vector<real_t>> inputSet = { { 1, 2, 3, 4, 5, 6, 7, 8 }, { 0, 0, 0, 0, 1, 1, 1, 1 } };
std::vector<double> outputSet = { 0, 0, 0, 0, 1, 1, 1, 1 }; std::vector<real_t> outputSet = { 0, 0, 0, 0, 1, 1, 1, 1 };
MLPPCLogLogReg model(alg.transpose(inputSet), outputSet); MLPPCLogLogReg model(alg.transpose(inputSet), outputSet);
model.SGD(0.1, 10000, ui); model.SGD(0.1, 10000, ui);
alg.printVector(model.modelSetTest(alg.transpose(inputSet))); alg.printVector(model.modelSetTest(alg.transpose(inputSet)));
@ -338,8 +338,8 @@ void MLPPTests::test_exp_reg_regression(bool ui) {
MLPPLinAlg alg; MLPPLinAlg alg;
// EXPREG REGRESSION // EXPREG REGRESSION
std::vector<std::vector<double>> inputSet = { { 0, 1, 2, 3, 4 } }; std::vector<std::vector<real_t>> inputSet = { { 0, 1, 2, 3, 4 } };
std::vector<double> outputSet = { 1, 2, 4, 8, 16 }; std::vector<real_t> outputSet = { 1, 2, 4, 8, 16 };
MLPPExpReg model(alg.transpose(inputSet), outputSet); MLPPExpReg model(alg.transpose(inputSet), outputSet);
model.SGD(0.001, 10000, ui); model.SGD(0.001, 10000, ui);
alg.printVector(model.modelSetTest(alg.transpose(inputSet))); alg.printVector(model.modelSetTest(alg.transpose(inputSet)));
@ -349,8 +349,8 @@ void MLPPTests::test_tanh_regression(bool ui) {
MLPPLinAlg alg; MLPPLinAlg alg;
// TANH REGRESSION // TANH REGRESSION
std::vector<std::vector<double>> inputSet = { { 4, 3, 0, -3, -4 }, { 0, 0, 0, 1, 1 } }; std::vector<std::vector<real_t>> inputSet = { { 4, 3, 0, -3, -4 }, { 0, 0, 0, 1, 1 } };
std::vector<double> outputSet = { 1, 1, 0, -1, -1 }; std::vector<real_t> outputSet = { 1, 1, 0, -1, -1 };
MLPPTanhReg model(alg.transpose(inputSet), outputSet); MLPPTanhReg model(alg.transpose(inputSet), outputSet);
model.SGD(0.1, 10000, ui); model.SGD(0.1, 10000, ui);
alg.printVector(model.modelSetTest(alg.transpose(inputSet))); alg.printVector(model.modelSetTest(alg.transpose(inputSet)));
@ -387,9 +387,9 @@ void MLPPTests::test_mlp(bool ui) {
MLPPLinAlg alg; MLPPLinAlg alg;
// MLP // MLP
std::vector<std::vector<double>> inputSet = { { 0, 0, 1, 1 }, { 0, 1, 0, 1 } }; std::vector<std::vector<real_t>> inputSet = { { 0, 0, 1, 1 }, { 0, 1, 0, 1 } };
inputSet = alg.transpose(inputSet); inputSet = alg.transpose(inputSet);
std::vector<double> outputSet = { 0, 1, 1, 0 }; std::vector<real_t> outputSet = { 0, 1, 1, 0 };
MLPPMLP model(inputSet, outputSet, 2); MLPPMLP model(inputSet, outputSet, 2);
model.gradientDescent(0.1, 10000, ui); model.gradientDescent(0.1, 10000, ui);
@ -412,7 +412,7 @@ void MLPPTests::test_autoencoder(bool ui) {
MLPPLinAlg alg; MLPPLinAlg alg;
// AUTOENCODER // AUTOENCODER
std::vector<std::vector<double>> inputSet = { { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, { 3, 5, 9, 12, 15, 18, 21, 24, 27, 30 } }; std::vector<std::vector<real_t>> inputSet = { { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, { 3, 5, 9, 12, 15, 18, 21, 24, 27, 30 } };
MLPPAutoEncoder model(alg.transpose(inputSet), 5); MLPPAutoEncoder model(alg.transpose(inputSet), 5);
model.SGD(0.001, 300000, ui); model.SGD(0.001, 300000, ui);
alg.printMatrix(model.modelSetTest(alg.transpose(inputSet))); alg.printMatrix(model.modelSetTest(alg.transpose(inputSet)));
@ -425,8 +425,8 @@ void MLPPTests::test_dynamically_sized_ann(bool ui) {
// Possible Weight Init Methods: Default, Uniform, HeNormal, HeUniform, XavierNormal, XavierUniform // Possible Weight Init Methods: Default, Uniform, HeNormal, HeUniform, XavierNormal, XavierUniform
// Possible Activations: Linear, Sigmoid, Swish, Softplus, Softsign, CLogLog, Ar{Sinh, Cosh, Tanh, Csch, Sech, Coth}, GaussianCDF, GELU, UnitStep // Possible Activations: Linear, Sigmoid, Swish, Softplus, Softsign, CLogLog, Ar{Sinh, Cosh, Tanh, Csch, Sech, Coth}, GaussianCDF, GELU, UnitStep
// Possible Loss Functions: MSE, RMSE, MBE, LogLoss, CrossEntropy, HingeLoss // Possible Loss Functions: MSE, RMSE, MBE, LogLoss, CrossEntropy, HingeLoss
std::vector<std::vector<double>> inputSet = { { 0, 0, 1, 1 }, { 0, 1, 0, 1 } }; std::vector<std::vector<real_t>> inputSet = { { 0, 0, 1, 1 }, { 0, 1, 0, 1 } };
std::vector<double> outputSet = { 0, 1, 1, 0 }; std::vector<real_t> outputSet = { 0, 1, 1, 0 };
MLPPANN ann(alg.transpose(inputSet), outputSet); MLPPANN ann(alg.transpose(inputSet), outputSet);
ann.addLayer(2, "Cosh"); ann.addLayer(2, "Cosh");
ann.addOutputLayer("Sigmoid", "LogLoss"); ann.addOutputLayer("Sigmoid", "LogLoss");
@ -448,7 +448,7 @@ void MLPPTests::test_wgan(bool ui) {
//MLPPData data; //MLPPData data;
//MLPPConvolutions conv; //MLPPConvolutions conv;
std::vector<std::vector<double>> outputSet = { std::vector<std::vector<real_t>> outputSet = {
{ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20 }, { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20 },
{ 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40 } { 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40 }
}; };
@ -465,8 +465,8 @@ void MLPPTests::test_wgan(bool ui) {
void MLPPTests::test_ann(bool ui) { void MLPPTests::test_ann(bool ui) {
MLPPLinAlg alg; MLPPLinAlg alg;
std::vector<std::vector<double>> inputSet = { { 0, 0 }, { 0, 1 }, { 1, 0 }, { 1, 1 } }; // XOR std::vector<std::vector<real_t>> inputSet = { { 0, 0 }, { 0, 1 }, { 1, 0 }, { 1, 1 } }; // XOR
std::vector<double> outputSet = { 0, 1, 1, 0 }; std::vector<real_t> outputSet = { 0, 1, 1, 0 };
MLPPANN ann(inputSet, outputSet); MLPPANN ann(inputSet, outputSet);
ann.addLayer(5, "Sigmoid"); ann.addLayer(5, "Sigmoid");
@ -474,7 +474,7 @@ void MLPPTests::test_ann(bool ui) {
ann.addOutputLayer("Sigmoid", "LogLoss"); ann.addOutputLayer("Sigmoid", "LogLoss");
ann.gradientDescent(1, 20000, ui); ann.gradientDescent(1, 20000, ui);
std::vector<double> predictions = ann.modelSetTest(inputSet); std::vector<real_t> predictions = ann.modelSetTest(inputSet);
alg.printVector(predictions); // Testing out the model's preds for train set. alg.printVector(predictions); // Testing out the model's preds for train set.
std::cout << "ACCURACY: " << 100 * ann.score() << "%" << std::endl; // Accuracy. std::cout << "ACCURACY: " << 100 * ann.score() << "%" << std::endl; // Accuracy.
} }
@ -483,8 +483,8 @@ void MLPPTests::test_dynamically_sized_mann(bool ui) {
MLPPData data; MLPPData data;
// DYNAMICALLY SIZED MANN (Multidimensional Output ANN) // DYNAMICALLY SIZED MANN (Multidimensional Output ANN)
std::vector<std::vector<double>> inputSet = { { 1, 2, 3 }, { 2, 4, 6 }, { 3, 6, 9 }, { 4, 8, 12 } }; std::vector<std::vector<real_t>> inputSet = { { 1, 2, 3 }, { 2, 4, 6 }, { 3, 6, 9 }, { 4, 8, 12 } };
std::vector<std::vector<double>> outputSet = { { 1, 5 }, { 2, 10 }, { 3, 15 }, { 4, 20 } }; std::vector<std::vector<real_t>> outputSet = { { 1, 5 }, { 2, 10 }, { 3, 15 }, { 4, 20 } };
MLPPMANN mann(inputSet, outputSet); MLPPMANN mann(inputSet, outputSet);
mann.addOutputLayer("Linear", "MSE"); mann.addOutputLayer("Linear", "MSE");
@ -497,8 +497,8 @@ void MLPPTests::test_train_test_split_mann(bool ui) {
MLPPData data; MLPPData data;
// TRAIN TEST SPLIT CHECK // TRAIN TEST SPLIT CHECK
std::vector<std::vector<double>> inputSet1 = { { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, { 3, 5, 9, 12, 15, 18, 21, 24, 27, 30 } }; std::vector<std::vector<real_t>> inputSet1 = { { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, { 3, 5, 9, 12, 15, 18, 21, 24, 27, 30 } };
std::vector<std::vector<double>> outputSet1 = { { 2, 4, 6, 8, 10, 12, 14, 16, 18, 20 } }; std::vector<std::vector<real_t>> outputSet1 = { { 2, 4, 6, 8, 10, 12, 14, 16, 18, 20 } };
Ref<MLPPDataComplex> d; Ref<MLPPDataComplex> d;
d.instance(); d.instance();
@ -525,8 +525,8 @@ void MLPPTests::test_naive_bayes() {
MLPPLinAlg alg; MLPPLinAlg alg;
// NAIVE BAYES // NAIVE BAYES
std::vector<std::vector<double>> inputSet = { { 1, 1, 1, 1, 1 }, { 0, 0, 1, 1, 1 }, { 0, 0, 1, 0, 1 } }; std::vector<std::vector<real_t>> inputSet = { { 1, 1, 1, 1, 1 }, { 0, 0, 1, 1, 1 }, { 0, 0, 1, 0, 1 } };
std::vector<double> outputSet = { 0, 1, 0, 1, 1 }; std::vector<real_t> outputSet = { 0, 1, 0, 1, 1 };
MLPPMultinomialNB MNB(alg.transpose(inputSet), outputSet, 2); MLPPMultinomialNB MNB(alg.transpose(inputSet), outputSet, 2);
alg.printVector(MNB.modelSetTest(alg.transpose(inputSet))); alg.printVector(MNB.modelSetTest(alg.transpose(inputSet)));
@ -541,7 +541,7 @@ void MLPPTests::test_k_means(bool ui) {
MLPPLinAlg alg; MLPPLinAlg alg;
// KMeans // KMeans
std::vector<std::vector<double>> inputSet = { { 32, 0, 7 }, { 2, 28, 17 }, { 0, 9, 23 } }; std::vector<std::vector<real_t>> inputSet = { { 32, 0, 7 }, { 2, 28, 17 }, { 0, 9, 23 } };
MLPPKMeans kmeans(inputSet, 3, "KMeans++"); MLPPKMeans kmeans(inputSet, 3, "KMeans++");
kmeans.train(3, ui); kmeans.train(3, ui);
std::cout << std::endl; std::cout << std::endl;
@ -553,8 +553,8 @@ void MLPPTests::test_knn(bool ui) {
MLPPLinAlg alg; MLPPLinAlg alg;
// kNN // kNN
std::vector<std::vector<double>> inputSet = { { 1, 2, 3, 4, 5, 6, 7, 8 }, { 0, 0, 0, 0, 1, 1, 1, 1 } }; std::vector<std::vector<real_t>> inputSet = { { 1, 2, 3, 4, 5, 6, 7, 8 }, { 0, 0, 0, 0, 1, 1, 1, 1 } };
std::vector<double> outputSet = { 0, 0, 0, 0, 1, 1, 1, 1 }; std::vector<real_t> outputSet = { 0, 0, 0, 0, 1, 1, 1, 1 };
MLPPKNN knn(alg.transpose(inputSet), outputSet, 8); MLPPKNN knn(alg.transpose(inputSet), outputSet, 8);
alg.printVector(knn.modelSetTest(alg.transpose(inputSet))); alg.printVector(knn.modelSetTest(alg.transpose(inputSet)));
std::cout << "ACCURACY: " << 100 * knn.score() << "%" << std::endl; std::cout << "ACCURACY: " << 100 * knn.score() << "%" << std::endl;
@ -566,18 +566,18 @@ void MLPPTests::test_convolution_tensors_etc() {
MLPPConvolutions conv; MLPPConvolutions conv;
// CONVOLUTION, POOLING, ETC.. // CONVOLUTION, POOLING, ETC..
std::vector<std::vector<double>> input = { std::vector<std::vector<real_t>> input = {
{ 1 }, { 1 },
}; };
std::vector<std::vector<std::vector<double>>> tensorSet; std::vector<std::vector<std::vector<real_t>>> tensorSet;
tensorSet.push_back(input); tensorSet.push_back(input);
tensorSet.push_back(input); tensorSet.push_back(input);
tensorSet.push_back(input); tensorSet.push_back(input);
alg.printTensor(data.rgb2xyz(tensorSet)); alg.printTensor(data.rgb2xyz(tensorSet));
std::vector<std::vector<double>> input2 = { std::vector<std::vector<real_t>> input2 = {
{ 62, 55, 55, 54, 49, 48, 47, 55 }, { 62, 55, 55, 54, 49, 48, 47, 55 },
{ 62, 57, 54, 52, 48, 47, 48, 53 }, { 62, 57, 54, 52, 48, 47, 48, 53 },
{ 61, 60, 52, 49, 48, 47, 49, 54 }, { 61, 60, 52, 49, 48, 47, 49, 54 },
@ -595,19 +595,19 @@ void MLPPTests::test_convolution_tensors_etc() {
alg.printMatrix(conv.convolve(input2, conv.getPrewittVertical(), 1)); // Can use padding alg.printMatrix(conv.convolve(input2, conv.getPrewittVertical(), 1)); // Can use padding
alg.printMatrix(conv.pool(input2, 4, 4, "Max")); // Can use Max, Min, or Average pooling. alg.printMatrix(conv.pool(input2, 4, 4, "Max")); // Can use Max, Min, or Average pooling.
std::vector<std::vector<std::vector<double>>> tensorSet2; std::vector<std::vector<std::vector<real_t>>> tensorSet2;
tensorSet2.push_back(input2); tensorSet2.push_back(input2);
tensorSet2.push_back(input2); tensorSet2.push_back(input2);
alg.printVector(conv.globalPool(tensorSet2, "Average")); // Can use Max, Min, or Average global pooling. alg.printVector(conv.globalPool(tensorSet2, "Average")); // Can use Max, Min, or Average global pooling.
std::vector<std::vector<double>> laplacian = { { 1, 1, 1 }, { 1, -4, 1 }, { 1, 1, 1 } }; std::vector<std::vector<real_t>> laplacian = { { 1, 1, 1 }, { 1, -4, 1 }, { 1, 1, 1 } };
alg.printMatrix(conv.convolve(conv.gaussianFilter2D(5, 1), laplacian, 1)); alg.printMatrix(conv.convolve(conv.gaussianFilter2D(5, 1), laplacian, 1));
} }
void MLPPTests::test_pca_svd_eigenvalues_eigenvectors(bool ui) { void MLPPTests::test_pca_svd_eigenvalues_eigenvectors(bool ui) {
MLPPLinAlg alg; MLPPLinAlg alg;
// PCA, SVD, eigenvalues & eigenvectors // PCA, SVD, eigenvalues & eigenvectors
std::vector<std::vector<double>> inputSet = { { 1, 1 }, { 1, 1 } }; std::vector<std::vector<real_t>> inputSet = { { 1, 1 }, { 1, 1 } };
MLPPLinAlg::EigenResult eigen = alg.eigen(inputSet); MLPPLinAlg::EigenResult eigen = alg.eigen(inputSet);
@ -675,7 +675,7 @@ void MLPPTests::test_nlp_and_data(bool ui) {
//alg.printMatrix(data.BOW(textArchive, "Default")); //alg.printMatrix(data.BOW(textArchive, "Default"));
std::cout << std::endl; std::cout << std::endl;
std::vector<std::vector<double>> inputSet = { { 1, 2 }, { 2, 3 }, { 3, 4 }, { 4, 5 }, { 5, 6 } }; std::vector<std::vector<real_t>> inputSet = { { 1, 2 }, { 2, 3 }, { 3, 4 }, { 4, 5 }, { 5, 6 } };
std::cout << "Feature Scaling Example:" << std::endl; std::cout << "Feature Scaling Example:" << std::endl;
alg.printMatrix(data.featureScaling(inputSet)); alg.printMatrix(data.featureScaling(inputSet));
std::cout << std::endl; std::cout << std::endl;
@ -692,7 +692,8 @@ void MLPPTests::test_outlier_finder(bool ui) {
MLPPLinAlg alg; MLPPLinAlg alg;
// Outlier Finder // Outlier Finder
std::vector<double> inputSet = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 23554332523523 }; //std::vector<real_t> inputSet = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 23554332523523 };
std::vector<real_t> inputSet = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 23554332 };
MLPPOutlierFinder outlierFinder(2); // Any datapoint outside of 2 stds from the mean is marked as an outlier. MLPPOutlierFinder outlierFinder(2); // Any datapoint outside of 2 stds from the mean is marked as an outlier.
alg.printVector(outlierFinder.modelTest(inputSet)); alg.printVector(outlierFinder.modelTest(inputSet));
} }
@ -702,15 +703,15 @@ void MLPPTests::test_new_math_functions() {
MLPPData data; MLPPData data;
// Testing new Functions // Testing new Functions
double z_s = 0.001; real_t z_s = 0.001;
std::cout << avn.logit(z_s) << std::endl; std::cout << avn.logit(z_s) << std::endl;
std::cout << avn.logit(z_s, 1) << std::endl; std::cout << avn.logit(z_s, 1) << std::endl;
std::vector<double> z_v = { 0.001 }; std::vector<real_t> z_v = { 0.001 };
alg.printVector(avn.logit(z_v)); alg.printVector(avn.logit(z_v));
alg.printVector(avn.logit(z_v, 1)); alg.printVector(avn.logit(z_v, 1));
std::vector<std::vector<double>> Z_m = { { 0.001 } }; std::vector<std::vector<real_t>> Z_m = { { 0.001 } };
alg.printMatrix(avn.logit(Z_m)); alg.printMatrix(avn.logit(Z_m));
alg.printMatrix(avn.logit(Z_m, 1)); alg.printMatrix(avn.logit(Z_m, 1));
@ -721,18 +722,18 @@ void MLPPTests::test_new_math_functions() {
alg.printMatrix(alg.matrixPower({ { 5, 5 }, { 5, 5 } }, 2)); alg.printMatrix(alg.matrixPower({ { 5, 5 }, { 5, 5 } }, 2));
alg.printVector(alg.solve({ { 1, 1 }, { 1.5, 4.0 } }, { 2200, 5050 })); alg.printVector(alg.solve({ { 1, 1 }, { 1.5, 4.0 } }, { 2200, 5050 }));
std::vector<std::vector<double>> matrixOfCubes = { { 1, 2, 64, 27 } }; std::vector<std::vector<real_t>> matrixOfCubes = { { 1, 2, 64, 27 } };
std::vector<double> vectorOfCubes = { 1, 2, 64, 27 }; std::vector<real_t> vectorOfCubes = { 1, 2, 64, 27 };
alg.printMatrix(alg.cbrt(matrixOfCubes)); alg.printMatrix(alg.cbrt(matrixOfCubes));
alg.printVector(alg.cbrt(vectorOfCubes)); alg.printVector(alg.cbrt(vectorOfCubes));
std::cout << alg.max({ { 1, 2, 3, 4, 5 }, { 6, 5, 3, 4, 1 }, { 9, 9, 9, 9, 9 } }) << std::endl; std::cout << alg.max({ { 1, 2, 3, 4, 5 }, { 6, 5, 3, 4, 1 }, { 9, 9, 9, 9, 9 } }) << std::endl;
std::cout << alg.min({ { 1, 2, 3, 4, 5 }, { 6, 5, 3, 4, 1 }, { 9, 9, 9, 9, 9 } }) << std::endl; std::cout << alg.min({ { 1, 2, 3, 4, 5 }, { 6, 5, 3, 4, 1 }, { 9, 9, 9, 9, 9 } }) << std::endl;
//std::vector<double> chicken; //std::vector<real_t> chicken;
//data.getImage("../../Data/apple.jpeg", chicken); //data.getImage("../../Data/apple.jpeg", chicken);
//alg.printVector(chicken); //alg.printVector(chicken);
std::vector<std::vector<double>> P = { { 12, -51, 4 }, { 6, 167, -68 }, { -4, 24, -41 } }; std::vector<std::vector<real_t>> P = { { 12, -51, 4 }, { 6, 167, -68 }, { -4, 24, -41 } };
alg.printMatrix(P); alg.printMatrix(P);
alg.printMatrix(alg.gramSchmidtProcess(P)); alg.printMatrix(alg.gramSchmidtProcess(P));
@ -752,7 +753,7 @@ void MLPPTests::test_positive_definiteness_checker() {
//MLPPConvolutions conv; //MLPPConvolutions conv;
// Checking positive-definiteness checker. For Cholesky Decomp. // Checking positive-definiteness checker. For Cholesky Decomp.
std::vector<std::vector<double>> A = { std::vector<std::vector<real_t>> A = {
{ 1, -1, -1, -1 }, { 1, -1, -1, -1 },
{ -1, 2, 2, 2 }, { -1, 2, 2, 2 },
{ -1, 2, 3, 1 }, { -1, 2, 3, 1 },
@ -765,19 +766,19 @@ void MLPPTests::test_positive_definiteness_checker() {
alg.printMatrix(chres.Lt); alg.printMatrix(chres.Lt);
} }
// double f(double x){ // real_t f(real_t x){
// return x*x*x + 2*x - 2; // return x*x*x + 2*x - 2;
// } // }
double f(double x) { real_t f(real_t x) {
return sin(x); return sin(x);
} }
double f_prime(double x) { real_t f_prime(real_t x) {
return 2 * x; return 2 * x;
} }
double f_prime_2var(std::vector<double> x) { real_t f_prime_2var(std::vector<real_t> x) {
return 2 * x[0] + x[1]; return 2 * x[0] + x[1];
} }
/* /*
@ -787,7 +788,7 @@ double f_prime_2var(std::vector<double> x) {
y''(2) = 12 y''(2) = 12
*/ */
// double f_mv(std::vector<double> x){ // real_t f_mv(std::vector<real_t> x){
// return x[0] * x[0] + x[0] * x[1] * x[1] + x[1] + 5; // return x[0] * x[0] + x[0] * x[1] * x[1] + x[1] + 5;
// } // }
@ -798,7 +799,7 @@ double f_prime_2var(std::vector<double> x) {
∂^2f/∂x∂y = 2 ∂^2f/∂x∂y = 2
*/ */
double f_mv(std::vector<double> x) { real_t f_mv(std::vector<real_t> x) {
return x[0] * x[0] * x[0] + x[0] + x[1] * x[1] * x[1] * x[0] + x[2] * x[2] * x[1]; return x[0] * x[0] * x[0] + x[0] + x[1] * x[1] * x[1] * x[0] + x[2] * x[2] * x[1];
} }
@ -865,7 +866,7 @@ void MLPPTests::test_numerical_analysis() {
std::cout << numAn.laplacian(f_mv, { 1, 1, 1 }) << std::endl; std::cout << numAn.laplacian(f_mv, { 1, 1, 1 }) << std::endl;
std::vector<std::vector<std::vector<double>>> tensor; std::vector<std::vector<std::vector<real_t>>> tensor;
tensor.push_back({ { 1, 2 }, { 1, 2 }, { 1, 2 } }); tensor.push_back({ { 1, 2 }, { 1, 2 }, { 1, 2 } });
tensor.push_back({ { 1, 2 }, { 1, 2 }, { 1, 2 } }); tensor.push_back({ { 1, 2 }, { 1, 2 }, { 1, 2 } });
@ -877,7 +878,7 @@ void MLPPTests::test_numerical_analysis() {
std::cout << numAn.eulerianMethod(f_prime, { 1, 1 }, 1.5, 0.000001) << std::endl; std::cout << numAn.eulerianMethod(f_prime, { 1, 1 }, 1.5, 0.000001) << std::endl;
std::cout << numAn.eulerianMethod(f_prime_2var, { 2, 3 }, 2.5, 0.00000001) << std::endl; std::cout << numAn.eulerianMethod(f_prime_2var, { 2, 3 }, 2.5, 0.00000001) << std::endl;
std::vector<std::vector<double>> A = { std::vector<std::vector<real_t>> A = {
{ 1, 0, 0, 0 }, { 1, 0, 0, 0 },
{ 0, 0, 0, 0 }, { 0, 0, 0, 0 },
{ 0, 0, 0, 0 }, { 0, 0, 0, 0 },
@ -898,8 +899,8 @@ void MLPPTests::test_numerical_analysis() {
std::cout << std::endl; std::cout << std::endl;
} // Harris detector works. Life is good! } // Harris detector works. Life is good!
std::vector<double> a = { 3, 4, 4 }; std::vector<real_t> a = { 3, 4, 4 };
std::vector<double> b = { 4, 4, 4 }; std::vector<real_t> b = { 4, 4, 4 };
alg.printVector(alg.cross(a, b)); alg.printVector(alg.cross(a, b));
} }
void MLPPTests::test_support_vector_classification_kernel(bool ui) { void MLPPTests::test_support_vector_classification_kernel(bool ui) {
@ -913,16 +914,16 @@ void MLPPTests::test_support_vector_classification_kernel(bool ui) {
kernelSVM.gradientDescent(0.0001, 20, ui); kernelSVM.gradientDescent(0.0001, 20, ui);
std::cout << "SCORE: " << kernelSVM.score() << std::endl; std::cout << "SCORE: " << kernelSVM.score() << std::endl;
std::vector<std::vector<double>> linearlyIndependentMat = { std::vector<std::vector<real_t>> linearlyIndependentMat = {
{ 1, 2, 3, 4 }, { 1, 2, 3, 4 },
{ 234538495, 4444, 6111, 55 } { 2345384, 4444, 6111, 55 }
}; };
std::cout << "True of false: linearly independent?: " << std::boolalpha << alg.linearIndependenceChecker(linearlyIndependentMat) << std::endl; std::cout << "True of false: linearly independent?: " << std::boolalpha << alg.linearIndependenceChecker(linearlyIndependentMat) << std::endl;
} }
void MLPPTests::test_mlpp_vector() { void MLPPTests::test_mlpp_vector() {
std::vector<double> a = { 4, 3, 1, 3 }; std::vector<real_t> a = { 4, 3, 1, 3 };
Ref<MLPPVector> rv; Ref<MLPPVector> rv;
rv.instance(); rv.instance();
@ -939,7 +940,7 @@ void MLPPTests::test_mlpp_vector() {
is_approx_equals_vec(rv, rv2, "re-set_from_std_vectors test."); is_approx_equals_vec(rv, rv2, "re-set_from_std_vectors test.");
} }
void MLPPTests::test_mlpp_matrix() { void MLPPTests::test_mlpp_matrix() {
std::vector<std::vector<double>> A = { std::vector<std::vector<real_t>> A = {
{ 1, 0, 0, 0 }, { 1, 0, 0, 0 },
{ 0, 1, 0, 0 }, { 0, 1, 0, 0 },
{ 0, 0, 1, 0 }, { 0, 0, 1, 0 },
@ -961,13 +962,13 @@ void MLPPTests::test_mlpp_matrix() {
is_approx_equals_mat(rmat, rmat2, "re-set_from_std_vectors test."); is_approx_equals_mat(rmat, rmat2, "re-set_from_std_vectors test.");
} }
void MLPPTests::is_approx_equalsd(double a, double b, const String &str) { void MLPPTests::is_approx_equalsd(real_t a, real_t b, const String &str) {
if (!Math::is_equal_approx(a, b)) { if (!Math::is_equal_approx(a, b)) {
ERR_PRINT("TEST FAILED: " + str + " Got: " + String::num(a) + " Should be: " + String::num(b)); ERR_PRINT("TEST FAILED: " + str + " Got: " + String::num(a) + " Should be: " + String::num(b));
} }
} }
void MLPPTests::is_approx_equals_dvec(const Vector<double> &a, const Vector<double> &b, const String &str) { void MLPPTests::is_approx_equals_dvec(const Vector<real_t> &a, const Vector<real_t> &b, const String &str) {
if (a.size() != b.size()) { if (a.size() != b.size()) {
goto IAEDVEC_FAILED; goto IAEDVEC_FAILED;
} }
@ -1003,7 +1004,7 @@ IAEDVEC_FAILED:
ERR_PRINT(fail_str); ERR_PRINT(fail_str);
} }
String vmat_to_str(const Vector<Vector<double>> &a) { String vmat_to_str(const Vector<Vector<real_t>> &a) {
String str; String str;
str += "[ \n"; str += "[ \n";
@ -1011,7 +1012,7 @@ String vmat_to_str(const Vector<Vector<double>> &a) {
for (int i = 0; i < a.size(); ++i) { for (int i = 0; i < a.size(); ++i) {
str += " [ "; str += " [ ";
const Vector<double> &aa = a[i]; const Vector<real_t> &aa = a[i];
for (int j = 0; j < aa.size(); ++j) { for (int j = 0; j < aa.size(); ++j) {
str += String::num(aa[j]); str += String::num(aa[j]);
@ -1026,14 +1027,14 @@ String vmat_to_str(const Vector<Vector<double>> &a) {
return str; return str;
} }
void MLPPTests::is_approx_equals_dmat(const Vector<Vector<double>> &a, const Vector<Vector<double>> &b, const String &str) { void MLPPTests::is_approx_equals_dmat(const Vector<Vector<real_t>> &a, const Vector<Vector<real_t>> &b, const String &str) {
if (a.size() != b.size()) { if (a.size() != b.size()) {
goto IAEDMAT_FAILED; goto IAEDMAT_FAILED;
} }
for (int i = 0; i < a.size(); ++i) { for (int i = 0; i < a.size(); ++i) {
const Vector<double> &aa = a[i]; const Vector<real_t> &aa = a[i];
const Vector<double> &bb = b[i]; const Vector<real_t> &bb = b[i];
if (aa.size() != bb.size()) { if (aa.size() != bb.size()) {
goto IAEDMAT_FAILED; goto IAEDMAT_FAILED;
@ -1066,8 +1067,8 @@ void MLPPTests::is_approx_equals_mat(Ref<MLPPMatrix> a, Ref<MLPPMatrix> b, const
int ds = a->data_size(); int ds = a->data_size();
const double *aa = a->ptr(); const real_t *aa = a->ptr();
const double *bb = b->ptr(); const real_t *bb = b->ptr();
if (a->size() != b->size()) { if (a->size() != b->size()) {
goto IAEMAT_FAILED; goto IAEMAT_FAILED;

8
test/mlpp_tests.h
View File

@ -4,6 +4,8 @@
// TODO port this class to use the test module once it's working // TODO port this class to use the test module once it's working
// Also don't forget to remove it's bindings // Also don't forget to remove it's bindings
#include "core/math/math_defs.h"
#include "core/containers/vector.h" #include "core/containers/vector.h"
#include "core/object/reference.h" #include "core/object/reference.h"
@ -64,9 +66,9 @@ public:
void test_mlpp_vector(); void test_mlpp_vector();
void test_mlpp_matrix(); void test_mlpp_matrix();
void is_approx_equalsd(double a, double b, const String &str); void is_approx_equalsd(real_t a, real_t b, const String &str);
void is_approx_equals_dvec(const Vector<double> &a, const Vector<double> &b, const String &str); void is_approx_equals_dvec(const Vector<real_t> &a, const Vector<real_t> &b, const String &str);
void is_approx_equals_dmat(const Vector<Vector<double>> &a, const Vector<Vector<double>> &b, const String &str); void is_approx_equals_dmat(const Vector<Vector<real_t>> &a, const Vector<Vector<real_t>> &b, const String &str);
void is_approx_equals_mat(Ref<MLPPMatrix> a, Ref<MLPPMatrix> b, const String &str); void is_approx_equals_mat(Ref<MLPPMatrix> a, Ref<MLPPMatrix> b, const String &str);
void is_approx_equals_vec(Ref<MLPPVector> a, Ref<MLPPVector> b, const String &str); void is_approx_equals_vec(Ref<MLPPVector> a, Ref<MLPPVector> b, const String &str);