Relintai/pmlpp
1
0
Fork 0
mirror of https://github.com/Relintai/pmlpp.git synced 2025-04-01 01:55:38 +02:00
Code Issues Packages Projects Releases Wiki Activity

Use real_t instead of doubles.

Browse Source
This commit is contained in:
Relintai 2023-01-27 13:01:16 +01:00
parent 7a291b451a
commit 4deb34c852
84 changed files with 2663 additions and 2579 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README_ORIG.md
View File

@ -30,7 +30,7 @@ g++ main.cpp /usr/local/lib/MLPP.so --std=c++17
```
## Usage
Please note that ML++ uses the ```std::vector<double>``` data type for emulating vectors, and the ```std::vector<std::vector<double>>``` data type for emulating matrices.
Please note that ML++ uses the ```std::vector<real_t>``` data type for emulating vectors, and the ```std::vector<std::vector<real_t>>``` data type for emulating matrices.
Begin by including the respective header file of your choice.
```cpp

162
main.cpp
View File

@ -52,19 +52,19 @@
#include "MLPP/Transforms/Transforms.hpp"
// double f(double x){
// real_t f(real_t x){
// return x*x*x + 2*x - 2;
// }
double f(double x){
real_t f(real_t x){
return sin(x);
}
double f_prime(double x){
real_t f_prime(real_t x){
return 2 * x;
}
double f_prime_2var(std::vector<double> x){
real_t f_prime_2var(std::vector<real_t> x){
return 2 * x[0] + x[1];
}
/*
@ -74,7 +74,7 @@ double f_prime_2var(std::vector<double> x){
y''(2) = 12
*/
// double f_mv(std::vector<double> x){
// real_t f_mv(std::vector<real_t> x){
// return x[0] * x[0] + x[0] * x[1] * x[1] + x[1] + 5;
// }
@ -85,7 +85,7 @@ double f_prime_2var(std::vector<double> x){
∂^2f/∂x∂y = 2
*/
double f_mv(std::vector<double> x){
real_t f_mv(std::vector<real_t> x){
return x[0] * x[0] * x[0] + x[0] + x[1] * x[1] * x[1] * x[0] + x[2] * x[2] * x[1];
}
@ -128,30 +128,30 @@ int main() {
MLPPConvolutions conv;
// DATA SETS
// std::vector<std::vector<double>> inputSet = {{1,2,3,4,5,6,7,8,9,10}, {3,5,9,12,15,18,21,24,27,30}};
// std::vector<double> outputSet = {2,4,6,8,10,12,14,16,18,20};
// std::vector<std::vector<real_t>> inputSet = {{1,2,3,4,5,6,7,8,9,10}, {3,5,9,12,15,18,21,24,27,30}};
// std::vector<real_t> outputSet = {2,4,6,8,10,12,14,16,18,20};
// std::vector<std::vector<double>> inputSet = {{1,2,3,4,5,6,7,8}, {0,0,0,0,1,1,1,1}};
// std::vector<double> outputSet = {0,0,0,0,1,1,1,1};
// std::vector<std::vector<real_t>> inputSet = {{1,2,3,4,5,6,7,8}, {0,0,0,0,1,1,1,1}};
// std::vector<real_t> outputSet = {0,0,0,0,1,1,1,1};
// std::vector<std::vector<double>> inputSet = {{4,3,0,-3,-4}, {0,0,0,1,1}};
// std::vector<double> outputSet = {1,1,0,-1,-1};
// std::vector<std::vector<real_t>> inputSet = {{4,3,0,-3,-4}, {0,0,0,1,1}};
// std::vector<real_t> outputSet = {1,1,0,-1,-1};
// std::vector<std::vector<double>> inputSet = {{0,1,2,3,4}};
// std::vector<double> outputSet = {1,2,4,8,16};
// std::vector<std::vector<real_t>> inputSet = {{0,1,2,3,4}};
// std::vector<real_t> outputSet = {1,2,4,8,16};
//std::vector<std::vector<double>> inputSet = {{32, 0, 7}, {2, 28, 17}, {0, 9, 23}};
//std::vector<std::vector<real_t>> inputSet = {{32, 0, 7}, {2, 28, 17}, {0, 9, 23}};
// std::vector<std::vector<double>> inputSet = {{1,1,0,0,1}, {0,0,1,1,1}, {0,1,1,0,1}};
// std::vector<double> outputSet = {0,1,0,1,1};
// std::vector<std::vector<real_t>> inputSet = {{1,1,0,0,1}, {0,0,1,1,1}, {0,1,1,0,1}};
// std::vector<real_t> outputSet = {0,1,0,1,1};
// std::vector<std::vector<double>> inputSet = {{0,0,1,1}, {0,1,0,1}};
// std::vector<double> outputSet = {0,1,1,0};
// std::vector<std::vector<real_t>> inputSet = {{0,0,1,1}, {0,1,0,1}};
// std::vector<real_t> outputSet = {0,1,1,0};
// // STATISTICS
// std::vector<double> x = {1,2,3,4,5,6,7,8,9,10};
// std::vector<double> y = {10,9,8,7,6,5,4,3,2,1};
// std::vector<double> w = {0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1};
// std::vector<real_t> x = {1,2,3,4,5,6,7,8,9,10};
// std::vector<real_t> y = {10,9,8,7,6,5,4,3,2,1};
// std::vector<real_t> w = {0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1};
// std::cout << "Arithmetic Mean: " << stat.mean(x) << std::endl;
// std::cout << "Median: " << stat.median(x) << std::endl;
@ -184,16 +184,16 @@ int main() {
// std::cout << "Absolute Average Deviation: " << stat.absAvgDeviation(x) << std::endl;
// LINEAR ALGEBRA
// std::vector<std::vector<double>> square = {{1, 1}, {-1, 1}, {1, -1}, {-1, -1}};
// std::vector<std::vector<real_t>> square = {{1, 1}, {-1, 1}, {1, -1}, {-1, -1}};
// alg.printMatrix(alg.rotate(square, M_PI/4));
// std::vector<std::vector<double>> A = {
// std::vector<std::vector<real_t>> A = {
// {1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
// {1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
// };
// std::vector<double> a = {4, 3, 1, 3};
// std::vector<double> b = {3, 5, 6, 1};
// std::vector<real_t> a = {4, 3, 1, 3};
// std::vector<real_t> b = {3, 5, 6, 1};
// alg.printMatrix(alg.matmult(alg.transpose(A), A));
// std::cout << std::endl;
@ -226,8 +226,8 @@ int main() {
// const int TRIAL_NUM = 1000;
// double scoreSGD = 0;
// double scoreADAM = 0;
// real_t scoreSGD = 0;
// real_t scoreADAM = 0;
// for(int i = 0; i < TRIAL_NUM; i++){
// LinReg model(alg.transpose(inputSet), outputSet);
// model.MBGD(0.001, 5, 1, 0);
@ -269,8 +269,8 @@ int main() {
// std::cout << "ACCURACY: " << 100 * model.score() << "%" << std::endl;
// // PROBIT REGRESSION
// std::vector<std::vector<double>> inputSet;
// std::vector<double> outputSet;
// std::vector<std::vector<real_t>> inputSet;
// std::vector<real_t> outputSet;
// data.setData(30, "/Users/marcmelikyan/Desktop/Data/BreastCancer.csv", inputSet, outputSet);
// ProbitReg model(inputSet, outputSet);
// model.SGD(0.001, 10000, 1);
@ -278,24 +278,24 @@ int main() {
// std::cout << "ACCURACY: " << 100 * model.score() << "%" << std::endl;
// // CLOGLOG REGRESSION
// std::vector<std::vector<double>> inputSet = {{1,2,3,4,5,6,7,8}, {0,0,0,0,1,1,1,1}};
// std::vector<double> outputSet = {0,0,0,0,1,1,1,1};
// std::vector<std::vector<real_t>> inputSet = {{1,2,3,4,5,6,7,8}, {0,0,0,0,1,1,1,1}};
// std::vector<real_t> outputSet = {0,0,0,0,1,1,1,1};
// CLogLogReg model(alg.transpose(inputSet), outputSet);
// model.SGD(0.1, 10000, 0);
// alg.printVector(model.modelSetTest(alg.transpose(inputSet)));
// std::cout << "ACCURACY: " << 100 * model.score() << "%" << std::endl;
// // EXPREG REGRESSION
// std::vector<std::vector<double>> inputSet = {{0,1,2,3,4}};
// std::vector<double> outputSet = {1,2,4,8,16};
// std::vector<std::vector<real_t>> inputSet = {{0,1,2,3,4}};
// std::vector<real_t> outputSet = {1,2,4,8,16};
// ExpReg model(alg.transpose(inputSet), outputSet);
// model.SGD(0.001, 10000, 0);
// alg.printVector(model.modelSetTest(alg.transpose(inputSet)));
// std::cout << "ACCURACY: " << 100 * model.score() << "%" << std::endl;
// // TANH REGRESSION
// std::vector<std::vector<double>> inputSet = {{4,3,0,-3,-4}, {0,0,0,1,1}};
// std::vector<double> outputSet = {1,1,0,-1,-1};
// std::vector<std::vector<real_t>> inputSet = {{4,3,0,-3,-4}, {0,0,0,1,1}};
// std::vector<real_t> outputSet = {1,1,0,-1,-1};
// TanhReg model(alg.transpose(inputSet), outputSet);
// model.SGD(0.1, 10000, 0);
// alg.printVector(model.modelSetTest(alg.transpose(inputSet)));
@ -320,9 +320,9 @@ int main() {
// alg.printMatrix(model.modelSetTest(inputSet));
// // MLP
// std::vector<std::vector<double>> inputSet = {{0,0,1,1}, {0,1,0,1}};
// std::vector<std::vector<real_t>> inputSet = {{0,0,1,1}, {0,1,0,1}};
// inputSet = alg.transpose(inputSet);
// std::vector<double> outputSet = {0,1,1,0};
// std::vector<real_t> outputSet = {0,1,1,0};
// MLP model(inputSet, outputSet, 2);
// model.gradientDescent(0.1, 10000, 0);
@ -337,7 +337,7 @@ int main() {
// std::cout << "ACCURACY: " << 100 * model.score() << "%" << std::endl;
// // AUTOENCODER
// std::vector<std::vector<double>> inputSet = {{1,2,3,4,5,6,7,8,9,10}, {3,5,9,12,15,18,21,24,27,30}};
// std::vector<std::vector<real_t>> inputSet = {{1,2,3,4,5,6,7,8,9,10}, {3,5,9,12,15,18,21,24,27,30}};
// AutoEncoder model(alg.transpose(inputSet), 5);
// model.SGD(0.001, 300000, 0);
// alg.printMatrix(model.modelSetTest(alg.transpose(inputSet)));
@ -347,8 +347,8 @@ int main() {
// Possible Weight Init Methods: Default, Uniform, HeNormal, HeUniform, XavierNormal, XavierUniform
// Possible Activations: Linear, Sigmoid, Swish, Softplus, Softsign, CLogLog, Ar{Sinh, Cosh, Tanh, Csch, Sech, Coth}, GaussianCDF, GELU, UnitStep
// Possible Loss Functions: MSE, RMSE, MBE, LogLoss, CrossEntropy, HingeLoss
// std::vector<std::vector<double>> inputSet = {{0,0,1,1}, {0,1,0,1}};
// std::vector<double> outputSet = {0,1,1,0};
// std::vector<std::vector<real_t>> inputSet = {{0,0,1,1}, {0,1,0,1}};
// std::vector<real_t> outputSet = {0,1,1,0};
// ANN ann(alg.transpose(inputSet), outputSet);
// ann.addLayer(2, "Cosh");
// ann.addOutputLayer("Sigmoid", "LogLoss");
@ -363,7 +363,7 @@ int main() {
// alg.printVector(ann.modelSetTest(alg.transpose(inputSet)));
// std::cout << "ACCURACY: " << 100 * ann.score() << "%" << std::endl;
std::vector<std::vector<double>> outputSet = {{1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20},
std::vector<std::vector<real_t>> outputSet = {{1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20},
{2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40}};
WGAN gan(2, alg.transpose(outputSet)); // our gan is a wasserstein gan (wgan)
@ -376,8 +376,8 @@ int main() {
alg.printMatrix(gan.generateExample(100));
// typedef std::vector<std::vector<double>> Matrix;
// typedef std::vector<double> Vector;
// typedef std::vector<std::vector<real_t>> Matrix;
// typedef std::vector<real_t> Vector;
// Matrix inputSet = {{0,0}, {0,1}, {1,0}, {1,1}}; // XOR
// Vector outputSet = {0,1,1,0};
@ -393,8 +393,8 @@ int main() {
// std::cout << "ACCURACY: " << 100 * ann.score() << "%" << std::endl; // Accuracy.
// // DYNAMICALLY SIZED MANN (Multidimensional Output ANN)
// std::vector<std::vector<double>> inputSet = {{1,2,3},{2,4,6},{3,6,9},{4,8,12}};
// std::vector<std::vector<double>> outputSet = {{1,5}, {2,10}, {3,15}, {4,20}};
// std::vector<std::vector<real_t>> inputSet = {{1,2,3},{2,4,6},{3,6,9},{4,8,12}};
// std::vector<std::vector<real_t>> outputSet = {{1,5}, {2,10}, {3,15}, {4,20}};
// MANN mann(inputSet, outputSet);
// mann.addOutputLayer("Linear", "MSE");
@ -402,14 +402,14 @@ int main() {
// alg.printMatrix(mann.modelSetTest(inputSet));
// std::cout << "ACCURACY: " << 100 * mann.score() << "%" << std::endl;
// std::vector<std::vector<double>> inputSet;
// std::vector<double> tempOutputSet;
// std::vector<std::vector<real_t>> inputSet;
// std::vector<real_t> tempOutputSet;
// data.setData(4, "/Users/marcmelikyan/Desktop/Data/Iris.csv", inputSet, tempOutputSet);
// std::vector<std::vector<double>> outputSet = data.oneHotRep(tempOutputSet, 3);
// std::vector<std::vector<real_t>> outputSet = data.oneHotRep(tempOutputSet, 3);
// TRAIN TEST SPLIT CHECK
// std::vector<std::vector<double>> inputSet1 = {{1,2,3,4,5,6,7,8,9,10}, {3,5,9,12,15,18,21,24,27,30}};
// std::vector<std::vector<double>> outputSet1 = {{2,4,6,8,10,12,14,16,18,20}};
// std::vector<std::vector<real_t>> inputSet1 = {{1,2,3,4,5,6,7,8,9,10}, {3,5,9,12,15,18,21,24,27,30}};
// std::vector<std::vector<real_t>> outputSet1 = {{2,4,6,8,10,12,14,16,18,20}};
// auto [inputSet, outputSet, inputTestSet, outputTestSet] = data.trainTestSplit(alg.transpose(inputSet1), alg.transpose(outputSet1), 0.2);
// alg.printMatrix(inputSet);
// alg.printMatrix(outputSet);
@ -428,8 +428,8 @@ int main() {
// std::cout << "ACCURACY: " << 100 * mann.score() << "%" << std::endl;
// // NAIVE BAYES
// std::vector<std::vector<double>> inputSet = {{1,1,1,1,1}, {0,0,1,1,1}, {0,0,1,0,1}};
// std::vector<double> outputSet = {0,1,0,1,1};
// std::vector<std::vector<real_t>> inputSet = {{1,1,1,1,1}, {0,0,1,1,1}, {0,0,1,0,1}};
// std::vector<real_t> outputSet = {0,1,0,1,1};
// MultinomialNB MNB(alg.transpose(inputSet), outputSet, 2);
// alg.printVector(MNB.modelSetTest(alg.transpose(inputSet)));
@ -441,7 +441,7 @@ int main() {
// alg.printVector(GNB.modelSetTest(alg.transpose(inputSet)));
// // KMeans
// std::vector<std::vector<double>> inputSet = {{32, 0, 7}, {2, 28, 17}, {0, 9, 23}};
// std::vector<std::vector<real_t>> inputSet = {{32, 0, 7}, {2, 28, 17}, {0, 9, 23}};
// KMeans kmeans(inputSet, 3, "KMeans++");
// kmeans.train(3, 1);
// std::cout << std::endl;
@ -450,26 +450,26 @@ int main() {
// alg.printVector(kmeans.silhouette_scores());
// // kNN
// std::vector<std::vector<double>> inputSet = {{1,2,3,4,5,6,7,8}, {0,0,0,0,1,1,1,1}};
// std::vector<double> outputSet = {0,0,0,0,1,1,1,1};
// std::vector<std::vector<real_t>> inputSet = {{1,2,3,4,5,6,7,8}, {0,0,0,0,1,1,1,1}};
// std::vector<real_t> outputSet = {0,0,0,0,1,1,1,1};
// kNN knn(alg.transpose(inputSet), outputSet, 8);
// alg.printVector(knn.modelSetTest(alg.transpose(inputSet)));
// std::cout << "ACCURACY: " << 100 * knn.score() << "%" << std::endl;
// // CONVOLUTION, POOLING, ETC..
// std::vector<std::vector<double>> input = {
// std::vector<std::vector<real_t>> input = {
// {1},
// };
// std::vector<std::vector<std::vector<double>>> tensorSet;
// std::vector<std::vector<std::vector<real_t>>> tensorSet;
// tensorSet.push_back(input);
// tensorSet.push_back(input);
// tensorSet.push_back(input);
// alg.printTensor(data.rgb2xyz(tensorSet));
// std::vector<std::vector<double>> input = {
// std::vector<std::vector<real_t>> input = {
// {62,55,55,54,49,48,47,55},
// {62,57,54,52,48,47,48,53},
// {61,60,52,49,48,47,49,54},
@ -487,17 +487,17 @@ int main() {
// alg.printMatrix(conv.convolve(input, conv.getPrewittVertical(), 1)); // Can use padding
// alg.printMatrix(conv.pool(input, 4, 4, "Max")); // Can use Max, Min, or Average pooling.
// std::vector<std::vector<std::vector<double>>> tensorSet;
// std::vector<std::vector<std::vector<real_t>>> tensorSet;
// tensorSet.push_back(input);
// tensorSet.push_back(input);
// alg.printVector(conv.globalPool(tensorSet, "Average")); // Can use Max, Min, or Average global pooling.
// std::vector<std::vector<double>> laplacian = {{1, 1, 1}, {1, -4, 1}, {1, 1, 1}};
// std::vector<std::vector<real_t>> laplacian = {{1, 1, 1}, {1, -4, 1}, {1, 1, 1}};
// alg.printMatrix(conv.convolve(conv.gaussianFilter2D(5, 1), laplacian, 1));
// // PCA, SVD, eigenvalues & eigenvectors
// std::vector<std::vector<double>> inputSet = {{1,1}, {1,1}};
// std::vector<std::vector<real_t>> inputSet = {{1,1}, {1,1}};
// auto [Eigenvectors, Eigenvalues] = alg.eig(inputSet);
// std::cout << "Eigenvectors:" << std::endl;
// alg.printMatrix(Eigenvectors);
@ -547,7 +547,7 @@ int main() {
// std::cout << std::endl;
// std::vector<std::vector<double>> inputSet = {{1,2},{2,3},{3,4},{4,5},{5,6}};
// std::vector<std::vector<real_t>> inputSet = {{1,2},{2,3},{3,4},{4,5},{5,6}};
// std::cout << "Feature Scaling Example:" << std::endl;
// alg.printMatrix(data.featureScaling(inputSet));
// std::cout << std::endl;
@ -561,20 +561,20 @@ int main() {
// std::cout << std::endl;
// // Outlier Finder
// std::vector<double> inputSet = {1,2,3,4,5,6,7,8,9,23554332523523};
// std::vector<real_t> inputSet = {1,2,3,4,5,6,7,8,9,23554332523523};
// OutlierFinder outlierFinder(2); // Any datapoint outside of 2 stds from the mean is marked as an outlier.
// alg.printVector(outlierFinder.modelTest(inputSet));
// // Testing new Functions
// double z_s = 0.001;
// real_t z_s = 0.001;
// std::cout << avn.logit(z_s) << std::endl;
// std::cout << avn.logit(z_s, 1) << std::endl;
// std::vector<double> z_v = {0.001};
// std::vector<real_t> z_v = {0.001};
// alg.printVector(avn.logit(z_v));
// alg.printVector(avn.logit(z_v, 1));
// std::vector<std::vector<double>> Z_m = {{0.001}};
// std::vector<std::vector<real_t>> Z_m = {{0.001}};
// alg.printMatrix(avn.logit(Z_m));
// alg.printMatrix(avn.logit(Z_m, 1));
@ -585,18 +585,18 @@ int main() {
// alg.printMatrix(alg.matrixPower({{5,5},{5,5}}, 2));
// alg.printVector(alg.solve({{1,1}, {1.5, 4.0}}, {2200, 5050}));
// std::vector<std::vector<double>> matrixOfCubes = {{1,2,64,27}};
// std::vector<double> vectorOfCubes = {1,2,64,27};
// std::vector<std::vector<real_t>> matrixOfCubes = {{1,2,64,27}};
// std::vector<real_t> vectorOfCubes = {1,2,64,27};
// alg.printMatrix(alg.cbrt(matrixOfCubes));
// alg.printVector(alg.cbrt(vectorOfCubes));
// std::cout << alg.max({{1,2,3,4,5}, {6,5,3,4,1}, {9,9,9,9,9}}) << std::endl;
// std::cout << alg.min({{1,2,3,4,5}, {6,5,3,4,1}, {9,9,9,9,9}}) << std::endl;
// std::vector<double> chicken;
// std::vector<real_t> chicken;
// data.getImage("../../Data/apple.jpeg", chicken);
// alg.printVector(chicken);
// std::vector<std::vector<double>> P = {{12, -51, 4}, {6, 167, -68}, {-4, 24, -41}};
// std::vector<std::vector<real_t>> P = {{12, -51, 4}, {6, 167, -68}, {-4, 24, -41}};
// alg.printMatrix(P);
// alg.printMatrix(alg.gramSchmidtProcess(P));
@ -608,7 +608,7 @@ int main() {
// alg.printMatrix(R);
// // Checking positive-definiteness checker. For Cholesky Decomp.
// std::vector<std::vector<double>> A =
// std::vector<std::vector<real_t>> A =
// {
// {1,-1,-1,-1},
// {-1,2,2,2},
@ -653,7 +653,7 @@ int main() {
// std::cout << numAn.laplacian(f_mv, {1,1,1}) << std::endl;
// std::vector<std::vector<std::vector<double>>> tensor;
// std::vector<std::vector<std::vector<real_t>>> tensor;
// tensor.push_back({{1,2}, {1,2}, {1,2}});
// tensor.push_back({{1,2}, {1,2}, {1,2}});
@ -672,7 +672,7 @@ int main() {
// alg.printMatrix(conv.gradOrientation(A));
// std::vector<std::vector<double>> A =
// std::vector<std::vector<real_t>> A =
// {
// {1,0,0,0},
// {0,0,0,0},
@ -689,23 +689,23 @@ int main() {
// std::cout << std::endl;
// } // Harris detector works. Life is good!
// std::vector<double> a = {3,4,4};
// std::vector<double> b = {4,4,4};
// std::vector<real_t> a = {3,4,4};
// std::vector<real_t> b = {4,4,4};
// alg.printVector(alg.cross(a,b));
//SUPPORT VECTOR CLASSIFICATION (kernel method)
// std::vector<std::vector<double>> inputSet;
// std::vector<double> outputSet;
// std::vector<std::vector<real_t>> inputSet;
// std::vector<real_t> outputSet;
// data.setData(30, "/Users/marcmelikyan/Desktop/Data/BreastCancerSVM.csv", inputSet, outputSet);
// std::vector<std::vector<double>> inputSet;
// std::vector<double> outputSet;
// std::vector<std::vector<real_t>> inputSet;
// std::vector<real_t> outputSet;
// data.setData(4, "/Users/marcmelikyan/Desktop/Data/IrisSVM.csv", inputSet, outputSet);
// DualSVC kernelSVM(inputSet, outputSet, 1000);
// kernelSVM.gradientDescent(0.0001, 20, 1);
// std::vector<std::vector<double>> linearlyIndependentMat =
// std::vector<std::vector<real_t>> linearlyIndependentMat =
// {
// {1,2,3,4},

274
mlpp/activation/activation.cpp
View File

@ -10,14 +10,14 @@
#include <cmath>
#include <iostream>
double MLPPActivation::linear(double z, bool deriv) {
real_t MLPPActivation::linear(real_t z, bool deriv) {
if (deriv) {
return 1;
}
return z;
}
std::vector<double> MLPPActivation::linear(std::vector<double> z, bool deriv) {
std::vector<real_t> MLPPActivation::linear(std::vector<real_t> z, bool deriv) {
if (deriv) {
MLPPLinAlg alg;
return alg.onevec(z.size());
@ -25,7 +25,7 @@ std::vector<double> MLPPActivation::linear(std::vector<double> z, bool deriv) {
return z;
}
std::vector<std::vector<double>> MLPPActivation::linear(std::vector<std::vector<double>> z, bool deriv) {
std::vector<std::vector<real_t>> MLPPActivation::linear(std::vector<std::vector<real_t>> z, bool deriv) {
if (deriv) {
MLPPLinAlg alg;
return alg.onemat(z.size(), z[0].size());
@ -33,14 +33,14 @@ std::vector<std::vector<double>> MLPPActivation::linear(std::vector<std::vector<
return z;
}
double MLPPActivation::sigmoid(double z, bool deriv) {
real_t MLPPActivation::sigmoid(real_t z, bool deriv) {
if (deriv) {
return sigmoid(z) * (1 - sigmoid(z));
}
return 1 / (1 + exp(-z));
}
std::vector<double> MLPPActivation::sigmoid(std::vector<double> z, bool deriv) {
std::vector<real_t> MLPPActivation::sigmoid(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.subtraction(sigmoid(z), alg.hadamard_product(sigmoid(z), sigmoid(z)));
@ -48,7 +48,7 @@ std::vector<double> MLPPActivation::sigmoid(std::vector<double> z, bool deriv) {
return alg.elementWiseDivision(alg.onevec(z.size()), alg.addition(alg.onevec(z.size()), alg.exp(alg.scalarMultiply(-1, z))));
}
std::vector<std::vector<double>> MLPPActivation::sigmoid(std::vector<std::vector<double>> z, bool deriv) {
std::vector<std::vector<real_t>> MLPPActivation::sigmoid(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.subtraction(sigmoid(z), alg.hadamard_product(sigmoid(z), sigmoid(z)));
@ -56,12 +56,12 @@ std::vector<std::vector<double>> MLPPActivation::sigmoid(std::vector<std::vector
return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.addition(alg.onemat(z.size(), z[0].size()), alg.exp(alg.scalarMultiply(-1, z))));
}
std::vector<double> MLPPActivation::softmax(std::vector<double> z, bool deriv) {
std::vector<real_t> MLPPActivation::softmax(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg;
std::vector<double> a;
std::vector<real_t> a;
a.resize(z.size());
std::vector<double> expZ = alg.exp(z);
double sum = 0;
std::vector<real_t> expZ = alg.exp(z);
real_t sum = 0;
for (int i = 0; i < z.size(); i++) {
sum += expZ[i];
@ -72,9 +72,9 @@ std::vector<double> MLPPActivation::softmax(std::vector<double> z, bool deriv) {
return a;
}
std::vector<std::vector<double>> MLPPActivation::softmax(std::vector<std::vector<double>> z, bool deriv) {
std::vector<std::vector<real_t>> MLPPActivation::softmax(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg;
std::vector<std::vector<double>> a;
std::vector<std::vector<real_t>> a;
a.resize(z.size());
for (int i = 0; i < z.size(); i++) {
@ -83,18 +83,18 @@ std::vector<std::vector<double>> MLPPActivation::softmax(std::vector<std::vector
return a;
}
std::vector<double> MLPPActivation::adjSoftmax(std::vector<double> z) {
std::vector<real_t> MLPPActivation::adjSoftmax(std::vector<real_t> z) {
MLPPLinAlg alg;
std::vector<double> a;
double C = -*std::max_element(z.begin(), z.end());
std::vector<real_t> a;
real_t C = -*std::max_element(z.begin(), z.end());
z = alg.scalarAdd(C, z);
return softmax(z);
}
std::vector<std::vector<double>> MLPPActivation::adjSoftmax(std::vector<std::vector<double>> z) {
std::vector<std::vector<real_t>> MLPPActivation::adjSoftmax(std::vector<std::vector<real_t>> z) {
MLPPLinAlg alg;
std::vector<std::vector<double>> a;
std::vector<std::vector<real_t>> a;
a.resize(z.size());
for (int i = 0; i < z.size(); i++) {
@ -103,10 +103,10 @@ std::vector<std::vector<double>> MLPPActivation::adjSoftmax(std::vector<std::vec
return a;
}
std::vector<std::vector<double>> MLPPActivation::softmaxDeriv(std::vector<double> z) {
std::vector<std::vector<real_t>> MLPPActivation::softmaxDeriv(std::vector<real_t> z) {
MLPPLinAlg alg;
std::vector<std::vector<double>> deriv;
std::vector<double> a = softmax(z);
std::vector<std::vector<real_t>> deriv;
std::vector<real_t> a = softmax(z);
deriv.resize(a.size());
for (int i = 0; i < deriv.size(); i++) {
deriv[i].resize(a.size());
@ -123,10 +123,10 @@ std::vector<std::vector<double>> MLPPActivation::softmaxDeriv(std::vector<double
return deriv;
}
std::vector<std::vector<std::vector<double>>> MLPPActivation::softmaxDeriv(std::vector<std::vector<double>> z) {
std::vector<std::vector<std::vector<real_t>>> MLPPActivation::softmaxDeriv(std::vector<std::vector<real_t>> z) {
MLPPLinAlg alg;
std::vector<std::vector<std::vector<double>>> deriv;
std::vector<std::vector<double>> a = softmax(z);
std::vector<std::vector<std::vector<real_t>>> deriv;
std::vector<std::vector<real_t>> a = softmax(z);
deriv.resize(a.size());
for (int i = 0; i < deriv.size(); i++) {
@ -144,14 +144,14 @@ std::vector<std::vector<std::vector<double>>> MLPPActivation::softmaxDeriv(std::
return deriv;
}
double MLPPActivation::softplus(double z, bool deriv) {
real_t MLPPActivation::softplus(real_t z, bool deriv) {
if (deriv) {
return sigmoid(z);
}
return std::log(1 + exp(z));
}
std::vector<double> MLPPActivation::softplus(std::vector<double> z, bool deriv) {
std::vector<real_t> MLPPActivation::softplus(std::vector<real_t> z, bool deriv) {
if (deriv) {
return sigmoid(z);
}
@ -159,7 +159,7 @@ std::vector<double> MLPPActivation::softplus(std::vector<double> z, bool deriv)
return alg.log(alg.addition(alg.onevec(z.size()), alg.exp(z)));
}
std::vector<std::vector<double>> MLPPActivation::softplus(std::vector<std::vector<double>> z, bool deriv) {
std::vector<std::vector<real_t>> MLPPActivation::softplus(std::vector<std::vector<real_t>> z, bool deriv) {
if (deriv) {
return sigmoid(z);
}
@ -167,14 +167,14 @@ std::vector<std::vector<double>> MLPPActivation::softplus(std::vector<std::vecto
return alg.log(alg.addition(alg.onemat(z.size(), z[0].size()), alg.exp(z)));
}
double MLPPActivation::softsign(double z, bool deriv) {
real_t MLPPActivation::softsign(real_t z, bool deriv) {
if (deriv) {
return 1 / ((1 + abs(z)) * (1 + abs(z)));
}
return z / (1 + abs(z));
}
std::vector<double> MLPPActivation::softsign(std::vector<double> z, bool deriv) {
std::vector<real_t> MLPPActivation::softsign(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.elementWiseDivision(alg.onevec(z.size()), alg.exponentiate(alg.addition(alg.onevec(z.size()), alg.abs(z)), 2));
@ -182,7 +182,7 @@ std::vector<double> MLPPActivation::softsign(std::vector<double> z, bool deriv)
return alg.elementWiseDivision(z, alg.addition(alg.onevec(z.size()), alg.abs(z)));
}
std::vector<std::vector<double>> MLPPActivation::softsign(std::vector<std::vector<double>> z, bool deriv) {
std::vector<std::vector<real_t>> MLPPActivation::softsign(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.exponentiate(alg.addition(alg.onemat(z.size(), z[0].size()), alg.abs(z)), 2));
@ -190,14 +190,14 @@ std::vector<std::vector<double>> MLPPActivation::softsign(std::vector<std::vecto
return alg.elementWiseDivision(z, alg.addition(alg.onemat(z.size(), z[0].size()), alg.abs(z)));
}
double MLPPActivation::gaussianCDF(double z, bool deriv) {
real_t MLPPActivation::gaussianCDF(real_t z, bool deriv) {
if (deriv) {
return (1 / sqrt(2 * M_PI)) * exp(-z * z / 2);
}
return 0.5 * (1 + erf(z / sqrt(2)));
}
std::vector<double> MLPPActivation::gaussianCDF(std::vector<double> z, bool deriv) {
std::vector<real_t> MLPPActivation::gaussianCDF(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.scalarMultiply(1 / sqrt(2 * M_PI), alg.exp(alg.scalarMultiply(-1 / 2, alg.hadamard_product(z, z))));
@ -205,7 +205,7 @@ std::vector<double> MLPPActivation::gaussianCDF(std::vector<double> z, bool deri
return alg.scalarMultiply(0.5, alg.addition(alg.onevec(z.size()), alg.erf(alg.scalarMultiply(1 / sqrt(2), z))));
}
std::vector<std::vector<double>> MLPPActivation::gaussianCDF(std::vector<std::vector<double>> z, bool deriv) {
std::vector<std::vector<real_t>> MLPPActivation::gaussianCDF(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.scalarMultiply(1 / sqrt(2 * M_PI), alg.exp(alg.scalarMultiply(-1 / 2, alg.hadamard_product(z, z))));
@ -213,14 +213,14 @@ std::vector<std::vector<double>> MLPPActivation::gaussianCDF(std::vector<std::ve
return alg.scalarMultiply(0.5, alg.addition(alg.onemat(z.size(), z[0].size()), alg.erf(alg.scalarMultiply(1 / sqrt(2), z))));
}
double MLPPActivation::cloglog(double z, bool deriv) {
real_t MLPPActivation::cloglog(real_t z, bool deriv) {
if (deriv) {
return exp(z - exp(z));
}
return 1 - exp(-exp(z));
}
std::vector<double> MLPPActivation::cloglog(std::vector<double> z, bool deriv) {
std::vector<real_t> MLPPActivation::cloglog(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.exp(alg.scalarMultiply(-1, alg.exp(z)));
@ -228,7 +228,7 @@ std::vector<double> MLPPActivation::cloglog(std::vector<double> z, bool deriv) {
return alg.scalarMultiply(-1, alg.scalarAdd(-1, alg.exp(alg.scalarMultiply(-1, alg.exp(z)))));
}
std::vector<std::vector<double>> MLPPActivation::cloglog(std::vector<std::vector<double>> z, bool deriv) {
std::vector<std::vector<real_t>> MLPPActivation::cloglog(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.exp(alg.scalarMultiply(-1, alg.exp(z)));
@ -236,14 +236,14 @@ std::vector<std::vector<double>> MLPPActivation::cloglog(std::vector<std::vector
return alg.scalarMultiply(-1, alg.scalarAdd(-1, alg.exp(alg.scalarMultiply(-1, alg.exp(z)))));
}
double MLPPActivation::logit(double z, bool deriv) {
real_t MLPPActivation::logit(real_t z, bool deriv) {
if (deriv) {
return 1 / z - 1 / (z - 1);
}
return std::log(z / (1 - z));
}
std::vector<double> MLPPActivation::logit(std::vector<double> z, bool deriv) {
std::vector<real_t> MLPPActivation::logit(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.subtraction(alg.elementWiseDivision(alg.onevec(z.size()), z), alg.elementWiseDivision(alg.onevec(z.size()), alg.subtraction(z, alg.onevec(z.size()))));
@ -251,7 +251,7 @@ std::vector<double> MLPPActivation::logit(std::vector<double> z, bool deriv) {
return alg.log(alg.elementWiseDivision(z, alg.subtraction(alg.onevec(z.size()), z)));
}
std::vector<std::vector<double>> MLPPActivation::logit(std::vector<std::vector<double>> z, bool deriv) {
std::vector<std::vector<real_t>> MLPPActivation::logit(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.subtraction(alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), z), alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.subtraction(z, alg.onemat(z.size(), z[0].size()))));
@ -259,23 +259,23 @@ std::vector<std::vector<double>> MLPPActivation::logit(std::vector<std::vector<d
return alg.log(alg.elementWiseDivision(z, alg.subtraction(alg.onemat(z.size(), z[0].size()), z)));
}
double MLPPActivation::unitStep(double z, bool deriv) {
real_t MLPPActivation::unitStep(real_t z, bool deriv) {
if (deriv) {
return 0;
}
return z < 0 ? 0 : 1;
}
std::vector<double> MLPPActivation::unitStep(std::vector<double> z, bool deriv) {
std::vector<real_t> MLPPActivation::unitStep(std::vector<real_t> z, bool deriv) {
if (deriv) {
std::vector<double> deriv;
std::vector<real_t> deriv;
deriv.resize(z.size());
for (int i = 0; i < z.size(); i++) {
deriv[i] = unitStep(z[i], 1);
}
return deriv;
}
std::vector<double> a;
std::vector<real_t> a;
a.resize(z.size());
for (int i = 0; i < a.size(); i++) {
@ -284,16 +284,16 @@ std::vector<double> MLPPActivation::unitStep(std::vector<double> z, bool deriv)
return a;
}
std::vector<std::vector<double>> MLPPActivation::unitStep(std::vector<std::vector<double>> z, bool deriv) {
std::vector<std::vector<real_t>> MLPPActivation::unitStep(std::vector<std::vector<real_t>> z, bool deriv) {
if (deriv) {
std::vector<std::vector<double>> deriv;
std::vector<std::vector<real_t>> deriv;
deriv.resize(z.size());
for (int i = 0; i < z.size(); i++) {
deriv[i] = unitStep(z[i], 1);
}
return deriv;
}
std::vector<std::vector<double>> a;
std::vector<std::vector<real_t>> a;
a.resize(z.size());
for (int i = 0; i < a.size(); i++) {
@ -302,14 +302,14 @@ std::vector<std::vector<double>> MLPPActivation::unitStep(std::vector<std::vecto
return a;
}
double MLPPActivation::swish(double z, bool deriv) {
real_t MLPPActivation::swish(real_t z, bool deriv) {
if (deriv) {
return swish(z) + sigmoid(z) * (1 - swish(z));
}
return z * sigmoid(z);
}
std::vector<double> MLPPActivation::swish(std::vector<double> z, bool deriv) {
std::vector<real_t> MLPPActivation::swish(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
alg.addition(swish(z), alg.subtraction(sigmoid(z), alg.hadamard_product(sigmoid(z), swish(z))));
@ -317,7 +317,7 @@ std::vector<double> MLPPActivation::swish(std::vector<double> z, bool deriv) {
return alg.hadamard_product(z, sigmoid(z));
}
std::vector<std::vector<double>> MLPPActivation::swish(std::vector<std::vector<double>> z, bool deriv) {
std::vector<std::vector<real_t>> MLPPActivation::swish(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
alg.addition(swish(z), alg.subtraction(sigmoid(z), alg.hadamard_product(sigmoid(z), swish(z))));
@ -325,14 +325,14 @@ std::vector<std::vector<double>> MLPPActivation::swish(std::vector<std::vector<d
return alg.hadamard_product(z, sigmoid(z));
}
double MLPPActivation::mish(double z, bool deriv) {
real_t MLPPActivation::mish(real_t z, bool deriv) {
if (deriv) {
return sech(softplus(z)) * sech(softplus(z)) * z * sigmoid(z) + mish(z) / z;
}
return z * tanh(softplus(z));
}
std::vector<double> MLPPActivation::mish(std::vector<double> z, bool deriv) {
std::vector<real_t> MLPPActivation::mish(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.addition(alg.hadamard_product(alg.hadamard_product(alg.hadamard_product(sech(softplus(z)), sech(softplus(z))), z), sigmoid(z)), alg.elementWiseDivision(mish(z), z));
@ -340,7 +340,7 @@ std::vector<double> MLPPActivation::mish(std::vector<double> z, bool deriv) {
return alg.hadamard_product(z, tanh(softplus(z)));
}
std::vector<std::vector<double>> MLPPActivation::mish(std::vector<std::vector<double>> z, bool deriv) {
std::vector<std::vector<real_t>> MLPPActivation::mish(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.addition(alg.hadamard_product(alg.hadamard_product(alg.hadamard_product(sech(softplus(z)), sech(softplus(z))), z), sigmoid(z)), alg.elementWiseDivision(mish(z), z));
@ -348,14 +348,14 @@ std::vector<std::vector<double>> MLPPActivation::mish(std::vector<std::vector<do
return alg.hadamard_product(z, tanh(softplus(z)));
}
double MLPPActivation::sinc(double z, bool deriv) {
real_t MLPPActivation::sinc(real_t z, bool deriv) {
if (deriv) {
return (z * std::cos(z) - std::sin(z)) / (z * z);
}
return std::sin(z) / z;
}
std::vector<double> MLPPActivation::sinc(std::vector<double> z, bool deriv) {
std::vector<real_t> MLPPActivation::sinc(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.elementWiseDivision(alg.subtraction(alg.hadamard_product(z, alg.cos(z)), alg.sin(z)), alg.hadamard_product(z, z));
@ -363,7 +363,7 @@ std::vector<double> MLPPActivation::sinc(std::vector<double> z, bool deriv) {
return alg.elementWiseDivision(alg.sin(z), z);
}
std::vector<std::vector<double>> MLPPActivation::sinc(std::vector<std::vector<double>> z, bool deriv) {
std::vector<std::vector<real_t>> MLPPActivation::sinc(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.elementWiseDivision(alg.subtraction(alg.hadamard_product(z, alg.cos(z)), alg.sin(z)), alg.hadamard_product(z, z));
@ -371,7 +371,7 @@ std::vector<std::vector<double>> MLPPActivation::sinc(std::vector<std::vector<do
return alg.elementWiseDivision(alg.sin(z), z);
}
double MLPPActivation::RELU(double z, bool deriv) {
real_t MLPPActivation::RELU(real_t z, bool deriv) {
if (deriv) {
if (z <= 0) {
return 0;
@ -382,16 +382,16 @@ double MLPPActivation::RELU(double z, bool deriv) {
return fmax(0, z);
}
std::vector<double> MLPPActivation::RELU(std::vector<double> z, bool deriv) {
std::vector<real_t> MLPPActivation::RELU(std::vector<real_t> z, bool deriv) {
if (deriv) {
std::vector<double> deriv;
std::vector<real_t> deriv;
deriv.resize(z.size());
for (int i = 0; i < z.size(); i++) {
deriv[i] = RELU(z[i], 1);
}
return deriv;
}
std::vector<double> a;
std::vector<real_t> a;
a.resize(z.size());
for (int i = 0; i < a.size(); i++) {
@ -400,16 +400,16 @@ std::vector<double> MLPPActivation::RELU(std::vector<double> z, bool deriv) {
return a;
}
std::vector<std::vector<double>> MLPPActivation::RELU(std::vector<std::vector<double>> z, bool deriv) {
std::vector<std::vector<real_t>> MLPPActivation::RELU(std::vector<std::vector<real_t>> z, bool deriv) {
if (deriv) {
std::vector<std::vector<double>> deriv;
std::vector<std::vector<real_t>> deriv;
deriv.resize(z.size());
for (int i = 0; i < z.size(); i++) {
deriv[i] = RELU(z[i], 1);
}
return deriv;
}
std::vector<std::vector<double>> a;
std::vector<std::vector<real_t>> a;
a.resize(z.size());
for (int i = 0; i < a.size(); i++) {
@ -418,7 +418,7 @@ std::vector<std::vector<double>> MLPPActivation::RELU(std::vector<std::vector<do
return a;
}
double MLPPActivation::leakyReLU(double z, double c, bool deriv) {
real_t MLPPActivation::leakyReLU(real_t z, real_t c, bool deriv) {
if (deriv) {
if (z <= 0) {
return c;
@ -429,16 +429,16 @@ double MLPPActivation::leakyReLU(double z, double c, bool deriv) {
return fmax(c * z, z);
}
std::vector<double> MLPPActivation::leakyReLU(std::vector<double> z, double c, bool deriv) {
std::vector<real_t> MLPPActivation::leakyReLU(std::vector<real_t> z, real_t c, bool deriv) {
if (deriv) {
std::vector<double> deriv;
std::vector<real_t> deriv;
deriv.resize(z.size());
for (int i = 0; i < z.size(); i++) {
deriv[i] = leakyReLU(z[i], c, 1);
}
return deriv;
}
std::vector<double> a;
std::vector<real_t> a;
a.resize(z.size());
for (int i = 0; i < a.size(); i++) {
@ -447,16 +447,16 @@ std::vector<double> MLPPActivation::leakyReLU(std::vector<double> z, double c, b
return a;
}
std::vector<std::vector<double>> MLPPActivation::leakyReLU(std::vector<std::vector<double>> z, double c, bool deriv) {
std::vector<std::vector<real_t>> MLPPActivation::leakyReLU(std::vector<std::vector<real_t>> z, real_t c, bool deriv) {
if (deriv) {
std::vector<std::vector<double>> deriv;
std::vector<std::vector<real_t>> deriv;
deriv.resize(z.size());
for (int i = 0; i < z.size(); i++) {
deriv[i] = leakyReLU(z[i], c, 1);
}
return deriv;
}
std::vector<std::vector<double>> a;
std::vector<std::vector<real_t>> a;
a.resize(z.size());
for (int i = 0; i < a.size(); i++) {
@ -465,7 +465,7 @@ std::vector<std::vector<double>> MLPPActivation::leakyReLU(std::vector<std::vect
return a;
}
double MLPPActivation::ELU(double z, double c, bool deriv) {
real_t MLPPActivation::ELU(real_t z, real_t c, bool deriv) {
if (deriv) {
if (z <= 0) {
return c * exp(z);
@ -480,16 +480,16 @@ double MLPPActivation::ELU(double z, double c, bool deriv) {
}
}
std::vector<double> MLPPActivation::ELU(std::vector<double> z, double c, bool deriv) {
std::vector<real_t> MLPPActivation::ELU(std::vector<real_t> z, real_t c, bool deriv) {
if (deriv) {
std::vector<double> deriv;
std::vector<real_t> deriv;
deriv.resize(z.size());
for (int i = 0; i < z.size(); i++) {
deriv[i] = ELU(z[i], c, 1);
}
return deriv;
}
std::vector<double> a;
std::vector<real_t> a;
a.resize(z.size());
for (int i = 0; i < a.size(); i++) {
@ -498,16 +498,16 @@ std::vector<double> MLPPActivation::ELU(std::vector<double> z, double c, bool de
return a;
}
std::vector<std::vector<double>> MLPPActivation::ELU(std::vector<std::vector<double>> z, double c, bool deriv) {
std::vector<std::vector<real_t>> MLPPActivation::ELU(std::vector<std::vector<real_t>> z, real_t c, bool deriv) {
if (deriv) {
std::vector<std::vector<double>> deriv;
std::vector<std::vector<real_t>> deriv;
deriv.resize(z.size());
for (int i = 0; i < z.size(); i++) {
deriv[i] = ELU(z[i], c, 1);
}
return deriv;
}
std::vector<std::vector<double>> a;
std::vector<std::vector<real_t>> a;
a.resize(z.size());
for (int i = 0; i < a.size(); i++) {
@ -516,23 +516,23 @@ std::vector<std::vector<double>> MLPPActivation::ELU(std::vector<std::vector<dou
return a;
}
double MLPPActivation::SELU(double z, double lambda, double c, bool deriv) {
real_t MLPPActivation::SELU(real_t z, real_t lambda, real_t c, bool deriv) {
if (deriv) {
return ELU(z, c, 1);
}
return lambda * ELU(z, c);
}
std::vector<double> MLPPActivation::SELU(std::vector<double> z, double lambda, double c, bool deriv) {
std::vector<real_t> MLPPActivation::SELU(std::vector<real_t> z, real_t lambda, real_t c, bool deriv) {
if (deriv) {
std::vector<double> deriv;
std::vector<real_t> deriv;
deriv.resize(z.size());
for (int i = 0; i < z.size(); i++) {
deriv[i] = SELU(z[i], lambda, c, 1);
}
return deriv;
}
std::vector<double> a;
std::vector<real_t> a;
a.resize(z.size());
for (int i = 0; i < a.size(); i++) {
@ -541,16 +541,16 @@ std::vector<double> MLPPActivation::SELU(std::vector<double> z, double lambda, d
return a;
}
std::vector<std::vector<double>> MLPPActivation::SELU(std::vector<std::vector<double>> z, double lambda, double c, bool deriv) {
std::vector<std::vector<real_t>> MLPPActivation::SELU(std::vector<std::vector<real_t>> z, real_t lambda, real_t c, bool deriv) {
if (deriv) {
std::vector<std::vector<double>> deriv;
std::vector<std::vector<real_t>> deriv;
deriv.resize(z.size());
for (int i = 0; i < z.size(); i++) {
deriv[i] = SELU(z[i], lambda, c, 1);
}
return deriv;
}
std::vector<std::vector<double>> a;
std::vector<std::vector<real_t>> a;
a.resize(z.size());
for (int i = 0; i < a.size(); i++) {
@ -559,23 +559,23 @@ std::vector<std::vector<double>> MLPPActivation::SELU(std::vector<std::vector<do
return a;
}
double MLPPActivation::GELU(double z, bool deriv) {
real_t MLPPActivation::GELU(real_t z, bool deriv) {
if (deriv) {
return 0.5 * tanh(0.0356774 * std::pow(z, 3) + 0.797885 * z) + (0.0535161 * std::pow(z, 3) + 0.398942 * z) * std::pow(sech(0.0356774 * std::pow(z, 3) + 0.797885 * z), 2) + 0.5;
}
return 0.5 * z * (1 + tanh(sqrt(2 / M_PI) * (z + 0.044715 * std::pow(z, 3))));
}
std::vector<double> MLPPActivation::GELU(std::vector<double> z, bool deriv) {
std::vector<real_t> MLPPActivation::GELU(std::vector<real_t> z, bool deriv) {
if (deriv) {
std::vector<double> deriv;
std::vector<real_t> deriv;
deriv.resize(z.size());
for (int i = 0; i < z.size(); i++) {
deriv[i] = GELU(z[i], 1);
}
return deriv;
}
std::vector<double> a;
std::vector<real_t> a;
a.resize(z.size());
for (int i = 0; i < a.size(); i++) {
@ -584,16 +584,16 @@ std::vector<double> MLPPActivation::GELU(std::vector<double> z, bool deriv) {
return a;
}
std::vector<std::vector<double>> MLPPActivation::GELU(std::vector<std::vector<double>> z, bool deriv) {
std::vector<std::vector<real_t>> MLPPActivation::GELU(std::vector<std::vector<real_t>> z, bool deriv) {
if (deriv) {
std::vector<std::vector<double>> deriv;
std::vector<std::vector<real_t>> deriv;
deriv.resize(z.size());
for (int i = 0; i < z.size(); i++) {
deriv[i] = GELU(z[i], 1);
}
return deriv;
}
std::vector<std::vector<double>> a;
std::vector<std::vector<real_t>> a;
a.resize(z.size());
for (int i = 0; i < a.size(); i++) {
@ -602,7 +602,7 @@ std::vector<std::vector<double>> MLPPActivation::GELU(std::vector<std::vector<do
return a;
}
double MLPPActivation::sign(double z, bool deriv) {
real_t MLPPActivation::sign(real_t z, bool deriv) {
if (deriv) {
return 0;
}
@ -615,16 +615,16 @@ double MLPPActivation::sign(double z, bool deriv) {
}
}
std::vector<double> MLPPActivation::sign(std::vector<double> z, bool deriv) {
std::vector<real_t> MLPPActivation::sign(std::vector<real_t> z, bool deriv) {
if (deriv) {
std::vector<double> deriv;
std::vector<real_t> deriv;
deriv.resize(z.size());
for (int i = 0; i < z.size(); i++) {
deriv[i] = sign(z[i], 1);
}
return deriv;
}
std::vector<double> a;
std::vector<real_t> a;
a.resize(z.size());
for (int i = 0; i < a.size(); i++) {
@ -633,16 +633,16 @@ std::vector<double> MLPPActivation::sign(std::vector<double> z, bool deriv) {
return a;
}
std::vector<std::vector<double>> MLPPActivation::sign(std::vector<std::vector<double>> z, bool deriv) {
std::vector<std::vector<real_t>> MLPPActivation::sign(std::vector<std::vector<real_t>> z, bool deriv) {
if (deriv) {
std::vector<std::vector<double>> deriv;
std::vector<std::vector<real_t>> deriv;
deriv.resize(z.size());
for (int i = 0; i < z.size(); i++) {
deriv[i] = sign(z[i], 1);
}
return deriv;
}
std::vector<std::vector<double>> a;
std::vector<std::vector<real_t>> a;
a.resize(z.size());
for (int i = 0; i < a.size(); i++) {
@ -651,14 +651,14 @@ std::vector<std::vector<double>> MLPPActivation::sign(std::vector<std::vector<do
return a;
}
double MLPPActivation::sinh(double z, bool deriv) {
real_t MLPPActivation::sinh(real_t z, bool deriv) {
if (deriv) {
return cosh(z);
}
return 0.5 * (exp(z) - exp(-z));
}
std::vector<double> MLPPActivation::sinh(std::vector<double> z, bool deriv) {
std::vector<real_t> MLPPActivation::sinh(std::vector<real_t> z, bool deriv) {
if (deriv) {
return cosh(z);
}
@ -666,7 +666,7 @@ std::vector<double> MLPPActivation::sinh(std::vector<double> z, bool deriv) {
return alg.scalarMultiply(0.5, alg.subtraction(alg.exp(z), alg.exp(alg.scalarMultiply(-1, z))));
}
std::vector<std::vector<double>> MLPPActivation::sinh(std::vector<std::vector<double>> z, bool deriv) {
std::vector<std::vector<real_t>> MLPPActivation::sinh(std::vector<std::vector<real_t>> z, bool deriv) {
if (deriv) {
return cosh(z);
}
@ -674,14 +674,14 @@ std::vector<std::vector<double>> MLPPActivation::sinh(std::vector<std::vector<do
return alg.scalarMultiply(0.5, alg.subtraction(alg.exp(z), alg.exp(alg.scalarMultiply(-1, z))));
}
double MLPPActivation::cosh(double z, bool deriv) {
real_t MLPPActivation::cosh(real_t z, bool deriv) {
if (deriv) {
return sinh(z);
}
return 0.5 * (exp(z) + exp(-z));
}
std::vector<double> MLPPActivation::cosh(std::vector<double> z, bool deriv) {
std::vector<real_t> MLPPActivation::cosh(std::vector<real_t> z, bool deriv) {
if (deriv) {
return sinh(z);
}
@ -689,7 +689,7 @@ std::vector<double> MLPPActivation::cosh(std::vector<double> z, bool deriv) {
return alg.scalarMultiply(0.5, alg.addition(alg.exp(z), alg.exp(alg.scalarMultiply(-1, z))));
}
std::vector<std::vector<double>> MLPPActivation::cosh(std::vector<std::vector<double>> z, bool deriv) {
std::vector<std::vector<real_t>> MLPPActivation::cosh(std::vector<std::vector<real_t>> z, bool deriv) {
if (deriv) {
return sinh(z);
}
@ -697,14 +697,14 @@ std::vector<std::vector<double>> MLPPActivation::cosh(std::vector<std::vector<do
return alg.scalarMultiply(0.5, alg.addition(alg.exp(z), alg.exp(alg.scalarMultiply(-1, z))));
}
double MLPPActivation::tanh(double z, bool deriv) {
real_t MLPPActivation::tanh(real_t z, bool deriv) {
if (deriv) {
return 1 - tanh(z) * tanh(z);
}
return (exp(z) - exp(-z)) / (exp(z) + exp(-z));
}
std::vector<double> MLPPActivation::tanh(std::vector<double> z, bool deriv) {
std::vector<real_t> MLPPActivation::tanh(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.scalarMultiply(-1, alg.scalarAdd(-1, alg.hadamard_product(tanh(z), tanh(z))));
@ -712,7 +712,7 @@ std::vector<double> MLPPActivation::tanh(std::vector<double> z, bool deriv) {
return alg.elementWiseDivision(alg.subtraction(alg.exp(z), alg.exp(alg.scalarMultiply(-1, z))), alg.addition(alg.exp(z), alg.exp(alg.scalarMultiply(-1, z))));
}
std::vector<std::vector<double>> MLPPActivation::tanh(std::vector<std::vector<double>> z, bool deriv) {
std::vector<std::vector<real_t>> MLPPActivation::tanh(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.scalarMultiply(-1, alg.scalarAdd(-1, alg.hadamard_product(tanh(z), tanh(z))));
@ -721,14 +721,14 @@ std::vector<std::vector<double>> MLPPActivation::tanh(std::vector<std::vector<do
return alg.elementWiseDivision(alg.subtraction(alg.exp(z), alg.exp(alg.scalarMultiply(-1, z))), alg.addition(alg.exp(z), alg.exp(alg.scalarMultiply(-1, z))));
}
double MLPPActivation::csch(double z, bool deriv) {
real_t MLPPActivation::csch(real_t z, bool deriv) {
if (deriv) {
return -csch(z) * coth(z);
}
return 1 / sinh(z);
}
std::vector<double> MLPPActivation::csch(std::vector<double> z, bool deriv) {
std::vector<real_t> MLPPActivation::csch(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.hadamard_product(alg.scalarMultiply(-1, csch(z)), coth(z));
@ -736,7 +736,7 @@ std::vector<double> MLPPActivation::csch(std::vector<double> z, bool deriv) {
return alg.elementWiseDivision(alg.onevec(z.size()), sinh(z));
}
std::vector<std::vector<double>> MLPPActivation::csch(std::vector<std::vector<double>> z, bool deriv) {
std::vector<std::vector<real_t>> MLPPActivation::csch(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.hadamard_product(alg.scalarMultiply(-1, csch(z)), coth(z));
@ -744,41 +744,41 @@ std::vector<std::vector<double>> MLPPActivation::csch(std::vector<std::vector<do
return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), sinh(z));
}
double MLPPActivation::sech(double z, bool deriv) {
real_t MLPPActivation::sech(real_t z, bool deriv) {
if (deriv) {
return -sech(z) * tanh(z);
}
return 1 / cosh(z);
}
std::vector<double> MLPPActivation::sech(std::vector<double> z, bool deriv) {
std::vector<real_t> MLPPActivation::sech(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.hadamard_product(alg.scalarMultiply(-1, sech(z)), tanh(z));
}
return alg.elementWiseDivision(alg.onevec(z.size()), cosh(z));
// return activation(z, deriv, static_cast<void (*)(double, bool)>(&sech));
// return activation(z, deriv, static_cast<void (*)(real_t, bool)>(&sech));
}
std::vector<std::vector<double>> MLPPActivation::sech(std::vector<std::vector<double>> z, bool deriv) {
std::vector<std::vector<real_t>> MLPPActivation::sech(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.hadamard_product(alg.scalarMultiply(-1, sech(z)), tanh(z));
}
return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), cosh(z));
// return activation(z, deriv, static_cast<void (*)(double, bool)>(&sech));
// return activation(z, deriv, static_cast<void (*)(real_t, bool)>(&sech));
}
double MLPPActivation::coth(double z, bool deriv) {
real_t MLPPActivation::coth(real_t z, bool deriv) {
if (deriv) {
return -csch(z) * csch(z);
}
return 1 / tanh(z);
}
std::vector<double> MLPPActivation::coth(std::vector<double> z, bool deriv) {
std::vector<real_t> MLPPActivation::coth(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.hadamard_product(alg.scalarMultiply(-1, csch(z)), csch(z));
@ -786,7 +786,7 @@ std::vector<double> MLPPActivation::coth(std::vector<double> z, bool deriv) {
return alg.elementWiseDivision(alg.onevec(z.size()), tanh(z));
}
std::vector<std::vector<double>> MLPPActivation::coth(std::vector<std::vector<double>> z, bool deriv) {
std::vector<std::vector<real_t>> MLPPActivation::coth(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.hadamard_product(alg.scalarMultiply(-1, csch(z)), csch(z));
@ -794,14 +794,14 @@ std::vector<std::vector<double>> MLPPActivation::coth(std::vector<std::vector<do
return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), tanh(z));
}
double MLPPActivation::arsinh(double z, bool deriv) {
real_t MLPPActivation::arsinh(real_t z, bool deriv) {
if (deriv) {
return 1 / sqrt(z * z + 1);
}
return std::log(z + sqrt(z * z + 1));
}
std::vector<double> MLPPActivation::arsinh(std::vector<double> z, bool deriv) {
std::vector<real_t> MLPPActivation::arsinh(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.elementWiseDivision(alg.onevec(z.size()), alg.sqrt(alg.addition(alg.hadamard_product(z, z), alg.onevec(z.size()))));
@ -809,7 +809,7 @@ std::vector<double> MLPPActivation::arsinh(std::vector<double> z, bool deriv) {
return alg.log(alg.addition(z, alg.sqrt(alg.addition(alg.hadamard_product(z, z), alg.onevec(z.size())))));
}
std::vector<std::vector<double>> MLPPActivation::arsinh(std::vector<std::vector<double>> z, bool deriv) {
std::vector<std::vector<real_t>> MLPPActivation::arsinh(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.sqrt(alg.addition(alg.hadamard_product(z, z), alg.onemat(z.size(), z[0].size()))));
@ -817,14 +817,14 @@ std::vector<std::vector<double>> MLPPActivation::arsinh(std::vector<std::vector<
return alg.log(alg.addition(z, alg.sqrt(alg.addition(alg.hadamard_product(z, z), alg.onemat(z.size(), z[0].size())))));
}
double MLPPActivation::arcosh(double z, bool deriv) {
real_t MLPPActivation::arcosh(real_t z, bool deriv) {
if (deriv) {
return 1 / sqrt(z * z - 1);
}
return std::log(z + sqrt(z * z - 1));
}
std::vector<double> MLPPActivation::arcosh(std::vector<double> z, bool deriv) {
std::vector<real_t> MLPPActivation::arcosh(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.elementWiseDivision(alg.onevec(z.size()), alg.sqrt(alg.subtraction(alg.hadamard_product(z, z), alg.onevec(z.size()))));
@ -832,7 +832,7 @@ std::vector<double> MLPPActivation::arcosh(std::vector<double> z, bool deriv) {
return alg.log(alg.addition(z, alg.sqrt(alg.subtraction(alg.hadamard_product(z, z), alg.onevec(z.size())))));
}
std::vector<std::vector<double>> MLPPActivation::arcosh(std::vector<std::vector<double>> z, bool deriv) {
std::vector<std::vector<real_t>> MLPPActivation::arcosh(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.sqrt(alg.subtraction(alg.hadamard_product(z, z), alg.onemat(z.size(), z[0].size()))));
@ -840,14 +840,14 @@ std::vector<std::vector<double>> MLPPActivation::arcosh(std::vector<std::vector<
return alg.log(alg.addition(z, alg.sqrt(alg.subtraction(alg.hadamard_product(z, z), alg.onemat(z.size(), z[0].size())))));
}
double MLPPActivation::artanh(double z, bool deriv) {
real_t MLPPActivation::artanh(real_t z, bool deriv) {
if (deriv) {
return 1 / (1 - z * z);
}
return 0.5 * std::log((1 + z) / (1 - z));
}
std::vector<double> MLPPActivation::artanh(std::vector<double> z, bool deriv) {
std::vector<real_t> MLPPActivation::artanh(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.elementWiseDivision(alg.onevec(z.size()), alg.subtraction(alg.onevec(z.size()), alg.hadamard_product(z, z)));
@ -855,7 +855,7 @@ std::vector<double> MLPPActivation::artanh(std::vector<double> z, bool deriv) {
return alg.scalarMultiply(0.5, alg.log(alg.elementWiseDivision(alg.addition(alg.onevec(z.size()), z), alg.subtraction(alg.onevec(z.size()), z))));
}
std::vector<std::vector<double>> MLPPActivation::artanh(std::vector<std::vector<double>> z, bool deriv) {
std::vector<std::vector<real_t>> MLPPActivation::artanh(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.subtraction(alg.onemat(z.size(), z[0].size()), alg.hadamard_product(z, z)));
@ -863,14 +863,14 @@ std::vector<std::vector<double>> MLPPActivation::artanh(std::vector<std::vector<
return alg.scalarMultiply(0.5, alg.log(alg.elementWiseDivision(alg.addition(alg.onemat(z.size(), z[0].size()), z), alg.subtraction(alg.onemat(z.size(), z[0].size()), z))));
}
double MLPPActivation::arcsch(double z, bool deriv) {
real_t MLPPActivation::arcsch(real_t z, bool deriv) {
if (deriv) {
return -1 / ((z * z) * sqrt(1 + (1 / (z * z))));
}
return std::log(sqrt(1 + (1 / (z * z))) + (1 / z));
}
std::vector<double> MLPPActivation::arcsch(std::vector<double> z, bool deriv) {
std::vector<real_t> MLPPActivation::arcsch(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.elementWiseDivision(alg.full(z.size(), -1), alg.hadamard_product(alg.hadamard_product(z, z), alg.sqrt(alg.addition(alg.onevec(z.size()), alg.elementWiseDivision(alg.onevec(z.size()), alg.hadamard_product(z, z))))));
@ -878,7 +878,7 @@ std::vector<double> MLPPActivation::arcsch(std::vector<double> z, bool deriv) {
return alg.log(alg.addition(alg.sqrt(alg.addition(alg.onevec(z.size()), alg.elementWiseDivision(alg.onevec(z.size()), alg.hadamard_product(z, z)))), alg.elementWiseDivision(alg.onevec(z.size()), z)));
}
std::vector<std::vector<double>> MLPPActivation::arcsch(std::vector<std::vector<double>> z, bool deriv) {
std::vector<std::vector<real_t>> MLPPActivation::arcsch(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.elementWiseDivision(alg.full(z.size(), z[0].size(), -1), alg.hadamard_product(alg.hadamard_product(z, z), alg.sqrt(alg.addition(alg.onemat(z.size(), z[0].size()), alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.hadamard_product(z, z))))));
@ -886,14 +886,14 @@ std::vector<std::vector<double>> MLPPActivation::arcsch(std::vector<std::vector<
return alg.log(alg.addition(alg.sqrt(alg.addition(alg.onemat(z.size(), z[0].size()), alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.hadamard_product(z, z)))), alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), z)));
}
double MLPPActivation::arsech(double z, bool deriv) {
real_t MLPPActivation::arsech(real_t z, bool deriv) {
if (deriv) {
return -1 / (z * sqrt(1 - z * z));
}
return std::log((1 / z) + ((1 / z) + 1) * ((1 / z) - 1));
}
std::vector<double> MLPPActivation::arsech(std::vector<double> z, bool deriv) {
std::vector<real_t> MLPPActivation::arsech(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.elementWiseDivision(alg.full(z.size(), -1), alg.hadamard_product(z, alg.sqrt(alg.subtraction(alg.onevec(z.size()), alg.hadamard_product(z, z)))));
@ -901,7 +901,7 @@ std::vector<double> MLPPActivation::arsech(std::vector<double> z, bool deriv) {
return alg.log(alg.addition(alg.elementWiseDivision(alg.onevec(z.size()), z), alg.hadamard_product(alg.addition(alg.elementWiseDivision(alg.onevec(z.size()), z), alg.onevec(z.size())), alg.subtraction(alg.elementWiseDivision(alg.onevec(z.size()), z), alg.onevec(z.size())))));
}
std::vector<std::vector<double>> MLPPActivation::arsech(std::vector<std::vector<double>> z, bool deriv) {
std::vector<std::vector<real_t>> MLPPActivation::arsech(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.elementWiseDivision(alg.full(z.size(), z[0].size(), -1), alg.hadamard_product(z, alg.sqrt(alg.subtraction(alg.onemat(z.size(), z[0].size()), alg.hadamard_product(z, z)))));
@ -909,14 +909,14 @@ std::vector<std::vector<double>> MLPPActivation::arsech(std::vector<std::vector<
return alg.log(alg.addition(alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), z), alg.hadamard_product(alg.addition(alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), z), alg.onemat(z.size(), z[0].size())), alg.subtraction(alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), z), alg.onemat(z.size(), z[0].size())))));
}
double MLPPActivation::arcoth(double z, bool deriv) {
real_t MLPPActivation::arcoth(real_t z, bool deriv) {
if (deriv) {
return 1 / (1 - z * z);
}
return 0.5 * std::log((1 + z) / (z - 1));
}
std::vector<double> MLPPActivation::arcoth(std::vector<double> z, bool deriv) {
std::vector<real_t> MLPPActivation::arcoth(std::vector<real_t> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.elementWiseDivision(alg.onevec(z.size()), alg.subtraction(alg.onevec(z.size()), alg.hadamard_product(z, z)));
@ -924,7 +924,7 @@ std::vector<double> MLPPActivation::arcoth(std::vector<double> z, bool deriv) {
return alg.scalarMultiply(0.5, alg.log(alg.elementWiseDivision(alg.addition(alg.onevec(z.size()), z), alg.subtraction(z, alg.onevec(z.size())))));
}
std::vector<std::vector<double>> MLPPActivation::arcoth(std::vector<std::vector<double>> z, bool deriv) {
std::vector<std::vector<real_t>> MLPPActivation::arcoth(std::vector<std::vector<real_t>> z, bool deriv) {
MLPPLinAlg alg;
if (deriv) {
return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.subtraction(alg.onemat(z.size(), z[0].size()), alg.hadamard_product(z, z)));
@ -933,16 +933,16 @@ std::vector<std::vector<double>> MLPPActivation::arcoth(std::vector<std::vector<
}
// TO DO: Implement this template activation
std::vector<double> MLPPActivation::activation(std::vector<double> z, bool deriv, double (*function)(double, bool)) {
std::vector<real_t> MLPPActivation::activation(std::vector<real_t> z, bool deriv, real_t (*function)(real_t, bool)) {
if (deriv) {
std::vector<double> deriv;
std::vector<real_t> deriv;
deriv.resize(z.size());
for (int i = 0; i < z.size(); i++) {
deriv[i] = function(z[i], 1);
}
return deriv;
}
std::vector<double> a;
std::vector<real_t> a;
a.resize(z.size());
for (int i = 0; i < z.size(); i++) {
a[i] = function(z[i], deriv);

190
mlpp/activation/activation.h
View File

@ -8,136 +8,138 @@
// Created by Marc Melikyan on 1/16/21.
//
#include "core/math/math_defs.h"
#include <vector>
class MLPPActivation {
public:
double linear(double z, bool deriv = 0);
std::vector<double> linear(std::vector<double> z, bool deriv = 0);
std::vector<std::vector<double>> linear(std::vector<std::vector<double>> z, bool deriv = 0);
real_t linear(real_t z, bool deriv = 0);
std::vector<real_t> linear(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<real_t>> linear(std::vector<std::vector<real_t>> z, bool deriv = 0);
double sigmoid(double z, bool deriv = 0);
std::vector<double> sigmoid(std::vector<double> z, bool deriv = 0);
std::vector<std::vector<double>> sigmoid(std::vector<std::vector<double>> z, bool deriv = 0);
real_t sigmoid(real_t z, bool deriv = 0);
std::vector<real_t> sigmoid(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<real_t>> sigmoid(std::vector<std::vector<real_t>> z, bool deriv = 0);
std::vector<double> softmax(std::vector<double> z, bool deriv = 0);
std::vector<std::vector<double>> softmax(std::vector<std::vector<double>> z, bool deriv = 0);
std::vector<real_t> softmax(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<real_t>> softmax(std::vector<std::vector<real_t>> z, bool deriv = 0);
std::vector<double> adjSoftmax(std::vector<double> z);
std::vector<std::vector<double>> adjSoftmax(std::vector<std::vector<double>> z);
std::vector<real_t> adjSoftmax(std::vector<real_t> z);
std::vector<std::vector<real_t>> adjSoftmax(std::vector<std::vector<real_t>> z);
std::vector<std::vector<double>> softmaxDeriv(std::vector<double> z);
std::vector<std::vector<std::vector<double>>> softmaxDeriv(std::vector<std::vector<double>> z);
std::vector<std::vector<real_t>> softmaxDeriv(std::vector<real_t> z);
std::vector<std::vector<std::vector<real_t>>> softmaxDeriv(std::vector<std::vector<real_t>> z);
double softplus(double z, bool deriv = 0);
std::vector<double> softplus(std::vector<double> z, bool deriv = 0);
std::vector<std::vector<double>> softplus(std::vector<std::vector<double>> z, bool deriv = 0);
real_t softplus(real_t z, bool deriv = 0);
std::vector<real_t> softplus(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<real_t>> softplus(std::vector<std::vector<real_t>> z, bool deriv = 0);
double softsign(double z, bool deriv = 0);
std::vector<double> softsign(std::vector<double> z, bool deriv = 0);
std::vector<std::vector<double>> softsign(std::vector<std::vector<double>> z, bool deriv = 0);
real_t softsign(real_t z, bool deriv = 0);
std::vector<real_t> softsign(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<real_t>> softsign(std::vector<std::vector<real_t>> z, bool deriv = 0);
double gaussianCDF(double z, bool deriv = 0);
std::vector<double> gaussianCDF(std::vector<double> z, bool deriv = 0);
std::vector<std::vector<double>> gaussianCDF(std::vector<std::vector<double>> z, bool deriv = 0);
real_t gaussianCDF(real_t z, bool deriv = 0);
std::vector<real_t> gaussianCDF(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<real_t>> gaussianCDF(std::vector<std::vector<real_t>> z, bool deriv = 0);
double cloglog(double z, bool deriv = 0);
std::vector<double> cloglog(std::vector<double> z, bool deriv = 0);
std::vector<std::vector<double>> cloglog(std::vector<std::vector<double>> z, bool deriv = 0);
real_t cloglog(real_t z, bool deriv = 0);
std::vector<real_t> cloglog(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<real_t>> cloglog(std::vector<std::vector<real_t>> z, bool deriv = 0);
double logit(double z, bool deriv = 0);
std::vector<double> logit(std::vector<double> z, bool deriv = 0);
std::vector<std::vector<double>> logit(std::vector<std::vector<double>> z, bool deriv = 0);
real_t logit(real_t z, bool deriv = 0);
std::vector<real_t> logit(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<real_t>> logit(std::vector<std::vector<real_t>> z, bool deriv = 0);
double unitStep(double z, bool deriv = 0);
std::vector<double> unitStep(std::vector<double> z, bool deriv = 0);
std::vector<std::vector<double>> unitStep(std::vector<std::vector<double>> z, bool deriv = 0);
real_t unitStep(real_t z, bool deriv = 0);
std::vector<real_t> unitStep(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<real_t>> unitStep(std::vector<std::vector<real_t>> z, bool deriv = 0);
double swish(double z, bool deriv = 0);
std::vector<double> swish(std::vector<double> z, bool deriv = 0);
std::vector<std::vector<double>> swish(std::vector<std::vector<double>> z, bool deriv = 0);
real_t swish(real_t z, bool deriv = 0);
std::vector<real_t> swish(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<real_t>> swish(std::vector<std::vector<real_t>> z, bool deriv = 0);
double mish(double z, bool deriv = 0);
std::vector<double> mish(std::vector<double> z, bool deriv = 0);
std::vector<std::vector<double>> mish(std::vector<std::vector<double>> z, bool deriv = 0);
real_t mish(real_t z, bool deriv = 0);
std::vector<real_t> mish(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<real_t>> mish(std::vector<std::vector<real_t>> z, bool deriv = 0);
double sinc(double z, bool deriv = 0);
std::vector<double> sinc(std::vector<double> z, bool deriv = 0);
std::vector<std::vector<double>> sinc(std::vector<std::vector<double>> z, bool deriv = 0);
real_t sinc(real_t z, bool deriv = 0);
std::vector<real_t> sinc(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<real_t>> sinc(std::vector<std::vector<real_t>> z, bool deriv = 0);
double RELU(double z, bool deriv = 0);
std::vector<double> RELU(std::vector<double> z, bool deriv = 0);
std::vector<std::vector<double>> RELU(std::vector<std::vector<double>> z, bool deriv = 0);
real_t RELU(real_t z, bool deriv = 0);
std::vector<real_t> RELU(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<real_t>> RELU(std::vector<std::vector<real_t>> z, bool deriv = 0);
double leakyReLU(double z, double c, bool deriv = 0);
std::vector<double> leakyReLU(std::vector<double> z, double c, bool deriv = 0);
std::vector<std::vector<double>> leakyReLU(std::vector<std::vector<double>> z, double c, bool deriv = 0);
real_t leakyReLU(real_t z, real_t c, bool deriv = 0);
std::vector<real_t> leakyReLU(std::vector<real_t> z, real_t c, bool deriv = 0);
std::vector<std::vector<real_t>> leakyReLU(std::vector<std::vector<real_t>> z, real_t c, bool deriv = 0);
double ELU(double z, double c, bool deriv = 0);
std::vector<double> ELU(std::vector<double> z, double c, bool deriv = 0);
std::vector<std::vector<double>> ELU(std::vector<std::vector<double>> z, double c, bool deriv = 0);
real_t ELU(real_t z, real_t c, bool deriv = 0);
std::vector<real_t> ELU(std::vector<real_t> z, real_t c, bool deriv = 0);
std::vector<std::vector<real_t>> ELU(std::vector<std::vector<real_t>> z, real_t c, bool deriv = 0);
double SELU(double z, double lambda, double c, bool deriv = 0);
std::vector<double> SELU(std::vector<double> z, double lambda, double c, bool deriv = 0);
std::vector<std::vector<double>> SELU(std::vector<std::vector<double>>, double lambda, double c, bool deriv = 0);
real_t SELU(real_t z, real_t lambda, real_t c, bool deriv = 0);
std::vector<real_t> SELU(std::vector<real_t> z, real_t lambda, real_t c, bool deriv = 0);
std::vector<std::vector<real_t>> SELU(std::vector<std::vector<real_t>>, real_t lambda, real_t c, bool deriv = 0);
double GELU(double z, bool deriv = 0);
std::vector<double> GELU(std::vector<double> z, bool deriv = 0);
std::vector<std::vector<double>> GELU(std::vector<std::vector<double>> z, bool deriv = 0);
real_t GELU(real_t z, bool deriv = 0);
std::vector<real_t> GELU(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<real_t>> GELU(std::vector<std::vector<real_t>> z, bool deriv = 0);
double sign(double z, bool deriv = 0);
std::vector<double> sign(std::vector<double> z, bool deriv = 0);
std::vector<std::vector<double>> sign(std::vector<std::vector<double>> z, bool deriv = 0);
real_t sign(real_t z, bool deriv = 0);
std::vector<real_t> sign(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<real_t>> sign(std::vector<std::vector<real_t>> z, bool deriv = 0);
double sinh(double z, bool deriv = 0);
std::vector<double> sinh(std::vector<double> z, bool deriv = 0);
std::vector<std::vector<double>> sinh(std::vector<std::vector<double>> z, bool deriv = 0);
real_t sinh(real_t z, bool deriv = 0);
std::vector<real_t> sinh(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<real_t>> sinh(std::vector<std::vector<real_t>> z, bool deriv = 0);
double cosh(double z, bool deriv = 0);
std::vector<double> cosh(std::vector<double> z, bool deriv = 0);
std::vector<std::vector<double>> cosh(std::vector<std::vector<double>> z, bool deriv = 0);
real_t cosh(real_t z, bool deriv = 0);
std::vector<real_t> cosh(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<real_t>> cosh(std::vector<std::vector<real_t>> z, bool deriv = 0);
double tanh(double z, bool deriv = 0);
std::vector<double> tanh(std::vector<double> z, bool deriv = 0);
std::vector<std::vector<double>> tanh(std::vector<std::vector<double>> z, bool deriv = 0);
real_t tanh(real_t z, bool deriv = 0);
std::vector<real_t> tanh(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<real_t>> tanh(std::vector<std::vector<real_t>> z, bool deriv = 0);
double csch(double z, bool deriv = 0);
std::vector<double> csch(std::vector<double> z, bool deriv = 0);
std::vector<std::vector<double>> csch(std::vector<std::vector<double>> z, bool deriv = 0);
real_t csch(real_t z, bool deriv = 0);
std::vector<real_t> csch(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<real_t>> csch(std::vector<std::vector<real_t>> z, bool deriv = 0);
double sech(double z, bool deriv = 0);
std::vector<double> sech(std::vector<double> z, bool deriv = 0);
std::vector<std::vector<double>> sech(std::vector<std::vector<double>> z, bool deriv = 0);
real_t sech(real_t z, bool deriv = 0);
std::vector<real_t> sech(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<real_t>> sech(std::vector<std::vector<real_t>> z, bool deriv = 0);
double coth(double z, bool deriv = 0);
std::vector<double> coth(std::vector<double> z, bool deriv = 0);
std::vector<std::vector<double>> coth(std::vector<std::vector<double>> z, bool deriv = 0);
real_t coth(real_t z, bool deriv = 0);
std::vector<real_t> coth(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<real_t>> coth(std::vector<std::vector<real_t>> z, bool deriv = 0);
double arsinh(double z, bool deriv = 0);
std::vector<double> arsinh(std::vector<double> z, bool deriv = 0);
std::vector<std::vector<double>> arsinh(std::vector<std::vector<double>> z, bool deriv = 0);
real_t arsinh(real_t z, bool deriv = 0);
std::vector<real_t> arsinh(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<real_t>> arsinh(std::vector<std::vector<real_t>> z, bool deriv = 0);
double arcosh(double z, bool deriv = 0);
std::vector<double> arcosh(std::vector<double> z, bool deriv = 0);
std::vector<std::vector<double>> arcosh(std::vector<std::vector<double>> z, bool deriv = 0);
real_t arcosh(real_t z, bool deriv = 0);
std::vector<real_t> arcosh(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<real_t>> arcosh(std::vector<std::vector<real_t>> z, bool deriv = 0);
double artanh(double z, bool deriv = 0);
std::vector<double> artanh(std::vector<double> z, bool deriv = 0);
std::vector<std::vector<double>> artanh(std::vector<std::vector<double>> z, bool deriv = 0);
real_t artanh(real_t z, bool deriv = 0);
std::vector<real_t> artanh(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<real_t>> artanh(std::vector<std::vector<real_t>> z, bool deriv = 0);
double arcsch(double z, bool deriv = 0);
std::vector<double> arcsch(std::vector<double> z, bool deriv = 0);
std::vector<std::vector<double>> arcsch(std::vector<std::vector<double>> z, bool deriv = 0);
real_t arcsch(real_t z, bool deriv = 0);
std::vector<real_t> arcsch(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<real_t>> arcsch(std::vector<std::vector<real_t>> z, bool deriv = 0);
double arsech(double z, bool deriv = 0);
std::vector<double> arsech(std::vector<double> z, bool deriv = 0);
std::vector<std::vector<double>> arsech(std::vector<std::vector<double>> z, bool deriv = 0);
real_t arsech(real_t z, bool deriv = 0);
std::vector<real_t> arsech(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<real_t>> arsech(std::vector<std::vector<real_t>> z, bool deriv = 0);
double arcoth(double z, bool deriv = 0);
std::vector<double> arcoth(std::vector<double> z, bool deriv = 0);
std::vector<std::vector<double>> arcoth(std::vector<std::vector<double>> z, bool deriv = 0);
real_t arcoth(real_t z, bool deriv = 0);
std::vector<real_t> arcoth(std::vector<real_t> z, bool deriv = 0);
std::vector<std::vector<real_t>> arcoth(std::vector<std::vector<real_t>> z, bool deriv = 0);
std::vector<double> activation(std::vector<double> z, bool deriv, double (*function)(double, bool));
std::vector<real_t> activation(std::vector<real_t> z, bool deriv, real_t (*function)(real_t, bool));
private:
};

212
mlpp/ann/ann.cpp
View File

@ -15,7 +15,7 @@
#include <iostream>
#include <random>
MLPPANN::MLPPANN(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet) :
MLPPANN::MLPPANN(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet) :
inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), lrScheduler("None"), decayConstant(0), dropRate(0) {
}
@ -23,7 +23,7 @@ MLPPANN::~MLPPANN() {
delete outputLayer;
}
std::vector<double> MLPPANN::modelSetTest(std::vector<std::vector<double>> X) {
std::vector<real_t> MLPPANN::modelSetTest(std::vector<std::vector<real_t>> X) {
if (!network.empty()) {
network[0].input = X;
network[0].forwardPass();
@ -40,7 +40,7 @@ std::vector<double> MLPPANN::modelSetTest(std::vector<std::vector<double>> X) {
return outputLayer->a;
}
double MLPPANN::modelTest(std::vector<double> x) {
real_t MLPPANN::modelTest(std::vector<real_t> x) {
if (!network.empty()) {
network[0].Test(x);
for (int i = 1; i < network.size(); i++) {
@ -53,13 +53,13 @@ double MLPPANN::modelTest(std::vector<double> x) {
return outputLayer->a_test;
}
void MLPPANN::gradientDescent(double learning_rate, int max_epoch, bool UI) {
void MLPPANN::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
class MLPPCost cost;
MLPPLinAlg alg;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
forwardPass();
double initial_learning_rate = learning_rate;
real_t initial_learning_rate = learning_rate;
alg.printMatrix(network[network.size() - 1].weights);
while (true) {
@ -87,13 +87,13 @@ void MLPPANN::gradientDescent(double learning_rate, int max_epoch, bool UI) {
}
}
void MLPPANN::SGD(double learning_rate, int max_epoch, bool UI) {
void MLPPANN::SGD(real_t learning_rate, int max_epoch, bool UI) {
class MLPPCost cost;
MLPPLinAlg alg;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
double initial_learning_rate = learning_rate;
real_t initial_learning_rate = learning_rate;
while (true) {
learning_rate = applyLearningRateScheduler(initial_learning_rate, decayConstant, epoch, dropRate);
@ -103,7 +103,7 @@ void MLPPANN::SGD(double learning_rate, int max_epoch, bool UI) {
std::uniform_int_distribution<int> distribution(0, int(n - 1));
int outputIndex = distribution(generator);
std::vector<double> y_hat = modelSetTest({ inputSet[outputIndex] });
std::vector<real_t> y_hat = modelSetTest({ inputSet[outputIndex] });
cost_prev = Cost({ y_hat }, { outputSet[outputIndex] });
auto [cumulativeHiddenLayerWGrad, outputWGrad] = computeGradients(y_hat, { outputSet[outputIndex] });
@ -125,13 +125,13 @@ void MLPPANN::SGD(double learning_rate, int max_epoch, bool UI) {
forwardPass();
}
void MLPPANN::MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI) {
void MLPPANN::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI) {
class MLPPCost cost;
MLPPLinAlg alg;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
double initial_learning_rate = learning_rate;
real_t initial_learning_rate = learning_rate;
// Creating the mini-batches
int n_mini_batch = n / mini_batch_size;
@ -141,7 +141,7 @@ void MLPPANN::MBGD(double learning_rate, int max_epoch, int mini_batch_size, boo
while (true) {
learning_rate = applyLearningRateScheduler(initial_learning_rate, decayConstant, epoch, dropRate);
for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = modelSetTest(inputMiniBatches[i]);
std::vector<real_t> y_hat = modelSetTest(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]);
auto [cumulativeHiddenLayerWGrad, outputWGrad] = computeGradients(y_hat, outputMiniBatches[i]);
@ -163,13 +163,13 @@ void MLPPANN::MBGD(double learning_rate, int max_epoch, int mini_batch_size, boo
forwardPass();
}
void MLPPANN::Momentum(double learning_rate, int max_epoch, int mini_batch_size, double gamma, bool NAG, bool UI) {
void MLPPANN::Momentum(real_t learning_rate, int max_epoch, int mini_batch_size, real_t gamma, bool NAG, bool UI) {
class MLPPCost cost;
MLPPLinAlg alg;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
double initial_learning_rate = learning_rate;
real_t initial_learning_rate = learning_rate;
// Creating the mini-batches
int n_mini_batch = n / mini_batch_size;
@ -178,13 +178,13 @@ void MLPPANN::Momentum(double learning_rate, int max_epoch, int mini_batch_size,
auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch);
// Initializing necessary components for Adam.
std::vector<std::vector<std::vector<double>>> v_hidden;
std::vector<std::vector<std::vector<real_t>>> v_hidden;
std::vector<double> v_output;
std::vector<real_t> v_output;
while (true) {
learning_rate = applyLearningRateScheduler(initial_learning_rate, decayConstant, epoch, dropRate);
for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = modelSetTest(inputMiniBatches[i]);
std::vector<real_t> y_hat = modelSetTest(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]);
auto [cumulativeHiddenLayerWGrad, outputWGrad] = computeGradients(y_hat, outputMiniBatches[i]);
@ -220,13 +220,13 @@ void MLPPANN::Momentum(double learning_rate, int max_epoch, int mini_batch_size,
forwardPass();
}
void MLPPANN::Adagrad(double learning_rate, int max_epoch, int mini_batch_size, double e, bool UI) {
void MLPPANN::Adagrad(real_t learning_rate, int max_epoch, int mini_batch_size, real_t e, bool UI) {
class MLPPCost cost;
MLPPLinAlg alg;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
double initial_learning_rate = learning_rate;
real_t initial_learning_rate = learning_rate;
// Creating the mini-batches
int n_mini_batch = n / mini_batch_size;
@ -235,13 +235,13 @@ void MLPPANN::Adagrad(double learning_rate, int max_epoch, int mini_batch_size,
auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch);
// Initializing necessary components for Adam.
std::vector<std::vector<std::vector<double>>> v_hidden;
std::vector<std::vector<std::vector<real_t>>> v_hidden;
std::vector<double> v_output;
std::vector<real_t> v_output;
while (true) {
learning_rate = applyLearningRateScheduler(initial_learning_rate, decayConstant, epoch, dropRate);
for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = modelSetTest(inputMiniBatches[i]);
std::vector<real_t> y_hat = modelSetTest(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]);
auto [cumulativeHiddenLayerWGrad, outputWGrad] = computeGradients(y_hat, outputMiniBatches[i]);
@ -258,8 +258,8 @@ void MLPPANN::Adagrad(double learning_rate, int max_epoch, int mini_batch_size,
v_output = alg.addition(v_output, alg.exponentiate(outputWGrad, 2));
std::vector<std::vector<std::vector<double>>> hiddenLayerUpdations = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(cumulativeHiddenLayerWGrad, alg.scalarAdd(e, alg.sqrt(v_hidden))));
std::vector<double> outputLayerUpdation = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(outputWGrad, alg.scalarAdd(e, alg.sqrt(v_output))));
std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(cumulativeHiddenLayerWGrad, alg.scalarAdd(e, alg.sqrt(v_hidden))));
std::vector<real_t> outputLayerUpdation = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(outputWGrad, alg.scalarAdd(e, alg.sqrt(v_output))));
updateParameters(hiddenLayerUpdations, outputLayerUpdation, learning_rate); // subject to change. may want bias to have this matrix too.
y_hat = modelSetTest(inputMiniBatches[i]);
@ -276,13 +276,13 @@ void MLPPANN::Adagrad(double learning_rate, int max_epoch, int mini_batch_size,
forwardPass();
}
void MLPPANN::Adadelta(double learning_rate, int max_epoch, int mini_batch_size, double b1, double e, bool UI) {
void MLPPANN::Adadelta(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t e, bool UI) {
class MLPPCost cost;
MLPPLinAlg alg;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
double initial_learning_rate = learning_rate;
real_t initial_learning_rate = learning_rate;
// Creating the mini-batches
int n_mini_batch = n / mini_batch_size;
@ -291,13 +291,13 @@ void MLPPANN::Adadelta(double learning_rate, int max_epoch, int mini_batch_size,
auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch);
// Initializing necessary components for Adam.
std::vector<std::vector<std::vector<double>>> v_hidden;
std::vector<std::vector<std::vector<real_t>>> v_hidden;
std::vector<double> v_output;
std::vector<real_t> v_output;
while (true) {
learning_rate = applyLearningRateScheduler(initial_learning_rate, decayConstant, epoch, dropRate);
for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = modelSetTest(inputMiniBatches[i]);
std::vector<real_t> y_hat = modelSetTest(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]);
auto [cumulativeHiddenLayerWGrad, outputWGrad] = computeGradients(y_hat, outputMiniBatches[i]);
@ -314,8 +314,8 @@ void MLPPANN::Adadelta(double learning_rate, int max_epoch, int mini_batch_size,
v_output = alg.addition(v_output, alg.exponentiate(outputWGrad, 2));
std::vector<std::vector<std::vector<double>>> hiddenLayerUpdations = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(cumulativeHiddenLayerWGrad, alg.scalarAdd(e, alg.sqrt(v_hidden))));
std::vector<double> outputLayerUpdation = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(outputWGrad, alg.scalarAdd(e, alg.sqrt(v_output))));
std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(cumulativeHiddenLayerWGrad, alg.scalarAdd(e, alg.sqrt(v_hidden))));
std::vector<real_t> outputLayerUpdation = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(outputWGrad, alg.scalarAdd(e, alg.sqrt(v_output))));
updateParameters(hiddenLayerUpdations, outputLayerUpdation, learning_rate); // subject to change. may want bias to have this matrix too.
y_hat = modelSetTest(inputMiniBatches[i]);
@ -332,13 +332,13 @@ void MLPPANN::Adadelta(double learning_rate, int max_epoch, int mini_batch_size,
forwardPass();
}
void MLPPANN::Adam(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI) {
void MLPPANN::Adam(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t b2, real_t e, bool UI) {
class MLPPCost cost;
MLPPLinAlg alg;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
double initial_learning_rate = learning_rate;
real_t initial_learning_rate = learning_rate;
// Creating the mini-batches
int n_mini_batch = n / mini_batch_size;
@ -347,15 +347,15 @@ void MLPPANN::Adam(double learning_rate, int max_epoch, int mini_batch_size, dou
auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch);
// Initializing necessary components for Adam.
std::vector<std::vector<std::vector<double>>> m_hidden;
std::vector<std::vector<std::vector<double>>> v_hidden;
std::vector<std::vector<std::vector<real_t>>> m_hidden;
std::vector<std::vector<std::vector<real_t>>> v_hidden;
std::vector<double> m_output;
std::vector<double> v_output;
std::vector<real_t> m_output;
std::vector<real_t> v_output;
while (true) {
learning_rate = applyLearningRateScheduler(initial_learning_rate, decayConstant, epoch, dropRate);
for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = modelSetTest(inputMiniBatches[i]);
std::vector<real_t> y_hat = modelSetTest(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]);
auto [cumulativeHiddenLayerWGrad, outputWGrad] = computeGradients(y_hat, outputMiniBatches[i]);
@ -375,14 +375,14 @@ void MLPPANN::Adam(double learning_rate, int max_epoch, int mini_batch_size, dou
m_output = alg.addition(alg.scalarMultiply(b1, m_output), alg.scalarMultiply(1 - b1, outputWGrad));
v_output = alg.addition(alg.scalarMultiply(b2, v_output), alg.scalarMultiply(1 - b2, alg.exponentiate(outputWGrad, 2)));
std::vector<std::vector<std::vector<double>>> m_hidden_hat = alg.scalarMultiply(1 / (1 - std::pow(b1, epoch)), m_hidden);
std::vector<std::vector<std::vector<double>>> v_hidden_hat = alg.scalarMultiply(1 / (1 - std::pow(b2, epoch)), v_hidden);
std::vector<std::vector<std::vector<real_t>>> m_hidden_hat = alg.scalarMultiply(1 / (1 - std::pow(b1, epoch)), m_hidden);
std::vector<std::vector<std::vector<real_t>>> v_hidden_hat = alg.scalarMultiply(1 / (1 - std::pow(b2, epoch)), v_hidden);
std::vector<double> m_output_hat = alg.scalarMultiply(1 / (1 - std::pow(b1, epoch)), m_output);
std::vector<double> v_output_hat = alg.scalarMultiply(1 / (1 - std::pow(b2, epoch)), v_output);
std::vector<real_t> m_output_hat = alg.scalarMultiply(1 / (1 - std::pow(b1, epoch)), m_output);
std::vector<real_t> v_output_hat = alg.scalarMultiply(1 / (1 - std::pow(b2, epoch)), v_output);
std::vector<std::vector<std::vector<double>>> hiddenLayerUpdations = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(m_hidden_hat, alg.scalarAdd(e, alg.sqrt(v_hidden_hat))));
std::vector<double> outputLayerUpdation = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(m_output_hat, alg.scalarAdd(e, alg.sqrt(v_output_hat))));
std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(m_hidden_hat, alg.scalarAdd(e, alg.sqrt(v_hidden_hat))));
std::vector<real_t> outputLayerUpdation = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(m_output_hat, alg.scalarAdd(e, alg.sqrt(v_output_hat))));
updateParameters(hiddenLayerUpdations, outputLayerUpdation, learning_rate); // subject to change. may want bias to have this matrix too.
y_hat = modelSetTest(inputMiniBatches[i]);
@ -399,13 +399,13 @@ void MLPPANN::Adam(double learning_rate, int max_epoch, int mini_batch_size, dou
forwardPass();
}
void MLPPANN::Adamax(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI) {
void MLPPANN::Adamax(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t b2, real_t e, bool UI) {
class MLPPCost cost;
MLPPLinAlg alg;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
double initial_learning_rate = learning_rate;
real_t initial_learning_rate = learning_rate;
// Creating the mini-batches
int n_mini_batch = n / mini_batch_size;
@ -414,15 +414,15 @@ void MLPPANN::Adamax(double learning_rate, int max_epoch, int mini_batch_size, d
auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch);
// Initializing necessary components for Adam.
std::vector<std::vector<std::vector<double>>> m_hidden;
std::vector<std::vector<std::vector<double>>> u_hidden;
std::vector<std::vector<std::vector<real_t>>> m_hidden;
std::vector<std::vector<std::vector<real_t>>> u_hidden;
std::vector<double> m_output;
std::vector<double> u_output;
std::vector<real_t> m_output;
std::vector<real_t> u_output;
while (true) {
learning_rate = applyLearningRateScheduler(initial_learning_rate, decayConstant, epoch, dropRate);
for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = modelSetTest(inputMiniBatches[i]);
std::vector<real_t> y_hat = modelSetTest(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]);
auto [cumulativeHiddenLayerWGrad, outputWGrad] = computeGradients(y_hat, outputMiniBatches[i]);
@ -442,12 +442,12 @@ void MLPPANN::Adamax(double learning_rate, int max_epoch, int mini_batch_size, d
m_output = alg.addition(alg.scalarMultiply(b1, m_output), alg.scalarMultiply(1 - b1, outputWGrad));
u_output = alg.max(alg.scalarMultiply(b2, u_output), alg.abs(outputWGrad));
std::vector<std::vector<std::vector<double>>> m_hidden_hat = alg.scalarMultiply(1 / (1 - std::pow(b1, epoch)), m_hidden);
std::vector<std::vector<std::vector<real_t>>> m_hidden_hat = alg.scalarMultiply(1 / (1 - std::pow(b1, epoch)), m_hidden);
std::vector<double> m_output_hat = alg.scalarMultiply(1 / (1 - std::pow(b1, epoch)), m_output);
std::vector<real_t> m_output_hat = alg.scalarMultiply(1 / (1 - std::pow(b1, epoch)), m_output);
std::vector<std::vector<std::vector<double>>> hiddenLayerUpdations = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(m_hidden_hat, alg.scalarAdd(e, u_hidden)));
std::vector<double> outputLayerUpdation = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(m_output_hat, alg.scalarAdd(e, u_output)));
std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(m_hidden_hat, alg.scalarAdd(e, u_hidden)));
std::vector<real_t> outputLayerUpdation = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(m_output_hat, alg.scalarAdd(e, u_output)));
updateParameters(hiddenLayerUpdations, outputLayerUpdation, learning_rate); // subject to change. may want bias to have this matrix too.
y_hat = modelSetTest(inputMiniBatches[i]);
@ -464,13 +464,13 @@ void MLPPANN::Adamax(double learning_rate, int max_epoch, int mini_batch_size, d
forwardPass();
}
void MLPPANN::Nadam(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI) {
void MLPPANN::Nadam(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t b2, real_t e, bool UI) {
class MLPPCost cost;
MLPPLinAlg alg;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
double initial_learning_rate = learning_rate;
real_t initial_learning_rate = learning_rate;
// Creating the mini-batches
int n_mini_batch = n / mini_batch_size;
@ -479,16 +479,16 @@ void MLPPANN::Nadam(double learning_rate, int max_epoch, int mini_batch_size, do
auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch);
// Initializing necessary components for Adam.
std::vector<std::vector<std::vector<double>>> m_hidden;
std::vector<std::vector<std::vector<double>>> v_hidden;
std::vector<std::vector<std::vector<double>>> m_hidden_final;
std::vector<std::vector<std::vector<real_t>>> m_hidden;
std::vector<std::vector<std::vector<real_t>>> v_hidden;
std::vector<std::vector<std::vector<real_t>>> m_hidden_final;
std::vector<double> m_output;
std::vector<double> v_output;
std::vector<real_t> m_output;
std::vector<real_t> v_output;
while (true) {
learning_rate = applyLearningRateScheduler(initial_learning_rate, decayConstant, epoch, dropRate);
for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = modelSetTest(inputMiniBatches[i]);
std::vector<real_t> y_hat = modelSetTest(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]);
auto [cumulativeHiddenLayerWGrad, outputWGrad] = computeGradients(y_hat, outputMiniBatches[i]);
@ -508,16 +508,16 @@ void MLPPANN::Nadam(double learning_rate, int max_epoch, int mini_batch_size, do
m_output = alg.addition(alg.scalarMultiply(b1, m_output), alg.scalarMultiply(1 - b1, outputWGrad));
v_output = alg.addition(alg.scalarMultiply(b2, v_output), alg.scalarMultiply(1 - b2, alg.exponentiate(outputWGrad, 2)));
std::vector<std::vector<std::vector<double>>> m_hidden_hat = alg.scalarMultiply(1 / (1 - std::pow(b1, epoch)), m_hidden);
std::vector<std::vector<std::vector<double>>> v_hidden_hat = alg.scalarMultiply(1 / (1 - std::pow(b2, epoch)), v_hidden);
std::vector<std::vector<std::vector<double>>> m_hidden_final = alg.addition(alg.scalarMultiply(b1, m_hidden_hat), alg.scalarMultiply((1 - b1) / (1 - std::pow(b1, epoch)), cumulativeHiddenLayerWGrad));
std::vector<std::vector<std::vector<real_t>>> m_hidden_hat = alg.scalarMultiply(1 / (1 - std::pow(b1, epoch)), m_hidden);
std::vector<std::vector<std::vector<real_t>>> v_hidden_hat = alg.scalarMultiply(1 / (1 - std::pow(b2, epoch)), v_hidden);
std::vector<std::vector<std::vector<real_t>>> m_hidden_final = alg.addition(alg.scalarMultiply(b1, m_hidden_hat), alg.scalarMultiply((1 - b1) / (1 - std::pow(b1, epoch)), cumulativeHiddenLayerWGrad));
std::vector<double> m_output_hat = alg.scalarMultiply(1 / (1 - std::pow(b1, epoch)), m_output);
std::vector<double> v_output_hat = alg.scalarMultiply(1 / (1 - std::pow(b2, epoch)), v_output);
std::vector<double> m_output_final = alg.addition(alg.scalarMultiply(b1, m_output_hat), alg.scalarMultiply((1 - b1) / (1 - std::pow(b1, epoch)), outputWGrad));
std::vector<real_t> m_output_hat = alg.scalarMultiply(1 / (1 - std::pow(b1, epoch)), m_output);
std::vector<real_t> v_output_hat = alg.scalarMultiply(1 / (1 - std::pow(b2, epoch)), v_output);
std::vector<real_t> m_output_final = alg.addition(alg.scalarMultiply(b1, m_output_hat), alg.scalarMultiply((1 - b1) / (1 - std::pow(b1, epoch)), outputWGrad));
std::vector<std::vector<std::vector<double>>> hiddenLayerUpdations = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(m_hidden_final, alg.scalarAdd(e, alg.sqrt(v_hidden_hat))));
std::vector<double> outputLayerUpdation = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(m_output_final, alg.scalarAdd(e, alg.sqrt(v_output_hat))));
std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(m_hidden_final, alg.scalarAdd(e, alg.sqrt(v_hidden_hat))));
std::vector<real_t> outputLayerUpdation = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(m_output_final, alg.scalarAdd(e, alg.sqrt(v_output_hat))));
updateParameters(hiddenLayerUpdations, outputLayerUpdation, learning_rate); // subject to change. may want bias to have this matrix too.
y_hat = modelSetTest(inputMiniBatches[i]);
@ -534,13 +534,13 @@ void MLPPANN::Nadam(double learning_rate, int max_epoch, int mini_batch_size, do
forwardPass();
}
void MLPPANN::AMSGrad(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI) {
void MLPPANN::AMSGrad(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t b2, real_t e, bool UI) {
class MLPPCost cost;
MLPPLinAlg alg;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
double initial_learning_rate = learning_rate;
real_t initial_learning_rate = learning_rate;
// Creating the mini-batches
int n_mini_batch = n / mini_batch_size;
@ -549,19 +549,19 @@ void MLPPANN::AMSGrad(double learning_rate, int max_epoch, int mini_batch_size,
auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch);
// Initializing necessary components for Adam.
std::vector<std::vector<std::vector<double>>> m_hidden;
std::vector<std::vector<std::vector<double>>> v_hidden;
std::vector<std::vector<std::vector<real_t>>> m_hidden;
std::vector<std::vector<std::vector<real_t>>> v_hidden;
std::vector<std::vector<std::vector<double>>> v_hidden_hat;
std::vector<std::vector<std::vector<real_t>>> v_hidden_hat;
std::vector<double> m_output;
std::vector<double> v_output;
std::vector<real_t> m_output;
std::vector<real_t> v_output;
std::vector<double> v_output_hat;
std::vector<real_t> v_output_hat;
while (true) {
learning_rate = applyLearningRateScheduler(initial_learning_rate, decayConstant, epoch, dropRate);
for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = modelSetTest(inputMiniBatches[i]);
std::vector<real_t> y_hat = modelSetTest(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]);
auto [cumulativeHiddenLayerWGrad, outputWGrad] = computeGradients(y_hat, outputMiniBatches[i]);
@ -587,8 +587,8 @@ void MLPPANN::AMSGrad(double learning_rate, int max_epoch, int mini_batch_size,
v_output_hat = alg.max(v_output_hat, v_output);
std::vector<std::vector<std::vector<double>>> hiddenLayerUpdations = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(m_hidden, alg.scalarAdd(e, alg.sqrt(v_hidden_hat))));
std::vector<double> outputLayerUpdation = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(m_output, alg.scalarAdd(e, alg.sqrt(v_output_hat))));
std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(m_hidden, alg.scalarAdd(e, alg.sqrt(v_hidden_hat))));
std::vector<real_t> outputLayerUpdation = alg.scalarMultiply(learning_rate / n, alg.elementWiseDivision(m_output, alg.scalarAdd(e, alg.sqrt(v_output_hat))));
updateParameters(hiddenLayerUpdations, outputLayerUpdation, learning_rate); // subject to change. may want bias to have this matrix too.
y_hat = modelSetTest(inputMiniBatches[i]);
@ -605,7 +605,7 @@ void MLPPANN::AMSGrad(double learning_rate, int max_epoch, int mini_batch_size,
forwardPass();
}
double MLPPANN::score() {
real_t MLPPANN::score() {
MLPPUtilities util;
forwardPass();
return util.performance(y_hat, outputSet);
@ -624,12 +624,12 @@ void MLPPANN::save(std::string fileName) {
}
}
void MLPPANN::setLearningRateScheduler(std::string type, double decayConstant) {
void MLPPANN::setLearningRateScheduler(std::string type, real_t decayConstant) {
lrScheduler = type;
MLPPANN::decayConstant = decayConstant;
}
void MLPPANN::setLearningRateScheduler(std::string type, double decayConstant, double dropRate) {
void MLPPANN::setLearningRateScheduler(std::string type, real_t decayConstant, real_t dropRate) {
lrScheduler = type;
MLPPANN::decayConstant = decayConstant;
MLPPANN::dropRate = dropRate;
@ -637,7 +637,7 @@ void MLPPANN::setLearningRateScheduler(std::string type, double decayConstant, d
// https://en.wikipedia.org/wiki/Learning_rate
// Learning Rate Decay (C2W2L09) - Andrew Ng - Deep Learning Specialization
double MLPPANN::applyLearningRateScheduler(double learningRate, double decayConstant, double epoch, double dropRate) {
real_t MLPPANN::applyLearningRateScheduler(real_t learningRate, real_t decayConstant, real_t epoch, real_t dropRate) {
if (lrScheduler == "Time") {
return learningRate / (1 + decayConstant * epoch);
} else if (lrScheduler == "Epoch") {
@ -650,7 +650,7 @@ double MLPPANN::applyLearningRateScheduler(double learningRate, double decayCons
return learningRate;
}
void MLPPANN::addLayer(int n_hidden, std::string activation, std::string weightInit, std::string reg, double lambda, double alpha) {
void MLPPANN::addLayer(int n_hidden, std::string activation, std::string weightInit, std::string reg, real_t lambda, real_t alpha) {
if (network.empty()) {
network.push_back(MLPPHiddenLayer(n_hidden, activation, inputSet, weightInit, reg, lambda, alpha));
network[0].forwardPass();
@ -660,7 +660,7 @@ void MLPPANN::addLayer(int n_hidden, std::string activation, std::string weightI
}
}
void MLPPANN::addOutputLayer(std::string activation, std::string loss, std::string weightInit, std::string reg, double lambda, double alpha) {
void MLPPANN::addOutputLayer(std::string activation, std::string loss, std::string weightInit, std::string reg, real_t lambda, real_t alpha) {
MLPPLinAlg alg;
if (!network.empty()) {
outputLayer = new MLPPOutputLayer(network[network.size() - 1].n_hidden, activation, loss, network[network.size() - 1].a, weightInit, reg, lambda, alpha);
@ -669,10 +669,10 @@ void MLPPANN::addOutputLayer(std::string activation, std::string loss, std::stri
}
}
double MLPPANN::Cost(std::vector<double> y_hat, std::vector<double> y) {
real_t MLPPANN::Cost(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPReg regularization;
class MLPPCost cost;
double totalRegTerm = 0;
real_t totalRegTerm = 0;
auto cost_function = outputLayer->cost_map[outputLayer->cost];
if (!network.empty()) {
@ -700,7 +700,7 @@ void MLPPANN::forwardPass() {
y_hat = outputLayer->a;
}
void MLPPANN::updateParameters(std::vector<std::vector<std::vector<double>>> hiddenLayerUpdations, std::vector<double> outputLayerUpdation, double learning_rate) {
void MLPPANN::updateParameters(std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations, std::vector<real_t> outputLayerUpdation, real_t learning_rate) {
MLPPLinAlg alg;
outputLayer->weights = alg.subtraction(outputLayer->weights, outputLayerUpdation);
@ -717,39 +717,39 @@ void MLPPANN::updateParameters(std::vector<std::vector<std::vector<double>>> hid
}
}
std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<double>> MLPPANN::computeGradients(std::vector<double> y_hat, std::vector<double> outputSet) {
std::tuple<std::vector<std::vector<std::vector<real_t>>>, std::vector<real_t>> MLPPANN::computeGradients(std::vector<real_t> y_hat, std::vector<real_t> outputSet) {
// std::cout << "BEGIN" << std::endl;
class MLPPCost cost;
MLPPActivation avn;
MLPPLinAlg alg;
MLPPReg regularization;
std::vector<std::vector<std::vector<double>>> cumulativeHiddenLayerWGrad; // Tensor containing ALL hidden grads.
std::vector<std::vector<std::vector<real_t>>> cumulativeHiddenLayerWGrad; // Tensor containing ALL hidden grads.
auto costDeriv = outputLayer->costDeriv_map[outputLayer->cost];
auto outputAvn = outputLayer->activation_map[outputLayer->activation];
outputLayer->delta = alg.hadamard_product((cost.*costDeriv)(y_hat, outputSet), (avn.*outputAvn)(outputLayer->z, 1));
std::vector<double> outputWGrad = alg.mat_vec_mult(alg.transpose(outputLayer->input), outputLayer->delta);
std::vector<real_t> outputWGrad = alg.mat_vec_mult(alg.transpose(outputLayer->input), outputLayer->delta);
outputWGrad = alg.addition(outputWGrad, regularization.regDerivTerm(outputLayer->weights, outputLayer->lambda, outputLayer->alpha, outputLayer->reg));
if (!network.empty()) {
auto hiddenLayerAvn = network[network.size() - 1].activation_map[network[network.size() - 1].activation];
network[network.size() - 1].delta = alg.hadamard_product(alg.outerProduct(outputLayer->delta, outputLayer->weights), (avn.*hiddenLayerAvn)(network[network.size() - 1].z, 1));
std::vector<std::vector<double>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[network.size() - 1].input), network[network.size() - 1].delta);
std::vector<std::vector<real_t>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[network.size() - 1].input), network[network.size() - 1].delta);
cumulativeHiddenLayerWGrad.push_back(alg.addition(hiddenLayerWGrad, regularization.regDerivTerm(network[network.size() - 1].weights, network[network.size() - 1].lambda, network[network.size() - 1].alpha, network[network.size() - 1].reg))); // Adding to our cumulative hidden layer grads. Maintain reg terms as well.
for (int i = network.size() - 2; i >= 0; i--) {
auto hiddenLayerAvn = network[i].activation_map[network[i].activation];
network[i].delta = alg.hadamard_product(alg.matmult(network[i + 1].delta, alg.transpose(network[i + 1].weights)), (avn.*hiddenLayerAvn)(network[i].z, 1));
std::vector<std::vector<double>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[i].input), network[i].delta);
std::vector<std::vector<real_t>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[i].input), network[i].delta);
cumulativeHiddenLayerWGrad.push_back(alg.addition(hiddenLayerWGrad, regularization.regDerivTerm(network[i].weights, network[i].lambda, network[i].alpha, network[i].reg))); // Adding to our cumulative hidden layer grads. Maintain reg terms as well.
}
}
return { cumulativeHiddenLayerWGrad, outputWGrad };
}
void MLPPANN::UI(int epoch, double cost_prev, std::vector<double> y_hat, std::vector<double> outputSet) {
void MLPPANN::UI(int epoch, real_t cost_prev, std::vector<real_t> y_hat, std::vector<real_t> outputSet) {
MLPPUtilities::CostInfo(epoch, cost_prev, Cost(y_hat, outputSet));
std::cout << "Layer " << network.size() + 1 << ": " << std::endl;
MLPPUtilities::UI(outputLayer->weights, outputLayer->bias);

58
mlpp/ann/ann.h
View File

@ -7,6 +7,8 @@
// Created by Marc Melikyan on 11/4/20.
//
#include "core/math/math_defs.h"
#include "../hidden_layer/hidden_layer.h"
#include "../output_layer/output_layer.h"
@ -16,43 +18,43 @@
class MLPPANN {
public:
MLPPANN(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet);
MLPPANN(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet);
~MLPPANN();
std::vector<double> modelSetTest(std::vector<std::vector<double>> X);
double modelTest(std::vector<double> x);
void gradientDescent(double learning_rate, int max_epoch, bool UI = 1);
void SGD(double learning_rate, int max_epoch, bool UI = 1);
void MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
void Momentum(double learning_rate, int max_epoch, int mini_batch_size, double gamma, bool NAG, bool UI = 1);
void Adagrad(double learning_rate, int max_epoch, int mini_batch_size, double e, bool UI = 1);
void Adadelta(double learning_rate, int max_epoch, int mini_batch_size, double b1, double e, bool UI = 1);
void Adam(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI = 1);
void Adamax(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI = 1);
void Nadam(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI = 1);
void AMSGrad(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI = 1);
double score();
std::vector<real_t> modelSetTest(std::vector<std::vector<real_t>> X);
real_t modelTest(std::vector<real_t> x);
void gradientDescent(real_t learning_rate, int max_epoch, bool UI = 1);
void SGD(real_t learning_rate, int max_epoch, bool UI = 1);
void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
void Momentum(real_t learning_rate, int max_epoch, int mini_batch_size, real_t gamma, bool NAG, bool UI = 1);
void Adagrad(real_t learning_rate, int max_epoch, int mini_batch_size, real_t e, bool UI = 1);
void Adadelta(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t e, bool UI = 1);
void Adam(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t b2, real_t e, bool UI = 1);
void Adamax(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t b2, real_t e, bool UI = 1);
void Nadam(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t b2, real_t e, bool UI = 1);
void AMSGrad(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t b2, real_t e, bool UI = 1);
real_t score();
void save(std::string fileName);
void setLearningRateScheduler(std::string type, double decayConstant);
void setLearningRateScheduler(std::string type, double decayConstant, double dropRate);
void setLearningRateScheduler(std::string type, real_t decayConstant);
void setLearningRateScheduler(std::string type, real_t decayConstant, real_t dropRate);
void addLayer(int n_hidden, std::string activation, std::string weightInit = "Default", std::string reg = "None", double lambda = 0.5, double alpha = 0.5);
void addOutputLayer(std::string activation, std::string loss, std::string weightInit = "Default", std::string reg = "None", double lambda = 0.5, double alpha = 0.5);
void addLayer(int n_hidden, std::string activation, std::string weightInit = "Default", std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
void addOutputLayer(std::string activation, std::string loss, std::string weightInit = "Default", std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
private:
double applyLearningRateScheduler(double learningRate, double decayConstant, double epoch, double dropRate);
real_t applyLearningRateScheduler(real_t learningRate, real_t decayConstant, real_t epoch, real_t dropRate);
double Cost(std::vector<double> y_hat, std::vector<double> y);
real_t Cost(std::vector<real_t> y_hat, std::vector<real_t> y);
void forwardPass();
void updateParameters(std::vector<std::vector<std::vector<double>>> hiddenLayerUpdations, std::vector<double> outputLayerUpdation, double learning_rate);
std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<double>> computeGradients(std::vector<double> y_hat, std::vector<double> outputSet);
void updateParameters(std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations, std::vector<real_t> outputLayerUpdation, real_t learning_rate);
std::tuple<std::vector<std::vector<std::vector<real_t>>>, std::vector<real_t>> computeGradients(std::vector<real_t> y_hat, std::vector<real_t> outputSet);
void UI(int epoch, double cost_prev, std::vector<double> y_hat, std::vector<double> outputSet);
void UI(int epoch, real_t cost_prev, std::vector<real_t> y_hat, std::vector<real_t> outputSet);
std::vector<std::vector<double>> inputSet;
std::vector<double> outputSet;
std::vector<double> y_hat;
std::vector<std::vector<real_t>> inputSet;
std::vector<real_t> outputSet;
std::vector<real_t> y_hat;
std::vector<MLPPHiddenLayer> network;
MLPPOutputLayer *outputLayer;
@ -61,8 +63,8 @@ private:
int k;
std::string lrScheduler;
double decayConstant;
double dropRate;
real_t decayConstant;
real_t dropRate;
};
#endif /* ANN_hpp */

82
mlpp/auto_encoder/auto_encoder.cpp
View File

@ -13,7 +13,7 @@
#include <iostream>
#include <random>
MLPPAutoEncoder::MLPPAutoEncoder(std::vector<std::vector<double>> inputSet, int n_hidden) :
MLPPAutoEncoder::MLPPAutoEncoder(std::vector<std::vector<real_t>> inputSet, int n_hidden) :
inputSet(inputSet), n_hidden(n_hidden), n(inputSet.size()), k(inputSet[0].size()) {
MLPPActivation avn;
y_hat.resize(inputSet.size());
@ -24,18 +24,18 @@ MLPPAutoEncoder::MLPPAutoEncoder(std::vector<std::vector<double>> inputSet, int
bias2 = MLPPUtilities::biasInitialization(k);
}
std::vector<std::vector<double>> MLPPAutoEncoder::modelSetTest(std::vector<std::vector<double>> X) {
std::vector<std::vector<real_t>> MLPPAutoEncoder::modelSetTest(std::vector<std::vector<real_t>> X) {
return Evaluate(X);
}
std::vector<double> MLPPAutoEncoder::modelTest(std::vector<double> x) {
std::vector<real_t> MLPPAutoEncoder::modelTest(std::vector<real_t> x) {
return Evaluate(x);
}
void MLPPAutoEncoder::gradientDescent(double learning_rate, int max_epoch, bool UI) {
void MLPPAutoEncoder::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
MLPPActivation avn;
MLPPLinAlg alg;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
forwardPass();
@ -43,10 +43,10 @@ void MLPPAutoEncoder::gradientDescent(double learning_rate, int max_epoch, bool
cost_prev = Cost(y_hat, inputSet);
// Calculating the errors
std::vector<std::vector<double>> error = alg.subtraction(y_hat, inputSet);
std::vector<std::vector<real_t>> error = alg.subtraction(y_hat, inputSet);
// Calculating the weight/bias gradients for layer 2
std::vector<std::vector<double>> D2_1 = alg.matmult(alg.transpose(a2), error);
std::vector<std::vector<real_t>> D2_1 = alg.matmult(alg.transpose(a2), error);
// weights and bias updation for layer 2
weights2 = alg.subtraction(weights2, alg.scalarMultiply(learning_rate / n, D2_1));
@ -56,11 +56,11 @@ void MLPPAutoEncoder::gradientDescent(double learning_rate, int max_epoch, bool
//Calculating the weight/bias for layer 1
std::vector<std::vector<double>> D1_1 = alg.matmult(error, alg.transpose(weights2));
std::vector<std::vector<real_t>> D1_1 = alg.matmult(error, alg.transpose(weights2));
std::vector<std::vector<double>> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1));
std::vector<std::vector<real_t>> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1));
std::vector<std::vector<double>> D1_3 = alg.matmult(alg.transpose(inputSet), D1_2);
std::vector<std::vector<real_t>> D1_3 = alg.matmult(alg.transpose(inputSet), D1_2);
// weight an bias updation for layer 1
weights1 = alg.subtraction(weights1, alg.scalarMultiply(learning_rate / n, D1_3));
@ -85,10 +85,10 @@ void MLPPAutoEncoder::gradientDescent(double learning_rate, int max_epoch, bool
}
}
void MLPPAutoEncoder::SGD(double learning_rate, int max_epoch, bool UI) {
void MLPPAutoEncoder::SGD(real_t learning_rate, int max_epoch, bool UI) {
MLPPActivation avn;
MLPPLinAlg alg;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
while (true) {
@ -97,22 +97,22 @@ void MLPPAutoEncoder::SGD(double learning_rate, int max_epoch, bool UI) {
std::uniform_int_distribution<int> distribution(0, int(n - 1));
int outputIndex = distribution(generator);
std::vector<double> y_hat = Evaluate(inputSet[outputIndex]);
std::vector<real_t> y_hat = Evaluate(inputSet[outputIndex]);
auto [z2, a2] = propagate(inputSet[outputIndex]);
cost_prev = Cost({ y_hat }, { inputSet[outputIndex] });
std::vector<double> error = alg.subtraction(y_hat, inputSet[outputIndex]);
std::vector<real_t> error = alg.subtraction(y_hat, inputSet[outputIndex]);
// Weight updation for layer 2
std::vector<std::vector<double>> D2_1 = alg.outerProduct(error, a2);
std::vector<std::vector<real_t>> D2_1 = alg.outerProduct(error, a2);
weights2 = alg.subtraction(weights2, alg.scalarMultiply(learning_rate, alg.transpose(D2_1)));
// Bias updation for layer 2
bias2 = alg.subtraction(bias2, alg.scalarMultiply(learning_rate, error));
// Weight updation for layer 1
std::vector<double> D1_1 = alg.mat_vec_mult(weights2, error);
std::vector<double> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1));
std::vector<std::vector<double>> D1_3 = alg.outerProduct(inputSet[outputIndex], D1_2);
std::vector<real_t> D1_1 = alg.mat_vec_mult(weights2, error);
std::vector<real_t> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1));
std::vector<std::vector<real_t>> D1_3 = alg.outerProduct(inputSet[outputIndex], D1_2);
weights1 = alg.subtraction(weights1, alg.scalarMultiply(learning_rate, D1_3));
// Bias updation for layer 1
@ -136,28 +136,28 @@ void MLPPAutoEncoder::SGD(double learning_rate, int max_epoch, bool UI) {
forwardPass();
}
void MLPPAutoEncoder::MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI) {
void MLPPAutoEncoder::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI) {
MLPPActivation avn;
MLPPLinAlg alg;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
// Creating the mini-batches
int n_mini_batch = n / mini_batch_size;
std::vector<std::vector<std::vector<double>>> inputMiniBatches = MLPPUtilities::createMiniBatches(inputSet, n_mini_batch);
std::vector<std::vector<std::vector<real_t>>> inputMiniBatches = MLPPUtilities::createMiniBatches(inputSet, n_mini_batch);
while (true) {
for (int i = 0; i < n_mini_batch; i++) {
std::vector<std::vector<double>> y_hat = Evaluate(inputMiniBatches[i]);
std::vector<std::vector<real_t>> y_hat = Evaluate(inputMiniBatches[i]);
auto [z2, a2] = propagate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, inputMiniBatches[i]);
// Calculating the errors
std::vector<std::vector<double>> error = alg.subtraction(y_hat, inputMiniBatches[i]);
std::vector<std::vector<real_t>> error = alg.subtraction(y_hat, inputMiniBatches[i]);
// Calculating the weight/bias gradients for layer 2
std::vector<std::vector<double>> D2_1 = alg.matmult(alg.transpose(a2), error);
std::vector<std::vector<real_t>> D2_1 = alg.matmult(alg.transpose(a2), error);
// weights and bias updation for layer 2
weights2 = alg.subtraction(weights2, alg.scalarMultiply(learning_rate / inputMiniBatches[i].size(), D2_1));
@ -167,11 +167,11 @@ void MLPPAutoEncoder::MBGD(double learning_rate, int max_epoch, int mini_batch_s
//Calculating the weight/bias for layer 1
std::vector<std::vector<double>> D1_1 = alg.matmult(error, alg.transpose(weights2));
std::vector<std::vector<real_t>> D1_1 = alg.matmult(error, alg.transpose(weights2));
std::vector<std::vector<double>> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1));
std::vector<std::vector<real_t>> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1));
std::vector<std::vector<double>> D1_3 = alg.matmult(alg.transpose(inputMiniBatches[i]), D1_2);
std::vector<std::vector<real_t>> D1_3 = alg.matmult(alg.transpose(inputMiniBatches[i]), D1_2);
// weight an bias updation for layer 1
weights1 = alg.subtraction(weights1, alg.scalarMultiply(learning_rate / inputMiniBatches[i].size(), D1_3));
@ -196,7 +196,7 @@ void MLPPAutoEncoder::MBGD(double learning_rate, int max_epoch, int mini_batch_s
forwardPass();
}
double MLPPAutoEncoder::score() {
real_t MLPPAutoEncoder::score() {
MLPPUtilities util;
return util.performance(y_hat, inputSet);
}
@ -207,40 +207,40 @@ void MLPPAutoEncoder::save(std::string fileName) {
util.saveParameters(fileName, weights2, bias2, 1, 2);
}
double MLPPAutoEncoder::Cost(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) {
real_t MLPPAutoEncoder::Cost(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
class MLPPCost cost;
return cost.MSE(y_hat, inputSet);
}
std::vector<std::vector<double>> MLPPAutoEncoder::Evaluate(std::vector<std::vector<double>> X) {
std::vector<std::vector<real_t>> MLPPAutoEncoder::Evaluate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg;
MLPPActivation avn;
std::vector<std::vector<double>> z2 = alg.mat_vec_add(alg.matmult(X, weights1), bias1);
std::vector<std::vector<double>> a2 = avn.sigmoid(z2);
std::vector<std::vector<real_t>> z2 = alg.mat_vec_add(alg.matmult(X, weights1), bias1);
std::vector<std::vector<real_t>> a2 = avn.sigmoid(z2);
return alg.mat_vec_add(alg.matmult(a2, weights2), bias2);
}
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> MLPPAutoEncoder::propagate(std::vector<std::vector<double>> X) {
std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> MLPPAutoEncoder::propagate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg;
MLPPActivation avn;
std::vector<std::vector<double>> z2 = alg.mat_vec_add(alg.matmult(X, weights1), bias1);
std::vector<std::vector<double>> a2 = avn.sigmoid(z2);
std::vector<std::vector<real_t>> z2 = alg.mat_vec_add(alg.matmult(X, weights1), bias1);
std::vector<std::vector<real_t>> a2 = avn.sigmoid(z2);
return { z2, a2 };
}
std::vector<double> MLPPAutoEncoder::Evaluate(std::vector<double> x) {
std::vector<real_t> MLPPAutoEncoder::Evaluate(std::vector<real_t> x) {
MLPPLinAlg alg;
MLPPActivation avn;
std::vector<double> z2 = alg.addition(alg.mat_vec_mult(alg.transpose(weights1), x), bias1);
std::vector<double> a2 = avn.sigmoid(z2);
std::vector<real_t> z2 = alg.addition(alg.mat_vec_mult(alg.transpose(weights1), x), bias1);
std::vector<real_t> a2 = avn.sigmoid(z2);
return alg.addition(alg.mat_vec_mult(alg.transpose(weights2), a2), bias2);
}
std::tuple<std::vector<double>, std::vector<double>> MLPPAutoEncoder::propagate(std::vector<double> x) {
std::tuple<std::vector<real_t>, std::vector<real_t>> MLPPAutoEncoder::propagate(std::vector<real_t> x) {
MLPPLinAlg alg;
MLPPActivation avn;
std::vector<double> z2 = alg.addition(alg.mat_vec_mult(alg.transpose(weights1), x), bias1);
std::vector<double> a2 = avn.sigmoid(z2);
std::vector<real_t> z2 = alg.addition(alg.mat_vec_mult(alg.transpose(weights1), x), bias1);
std::vector<real_t> a2 = avn.sigmoid(z2);
return { z2, a2 };
}

42
mlpp/auto_encoder/auto_encoder.h
View File

@ -8,41 +8,43 @@
// Created by Marc Melikyan on 11/4/20.
//
#include "core/math/math_defs.h"
#include <string>
#include <tuple>
#include <vector>
class MLPPAutoEncoder {
public:
MLPPAutoEncoder(std::vector<std::vector<double>> inputSet, int n_hidden);
std::vector<std::vector<double>> modelSetTest(std::vector<std::vector<double>> X);
std::vector<double> modelTest(std::vector<double> x);
void gradientDescent(double learning_rate, int max_epoch, bool UI = 1);
void SGD(double learning_rate, int max_epoch, bool UI = 1);
void MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
double score();
MLPPAutoEncoder(std::vector<std::vector<real_t>> inputSet, int n_hidden);
std::vector<std::vector<real_t>> modelSetTest(std::vector<std::vector<real_t>> X);
std::vector<real_t> modelTest(std::vector<real_t> x);
void gradientDescent(real_t learning_rate, int max_epoch, bool UI = 1);
void SGD(real_t learning_rate, int max_epoch, bool UI = 1);
void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
real_t score();
void save(std::string fileName);
private:
double Cost(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y);
real_t Cost(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
std::vector<std::vector<double>> Evaluate(std::vector<std::vector<double>> X);
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> propagate(std::vector<std::vector<double>> X);
std::vector<double> Evaluate(std::vector<double> x);
std::tuple<std::vector<double>, std::vector<double>> propagate(std::vector<double> x);
std::vector<std::vector<real_t>> Evaluate(std::vector<std::vector<real_t>> X);
std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> propagate(std::vector<std::vector<real_t>> X);
std::vector<real_t> Evaluate(std::vector<real_t> x);
std::tuple<std::vector<real_t>, std::vector<real_t>> propagate(std::vector<real_t> x);
void forwardPass();
std::vector<std::vector<double>> inputSet;
std::vector<std::vector<double>> y_hat;
std::vector<std::vector<real_t>> inputSet;
std::vector<std::vector<real_t>> y_hat;
std::vector<std::vector<double>> weights1;
std::vector<std::vector<double>> weights2;
std::vector<std::vector<real_t>> weights1;
std::vector<std::vector<real_t>> weights2;
std::vector<double> bias1;
std::vector<double> bias2;
std::vector<real_t> bias1;
std::vector<real_t> bias2;
std::vector<std::vector<double>> z2;
std::vector<std::vector<double>> a2;
std::vector<std::vector<real_t>> z2;
std::vector<std::vector<real_t>> a2;
int n;
int k;

22
mlpp/bernoulli_nb/bernoulli_nb.cpp
View File

@ -12,23 +12,23 @@
#include <iostream>
#include <random>
MLPPBernoulliNB::MLPPBernoulliNB(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet) :
MLPPBernoulliNB::MLPPBernoulliNB(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet) :
inputSet(inputSet), outputSet(outputSet), class_num(2) {
y_hat.resize(outputSet.size());
Evaluate();
}
std::vector<double> MLPPBernoulliNB::modelSetTest(std::vector<std::vector<double>> X) {
std::vector<double> y_hat;
std::vector<real_t> MLPPBernoulliNB::modelSetTest(std::vector<std::vector<real_t>> X) {
std::vector<real_t> y_hat;
for (int i = 0; i < X.size(); i++) {
y_hat.push_back(modelTest(X[i]));
}
return y_hat;
}
double MLPPBernoulliNB::modelTest(std::vector<double> x) {
double score_0 = 1;
double score_1 = 1;
real_t MLPPBernoulliNB::modelTest(std::vector<real_t> x) {
real_t score_0 = 1;
real_t score_1 = 1;
std::vector<int> foundIndices;
@ -68,7 +68,7 @@ double MLPPBernoulliNB::modelTest(std::vector<double> x) {
}
}
double MLPPBernoulliNB::score() {
real_t MLPPBernoulliNB::score() {
MLPPUtilities util;
return util.performance(y_hat, outputSet);
}
@ -76,7 +76,7 @@ double MLPPBernoulliNB::score() {
void MLPPBernoulliNB::computeVocab() {
MLPPLinAlg alg;
MLPPData data;
vocab = data.vecToSet<double>(alg.flatten(inputSet));
vocab = data.vecToSet<real_t>(alg.flatten(inputSet));
}
void MLPPBernoulliNB::computeTheta() {
@ -110,10 +110,10 @@ void MLPPBernoulliNB::computeTheta() {
void MLPPBernoulliNB::Evaluate() {
for (int i = 0; i < outputSet.size(); i++) {
// Pr(B | A) * Pr(A)
double score_0 = 1;
double score_1 = 1;
real_t score_0 = 1;
real_t score_1 = 1;
double sum = 0;
real_t sum = 0;
for (int i = 0; i < outputSet.size(); i++) {
if (outputSet[i] == 1) {
sum += outputSet[i];

24
mlpp/bernoulli_nb/bernoulli_nb.h
View File

@ -8,15 +8,17 @@
// Created by Marc Melikyan on 1/17/21.
//
#include "core/math/math_defs.h"
#include <map>
#include <vector>
class MLPPBernoulliNB {
public:
MLPPBernoulliNB(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet);
std::vector<double> modelSetTest(std::vector<std::vector<double>> X);
double modelTest(std::vector<double> x);
double score();
MLPPBernoulliNB(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet);
std::vector<real_t> modelSetTest(std::vector<std::vector<real_t>> X);
real_t modelTest(std::vector<real_t> x);
real_t score();
private:
void computeVocab();
@ -24,17 +26,17 @@ private:
void Evaluate();
// Model Params
double prior_1 = 0;
double prior_0 = 0;
real_t prior_1 = 0;
real_t prior_0 = 0;
std::vector<std::map<double, int>> theta;
std::vector<double> vocab;
std::vector<std::map<real_t, int>> theta;
std::vector<real_t> vocab;
int class_num;
// Datasets
std::vector<std::vector<double>> inputSet;
std::vector<double> outputSet;
std::vector<double> y_hat;
std::vector<std::vector<real_t>> inputSet;
std::vector<real_t> outputSet;
std::vector<real_t> y_hat;
};
#endif /* BernoulliNB_hpp */

50
mlpp/c_log_log_reg/c_log_log_reg.cpp
View File

@ -14,33 +14,33 @@
#include <iostream>
#include <random>
MLPPCLogLogReg::MLPPCLogLogReg(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, std::string reg, double lambda, double alpha) :
MLPPCLogLogReg::MLPPCLogLogReg(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, std::string reg, real_t lambda, real_t alpha) :
inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) {
y_hat.resize(n);
weights = MLPPUtilities::weightInitialization(k);
bias = MLPPUtilities::biasInitialization();
}
std::vector<double> MLPPCLogLogReg::modelSetTest(std::vector<std::vector<double>> X) {
std::vector<real_t> MLPPCLogLogReg::modelSetTest(std::vector<std::vector<real_t>> X) {
return Evaluate(X);
}
double MLPPCLogLogReg::modelTest(std::vector<double> x) {
real_t MLPPCLogLogReg::modelTest(std::vector<real_t> x) {
return Evaluate(x);
}
void MLPPCLogLogReg::gradientDescent(double learning_rate, int max_epoch, bool UI) {
void MLPPCLogLogReg::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
MLPPActivation avn;
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
forwardPass();
while (true) {
cost_prev = Cost(y_hat, outputSet);
std::vector<double> error = alg.subtraction(y_hat, outputSet);
std::vector<real_t> error = alg.subtraction(y_hat, outputSet);
// Calculating the weight gradients
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputSet), alg.hadamard_product(error, avn.cloglog(z, 1)))));
@ -63,18 +63,18 @@ void MLPPCLogLogReg::gradientDescent(double learning_rate, int max_epoch, bool U
}
}
void MLPPCLogLogReg::MLE(double learning_rate, int max_epoch, bool UI) {
void MLPPCLogLogReg::MLE(real_t learning_rate, int max_epoch, bool UI) {
MLPPActivation avn;
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
forwardPass();
while (true) {
cost_prev = Cost(y_hat, outputSet);
std::vector<double> error = alg.subtraction(y_hat, outputSet);
std::vector<real_t> error = alg.subtraction(y_hat, outputSet);
weights = alg.addition(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputSet), alg.hadamard_product(error, avn.cloglog(z, 1)))));
weights = regularization.regWeights(weights, lambda, alpha, reg);
@ -95,10 +95,10 @@ void MLPPCLogLogReg::MLE(double learning_rate, int max_epoch, bool UI) {
}
}
void MLPPCLogLogReg::SGD(double learning_rate, int max_epoch, bool UI) {
void MLPPCLogLogReg::SGD(real_t learning_rate, int max_epoch, bool UI) {
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
forwardPass();
@ -108,11 +108,11 @@ void MLPPCLogLogReg::SGD(double learning_rate, int max_epoch, bool UI) {
std::uniform_int_distribution<int> distribution(0, int(n - 1));
int outputIndex = distribution(generator);
double y_hat = Evaluate(inputSet[outputIndex]);
double z = propagate(inputSet[outputIndex]);
real_t y_hat = Evaluate(inputSet[outputIndex]);
real_t z = propagate(inputSet[outputIndex]);
cost_prev = Cost({ y_hat }, { outputSet[outputIndex] });
double error = y_hat - outputSet[outputIndex];
real_t error = y_hat - outputSet[outputIndex];
// Weight Updation
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate * error * exp(z - exp(z)), inputSet[outputIndex]));
@ -136,11 +136,11 @@ void MLPPCLogLogReg::SGD(double learning_rate, int max_epoch, bool UI) {
forwardPass();
}
void MLPPCLogLogReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI) {
void MLPPCLogLogReg::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI) {
MLPPActivation avn;
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
// Creating the mini-batches
@ -149,11 +149,11 @@ void MLPPCLogLogReg::MBGD(double learning_rate, int max_epoch, int mini_batch_si
while (true) {
for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = Evaluate(inputMiniBatches[i]);
std::vector<double> z = propagate(inputMiniBatches[i]);
std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]);
std::vector<real_t> z = propagate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]);
std::vector<double> error = alg.subtraction(y_hat, outputMiniBatches[i]);
std::vector<real_t> error = alg.subtraction(y_hat, outputMiniBatches[i]);
// Calculating the weight gradients
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), alg.hadamard_product(error, avn.cloglog(z, 1)))));
@ -179,35 +179,35 @@ void MLPPCLogLogReg::MBGD(double learning_rate, int max_epoch, int mini_batch_si
forwardPass();
}
double MLPPCLogLogReg::score() {
real_t MLPPCLogLogReg::score() {
MLPPUtilities util;
return util.performance(y_hat, outputSet);
}
double MLPPCLogLogReg::Cost(std::vector<double> y_hat, std::vector<double> y) {
real_t MLPPCLogLogReg::Cost(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPReg regularization;
class MLPPCost cost;
return cost.MSE(y_hat, y) + regularization.regTerm(weights, lambda, alpha, reg);
}
std::vector<double> MLPPCLogLogReg::Evaluate(std::vector<std::vector<double>> X) {
std::vector<real_t> MLPPCLogLogReg::Evaluate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg;
MLPPActivation avn;
return avn.cloglog(alg.scalarAdd(bias, alg.mat_vec_mult(X, weights)));
}
std::vector<double> MLPPCLogLogReg::propagate(std::vector<std::vector<double>> X) {
std::vector<real_t> MLPPCLogLogReg::propagate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg;
return alg.scalarAdd(bias, alg.mat_vec_mult(X, weights));
}
double MLPPCLogLogReg::Evaluate(std::vector<double> x) {
real_t MLPPCLogLogReg::Evaluate(std::vector<real_t> x) {
MLPPLinAlg alg;
MLPPActivation avn;
return avn.cloglog(alg.dot(weights, x) + bias);
}
double MLPPCLogLogReg::propagate(std::vector<double> x) {
real_t MLPPCLogLogReg::propagate(std::vector<real_t> x) {
MLPPLinAlg alg;
return alg.dot(weights, x) + bias;
}

44
mlpp/c_log_log_reg/c_log_log_reg.h
View File

@ -8,45 +8,47 @@
// Created by Marc Melikyan on 10/2/20.
//
#include "core/math/math_defs.h"
#include <string>
#include <vector>
class MLPPCLogLogReg {
public:
MLPPCLogLogReg(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, std::string reg = "None", double lambda = 0.5, double alpha = 0.5);
std::vector<double> modelSetTest(std::vector<std::vector<double>> X);
double modelTest(std::vector<double> x);
void gradientDescent(double learning_rate, int max_epoch, bool UI = 1);
void MLE(double learning_rate, int max_epoch, bool UI = 1);
void SGD(double learning_rate, int max_epoch, bool UI = 1);
void MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
double score();
MLPPCLogLogReg(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
std::vector<real_t> modelSetTest(std::vector<std::vector<real_t>> X);
real_t modelTest(std::vector<real_t> x);
void gradientDescent(real_t learning_rate, int max_epoch, bool UI = 1);
void MLE(real_t learning_rate, int max_epoch, bool UI = 1);
void SGD(real_t learning_rate, int max_epoch, bool UI = 1);
void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
real_t score();
private:
void weightInitialization(int k);
void biasInitialization();
double Cost(std::vector<double> y_hat, std::vector<double> y);
real_t Cost(std::vector<real_t> y_hat, std::vector<real_t> y);
std::vector<double> Evaluate(std::vector<std::vector<double>> X);
std::vector<double> propagate(std::vector<std::vector<double>> X);
double Evaluate(std::vector<double> x);
double propagate(std::vector<double> x);
std::vector<real_t> Evaluate(std::vector<std::vector<real_t>> X);
std::vector<real_t> propagate(std::vector<std::vector<real_t>> X);
real_t Evaluate(std::vector<real_t> x);
real_t propagate(std::vector<real_t> x);
void forwardPass();
std::vector<std::vector<double>> inputSet;
std::vector<double> outputSet;
std::vector<double> y_hat;
std::vector<double> z;
std::vector<double> weights;
double bias;
std::vector<std::vector<real_t>> inputSet;
std::vector<real_t> outputSet;
std::vector<real_t> y_hat;
std::vector<real_t> z;
std::vector<real_t> weights;
real_t bias;
int n;
int k;
// Regularization Params
std::string reg;
double lambda;
double alpha; /* This is the controlling param for Elastic Net*/
real_t lambda;
real_t alpha; /* This is the controlling param for Elastic Net*/
};
#endif /* CLogLogReg_hpp */

112
mlpp/convolutions/convolutions.cpp
View File

@ -14,15 +14,15 @@ MLPPConvolutions::MLPPConvolutions() :
prewittHorizontal({ { 1, 1, 1 }, { 0, 0, 0 }, { -1, -1, -1 } }), prewittVertical({ { 1, 0, -1 }, { 1, 0, -1 }, { 1, 0, -1 } }), sobelHorizontal({ { 1, 2, 1 }, { 0, 0, 0 }, { -1, -2, -1 } }), sobelVertical({ { -1, 0, 1 }, { -2, 0, 2 }, { -1, 0, 1 } }), scharrHorizontal({ { 3, 10, 3 }, { 0, 0, 0 }, { -3, -10, -3 } }), scharrVertical({ { 3, 0, -3 }, { 10, 0, -10 }, { 3, 0, -3 } }), robertsHorizontal({ { 0, 1 }, { -1, 0 } }), robertsVertical({ { 1, 0 }, { 0, -1 } }) {
}
std::vector<std::vector<double>> MLPPConvolutions::convolve(std::vector<std::vector<double>> input, std::vector<std::vector<double>> filter, int S, int P) {
std::vector<std::vector<real_t>> MLPPConvolutions::convolve(std::vector<std::vector<real_t>> input, std::vector<std::vector<real_t>> filter, int S, int P) {
MLPPLinAlg alg;
std::vector<std::vector<double>> featureMap;
std::vector<std::vector<real_t>> featureMap;
int N = input.size();
int F = filter.size();
int mapSize = (N - F + 2 * P) / S + 1; // This is computed as ⌊mapSize⌋ by def- thanks C++!
if (P != 0) {
std::vector<std::vector<double>> paddedInput;
std::vector<std::vector<real_t>> paddedInput;
paddedInput.resize(N + 2 * P);
for (int i = 0; i < paddedInput.size(); i++) {
paddedInput[i].resize(N + 2 * P);
@ -50,7 +50,7 @@ std::vector<std::vector<double>> MLPPConvolutions::convolve(std::vector<std::vec
for (int i = 0; i < mapSize; i++) {
for (int j = 0; j < mapSize; j++) {
std::vector<double> convolvingInput;
std::vector<real_t> convolvingInput;
for (int k = 0; k < F; k++) {
for (int p = 0; p < F; p++) {
if (i == 0 && j == 0) {
@ -70,9 +70,9 @@ std::vector<std::vector<double>> MLPPConvolutions::convolve(std::vector<std::vec
return featureMap;
}
std::vector<std::vector<std::vector<double>>> MLPPConvolutions::convolve(std::vector<std::vector<std::vector<double>>> input, std::vector<std::vector<std::vector<double>>> filter, int S, int P) {
std::vector<std::vector<std::vector<real_t>>> MLPPConvolutions::convolve(std::vector<std::vector<std::vector<real_t>>> input, std::vector<std::vector<std::vector<real_t>>> filter, int S, int P) {
MLPPLinAlg alg;
std::vector<std::vector<std::vector<double>>> featureMap;
std::vector<std::vector<std::vector<real_t>>> featureMap;
int N = input[0].size();
int F = filter[0].size();
int C = filter.size() / input.size();
@ -80,7 +80,7 @@ std::vector<std::vector<std::vector<double>>> MLPPConvolutions::convolve(std::ve
if (P != 0) {
for (int c = 0; c < input.size(); c++) {
std::vector<std::vector<double>> paddedInput;
std::vector<std::vector<real_t>> paddedInput;
paddedInput.resize(N + 2 * P);
for (int i = 0; i < paddedInput.size(); i++) {
paddedInput[i].resize(N + 2 * P);
@ -113,7 +113,7 @@ std::vector<std::vector<std::vector<double>>> MLPPConvolutions::convolve(std::ve
for (int c = 0; c < C; c++) {
for (int i = 0; i < mapSize; i++) {
for (int j = 0; j < mapSize; j++) {
std::vector<double> convolvingInput;
std::vector<real_t> convolvingInput;
for (int t = 0; t < input.size(); t++) {
for (int k = 0; k < F; k++) {
for (int p = 0; p < F; p++) {
@ -136,9 +136,9 @@ std::vector<std::vector<std::vector<double>>> MLPPConvolutions::convolve(std::ve
return featureMap;
}
std::vector<std::vector<double>> MLPPConvolutions::pool(std::vector<std::vector<double>> input, int F, int S, std::string type) {
std::vector<std::vector<real_t>> MLPPConvolutions::pool(std::vector<std::vector<real_t>> input, int F, int S, std::string type) {
MLPPLinAlg alg;
std::vector<std::vector<double>> pooledMap;
std::vector<std::vector<real_t>> pooledMap;
int N = input.size();
int mapSize = floor((N - F) / S + 1);
@ -149,7 +149,7 @@ std::vector<std::vector<double>> MLPPConvolutions::pool(std::vector<std::vector<
for (int i = 0; i < mapSize; i++) {
for (int j = 0; j < mapSize; j++) {
std::vector<double> poolingInput;
std::vector<real_t> poolingInput;
for (int k = 0; k < F; k++) {
for (int p = 0; p < F; p++) {
if (i == 0 && j == 0) {
@ -176,15 +176,15 @@ std::vector<std::vector<double>> MLPPConvolutions::pool(std::vector<std::vector<
return pooledMap;
}
std::vector<std::vector<std::vector<double>>> MLPPConvolutions::pool(std::vector<std::vector<std::vector<double>>> input, int F, int S, std::string type) {
std::vector<std::vector<std::vector<double>>> pooledMap;
std::vector<std::vector<std::vector<real_t>>> MLPPConvolutions::pool(std::vector<std::vector<std::vector<real_t>>> input, int F, int S, std::string type) {
std::vector<std::vector<std::vector<real_t>>> pooledMap;
for (int i = 0; i < input.size(); i++) {
pooledMap.push_back(pool(input[i], F, S, type));
}
return pooledMap;
}
double MLPPConvolutions::globalPool(std::vector<std::vector<double>> input, std::string type) {
real_t MLPPConvolutions::globalPool(std::vector<std::vector<real_t>> input, std::string type) {
MLPPLinAlg alg;
if (type == "Average") {
MLPPStat stat;
@ -196,21 +196,21 @@ double MLPPConvolutions::globalPool(std::vector<std::vector<double>> input, std:
}
}
std::vector<double> MLPPConvolutions::globalPool(std::vector<std::vector<std::vector<double>>> input, std::string type) {
std::vector<double> pooledMap;
std::vector<real_t> MLPPConvolutions::globalPool(std::vector<std::vector<std::vector<real_t>>> input, std::string type) {
std::vector<real_t> pooledMap;
for (int i = 0; i < input.size(); i++) {
pooledMap.push_back(globalPool(input[i], type));
}
return pooledMap;
}
double MLPPConvolutions::gaussian2D(double x, double y, double std) {
double std_sq = std * std;
real_t MLPPConvolutions::gaussian2D(real_t x, real_t y, real_t std) {
real_t std_sq = std * std;
return 1 / (2 * M_PI * std_sq) * std::exp(-(x * x + y * y) / 2 * std_sq);
}
std::vector<std::vector<double>> MLPPConvolutions::gaussianFilter2D(int size, double std) {
std::vector<std::vector<double>> filter;
std::vector<std::vector<real_t>> MLPPConvolutions::gaussianFilter2D(int size, real_t std) {
std::vector<std::vector<real_t>> filter;
filter.resize(size);
for (int i = 0; i < filter.size(); i++) {
filter[i].resize(size);
@ -229,8 +229,8 @@ been easier to carry out the calculation explicitly, mainly because it is more i
and also because my convolution algorithm is only built for filters with equally sized
heights and widths.
*/
std::vector<std::vector<double>> MLPPConvolutions::dx(std::vector<std::vector<double>> input) {
std::vector<std::vector<double>> deriv; // We assume a gray scale image.
std::vector<std::vector<real_t>> MLPPConvolutions::dx(std::vector<std::vector<real_t>> input) {
std::vector<std::vector<real_t>> deriv; // We assume a gray scale image.
deriv.resize(input.size());
for (int i = 0; i < deriv.size(); i++) {
deriv[i].resize(input[i].size());
@ -250,8 +250,8 @@ std::vector<std::vector<double>> MLPPConvolutions::dx(std::vector<std::vector<do
return deriv;
}
std::vector<std::vector<double>> MLPPConvolutions::dy(std::vector<std::vector<double>> input) {
std::vector<std::vector<double>> deriv;
std::vector<std::vector<real_t>> MLPPConvolutions::dy(std::vector<std::vector<real_t>> input) {
std::vector<std::vector<real_t>> deriv;
deriv.resize(input.size());
for (int i = 0; i < deriv.size(); i++) {
deriv[i].resize(input[i].size());
@ -271,22 +271,22 @@ std::vector<std::vector<double>> MLPPConvolutions::dy(std::vector<std::vector<do
return deriv;
}
std::vector<std::vector<double>> MLPPConvolutions::gradMagnitude(std::vector<std::vector<double>> input) {
std::vector<std::vector<real_t>> MLPPConvolutions::gradMagnitude(std::vector<std::vector<real_t>> input) {
MLPPLinAlg alg;
std::vector<std::vector<double>> xDeriv_2 = alg.hadamard_product(dx(input), dx(input));
std::vector<std::vector<double>> yDeriv_2 = alg.hadamard_product(dy(input), dy(input));
std::vector<std::vector<real_t>> xDeriv_2 = alg.hadamard_product(dx(input), dx(input));
std::vector<std::vector<real_t>> yDeriv_2 = alg.hadamard_product(dy(input), dy(input));
return alg.sqrt(alg.addition(xDeriv_2, yDeriv_2));
}
std::vector<std::vector<double>> MLPPConvolutions::gradOrientation(std::vector<std::vector<double>> input) {
std::vector<std::vector<double>> deriv;
std::vector<std::vector<real_t>> MLPPConvolutions::gradOrientation(std::vector<std::vector<real_t>> input) {
std::vector<std::vector<real_t>> deriv;
deriv.resize(input.size());
for (int i = 0; i < deriv.size(); i++) {
deriv[i].resize(input[i].size());
}
std::vector<std::vector<double>> xDeriv = dx(input);
std::vector<std::vector<double>> yDeriv = dy(input);
std::vector<std::vector<real_t>> xDeriv = dx(input);
std::vector<std::vector<real_t>> yDeriv = dy(input);
for (int i = 0; i < deriv.size(); i++) {
for (int j = 0; j < deriv[i].size(); j++) {
deriv[i][j] = std::atan2(yDeriv[i][j], xDeriv[i][j]);
@ -295,33 +295,33 @@ std::vector<std::vector<double>> MLPPConvolutions::gradOrientation(std::vector<s
return deriv;
}
std::vector<std::vector<std::vector<double>>> MLPPConvolutions::computeM(std::vector<std::vector<double>> input) {
double const SIGMA = 1;
double const GAUSSIAN_SIZE = 3;
std::vector<std::vector<std::vector<real_t>>> MLPPConvolutions::computeM(std::vector<std::vector<real_t>> input) {
real_t const SIGMA = 1;
real_t const GAUSSIAN_SIZE = 3;
double const GAUSSIAN_PADDING = ((input.size() - 1) + GAUSSIAN_SIZE - input.size()) / 2; // Convs must be same.
real_t const GAUSSIAN_PADDING = ((input.size() - 1) + GAUSSIAN_SIZE - input.size()) / 2; // Convs must be same.
std::cout << GAUSSIAN_PADDING << std::endl;
MLPPLinAlg alg;
std::vector<std::vector<double>> xDeriv = dx(input);
std::vector<std::vector<double>> yDeriv = dy(input);
std::vector<std::vector<real_t>> xDeriv = dx(input);
std::vector<std::vector<real_t>> yDeriv = dy(input);
std::vector<std::vector<double>> gaussianFilter = gaussianFilter2D(GAUSSIAN_SIZE, SIGMA); // Sigma of 1, size of 3.
std::vector<std::vector<double>> xxDeriv = convolve(alg.hadamard_product(xDeriv, xDeriv), gaussianFilter, 1, GAUSSIAN_PADDING);
std::vector<std::vector<double>> yyDeriv = convolve(alg.hadamard_product(yDeriv, yDeriv), gaussianFilter, 1, GAUSSIAN_PADDING);
std::vector<std::vector<double>> xyDeriv = convolve(alg.hadamard_product(xDeriv, yDeriv), gaussianFilter, 1, GAUSSIAN_PADDING);
std::vector<std::vector<real_t>> gaussianFilter = gaussianFilter2D(GAUSSIAN_SIZE, SIGMA); // Sigma of 1, size of 3.
std::vector<std::vector<real_t>> xxDeriv = convolve(alg.hadamard_product(xDeriv, xDeriv), gaussianFilter, 1, GAUSSIAN_PADDING);
std::vector<std::vector<real_t>> yyDeriv = convolve(alg.hadamard_product(yDeriv, yDeriv), gaussianFilter, 1, GAUSSIAN_PADDING);
std::vector<std::vector<real_t>> xyDeriv = convolve(alg.hadamard_product(xDeriv, yDeriv), gaussianFilter, 1, GAUSSIAN_PADDING);
std::vector<std::vector<std::vector<double>>> M = { xxDeriv, yyDeriv, xyDeriv };
std::vector<std::vector<std::vector<real_t>>> M = { xxDeriv, yyDeriv, xyDeriv };
return M;
}
std::vector<std::vector<std::string>> MLPPConvolutions::harrisCornerDetection(std::vector<std::vector<double>> input) {
double const k = 0.05; // Empirically determined wherein k -> [0.04, 0.06], though conventionally 0.05 is typically used as well.
std::vector<std::vector<std::string>> MLPPConvolutions::harrisCornerDetection(std::vector<std::vector<real_t>> input) {
real_t const k = 0.05; // Empirically determined wherein k -> [0.04, 0.06], though conventionally 0.05 is typically used as well.
MLPPLinAlg alg;
std::vector<std::vector<std::vector<double>>> M = computeM(input);
std::vector<std::vector<double>> det = alg.subtraction(alg.hadamard_product(M[0], M[1]), alg.hadamard_product(M[2], M[2]));
std::vector<std::vector<double>> trace = alg.addition(M[0], M[1]);
std::vector<std::vector<std::vector<real_t>>> M = computeM(input);
std::vector<std::vector<real_t>> det = alg.subtraction(alg.hadamard_product(M[0], M[1]), alg.hadamard_product(M[2], M[2]));
std::vector<std::vector<real_t>> trace = alg.addition(M[0], M[1]);
// The reason this is not a scalar is because xxDeriv, xyDeriv, yxDeriv, and yyDeriv are not scalars.
std::vector<std::vector<double>> r = alg.subtraction(det, alg.scalarMultiply(k, alg.hadamard_product(trace, trace)));
std::vector<std::vector<real_t>> r = alg.subtraction(det, alg.scalarMultiply(k, alg.hadamard_product(trace, trace)));
std::vector<std::vector<std::string>> imageTypes;
imageTypes.resize(r.size());
alg.printMatrix(r);
@ -340,34 +340,34 @@ std::vector<std::vector<std::string>> MLPPConvolutions::harrisCornerDetection(st
return imageTypes;
}
std::vector<std::vector<double>> MLPPConvolutions::getPrewittHorizontal() {
std::vector<std::vector<real_t>> MLPPConvolutions::getPrewittHorizontal() {
return prewittHorizontal;
}
std::vector<std::vector<double>> MLPPConvolutions::getPrewittVertical() {
std::vector<std::vector<real_t>> MLPPConvolutions::getPrewittVertical() {
return prewittVertical;
}
std::vector<std::vector<double>> MLPPConvolutions::getSobelHorizontal() {
std::vector<std::vector<real_t>> MLPPConvolutions::getSobelHorizontal() {
return sobelHorizontal;
}
std::vector<std::vector<double>> MLPPConvolutions::getSobelVertical() {
std::vector<std::vector<real_t>> MLPPConvolutions::getSobelVertical() {
return sobelVertical;
}
std::vector<std::vector<double>> MLPPConvolutions::getScharrHorizontal() {
std::vector<std::vector<real_t>> MLPPConvolutions::getScharrHorizontal() {
return scharrHorizontal;
}
std::vector<std::vector<double>> MLPPConvolutions::getScharrVertical() {
std::vector<std::vector<real_t>> MLPPConvolutions::getScharrVertical() {
return scharrVertical;
}
std::vector<std::vector<double>> MLPPConvolutions::getRobertsHorizontal() {
std::vector<std::vector<real_t>> MLPPConvolutions::getRobertsHorizontal() {
return robertsHorizontal;
}
std::vector<std::vector<double>> MLPPConvolutions::getRobertsVertical() {
std::vector<std::vector<real_t>> MLPPConvolutions::getRobertsVertical() {
return robertsVertical;
}

62
mlpp/convolutions/convolutions.h
View File

@ -5,46 +5,48 @@
#include <vector>
#include <string>
#include "core/math/math_defs.h"
class MLPPConvolutions {
public:
MLPPConvolutions();
std::vector<std::vector<double>> convolve(std::vector<std::vector<double>> input, std::vector<std::vector<double>> filter, int S, int P = 0);
std::vector<std::vector<std::vector<double>>> convolve(std::vector<std::vector<std::vector<double>>> input, std::vector<std::vector<std::vector<double>>> filter, int S, int P = 0);
std::vector<std::vector<double>> pool(std::vector<std::vector<double>> input, int F, int S, std::string type);
std::vector<std::vector<std::vector<double>>> pool(std::vector<std::vector<std::vector<double>>> input, int F, int S, std::string type);
double globalPool(std::vector<std::vector<double>> input, std::string type);
std::vector<double> globalPool(std::vector<std::vector<std::vector<double>>> input, std::string type);
std::vector<std::vector<real_t>> convolve(std::vector<std::vector<real_t>> input, std::vector<std::vector<real_t>> filter, int S, int P = 0);
std::vector<std::vector<std::vector<real_t>>> convolve(std::vector<std::vector<std::vector<real_t>>> input, std::vector<std::vector<std::vector<real_t>>> filter, int S, int P = 0);
std::vector<std::vector<real_t>> pool(std::vector<std::vector<real_t>> input, int F, int S, std::string type);
std::vector<std::vector<std::vector<real_t>>> pool(std::vector<std::vector<std::vector<real_t>>> input, int F, int S, std::string type);
real_t globalPool(std::vector<std::vector<real_t>> input, std::string type);
std::vector<real_t> globalPool(std::vector<std::vector<std::vector<real_t>>> input, std::string type);
double gaussian2D(double x, double y, double std);
std::vector<std::vector<double>> gaussianFilter2D(int size, double std);
real_t gaussian2D(real_t x, real_t y, real_t std);
std::vector<std::vector<real_t>> gaussianFilter2D(int size, real_t std);
std::vector<std::vector<double>> dx(std::vector<std::vector<double>> input);
std::vector<std::vector<double>> dy(std::vector<std::vector<double>> input);
std::vector<std::vector<real_t>> dx(std::vector<std::vector<real_t>> input);
std::vector<std::vector<real_t>> dy(std::vector<std::vector<real_t>> input);
std::vector<std::vector<double>> gradMagnitude(std::vector<std::vector<double>> input);
std::vector<std::vector<double>> gradOrientation(std::vector<std::vector<double>> input);
std::vector<std::vector<real_t>> gradMagnitude(std::vector<std::vector<real_t>> input);
std::vector<std::vector<real_t>> gradOrientation(std::vector<std::vector<real_t>> input);
std::vector<std::vector<std::vector<double>>> computeM(std::vector<std::vector<double>> input);
std::vector<std::vector<std::string>> harrisCornerDetection(std::vector<std::vector<double>> input);
std::vector<std::vector<std::vector<real_t>>> computeM(std::vector<std::vector<real_t>> input);
std::vector<std::vector<std::string>> harrisCornerDetection(std::vector<std::vector<real_t>> input);
std::vector<std::vector<double>> getPrewittHorizontal();
std::vector<std::vector<double>> getPrewittVertical();
std::vector<std::vector<double>> getSobelHorizontal();
std::vector<std::vector<double>> getSobelVertical();
std::vector<std::vector<double>> getScharrHorizontal();
std::vector<std::vector<double>> getScharrVertical();
std::vector<std::vector<double>> getRobertsHorizontal();
std::vector<std::vector<double>> getRobertsVertical();
std::vector<std::vector<real_t>> getPrewittHorizontal();
std::vector<std::vector<real_t>> getPrewittVertical();
std::vector<std::vector<real_t>> getSobelHorizontal();
std::vector<std::vector<real_t>> getSobelVertical();
std::vector<std::vector<real_t>> getScharrHorizontal();
std::vector<std::vector<real_t>> getScharrVertical();
std::vector<std::vector<real_t>> getRobertsHorizontal();
std::vector<std::vector<real_t>> getRobertsVertical();
private:
std::vector<std::vector<double>> prewittHorizontal;
std::vector<std::vector<double>> prewittVertical;
std::vector<std::vector<double>> sobelHorizontal;
std::vector<std::vector<double>> sobelVertical;
std::vector<std::vector<double>> scharrHorizontal;
std::vector<std::vector<double>> scharrVertical;
std::vector<std::vector<double>> robertsHorizontal;
std::vector<std::vector<double>> robertsVertical;
std::vector<std::vector<real_t>> prewittHorizontal;
std::vector<std::vector<real_t>> prewittVertical;
std::vector<std::vector<real_t>> sobelHorizontal;
std::vector<std::vector<real_t>> sobelVertical;
std::vector<std::vector<real_t>> scharrHorizontal;
std::vector<std::vector<real_t>> scharrVertical;
std::vector<std::vector<real_t>> robertsHorizontal;
std::vector<std::vector<real_t>> robertsVertical;
};
#endif // Convolutions_hpp

160
mlpp/cost/cost.cpp
View File

@ -11,16 +11,16 @@
#include <iostream>
double MLPPCost::MSE(std::vector<double> y_hat, std::vector<double> y) {
double sum = 0;
real_t MLPPCost::MSE(std::vector<real_t> y_hat, std::vector<real_t> y) {
real_t sum = 0;
for (int i = 0; i < y_hat.size(); i++) {
sum += (y_hat[i] - y[i]) * (y_hat[i] - y[i]);
}
return sum / 2 * y_hat.size();
}
double MLPPCost::MSE(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) {
double sum = 0;
real_t MLPPCost::MSE(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
real_t sum = 0;
for (int i = 0; i < y_hat.size(); i++) {
for (int j = 0; j < y_hat[i].size(); j++) {
sum += (y_hat[i][j] - y[i][j]) * (y_hat[i][j] - y[i][j]);
@ -29,26 +29,26 @@ double MLPPCost::MSE(std::vector<std::vector<double>> y_hat, std::vector<std::ve
return sum / 2 * y_hat.size();
}
std::vector<double> MLPPCost::MSEDeriv(std::vector<double> y_hat, std::vector<double> y) {
std::vector<real_t> MLPPCost::MSEDeriv(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPLinAlg alg;
return alg.subtraction(y_hat, y);
}
std::vector<std::vector<double>> MLPPCost::MSEDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) {
std::vector<std::vector<real_t>> MLPPCost::MSEDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
MLPPLinAlg alg;
return alg.subtraction(y_hat, y);
}
double MLPPCost::RMSE(std::vector<double> y_hat, std::vector<double> y) {
double sum = 0;
real_t MLPPCost::RMSE(std::vector<real_t> y_hat, std::vector<real_t> y) {
real_t sum = 0;
for (int i = 0; i < y_hat.size(); i++) {
sum += (y_hat[i] - y[i]) * (y_hat[i] - y[i]);
}
return sqrt(sum / y_hat.size());
}
double MLPPCost::RMSE(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) {
double sum = 0;
real_t MLPPCost::RMSE(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
real_t sum = 0;
for (int i = 0; i < y_hat.size(); i++) {
for (int j = 0; j < y_hat[i].size(); j++) {
sum += (y_hat[i][j] - y[i][j]) * (y_hat[i][j] - y[i][j]);
@ -57,26 +57,26 @@ double MLPPCost::RMSE(std::vector<std::vector<double>> y_hat, std::vector<std::v
return sqrt(sum / y_hat.size());
}
std::vector<double> MLPPCost::RMSEDeriv(std::vector<double> y_hat, std::vector<double> y) {
std::vector<real_t> MLPPCost::RMSEDeriv(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPLinAlg alg;
return alg.scalarMultiply(1 / (2 * sqrt(MSE(y_hat, y))), MSEDeriv(y_hat, y));
}
std::vector<std::vector<double>> MLPPCost::RMSEDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) {
std::vector<std::vector<real_t>> MLPPCost::RMSEDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
MLPPLinAlg alg;
return alg.scalarMultiply(1 / (2 / sqrt(MSE(y_hat, y))), MSEDeriv(y_hat, y));
}
double MLPPCost::MAE(std::vector<double> y_hat, std::vector<double> y) {
double sum = 0;
real_t MLPPCost::MAE(std::vector<real_t> y_hat, std::vector<real_t> y) {
real_t sum = 0;
for (int i = 0; i < y_hat.size(); i++) {
sum += abs((y_hat[i] - y[i]));
}
return sum / y_hat.size();
}
double MLPPCost::MAE(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) {
double sum = 0;
real_t MLPPCost::MAE(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
real_t sum = 0;
for (int i = 0; i < y_hat.size(); i++) {
for (int j = 0; j < y_hat[i].size(); j++) {
sum += abs((y_hat[i][j] - y[i][j]));
@ -85,8 +85,8 @@ double MLPPCost::MAE(std::vector<std::vector<double>> y_hat, std::vector<std::ve
return sum / y_hat.size();
}
std::vector<double> MLPPCost::MAEDeriv(std::vector<double> y_hat, std::vector<double> y) {
std::vector<double> deriv;
std::vector<real_t> MLPPCost::MAEDeriv(std::vector<real_t> y_hat, std::vector<real_t> y) {
std::vector<real_t> deriv;
deriv.resize(y_hat.size());
for (int i = 0; i < deriv.size(); i++) {
if (y_hat[i] < 0) {
@ -100,8 +100,8 @@ std::vector<double> MLPPCost::MAEDeriv(std::vector<double> y_hat, std::vector<do
return deriv;
}
std::vector<std::vector<double>> MLPPCost::MAEDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) {
std::vector<std::vector<double>> deriv;
std::vector<std::vector<real_t>> MLPPCost::MAEDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
std::vector<std::vector<real_t>> deriv;
deriv.resize(y_hat.size());
for (int i = 0; i < deriv.size(); i++) {
deriv.resize(y_hat[i].size());
@ -120,16 +120,16 @@ std::vector<std::vector<double>> MLPPCost::MAEDeriv(std::vector<std::vector<doub
return deriv;
}
double MLPPCost::MBE(std::vector<double> y_hat, std::vector<double> y) {
double sum = 0;
real_t MLPPCost::MBE(std::vector<real_t> y_hat, std::vector<real_t> y) {
real_t sum = 0;
for (int i = 0; i < y_hat.size(); i++) {
sum += (y_hat[i] - y[i]);
}
return sum / y_hat.size();
}
double MLPPCost::MBE(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) {
double sum = 0;
real_t MLPPCost::MBE(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
real_t sum = 0;
for (int i = 0; i < y_hat.size(); i++) {
for (int j = 0; j < y_hat[i].size(); j++) {
sum += (y_hat[i][j] - y[i][j]);
@ -138,19 +138,19 @@ double MLPPCost::MBE(std::vector<std::vector<double>> y_hat, std::vector<std::ve
return sum / y_hat.size();
}
std::vector<double> MLPPCost::MBEDeriv(std::vector<double> y_hat, std::vector<double> y) {
std::vector<real_t> MLPPCost::MBEDeriv(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPLinAlg alg;
return alg.onevec(y_hat.size());
}
std::vector<std::vector<double>> MLPPCost::MBEDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) {
std::vector<std::vector<real_t>> MLPPCost::MBEDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
MLPPLinAlg alg;
return alg.onemat(y_hat.size(), y_hat[0].size());
}
double MLPPCost::LogLoss(std::vector<double> y_hat, std::vector<double> y) {
double sum = 0;
double eps = 1e-8;
real_t MLPPCost::LogLoss(std::vector<real_t> y_hat, std::vector<real_t> y) {
real_t sum = 0;
real_t eps = 1e-8;
for (int i = 0; i < y_hat.size(); i++) {
sum += -(y[i] * std::log(y_hat[i] + eps) + (1 - y[i]) * std::log(1 - y_hat[i] + eps));
}
@ -158,9 +158,9 @@ double MLPPCost::LogLoss(std::vector<double> y_hat, std::vector<double> y) {
return sum / y_hat.size();
}
double MLPPCost::LogLoss(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) {
double sum = 0;
double eps = 1e-8;
real_t MLPPCost::LogLoss(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
real_t sum = 0;
real_t eps = 1e-8;
for (int i = 0; i < y_hat.size(); i++) {
for (int j = 0; j < y_hat[i].size(); j++) {
sum += -(y[i][j] * std::log(y_hat[i][j] + eps) + (1 - y[i][j]) * std::log(1 - y_hat[i][j] + eps));
@ -170,18 +170,18 @@ double MLPPCost::LogLoss(std::vector<std::vector<double>> y_hat, std::vector<std
return sum / y_hat.size();
}
std::vector<double> MLPPCost::LogLossDeriv(std::vector<double> y_hat, std::vector<double> y) {
std::vector<real_t> MLPPCost::LogLossDeriv(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPLinAlg alg;
return alg.addition(alg.scalarMultiply(-1, alg.elementWiseDivision(y, y_hat)), alg.elementWiseDivision(alg.scalarMultiply(-1, alg.scalarAdd(-1, y)), alg.scalarMultiply(-1, alg.scalarAdd(-1, y_hat))));
}
std::vector<std::vector<double>> MLPPCost::LogLossDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) {
std::vector<std::vector<real_t>> MLPPCost::LogLossDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
MLPPLinAlg alg;
return alg.addition(alg.scalarMultiply(-1, alg.elementWiseDivision(y, y_hat)), alg.elementWiseDivision(alg.scalarMultiply(-1, alg.scalarAdd(-1, y)), alg.scalarMultiply(-1, alg.scalarAdd(-1, y_hat))));
}
double MLPPCost::CrossEntropy(std::vector<double> y_hat, std::vector<double> y) {
double sum = 0;
real_t MLPPCost::CrossEntropy(std::vector<real_t> y_hat, std::vector<real_t> y) {
real_t sum = 0;
for (int i = 0; i < y_hat.size(); i++) {
sum += y[i] * std::log(y_hat[i]);
}
@ -189,8 +189,8 @@ double MLPPCost::CrossEntropy(std::vector<double> y_hat, std::vector<double> y)
return -1 * sum;
}
double MLPPCost::CrossEntropy(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) {
double sum = 0;
real_t MLPPCost::CrossEntropy(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
real_t sum = 0;
for (int i = 0; i < y_hat.size(); i++) {
for (int j = 0; j < y_hat[i].size(); j++) {
sum += y[i][j] * std::log(y_hat[i][j]);
@ -200,19 +200,19 @@ double MLPPCost::CrossEntropy(std::vector<std::vector<double>> y_hat, std::vecto
return -1 * sum;
}
std::vector<double> MLPPCost::CrossEntropyDeriv(std::vector<double> y_hat, std::vector<double> y) {
std::vector<real_t> MLPPCost::CrossEntropyDeriv(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPLinAlg alg;
return alg.scalarMultiply(-1, alg.elementWiseDivision(y, y_hat));
}
std::vector<std::vector<double>> MLPPCost::CrossEntropyDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) {
std::vector<std::vector<real_t>> MLPPCost::CrossEntropyDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
MLPPLinAlg alg;
return alg.scalarMultiply(-1, alg.elementWiseDivision(y, y_hat));
}
double MLPPCost::HuberLoss(std::vector<double> y_hat, std::vector<double> y, double delta) {
real_t MLPPCost::HuberLoss(std::vector<real_t> y_hat, std::vector<real_t> y, real_t delta) {
MLPPLinAlg alg;
double sum = 0;
real_t sum = 0;
for (int i = 0; i < y_hat.size(); i++) {
if (abs(y[i] - y_hat[i]) <= delta) {
sum += (y[i] - y_hat[i]) * (y[i] - y_hat[i]);
@ -223,9 +223,9 @@ double MLPPCost::HuberLoss(std::vector<double> y_hat, std::vector<double> y, dou
return sum;
}
double MLPPCost::HuberLoss(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y, double delta) {
real_t MLPPCost::HuberLoss(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y, real_t delta) {
MLPPLinAlg alg;
double sum = 0;
real_t sum = 0;
for (int i = 0; i < y_hat.size(); i++) {
for (int j = 0; j < y_hat[i].size(); j++) {
if (abs(y[i][j] - y_hat[i][j]) <= delta) {
@ -238,10 +238,10 @@ double MLPPCost::HuberLoss(std::vector<std::vector<double>> y_hat, std::vector<s
return sum;
}
std::vector<double> MLPPCost::HuberLossDeriv(std::vector<double> y_hat, std::vector<double> y, double delta) {
std::vector<real_t> MLPPCost::HuberLossDeriv(std::vector<real_t> y_hat, std::vector<real_t> y, real_t delta) {
MLPPLinAlg alg;
double sum = 0;
std::vector<double> deriv;
real_t sum = 0;
std::vector<real_t> deriv;
deriv.resize(y_hat.size());
for (int i = 0; i < y_hat.size(); i++) {
@ -258,10 +258,10 @@ std::vector<double> MLPPCost::HuberLossDeriv(std::vector<double> y_hat, std::vec
return deriv;
}
std::vector<std::vector<double>> MLPPCost::HuberLossDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y, double delta) {
std::vector<std::vector<real_t>> MLPPCost::HuberLossDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y, real_t delta) {
MLPPLinAlg alg;
double sum = 0;
std::vector<std::vector<double>> deriv;
real_t sum = 0;
std::vector<std::vector<real_t>> deriv;
deriv.resize(y_hat.size());
for (int i = 0; i < deriv.size(); i++) {
deriv[i].resize(y_hat[i].size());
@ -283,8 +283,8 @@ std::vector<std::vector<double>> MLPPCost::HuberLossDeriv(std::vector<std::vecto
return deriv;
}
double MLPPCost::HingeLoss(std::vector<double> y_hat, std::vector<double> y) {
double sum = 0;
real_t MLPPCost::HingeLoss(std::vector<real_t> y_hat, std::vector<real_t> y) {
real_t sum = 0;
for (int i = 0; i < y_hat.size(); i++) {
sum += fmax(0, 1 - y[i] * y_hat[i]);
}
@ -292,8 +292,8 @@ double MLPPCost::HingeLoss(std::vector<double> y_hat, std::vector<double> y) {
return sum / y_hat.size();
}
double MLPPCost::HingeLoss(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) {
double sum = 0;
real_t MLPPCost::HingeLoss(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
real_t sum = 0;
for (int i = 0; i < y_hat.size(); i++) {
for (int j = 0; j < y_hat[i].size(); j++) {
sum += fmax(0, 1 - y[i][j] * y_hat[i][j]);
@ -303,8 +303,8 @@ double MLPPCost::HingeLoss(std::vector<std::vector<double>> y_hat, std::vector<s
return sum / y_hat.size();
}
std::vector<double> MLPPCost::HingeLossDeriv(std::vector<double> y_hat, std::vector<double> y) {
std::vector<double> deriv;
std::vector<real_t> MLPPCost::HingeLossDeriv(std::vector<real_t> y_hat, std::vector<real_t> y) {
std::vector<real_t> deriv;
deriv.resize(y_hat.size());
for (int i = 0; i < y_hat.size(); i++) {
if (1 - y[i] * y_hat[i] > 0) {
@ -316,8 +316,8 @@ std::vector<double> MLPPCost::HingeLossDeriv(std::vector<double> y_hat, std::vec
return deriv;
}
std::vector<std::vector<double>> MLPPCost::HingeLossDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) {
std::vector<std::vector<double>> deriv;
std::vector<std::vector<real_t>> MLPPCost::HingeLossDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
std::vector<std::vector<real_t>> deriv;
for (int i = 0; i < y_hat.size(); i++) {
for (int j = 0; j < y_hat[i].size(); j++) {
if (1 - y[i][j] * y_hat[i][j] > 0) {
@ -330,16 +330,16 @@ std::vector<std::vector<double>> MLPPCost::HingeLossDeriv(std::vector<std::vecto
return deriv;
}
double MLPPCost::WassersteinLoss(std::vector<double> y_hat, std::vector<double> y) {
double sum = 0;
real_t MLPPCost::WassersteinLoss(std::vector<real_t> y_hat, std::vector<real_t> y) {
real_t sum = 0;
for (int i = 0; i < y_hat.size(); i++) {
sum += y_hat[i] * y[i];
}
return -sum / y_hat.size();
}
double MLPPCost::WassersteinLoss(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) {
double sum = 0;
real_t MLPPCost::WassersteinLoss(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
real_t sum = 0;
for (int i = 0; i < y_hat.size(); i++) {
for (int j = 0; j < y_hat[i].size(); j++) {
sum += y_hat[i][j] * y[i][j];
@ -348,59 +348,59 @@ double MLPPCost::WassersteinLoss(std::vector<std::vector<double>> y_hat, std::ve
return -sum / y_hat.size();
}
std::vector<double> MLPPCost::WassersteinLossDeriv(std::vector<double> y_hat, std::vector<double> y) {
std::vector<real_t> MLPPCost::WassersteinLossDeriv(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPLinAlg alg;
return alg.scalarMultiply(-1, y); // Simple.
}
std::vector<std::vector<double>> MLPPCost::WassersteinLossDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) {
std::vector<std::vector<real_t>> MLPPCost::WassersteinLossDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
MLPPLinAlg alg;
return alg.scalarMultiply(-1, y); // Simple.
}
double MLPPCost::HingeLoss(std::vector<double> y_hat, std::vector<double> y, std::vector<double> weights, double C) {
real_t MLPPCost::HingeLoss(std::vector<real_t> y_hat, std::vector<real_t> y, std::vector<real_t> weights, real_t C) {
MLPPLinAlg alg;
MLPPReg regularization;
return C * HingeLoss(y_hat, y) + regularization.regTerm(weights, 1, 0, "Ridge");
}
double MLPPCost::HingeLoss(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y, std::vector<std::vector<double>> weights, double C) {
real_t MLPPCost::HingeLoss(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y, std::vector<std::vector<real_t>> weights, real_t C) {
MLPPLinAlg alg;
MLPPReg regularization;
return C * HingeLoss(y_hat, y) + regularization.regTerm(weights, 1, 0, "Ridge");
}
std::vector<double> MLPPCost::HingeLossDeriv(std::vector<double> y_hat, std::vector<double> y, double C) {
std::vector<real_t> MLPPCost::HingeLossDeriv(std::vector<real_t> y_hat, std::vector<real_t> y, real_t C) {
MLPPLinAlg alg;
MLPPReg regularization;
return alg.scalarMultiply(C, HingeLossDeriv(y_hat, y));
}
std::vector<std::vector<double>> MLPPCost::HingeLossDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y, double C) {
std::vector<std::vector<real_t>> MLPPCost::HingeLossDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y, real_t C) {
MLPPLinAlg alg;
MLPPReg regularization;
return alg.scalarMultiply(C, HingeLossDeriv(y_hat, y));
}
double MLPPCost::dualFormSVM(std::vector<double> alpha, std::vector<std::vector<double>> X, std::vector<double> y) {
real_t MLPPCost::dualFormSVM(std::vector<real_t> alpha, std::vector<std::vector<real_t>> X, std::vector<real_t> y) {
MLPPLinAlg alg;
std::vector<std::vector<double>> Y = alg.diag(y); // Y is a diagnoal matrix. Y[i][j] = y[i] if i = i, else Y[i][j] = 0. Yt = Y.
std::vector<std::vector<double>> K = alg.matmult(X, alg.transpose(X)); // TO DO: DON'T forget to add non-linear kernelizations.
std::vector<std::vector<double>> Q = alg.matmult(alg.matmult(alg.transpose(Y), K), Y);
double alphaQ = alg.matmult(alg.matmult({ alpha }, Q), alg.transpose({ alpha }))[0][0];
std::vector<double> one = alg.onevec(alpha.size());
std::vector<std::vector<real_t>> Y = alg.diag(y); // Y is a diagnoal matrix. Y[i][j] = y[i] if i = i, else Y[i][j] = 0. Yt = Y.
std::vector<std::vector<real_t>> K = alg.matmult(X, alg.transpose(X)); // TO DO: DON'T forget to add non-linear kernelizations.
std::vector<std::vector<real_t>> Q = alg.matmult(alg.matmult(alg.transpose(Y), K), Y);
real_t alphaQ = alg.matmult(alg.matmult({ alpha }, Q), alg.transpose({ alpha }))[0][0];
std::vector<real_t> one = alg.onevec(alpha.size());
return -alg.dot(one, alpha) + 0.5 * alphaQ;
}
std::vector<double> MLPPCost::dualFormSVMDeriv(std::vector<double> alpha, std::vector<std::vector<double>> X, std::vector<double> y) {
std::vector<real_t> MLPPCost::dualFormSVMDeriv(std::vector<real_t> alpha, std::vector<std::vector<real_t>> X, std::vector<real_t> y) {
MLPPLinAlg alg;
std::vector<std::vector<double>> Y = alg.zeromat(y.size(), y.size());
std::vector<std::vector<real_t>> Y = alg.zeromat(y.size(), y.size());
for (int i = 0; i < y.size(); i++) {
Y[i][i] = y[i]; // Y is a diagnoal matrix. Y[i][j] = y[i] if i = i, else Y[i][j] = 0. Yt = Y.
}
std::vector<std::vector<double>> K = alg.matmult(X, alg.transpose(X)); // TO DO: DON'T forget to add non-linear kernelizations.
std::vector<std::vector<double>> Q = alg.matmult(alg.matmult(alg.transpose(Y), K), Y);
std::vector<double> alphaQDeriv = alg.mat_vec_mult(Q, alpha);
std::vector<double> one = alg.onevec(alpha.size());
std::vector<std::vector<real_t>> K = alg.matmult(X, alg.transpose(X)); // TO DO: DON'T forget to add non-linear kernelizations.
std::vector<std::vector<real_t>> Q = alg.matmult(alg.matmult(alg.transpose(Y), K), Y);
std::vector<real_t> alphaQDeriv = alg.mat_vec_mult(Q, alpha);
std::vector<real_t> one = alg.onevec(alpha.size());
return alg.subtraction(alphaQDeriv, one);
}

86
mlpp/cost/cost.h
View File

@ -8,76 +8,78 @@
// Created by Marc Melikyan on 1/16/21.
//
#include "core/math/math_defs.h"
#include <vector>
class MLPPCost {
public:
// Regression Costs
double MSE(std::vector<double> y_hat, std::vector<double> y);
double MSE(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y);
real_t MSE(std::vector<real_t> y_hat, std::vector<real_t> y);
real_t MSE(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
std::vector<double> MSEDeriv(std::vector<double> y_hat, std::vector<double> y);
std::vector<std::vector<double>> MSEDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y);
std::vector<real_t> MSEDeriv(std::vector<real_t> y_hat, std::vector<real_t> y);
std::vector<std::vector<real_t>> MSEDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
double RMSE(std::vector<double> y_hat, std::vector<double> y);
double RMSE(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y);
real_t RMSE(std::vector<real_t> y_hat, std::vector<real_t> y);
real_t RMSE(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
std::vector<double> RMSEDeriv(std::vector<double> y_hat, std::vector<double> y);
std::vector<std::vector<double>> RMSEDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y);
std::vector<real_t> RMSEDeriv(std::vector<real_t> y_hat, std::vector<real_t> y);
std::vector<std::vector<real_t>> RMSEDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
double MAE(std::vector<double> y_hat, std::vector<double> y);
double MAE(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y);
real_t MAE(std::vector<real_t> y_hat, std::vector<real_t> y);
real_t MAE(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
std::vector<double> MAEDeriv(std::vector<double> y_hat, std::vector<double> y);
std::vector<std::vector<double>> MAEDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y);
std::vector<real_t> MAEDeriv(std::vector<real_t> y_hat, std::vector<real_t> y);
std::vector<std::vector<real_t>> MAEDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
double MBE(std::vector<double> y_hat, std::vector<double> y);
double MBE(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y);
real_t MBE(std::vector<real_t> y_hat, std::vector<real_t> y);
real_t MBE(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
std::vector<double> MBEDeriv(std::vector<double> y_hat, std::vector<double> y);
std::vector<std::vector<double>> MBEDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y);
std::vector<real_t> MBEDeriv(std::vector<real_t> y_hat, std::vector<real_t> y);
std::vector<std::vector<real_t>> MBEDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
// Classification Costs
double LogLoss(std::vector<double> y_hat, std::vector<double> y);
double LogLoss(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y);
real_t LogLoss(std::vector<real_t> y_hat, std::vector<real_t> y);
real_t LogLoss(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
std::vector<double> LogLossDeriv(std::vector<double> y_hat, std::vector<double> y);
std::vector<std::vector<double>> LogLossDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y);
std::vector<real_t> LogLossDeriv(std::vector<real_t> y_hat, std::vector<real_t> y);
std::vector<std::vector<real_t>> LogLossDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
double CrossEntropy(std::vector<double> y_hat, std::vector<double> y);
double CrossEntropy(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y);
real_t CrossEntropy(std::vector<real_t> y_hat, std::vector<real_t> y);
real_t CrossEntropy(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
std::vector<double> CrossEntropyDeriv(std::vector<double> y_hat, std::vector<double> y);
std::vector<std::vector<double>> CrossEntropyDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y);
std::vector<real_t> CrossEntropyDeriv(std::vector<real_t> y_hat, std::vector<real_t> y);
std::vector<std::vector<real_t>> CrossEntropyDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
double HuberLoss(std::vector<double> y_hat, std::vector<double> y, double delta);
double HuberLoss(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y, double delta);
real_t HuberLoss(std::vector<real_t> y_hat, std::vector<real_t> y, real_t delta);
real_t HuberLoss(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y, real_t delta);
std::vector<double> HuberLossDeriv(std::vector<double> y_hat, std::vector<double> y, double delta);
std::vector<std::vector<double>> HuberLossDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y, double delta);
std::vector<real_t> HuberLossDeriv(std::vector<real_t> y_hat, std::vector<real_t> y, real_t delta);
std::vector<std::vector<real_t>> HuberLossDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y, real_t delta);
double HingeLoss(std::vector<double> y_hat, std::vector<double> y);
double HingeLoss(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y);
real_t HingeLoss(std::vector<real_t> y_hat, std::vector<real_t> y);
real_t HingeLoss(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
std::vector<double> HingeLossDeriv(std::vector<double> y_hat, std::vector<double> y);
std::vector<std::vector<double>> HingeLossDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y);
std::vector<real_t> HingeLossDeriv(std::vector<real_t> y_hat, std::vector<real_t> y);
std::vector<std::vector<real_t>> HingeLossDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
double HingeLoss(std::vector<double> y_hat, std::vector<double> y, std::vector<double> weights, double C);
double HingeLoss(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y, std::vector<std::vector<double>> weights, double C);
real_t HingeLoss(std::vector<real_t> y_hat, std::vector<real_t> y, std::vector<real_t> weights, real_t C);
real_t HingeLoss(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y, std::vector<std::vector<real_t>> weights, real_t C);
std::vector<double> HingeLossDeriv(std::vector<double> y_hat, std::vector<double> y, double C);
std::vector<std::vector<double>> HingeLossDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y, double C);
std::vector<real_t> HingeLossDeriv(std::vector<real_t> y_hat, std::vector<real_t> y, real_t C);
std::vector<std::vector<real_t>> HingeLossDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y, real_t C);
double WassersteinLoss(std::vector<double> y_hat, std::vector<double> y);
double WassersteinLoss(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y);
real_t WassersteinLoss(std::vector<real_t> y_hat, std::vector<real_t> y);
real_t WassersteinLoss(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
std::vector<double> WassersteinLossDeriv(std::vector<double> y_hat, std::vector<double> y);
std::vector<std::vector<double>> WassersteinLossDeriv(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y);
std::vector<real_t> WassersteinLossDeriv(std::vector<real_t> y_hat, std::vector<real_t> y);
std::vector<std::vector<real_t>> WassersteinLossDeriv(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
double dualFormSVM(std::vector<double> alpha, std::vector<std::vector<double>> X, std::vector<double> y); // TO DO: DON'T forget to add non-linear kernelizations.
real_t dualFormSVM(std::vector<real_t> alpha, std::vector<std::vector<real_t>> X, std::vector<real_t> y); // TO DO: DON'T forget to add non-linear kernelizations.
std::vector<double> dualFormSVMDeriv(std::vector<double> alpha, std::vector<std::vector<double>> X, std::vector<double> y);
std::vector<real_t> dualFormSVMDeriv(std::vector<real_t> alpha, std::vector<std::vector<real_t>> X, std::vector<real_t> y);
private:
};

210
mlpp/data/data.cpp
View File

@ -56,7 +56,7 @@ Ref<MLPPDataComplex> MLPPData::load_iris(const String &path) {
const int IRIS_SIZE = 4;
const int ONE_HOT_NUM = 3;
std::vector<double> tempOutputSet;
std::vector<real_t> tempOutputSet;
Ref<MLPPDataComplex> data;
data.instance();
@ -71,7 +71,7 @@ Ref<MLPPDataComplex> MLPPData::load_wine(const String &path) {
const int WINE_SIZE = 4;
const int ONE_HOT_NUM = 3;
std::vector<double> tempOutputSet;
std::vector<real_t> tempOutputSet;
Ref<MLPPDataComplex> data;
data.instance();
@ -86,8 +86,8 @@ Ref<MLPPDataComplex> MLPPData::load_mnist_train(const String &path) {
const int MNIST_SIZE = 784;
const int ONE_HOT_NUM = 10;
std::vector<std::vector<double>> inputSet;
std::vector<double> tempOutputSet;
std::vector<std::vector<real_t>> inputSet;
std::vector<real_t> tempOutputSet;
Ref<MLPPDataComplex> data;
data.instance();
@ -101,8 +101,8 @@ Ref<MLPPDataComplex> MLPPData::load_mnist_train(const String &path) {
Ref<MLPPDataComplex> MLPPData::load_mnist_test(const String &path) {
const int MNIST_SIZE = 784;
const int ONE_HOT_NUM = 10;
std::vector<std::vector<double>> inputSet;
std::vector<double> tempOutputSet;
std::vector<std::vector<real_t>> inputSet;
std::vector<real_t> tempOutputSet;
Ref<MLPPDataComplex> data;
data.instance();
@ -137,7 +137,7 @@ Ref<MLPPDataESimple> MLPPData::load_fires_and_crime(const String &path) {
// MULTIVARIATE SUPERVISED
void MLPPData::set_data_supervised(int k, const String &file_name, std::vector<std::vector<double>> &inputSet, std::vector<double> &outputSet) {
void MLPPData::set_data_supervised(int k, const String &file_name, std::vector<std::vector<real_t>> &inputSet, std::vector<real_t> &outputSet) {
MLPPLinAlg alg;
inputSet.resize(k);
@ -150,10 +150,10 @@ void MLPPData::set_data_supervised(int k, const String &file_name, std::vector<s
Vector<String> ll = file->get_csv_line();
for (int i = 0; i < k; ++i) {
inputSet[i].push_back(ll[i].to_double());
inputSet[i].push_back(static_cast<real_t>(ll[i].to_double()));
}
outputSet.push_back(ll[k].to_double());
outputSet.push_back(static_cast<real_t>(ll[k].to_double()));
}
inputSet = alg.transpose(inputSet);
@ -161,7 +161,7 @@ void MLPPData::set_data_supervised(int k, const String &file_name, std::vector<s
memdelete(file);
}
void MLPPData::set_data_unsupervised(int k, const String &file_name, std::vector<std::vector<double>> &inputSet) {
void MLPPData::set_data_unsupervised(int k, const String &file_name, std::vector<std::vector<real_t>> &inputSet) {
MLPPLinAlg alg;
inputSet.resize(k);
@ -174,7 +174,7 @@ void MLPPData::set_data_unsupervised(int k, const String &file_name, std::vector
Vector<String> ll = file->get_csv_line();
for (int i = 0; i < k; ++i) {
inputSet[i].push_back(ll[i].to_double());
inputSet[i].push_back(static_cast<real_t>(ll[i].to_double()));
}
}
@ -183,7 +183,7 @@ void MLPPData::set_data_unsupervised(int k, const String &file_name, std::vector
memdelete(file);
}
void MLPPData::set_data_simple(const String &file_name, std::vector<double> &inputSet, std::vector<double> &outputSet) {
void MLPPData::set_data_simple(const String &file_name, std::vector<real_t> &inputSet, std::vector<real_t> &outputSet) {
FileAccess *file = FileAccess::open(file_name, FileAccess::READ);
ERR_FAIL_COND(!file);
@ -192,15 +192,15 @@ void MLPPData::set_data_simple(const String &file_name, std::vector<double> &inp
Vector<String> ll = file->get_csv_line();
for (int i = 0; i < ll.size(); i += 2) {
inputSet.push_back(ll[i].to_double());
outputSet.push_back(ll[i + 1].to_double());
inputSet.push_back(static_cast<real_t>(ll[i].to_double()));
outputSet.push_back(static_cast<real_t>(ll[i + 1].to_double()));
}
}
memdelete(file);
}
MLPPData::SplitComplexData MLPPData::train_test_split(const Ref<MLPPDataComplex> &data, double test_size) {
MLPPData::SplitComplexData MLPPData::train_test_split(const Ref<MLPPDataComplex> &data, real_t test_size) {
SplitComplexData res;
res.train.instance();
@ -237,7 +237,7 @@ MLPPData::SplitComplexData MLPPData::train_test_split(const Ref<MLPPDataComplex>
return res;
}
Array MLPPData::train_test_split_bind(const Ref<MLPPDataComplex> &data, double test_size) {
Array MLPPData::train_test_split_bind(const Ref<MLPPDataComplex> &data, real_t test_size) {
SplitComplexData res = train_test_split(data, test_size);
Array arr;
@ -248,80 +248,80 @@ Array MLPPData::train_test_split_bind(const Ref<MLPPDataComplex> &data, double t
}
// Loading Datasets
std::tuple<std::vector<std::vector<double>>, std::vector<double>> MLPPData::loadBreastCancer() {
std::tuple<std::vector<std::vector<real_t>>, std::vector<real_t>> MLPPData::loadBreastCancer() {
const int BREAST_CANCER_SIZE = 30; // k = 30
std::vector<std::vector<double>> inputSet;
std::vector<double> outputSet;
std::vector<std::vector<real_t>> inputSet;
std::vector<real_t> outputSet;
setData(BREAST_CANCER_SIZE, "MLPP/Data/Datasets/BreastCancer.csv", inputSet, outputSet);
return { inputSet, outputSet };
}
std::tuple<std::vector<std::vector<double>>, std::vector<double>> MLPPData::loadBreastCancerSVC() {
std::tuple<std::vector<std::vector<real_t>>, std::vector<real_t>> MLPPData::loadBreastCancerSVC() {
const int BREAST_CANCER_SIZE = 30; // k = 30
std::vector<std::vector<double>> inputSet;
std::vector<double> outputSet;
std::vector<std::vector<real_t>> inputSet;
std::vector<real_t> outputSet;
setData(BREAST_CANCER_SIZE, "MLPP/Data/Datasets/BreastCancerSVM.csv", inputSet, outputSet);
return { inputSet, outputSet };
}
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> MLPPData::loadIris() {
std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> MLPPData::loadIris() {
const int IRIS_SIZE = 4;
const int ONE_HOT_NUM = 3;
std::vector<std::vector<double>> inputSet;
std::vector<double> tempOutputSet;
std::vector<std::vector<real_t>> inputSet;
std::vector<real_t> tempOutputSet;
setData(IRIS_SIZE, "/Users/marcmelikyan/Desktop/Data/Iris.csv", inputSet, tempOutputSet);
std::vector<std::vector<double>> outputSet = oneHotRep(tempOutputSet, ONE_HOT_NUM);
std::vector<std::vector<real_t>> outputSet = oneHotRep(tempOutputSet, ONE_HOT_NUM);
return { inputSet, outputSet };
}
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> MLPPData::loadWine() {
std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> MLPPData::loadWine() {
const int WINE_SIZE = 4;
const int ONE_HOT_NUM = 3;
std::vector<std::vector<double>> inputSet;
std::vector<double> tempOutputSet;
std::vector<std::vector<real_t>> inputSet;
std::vector<real_t> tempOutputSet;
setData(WINE_SIZE, "MLPP/Data/Datasets/Iris.csv", inputSet, tempOutputSet);
std::vector<std::vector<double>> outputSet = oneHotRep(tempOutputSet, ONE_HOT_NUM);
std::vector<std::vector<real_t>> outputSet = oneHotRep(tempOutputSet, ONE_HOT_NUM);
return { inputSet, outputSet };
}
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> MLPPData::loadMnistTrain() {
std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> MLPPData::loadMnistTrain() {
const int MNIST_SIZE = 784;
const int ONE_HOT_NUM = 10;
std::vector<std::vector<double>> inputSet;
std::vector<double> tempOutputSet;
std::vector<std::vector<real_t>> inputSet;
std::vector<real_t> tempOutputSet;
setData(MNIST_SIZE, "MLPP/Data/Datasets/MnistTrain.csv", inputSet, tempOutputSet);
std::vector<std::vector<double>> outputSet = oneHotRep(tempOutputSet, ONE_HOT_NUM);
std::vector<std::vector<real_t>> outputSet = oneHotRep(tempOutputSet, ONE_HOT_NUM);
return { inputSet, outputSet };
}
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> MLPPData::loadMnistTest() {
std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> MLPPData::loadMnistTest() {
const int MNIST_SIZE = 784;
const int ONE_HOT_NUM = 10;
std::vector<std::vector<double>> inputSet;
std::vector<double> tempOutputSet;
std::vector<std::vector<real_t>> inputSet;
std::vector<real_t> tempOutputSet;
setData(MNIST_SIZE, "MLPP/Data/Datasets/MnistTest.csv", inputSet, tempOutputSet);
std::vector<std::vector<double>> outputSet = oneHotRep(tempOutputSet, ONE_HOT_NUM);
std::vector<std::vector<real_t>> outputSet = oneHotRep(tempOutputSet, ONE_HOT_NUM);
return { inputSet, outputSet };
}
std::tuple<std::vector<std::vector<double>>, std::vector<double>> MLPPData::loadCaliforniaHousing() {
std::tuple<std::vector<std::vector<real_t>>, std::vector<real_t>> MLPPData::loadCaliforniaHousing() {
const int CALIFORNIA_HOUSING_SIZE = 13; // k = 30
std::vector<std::vector<double>> inputSet;
std::vector<double> outputSet;
std::vector<std::vector<real_t>> inputSet;
std::vector<real_t> outputSet;
setData(CALIFORNIA_HOUSING_SIZE, "MLPP/Data/Datasets/CaliforniaHousing.csv", inputSet, outputSet);
return { inputSet, outputSet };
}
std::tuple<std::vector<double>, std::vector<double>> MLPPData::loadFiresAndCrime() {
std::vector<double> inputSet; // k is implicitly 1.
std::vector<double> outputSet;
std::tuple<std::vector<real_t>, std::vector<real_t>> MLPPData::loadFiresAndCrime() {
std::vector<real_t> inputSet; // k is implicitly 1.
std::vector<real_t> outputSet;
setData("MLPP/Data/Datasets/FiresAndCrime.csv", inputSet, outputSet);
return { inputSet, outputSet };
@ -330,15 +330,15 @@ std::tuple<std::vector<double>, std::vector<double>> MLPPData::loadFiresAndCrime
// Note that inputs and outputs should be pairs (technically), but this
// implementation will separate them. (My implementation keeps them tied together.)
// Not yet sure whether this is intentional or not (or it's something like a compiler specific difference)
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>, std::vector<std::vector<double>>, std::vector<std::vector<double>>> MLPPData::trainTestSplit(std::vector<std::vector<double>> inputSet, std::vector<std::vector<double>> outputSet, double testSize) {
std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> MLPPData::trainTestSplit(std::vector<std::vector<real_t>> inputSet, std::vector<std::vector<real_t>> outputSet, real_t testSize) {
std::random_device rd;
std::default_random_engine generator(rd());
std::shuffle(inputSet.begin(), inputSet.end(), generator); // inputSet random shuffle
std::shuffle(outputSet.begin(), outputSet.end(), generator); // outputSet random shuffle)
std::vector<std::vector<double>> inputTestSet;
std::vector<std::vector<double>> outputTestSet;
std::vector<std::vector<real_t>> inputTestSet;
std::vector<std::vector<real_t>> outputTestSet;
int testInputNumber = testSize * inputSet.size(); // implicit usage of floor
int testOutputNumber = testSize * outputSet.size(); // implicit usage of floor
@ -358,7 +358,7 @@ std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>, s
// MULTIVARIATE SUPERVISED
void MLPPData::setData(int k, std::string fileName, std::vector<std::vector<double>> &inputSet, std::vector<double> &outputSet) {
void MLPPData::setData(int k, std::string fileName, std::vector<std::vector<real_t>> &inputSet, std::vector<real_t> &outputSet) {
MLPPLinAlg alg;
std::string inputTemp;
std::string outputTemp;
@ -386,7 +386,7 @@ void MLPPData::setData(int k, std::string fileName, std::vector<std::vector<doub
dataFile.close();
}
void MLPPData::printData(std::vector<std::string> inputName, std::string outputName, std::vector<std::vector<double>> inputSet, std::vector<double> outputSet) {
void MLPPData::printData(std::vector<std::string> inputName, std::string outputName, std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet) {
MLPPLinAlg alg;
inputSet = alg.transpose(inputSet);
for (int i = 0; i < inputSet.size(); i++) {
@ -404,7 +404,7 @@ void MLPPData::printData(std::vector<std::string> inputName, std::string outputN
// UNSUPERVISED
void MLPPData::setData(int k, std::string fileName, std::vector<std::vector<double>> &inputSet) {
void MLPPData::setData(int k, std::string fileName, std::vector<std::vector<real_t>> &inputSet) {
MLPPLinAlg alg;
std::string inputTemp;
@ -428,7 +428,7 @@ void MLPPData::setData(int k, std::string fileName, std::vector<std::vector<doub
dataFile.close();
}
void MLPPData::printData(std::vector<std::string> inputName, std::vector<std::vector<double>> inputSet) {
void MLPPData::printData(std::vector<std::string> inputName, std::vector<std::vector<real_t>> inputSet) {
MLPPLinAlg alg;
inputSet = alg.transpose(inputSet);
for (int i = 0; i < inputSet.size(); i++) {
@ -441,7 +441,7 @@ void MLPPData::printData(std::vector<std::string> inputName, std::vector<std::ve
// SIMPLE
void MLPPData::setData(std::string fileName, std::vector<double> &inputSet, std::vector<double> &outputSet) {
void MLPPData::setData(std::string fileName, std::vector<real_t> &inputSet, std::vector<real_t> &outputSet) {
std::string inputTemp, outputTemp;
std::ifstream dataFile(fileName);
@ -464,7 +464,7 @@ void MLPPData::setData(std::string fileName, std::vector<double> &inputSet, std:
dataFile.close();
}
void MLPPData::printData(std::string &inputName, std::string &outputName, std::vector<double> &inputSet, std::vector<double> &outputSet) {
void MLPPData::printData(std::string &inputName, std::string &outputName, std::vector<real_t> &inputSet, std::vector<real_t> &outputSet) {
std::cout << inputName << std::endl;
for (int i = 0; i < inputSet.size(); i++) {
std::cout << inputSet[i] << std::endl;
@ -477,8 +477,8 @@ void MLPPData::printData(std::string &inputName, std::string &outputName, std::v
}
// Images
std::vector<std::vector<double>> MLPPData::rgb2gray(std::vector<std::vector<std::vector<double>>> input) {
std::vector<std::vector<double>> grayScale;
std::vector<std::vector<real_t>> MLPPData::rgb2gray(std::vector<std::vector<std::vector<real_t>>> input) {
std::vector<std::vector<real_t>> grayScale;
grayScale.resize(input[0].size());
for (int i = 0; i < grayScale.size(); i++) {
grayScale[i].resize(input[0][i].size());
@ -491,9 +491,9 @@ std::vector<std::vector<double>> MLPPData::rgb2gray(std::vector<std::vector<std:
return grayScale;
}
std::vector<std::vector<std::vector<double>>> MLPPData::rgb2ycbcr(std::vector<std::vector<std::vector<double>>> input) {
std::vector<std::vector<std::vector<real_t>>> MLPPData::rgb2ycbcr(std::vector<std::vector<std::vector<real_t>>> input) {
MLPPLinAlg alg;
std::vector<std::vector<std::vector<double>>> YCbCr;
std::vector<std::vector<std::vector<real_t>>> YCbCr;
YCbCr = alg.resize(YCbCr, input);
for (int i = 0; i < YCbCr[0].size(); i++) {
for (int j = 0; j < YCbCr[0][i].size(); j++) {
@ -507,19 +507,19 @@ std::vector<std::vector<std::vector<double>>> MLPPData::rgb2ycbcr(std::vector<st
// Conversion formulas available here:
// https://www.rapidtables.com/convert/color/rgb-to-hsv.html
std::vector<std::vector<std::vector<double>>> MLPPData::rgb2hsv(std::vector<std::vector<std::vector<double>>> input) {
std::vector<std::vector<std::vector<real_t>>> MLPPData::rgb2hsv(std::vector<std::vector<std::vector<real_t>>> input) {
MLPPLinAlg alg;
std::vector<std::vector<std::vector<double>>> HSV;
std::vector<std::vector<std::vector<real_t>>> HSV;
HSV = alg.resize(HSV, input);
for (int i = 0; i < HSV[0].size(); i++) {
for (int j = 0; j < HSV[0][i].size(); j++) {
double rPrime = input[0][i][j] / 255;
double gPrime = input[1][i][j] / 255;
double bPrime = input[2][i][j] / 255;
real_t rPrime = input[0][i][j] / 255;
real_t gPrime = input[1][i][j] / 255;
real_t bPrime = input[2][i][j] / 255;
double cMax = alg.max({ rPrime, gPrime, bPrime });
double cMin = alg.min({ rPrime, gPrime, bPrime });
double delta = cMax - cMin;
real_t cMax = alg.max({ rPrime, gPrime, bPrime });
real_t cMin = alg.min({ rPrime, gPrime, bPrime });
real_t delta = cMax - cMin;
// H calculation.
if (delta == 0) {
@ -549,19 +549,19 @@ std::vector<std::vector<std::vector<double>>> MLPPData::rgb2hsv(std::vector<std:
}
// http://machinethatsees.blogspot.com/2013/07/how-to-convert-rgb-to-xyz-or-vice-versa.html
std::vector<std::vector<std::vector<double>>> MLPPData::rgb2xyz(std::vector<std::vector<std::vector<double>>> input) {
std::vector<std::vector<std::vector<real_t>>> MLPPData::rgb2xyz(std::vector<std::vector<std::vector<real_t>>> input) {
MLPPLinAlg alg;
std::vector<std::vector<std::vector<double>>> XYZ;
std::vector<std::vector<std::vector<real_t>>> XYZ;
XYZ = alg.resize(XYZ, input);
std::vector<std::vector<double>> RGB2XYZ = { { 0.4124564, 0.3575761, 0.1804375 }, { 0.2126726, 0.7151522, 0.0721750 }, { 0.0193339, 0.1191920, 0.9503041 } };
std::vector<std::vector<real_t>> RGB2XYZ = { { 0.4124564, 0.3575761, 0.1804375 }, { 0.2126726, 0.7151522, 0.0721750 }, { 0.0193339, 0.1191920, 0.9503041 } };
return alg.vector_wise_tensor_product(input, RGB2XYZ);
}
std::vector<std::vector<std::vector<double>>> MLPPData::xyz2rgb(std::vector<std::vector<std::vector<double>>> input) {
std::vector<std::vector<std::vector<real_t>>> MLPPData::xyz2rgb(std::vector<std::vector<std::vector<real_t>>> input) {
MLPPLinAlg alg;
std::vector<std::vector<std::vector<double>>> XYZ;
std::vector<std::vector<std::vector<real_t>>> XYZ;
XYZ = alg.resize(XYZ, input);
std::vector<std::vector<double>> RGB2XYZ = alg.inverse({ { 0.4124564, 0.3575761, 0.1804375 }, { 0.2126726, 0.7151522, 0.0721750 }, { 0.0193339, 0.1191920, 0.9503041 } });
std::vector<std::vector<real_t>> RGB2XYZ = alg.inverse({ { 0.4124564, 0.3575761, 0.1804375 }, { 0.2126726, 0.7151522, 0.0721750 }, { 0.0193339, 0.1191920, 0.9503041 } });
return alg.vector_wise_tensor_product(input, RGB2XYZ);
}
@ -640,11 +640,11 @@ std::vector<std::string> MLPPData::segment(std::string text) {
return segmented_data;
}
std::vector<double> MLPPData::tokenize(std::string text) {
std::vector<real_t> MLPPData::tokenize(std::string text) {
int max_num = 0;
bool new_num = true;
std::vector<std::string> segmented_data = segment(text);
std::vector<double> tokenized_data;
std::vector<real_t> tokenized_data;
tokenized_data.resize(segmented_data.size());
for (int i = 0; i < segmented_data.size(); i++) {
for (int j = i - 1; j >= 0; j--) {
@ -710,7 +710,7 @@ std::string MLPPData::stemming(std::string text) {
return text;
}
std::vector<std::vector<double>> MLPPData::BOW(std::vector<std::string> sentences, std::string type) {
std::vector<std::vector<real_t>> MLPPData::BOW(std::vector<std::string> sentences, std::string type) {
/*
STEPS OF BOW:
1) To lowercase (done by removeStopWords function by def)
@ -729,7 +729,7 @@ std::vector<std::vector<double>> MLPPData::BOW(std::vector<std::string> sentence
segmented_sentences[i] = removeStopWords(sentences[i]);
}
std::vector<std::vector<double>> bow;
std::vector<std::vector<real_t>> bow;
bow.resize(sentences.size());
for (int i = 0; i < bow.size(); i++) {
@ -752,7 +752,7 @@ std::vector<std::vector<double>> MLPPData::BOW(std::vector<std::string> sentence
return bow;
}
std::vector<std::vector<double>> MLPPData::TFIDF(std::vector<std::string> sentences) {
std::vector<std::vector<real_t>> MLPPData::TFIDF(std::vector<std::string> sentences) {
MLPPLinAlg alg;
std::vector<std::string> wordList = removeNullByte(removeStopWords(createWordList(sentences)));
@ -763,7 +763,7 @@ std::vector<std::vector<double>> MLPPData::TFIDF(std::vector<std::string> senten
segmented_sentences[i] = removeStopWords(sentences[i]);
}
std::vector<std::vector<double>> TF;
std::vector<std::vector<real_t>> TF;
std::vector<int> frequency;
frequency.resize(wordList.size());
TF.resize(segmented_sentences.size());
@ -783,17 +783,17 @@ std::vector<std::vector<double>> MLPPData::TFIDF(std::vector<std::string> senten
}
}
}
TF[i] = alg.scalarMultiply(double(1) / double(segmented_sentences[i].size()), TF[i]);
TF[i] = alg.scalarMultiply(real_t(1) / real_t(segmented_sentences[i].size()), TF[i]);
}
std::vector<double> IDF;
std::vector<real_t> IDF;
IDF.resize(frequency.size());
for (int i = 0; i < IDF.size(); i++) {
IDF[i] = std::log((double)segmented_sentences.size() / (double)frequency[i]);
IDF[i] = std::log((real_t)segmented_sentences.size() / (real_t)frequency[i]);
}
std::vector<std::vector<double>> TFIDF;
std::vector<std::vector<real_t>> TFIDF;
TFIDF.resize(segmented_sentences.size());
for (int i = 0; i < TFIDF.size(); i++) {
TFIDF[i].resize(wordList.size());
@ -808,7 +808,7 @@ std::vector<std::vector<double>> MLPPData::TFIDF(std::vector<std::string> senten
return TFIDF;
}
std::tuple<std::vector<std::vector<double>>, std::vector<std::string>> MLPPData::word2Vec(std::vector<std::string> sentences, std::string type, int windowSize, int dimension, double learning_rate, int max_epoch) {
std::tuple<std::vector<std::vector<real_t>>, std::vector<std::string>> MLPPData::word2Vec(std::vector<std::string> sentences, std::string type, int windowSize, int dimension, real_t learning_rate, int max_epoch) {
std::vector<std::string> wordList = removeNullByte(removeStopWords(createWordList(sentences)));
std::vector<std::vector<std::string>> segmented_sentences;
@ -841,10 +841,10 @@ std::tuple<std::vector<std::vector<double>>, std::vector<std::string>> MLPPData:
inputStrings.insert(inputStrings.end(), outputStrings.begin(), outputStrings.end());
std::vector<std::vector<double>> BOW = MLPPData::BOW(inputStrings, "Binary");
std::vector<std::vector<real_t>> BOW = MLPPData::BOW(inputStrings, "Binary");
std::vector<std::vector<double>> inputSet;
std::vector<std::vector<double>> outputSet;
std::vector<std::vector<real_t>> inputSet;
std::vector<std::vector<real_t>> outputSet;
for (int i = 0; i < inputSize; i++) {
inputSet.push_back(BOW[i]);
@ -862,17 +862,17 @@ std::tuple<std::vector<std::vector<double>>, std::vector<std::string>> MLPPData:
}
model->gradientDescent(learning_rate, max_epoch, 1);
std::vector<std::vector<double>> wordEmbeddings = model->getEmbeddings();
std::vector<std::vector<real_t>> wordEmbeddings = model->getEmbeddings();
delete model;
return { wordEmbeddings, wordList };
}
struct WordsToVecResult {
std::vector<std::vector<double>> word_embeddings;
std::vector<std::vector<real_t>> word_embeddings;
std::vector<std::string> word_list;
};
MLPPData::WordsToVecResult MLPPData::word_to_vec(std::vector<std::string> sentences, std::string type, int windowSize, int dimension, double learning_rate, int max_epoch) {
MLPPData::WordsToVecResult MLPPData::word_to_vec(std::vector<std::string> sentences, std::string type, int windowSize, int dimension, real_t learning_rate, int max_epoch) {
WordsToVecResult res;
res.word_list = removeNullByte(removeStopWords(createWordList(sentences)));
@ -907,10 +907,10 @@ MLPPData::WordsToVecResult MLPPData::word_to_vec(std::vector<std::string> senten
inputStrings.insert(inputStrings.end(), outputStrings.begin(), outputStrings.end());
std::vector<std::vector<double>> BOW = MLPPData::BOW(inputStrings, "Binary");
std::vector<std::vector<real_t>> BOW = MLPPData::BOW(inputStrings, "Binary");
std::vector<std::vector<double>> inputSet;
std::vector<std::vector<double>> outputSet;
std::vector<std::vector<real_t>> inputSet;
std::vector<std::vector<real_t>> outputSet;
for (int i = 0; i < inputSize; i++) {
inputSet.push_back(BOW[i]);
@ -934,19 +934,19 @@ MLPPData::WordsToVecResult MLPPData::word_to_vec(std::vector<std::string> senten
return res;
}
std::vector<std::vector<double>> MLPPData::LSA(std::vector<std::string> sentences, int dim) {
std::vector<std::vector<real_t>> MLPPData::LSA(std::vector<std::string> sentences, int dim) {
MLPPLinAlg alg;
std::vector<std::vector<double>> docWordData = BOW(sentences, "Binary");
std::vector<std::vector<real_t>> docWordData = BOW(sentences, "Binary");
auto [U, S, Vt] = alg.SVD(docWordData);
std::vector<std::vector<double>> S_trunc = alg.zeromat(dim, dim);
std::vector<std::vector<double>> Vt_trunc;
std::vector<std::vector<real_t>> S_trunc = alg.zeromat(dim, dim);
std::vector<std::vector<real_t>> Vt_trunc;
for (int i = 0; i < dim; i++) {
S_trunc[i][i] = S[i][i];
Vt_trunc.push_back(Vt[i]);
}
std::vector<std::vector<double>> embeddings = alg.matmult(S_trunc, Vt_trunc);
std::vector<std::vector<real_t>> embeddings = alg.matmult(S_trunc, Vt_trunc);
return embeddings;
}
@ -977,10 +977,10 @@ void MLPPData::setInputNames(std::string fileName, std::vector<std::string> &inp
dataFile.close();
}
std::vector<std::vector<double>> MLPPData::featureScaling(std::vector<std::vector<double>> X) {
std::vector<std::vector<real_t>> MLPPData::featureScaling(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg;
X = alg.transpose(X);
std::vector<double> max_elements, min_elements;
std::vector<real_t> max_elements, min_elements;
max_elements.resize(X.size());
min_elements.resize(X.size());
@ -997,7 +997,7 @@ std::vector<std::vector<double>> MLPPData::featureScaling(std::vector<std::vecto
return alg.transpose(X);
}
std::vector<std::vector<double>> MLPPData::meanNormalization(std::vector<std::vector<double>> X) {
std::vector<std::vector<real_t>> MLPPData::meanNormalization(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg;
MLPPStat stat;
// (X_j - mu_j) / std_j, for every j
@ -1009,11 +1009,11 @@ std::vector<std::vector<double>> MLPPData::meanNormalization(std::vector<std::ve
return X;
}
std::vector<std::vector<double>> MLPPData::meanCentering(std::vector<std::vector<double>> X) {
std::vector<std::vector<real_t>> MLPPData::meanCentering(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg;
MLPPStat stat;
for (int i = 0; i < X.size(); i++) {
double mean_i = stat.mean(X[i]);
real_t mean_i = stat.mean(X[i]);
for (int j = 0; j < X[i].size(); j++) {
X[i][j] -= mean_i;
}
@ -1021,8 +1021,8 @@ std::vector<std::vector<double>> MLPPData::meanCentering(std::vector<std::vector
return X;
}
std::vector<std::vector<double>> MLPPData::oneHotRep(std::vector<double> tempOutputSet, int n_class) {
std::vector<std::vector<double>> outputSet;
std::vector<std::vector<real_t>> MLPPData::oneHotRep(std::vector<real_t> tempOutputSet, int n_class) {
std::vector<std::vector<real_t>> outputSet;
outputSet.resize(tempOutputSet.size());
for (int i = 0; i < tempOutputSet.size(); i++) {
for (int j = 0; j <= n_class - 1; j++) {
@ -1036,8 +1036,8 @@ std::vector<std::vector<double>> MLPPData::oneHotRep(std::vector<double> tempOut
return outputSet;
}
std::vector<double> MLPPData::reverseOneHot(std::vector<std::vector<double>> tempOutputSet) {
std::vector<double> outputSet;
std::vector<real_t> MLPPData::reverseOneHot(std::vector<std::vector<real_t>> tempOutputSet) {
std::vector<real_t> outputSet;
int n_class = tempOutputSet[0].size();
for (int i = 0; i < tempOutputSet.size(); i++) {
int current_class = 1;

88
mlpp/data/data.h
View File

@ -9,6 +9,8 @@
// Created by Marc Melikyan on 11/4/20.
//
#include "core/math/math_defs.h"
#include "core/string/ustring.h"
#include "core/variant/array.h"
@ -22,8 +24,8 @@ class MLPPDataESimple : public Reference {
GDCLASS(MLPPDataESimple, Reference);
public:
std::vector<double> input;
std::vector<double> output;
std::vector<real_t> input;
std::vector<real_t> output;
protected:
static void _bind_methods();
@ -33,8 +35,8 @@ class MLPPDataSimple : public Reference {
GDCLASS(MLPPDataSimple, Reference);
public:
std::vector<std::vector<double>> input;
std::vector<double> output;
std::vector<std::vector<real_t>> input;
std::vector<real_t> output;
protected:
static void _bind_methods();
@ -44,8 +46,8 @@ class MLPPDataComplex : public Reference {
GDCLASS(MLPPDataComplex, Reference);
public:
std::vector<std::vector<double>> input;
std::vector<std::vector<double>> output;
std::vector<std::vector<real_t>> input;
std::vector<std::vector<real_t>> output;
protected:
static void _bind_methods();
@ -65,48 +67,48 @@ public:
Ref<MLPPDataSimple> load_california_housing(const String &path);
Ref<MLPPDataESimple> load_fires_and_crime(const String &path);
void set_data_supervised(int k, const String &file_name, std::vector<std::vector<double>> &inputSet, std::vector<double> &outputSet);
void set_data_unsupervised(int k, const String &file_name, std::vector<std::vector<double>> &inputSet);
void set_data_simple(const String &file_name, std::vector<double> &inputSet, std::vector<double> &outputSet);
void set_data_supervised(int k, const String &file_name, std::vector<std::vector<real_t>> &inputSet, std::vector<real_t> &outputSet);
void set_data_unsupervised(int k, const String &file_name, std::vector<std::vector<real_t>> &inputSet);
void set_data_simple(const String &file_name, std::vector<real_t> &inputSet, std::vector<real_t> &outputSet);
struct SplitComplexData {
Ref<MLPPDataComplex> train;
Ref<MLPPDataComplex> test;
};
SplitComplexData train_test_split(const Ref<MLPPDataComplex> &data, double test_size);
Array train_test_split_bind(const Ref<MLPPDataComplex> &data, double test_size);
SplitComplexData train_test_split(const Ref<MLPPDataComplex> &data, real_t test_size);
Array train_test_split_bind(const Ref<MLPPDataComplex> &data, real_t test_size);
// Load Datasets
std::tuple<std::vector<std::vector<double>>, std::vector<double>> loadBreastCancer();
std::tuple<std::vector<std::vector<double>>, std::vector<double>> loadBreastCancerSVC();
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> loadIris();
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> loadWine();
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> loadMnistTrain();
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> loadMnistTest();
std::tuple<std::vector<std::vector<double>>, std::vector<double>> loadCaliforniaHousing();
std::tuple<std::vector<double>, std::vector<double>> loadFiresAndCrime();
std::tuple<std::vector<std::vector<real_t>>, std::vector<real_t>> loadBreastCancer();
std::tuple<std::vector<std::vector<real_t>>, std::vector<real_t>> loadBreastCancerSVC();
std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> loadIris();
std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> loadWine();
std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> loadMnistTrain();
std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> loadMnistTest();
std::tuple<std::vector<std::vector<real_t>>, std::vector<real_t>> loadCaliforniaHousing();
std::tuple<std::vector<real_t>, std::vector<real_t>> loadFiresAndCrime();
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>, std::vector<std::vector<double>>, std::vector<std::vector<double>>> trainTestSplit(std::vector<std::vector<double>> inputSet, std::vector<std::vector<double>> outputSet, double testSize);
std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> trainTestSplit(std::vector<std::vector<real_t>> inputSet, std::vector<std::vector<real_t>> outputSet, real_t testSize);
// Supervised
void setData(int k, std::string fileName, std::vector<std::vector<double>> &inputSet, std::vector<double> &outputSet);
void printData(std::vector<std::string> inputName, std::string outputName, std::vector<std::vector<double>> inputSet, std::vector<double> outputSet);
void setData(int k, std::string fileName, std::vector<std::vector<real_t>> &inputSet, std::vector<real_t> &outputSet);
void printData(std::vector<std::string> inputName, std::string outputName, std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet);
// Unsupervised
void setData(int k, std::string fileName, std::vector<std::vector<double>> &inputSet);
void printData(std::vector<std::string> inputName, std::vector<std::vector<double>> inputSet);
void setData(int k, std::string fileName, std::vector<std::vector<real_t>> &inputSet);
void printData(std::vector<std::string> inputName, std::vector<std::vector<real_t>> inputSet);
// Simple
void setData(std::string fileName, std::vector<double> &inputSet, std::vector<double> &outputSet);
void printData(std::string &inputName, std::string &outputName, std::vector<double> &inputSet, std::vector<double> &outputSet);
void setData(std::string fileName, std::vector<real_t> &inputSet, std::vector<real_t> &outputSet);
void printData(std::string &inputName, std::string &outputName, std::vector<real_t> &inputSet, std::vector<real_t> &outputSet);
// Images
std::vector<std::vector<double>> rgb2gray(std::vector<std::vector<std::vector<double>>> input);
std::vector<std::vector<std::vector<double>>> rgb2ycbcr(std::vector<std::vector<std::vector<double>>> input);
std::vector<std::vector<std::vector<double>>> rgb2hsv(std::vector<std::vector<std::vector<double>>> input);
std::vector<std::vector<std::vector<double>>> rgb2xyz(std::vector<std::vector<std::vector<double>>> input);
std::vector<std::vector<std::vector<double>>> xyz2rgb(std::vector<std::vector<std::vector<double>>> input);
std::vector<std::vector<real_t>> rgb2gray(std::vector<std::vector<std::vector<real_t>>> input);
std::vector<std::vector<std::vector<real_t>>> rgb2ycbcr(std::vector<std::vector<std::vector<real_t>>> input);
std::vector<std::vector<std::vector<real_t>>> rgb2hsv(std::vector<std::vector<std::vector<real_t>>> input);
std::vector<std::vector<std::vector<real_t>>> rgb2xyz(std::vector<std::vector<std::vector<real_t>>> input);
std::vector<std::vector<std::vector<real_t>>> xyz2rgb(std::vector<std::vector<std::vector<real_t>>> input);
// Text-Based & NLP
std::string toLower(std::string text);
@ -115,35 +117,35 @@ public:
std::vector<std::string> removeSpaces(std::vector<std::string> data);
std::vector<std::string> removeNullByte(std::vector<std::string> data);
std::vector<std::string> segment(std::string text);
std::vector<double> tokenize(std::string text);
std::vector<real_t> tokenize(std::string text);
std::vector<std::string> removeStopWords(std::string text);
std::vector<std::string> removeStopWords(std::vector<std::string> segmented_data);
std::string stemming(std::string text);
std::vector<std::vector<double>> BOW(std::vector<std::string> sentences, std::string = "Default");
std::vector<std::vector<double>> TFIDF(std::vector<std::string> sentences);
std::vector<std::vector<real_t>> BOW(std::vector<std::string> sentences, std::string = "Default");
std::vector<std::vector<real_t>> TFIDF(std::vector<std::string> sentences);
std::tuple<std::vector<std::vector<double>>, std::vector<std::string>> word2Vec(std::vector<std::string> sentences, std::string type, int windowSize, int dimension, double learning_rate, int max_epoch);
std::tuple<std::vector<std::vector<real_t>>, std::vector<std::string>> word2Vec(std::vector<std::string> sentences, std::string type, int windowSize, int dimension, real_t learning_rate, int max_epoch);
struct WordsToVecResult {
std::vector<std::vector<double>> word_embeddings;
std::vector<std::vector<real_t>> word_embeddings;
std::vector<std::string> word_list;
};
WordsToVecResult word_to_vec(std::vector<std::string> sentences, std::string type, int windowSize, int dimension, double learning_rate, int max_epoch);
WordsToVecResult word_to_vec(std::vector<std::string> sentences, std::string type, int windowSize, int dimension, real_t learning_rate, int max_epoch);
std::vector<std::vector<double>> LSA(std::vector<std::string> sentences, int dim);
std::vector<std::vector<real_t>> LSA(std::vector<std::string> sentences, int dim);
std::vector<std::string> createWordList(std::vector<std::string> sentences);
// Extra
void setInputNames(std::string fileName, std::vector<std::string> &inputNames);
std::vector<std::vector<double>> featureScaling(std::vector<std::vector<double>> X);
std::vector<std::vector<double>> meanNormalization(std::vector<std::vector<double>> X);
std::vector<std::vector<double>> meanCentering(std::vector<std::vector<double>> X);
std::vector<std::vector<double>> oneHotRep(std::vector<double> tempOutputSet, int n_class);
std::vector<double> reverseOneHot(std::vector<std::vector<double>> tempOutputSet);
std::vector<std::vector<real_t>> featureScaling(std::vector<std::vector<real_t>> X);
std::vector<std::vector<real_t>> meanNormalization(std::vector<std::vector<real_t>> X);
std::vector<std::vector<real_t>> meanCentering(std::vector<std::vector<real_t>> X);
std::vector<std::vector<real_t>> oneHotRep(std::vector<real_t> tempOutputSet, int n_class);
std::vector<real_t> reverseOneHot(std::vector<std::vector<real_t>> tempOutputSet);
template <class T>
std::vector<T> vecToSet(std::vector<T> inputSet) {

48
mlpp/dual_svc/dual_svc.cpp
View File

@ -15,7 +15,7 @@
#include <random>
MLPPDualSVC::MLPPDualSVC(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, double C, std::string kernel) :
MLPPDualSVC::MLPPDualSVC(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, real_t C, std::string kernel) :
inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), C(C), kernel(kernel) {
y_hat.resize(n);
bias = MLPPUtilities::biasInitialization();
@ -23,20 +23,20 @@ MLPPDualSVC::MLPPDualSVC(std::vector<std::vector<double>> inputSet, std::vector<
K = kernelFunction(inputSet, inputSet, kernel); // For now this is unused. When non-linear kernels are added, the K will be manipulated.
}
std::vector<double> MLPPDualSVC::modelSetTest(std::vector<std::vector<double>> X) {
std::vector<real_t> MLPPDualSVC::modelSetTest(std::vector<std::vector<real_t>> X) {
return Evaluate(X);
}
double MLPPDualSVC::modelTest(std::vector<double> x) {
real_t MLPPDualSVC::modelTest(std::vector<real_t> x) {
return Evaluate(x);
}
void MLPPDualSVC::gradientDescent(double learning_rate, int max_epoch, bool UI) {
void MLPPDualSVC::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
class MLPPCost cost;
MLPPActivation avn;
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
forwardPass();
@ -48,9 +48,9 @@ void MLPPDualSVC::gradientDescent(double learning_rate, int max_epoch, bool UI)
alphaProjection();
// Calculating the bias
double biasGradient = 0;
real_t biasGradient = 0;
for (int i = 0; i < alpha.size(); i++) {
double sum = 0;
real_t sum = 0;
if (alpha[i] < C && alpha[i] > 0) {
for (int j = 0; j < alpha.size(); j++) {
if (alpha[j] > 0) {
@ -79,13 +79,13 @@ void MLPPDualSVC::gradientDescent(double learning_rate, int max_epoch, bool UI)
}
}
// void MLPPDualSVC::SGD(double learning_rate, int max_epoch, bool UI){
// void MLPPDualSVC::SGD(real_t learning_rate, int max_epoch, bool UI){
// class MLPPCost cost;
// MLPPActivation avn;
// MLPPLinAlg alg;
// MLPPReg regularization;
// double cost_prev = 0;
// real_t cost_prev = 0;
// int epoch = 1;
// while(true){
@ -112,12 +112,12 @@ void MLPPDualSVC::gradientDescent(double learning_rate, int max_epoch, bool UI)
// forwardPass();
// }
// void MLPPDualSVC::MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI){
// void MLPPDualSVC::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI){
// class MLPPCost cost;
// MLPPActivation avn;
// MLPPLinAlg alg;
// MLPPReg regularization;
// double cost_prev = 0;
// real_t cost_prev = 0;
// int epoch = 1;
// // Creating the mini-batches
@ -126,8 +126,8 @@ void MLPPDualSVC::gradientDescent(double learning_rate, int max_epoch, bool UI)
// while(true){
// for(int i = 0; i < n_mini_batch; i++){
// std::vector<double> y_hat = Evaluate(inputMiniBatches[i]);
// std::vector<double> z = propagate(inputMiniBatches[i]);
// std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]);
// std::vector<real_t> z = propagate(inputMiniBatches[i]);
// cost_prev = Cost(z, outputMiniBatches[i], weights, C);
// // Calculating the weight gradients
@ -152,7 +152,7 @@ void MLPPDualSVC::gradientDescent(double learning_rate, int max_epoch, bool UI)
// forwardPass();
// }
double MLPPDualSVC::score() {
real_t MLPPDualSVC::score() {
MLPPUtilities util;
return util.performance(y_hat, outputSet);
}
@ -162,21 +162,21 @@ void MLPPDualSVC::save(std::string fileName) {
util.saveParameters(fileName, alpha, bias);
}
double MLPPDualSVC::Cost(std::vector<double> alpha, std::vector<std::vector<double>> X, std::vector<double> y) {
real_t MLPPDualSVC::Cost(std::vector<real_t> alpha, std::vector<std::vector<real_t>> X, std::vector<real_t> y) {
class MLPPCost cost;
return cost.dualFormSVM(alpha, X, y);
}
std::vector<double> MLPPDualSVC::Evaluate(std::vector<std::vector<double>> X) {
std::vector<real_t> MLPPDualSVC::Evaluate(std::vector<std::vector<real_t>> X) {
MLPPActivation avn;
return avn.sign(propagate(X));
}
std::vector<double> MLPPDualSVC::propagate(std::vector<std::vector<double>> X) {
std::vector<real_t> MLPPDualSVC::propagate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg;
std::vector<double> z;
std::vector<real_t> z;
for (int i = 0; i < X.size(); i++) {
double sum = 0;
real_t sum = 0;
for (int j = 0; j < alpha.size(); j++) {
if (alpha[j] != 0) {
sum += alpha[j] * outputSet[j] * alg.dot(inputSet[j], X[i]); // TO DO: DON'T forget to add non-linear kernelizations.
@ -188,14 +188,14 @@ std::vector<double> MLPPDualSVC::propagate(std::vector<std::vector<double>> X) {
return z;
}
double MLPPDualSVC::Evaluate(std::vector<double> x) {
real_t MLPPDualSVC::Evaluate(std::vector<real_t> x) {
MLPPActivation avn;
return avn.sign(propagate(x));
}
double MLPPDualSVC::propagate(std::vector<double> x) {
real_t MLPPDualSVC::propagate(std::vector<real_t> x) {
MLPPLinAlg alg;
double z = 0;
real_t z = 0;
for (int j = 0; j < alpha.size(); j++) {
if (alpha[j] != 0) {
z += alpha[j] * outputSet[j] * alg.dot(inputSet[j], x); // TO DO: DON'T forget to add non-linear kernelizations.
@ -223,14 +223,14 @@ void MLPPDualSVC::alphaProjection() {
}
}
double MLPPDualSVC::kernelFunction(std::vector<double> u, std::vector<double> v, std::string kernel) {
real_t MLPPDualSVC::kernelFunction(std::vector<real_t> u, std::vector<real_t> v, std::string kernel) {
MLPPLinAlg alg;
if (kernel == "Linear") {
return alg.dot(u, v);
} // warning: non-void function does not return a value in all control paths [-Wreturn-type]
}
std::vector<std::vector<double>> MLPPDualSVC::kernelFunction(std::vector<std::vector<double>> A, std::vector<std::vector<double>> B, std::string kernel) {
std::vector<std::vector<real_t>> MLPPDualSVC::kernelFunction(std::vector<std::vector<real_t>> A, std::vector<std::vector<real_t>> B, std::string kernel) {
MLPPLinAlg alg;
if (kernel == "Linear") {
return alg.matmult(inputSet, alg.transpose(inputSet));

54
mlpp/dual_svc/dual_svc.h
View File

@ -11,6 +11,8 @@
// http://ciml.info/dl/v0_99/ciml-v0_99-ch11.pdf
// Were excellent for the practical intution behind the dual formulation.
#include "core/math/math_defs.h"
#include <string>
#include <vector>
@ -18,52 +20,52 @@
class MLPPDualSVC {
public:
MLPPDualSVC(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, double C, std::string kernel = "Linear");
MLPPDualSVC(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, double C, std::string kernel, double p, double c);
MLPPDualSVC(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, real_t C, std::string kernel = "Linear");
MLPPDualSVC(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, real_t C, std::string kernel, real_t p, real_t c);
std::vector<double> modelSetTest(std::vector<std::vector<double>> X);
double modelTest(std::vector<double> x);
void gradientDescent(double learning_rate, int max_epoch, bool UI = 1);
void SGD(double learning_rate, int max_epoch, bool UI = 1);
void MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
double score();
std::vector<real_t> modelSetTest(std::vector<std::vector<real_t>> X);
real_t modelTest(std::vector<real_t> x);
void gradientDescent(real_t learning_rate, int max_epoch, bool UI = 1);
void SGD(real_t learning_rate, int max_epoch, bool UI = 1);
void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
real_t score();
void save(std::string fileName);
private:
void init();
double Cost(std::vector<double> alpha, std::vector<std::vector<double>> X, std::vector<double> y);
real_t Cost(std::vector<real_t> alpha, std::vector<std::vector<real_t>> X, std::vector<real_t> y);
std::vector<double> Evaluate(std::vector<std::vector<double>> X);
std::vector<double> propagate(std::vector<std::vector<double>> X);
double Evaluate(std::vector<double> x);
double propagate(std::vector<double> x);
std::vector<real_t> Evaluate(std::vector<std::vector<real_t>> X);
std::vector<real_t> propagate(std::vector<std::vector<real_t>> X);
real_t Evaluate(std::vector<real_t> x);
real_t propagate(std::vector<real_t> x);
void forwardPass();
void alphaProjection();
double kernelFunction(std::vector<double> v, std::vector<double> u, std::string kernel);
std::vector<std::vector<double>> kernelFunction(std::vector<std::vector<double>> U, std::vector<std::vector<double>> V, std::string kernel);
real_t kernelFunction(std::vector<real_t> v, std::vector<real_t> u, std::string kernel);
std::vector<std::vector<real_t>> kernelFunction(std::vector<std::vector<real_t>> U, std::vector<std::vector<real_t>> V, std::string kernel);
std::vector<std::vector<double>> inputSet;
std::vector<double> outputSet;
std::vector<double> z;
std::vector<double> y_hat;
double bias;
std::vector<std::vector<real_t>> inputSet;
std::vector<real_t> outputSet;
std::vector<real_t> z;
std::vector<real_t> y_hat;
real_t bias;
std::vector<double> alpha;
std::vector<std::vector<double>> K;
std::vector<real_t> alpha;
std::vector<std::vector<real_t>> K;
double C;
real_t C;
int n;
int k;
std::string kernel;
double p; // Poly
double c; // Poly
real_t p; // Poly
real_t c; // Poly
// UI Portion
void UI(int epoch, double cost_prev);
void UI(int epoch, real_t cost_prev);
};

68
mlpp/exp_reg/exp_reg.cpp
View File

@ -15,7 +15,7 @@
#include <random>
MLPPExpReg::MLPPExpReg(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, std::string reg, double lambda, double alpha) :
MLPPExpReg::MLPPExpReg(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, std::string reg, real_t lambda, real_t alpha) :
inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) {
y_hat.resize(n);
weights = MLPPUtilities::weightInitialization(k);
@ -23,41 +23,41 @@ MLPPExpReg::MLPPExpReg(std::vector<std::vector<double>> inputSet, std::vector<do
bias = MLPPUtilities::biasInitialization();
}
std::vector<double> MLPPExpReg::modelSetTest(std::vector<std::vector<double>> X) {
std::vector<real_t> MLPPExpReg::modelSetTest(std::vector<std::vector<real_t>> X) {
return Evaluate(X);
}
double MLPPExpReg::modelTest(std::vector<double> x) {
real_t MLPPExpReg::modelTest(std::vector<real_t> x) {
return Evaluate(x);
}
void MLPPExpReg::gradientDescent(double learning_rate, int max_epoch, bool UI) {
void MLPPExpReg::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
forwardPass();
while (true) {
cost_prev = Cost(y_hat, outputSet);
std::vector<double> error = alg.subtraction(y_hat, outputSet);
std::vector<real_t> error = alg.subtraction(y_hat, outputSet);
for (int i = 0; i < k; i++) {
// Calculating the weight gradient
double sum = 0;
real_t sum = 0;
for (int j = 0; j < n; j++) {
sum += error[j] * inputSet[j][i] * std::pow(weights[i], inputSet[j][i] - 1);
}
double w_gradient = sum / n;
real_t w_gradient = sum / n;
// Calculating the initial gradient
double sum2 = 0;
real_t sum2 = 0;
for (int j = 0; j < n; j++) {
sum2 += error[j] * std::pow(weights[i], inputSet[j][i]);
}
double i_gradient = sum2 / n;
real_t i_gradient = sum2 / n;
// Weight/initial updation
weights[i] -= learning_rate * w_gradient;
@ -66,11 +66,11 @@ void MLPPExpReg::gradientDescent(double learning_rate, int max_epoch, bool UI) {
weights = regularization.regWeights(weights, lambda, alpha, reg);
// Calculating the bias gradient
double sum = 0;
real_t sum = 0;
for (int j = 0; j < n; j++) {
sum += (y_hat[j] - outputSet[j]);
}
double b_gradient = sum / n;
real_t b_gradient = sum / n;
// bias updation
bias -= learning_rate * b_gradient;
@ -88,9 +88,9 @@ void MLPPExpReg::gradientDescent(double learning_rate, int max_epoch, bool UI) {
}
}
void MLPPExpReg::SGD(double learning_rate, int max_epoch, bool UI) {
void MLPPExpReg::SGD(real_t learning_rate, int max_epoch, bool UI) {
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
while (true) {
@ -99,14 +99,14 @@ void MLPPExpReg::SGD(double learning_rate, int max_epoch, bool UI) {
std::uniform_int_distribution<int> distribution(0, int(n - 1));
int outputIndex = distribution(generator);
double y_hat = Evaluate(inputSet[outputIndex]);
real_t y_hat = Evaluate(inputSet[outputIndex]);
cost_prev = Cost({ y_hat }, { outputSet[outputIndex] });
for (int i = 0; i < k; i++) {
// Calculating the weight gradients
double w_gradient = (y_hat - outputSet[outputIndex]) * inputSet[outputIndex][i] * std::pow(weights[i], inputSet[outputIndex][i] - 1);
double i_gradient = (y_hat - outputSet[outputIndex]) * std::pow(weights[i], inputSet[outputIndex][i]);
real_t w_gradient = (y_hat - outputSet[outputIndex]) * inputSet[outputIndex][i] * std::pow(weights[i], inputSet[outputIndex][i] - 1);
real_t i_gradient = (y_hat - outputSet[outputIndex]) * std::pow(weights[i], inputSet[outputIndex][i]);
// Weight/initial updation
weights[i] -= learning_rate * w_gradient;
@ -115,7 +115,7 @@ void MLPPExpReg::SGD(double learning_rate, int max_epoch, bool UI) {
weights = regularization.regWeights(weights, lambda, alpha, reg);
// Calculating the bias gradients
double b_gradient = (y_hat - outputSet[outputIndex]);
real_t b_gradient = (y_hat - outputSet[outputIndex]);
// Bias updation
bias -= learning_rate * b_gradient;
@ -134,10 +134,10 @@ void MLPPExpReg::SGD(double learning_rate, int max_epoch, bool UI) {
forwardPass();
}
void MLPPExpReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI) {
void MLPPExpReg::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI) {
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
// Creating the mini-batches
@ -146,25 +146,25 @@ void MLPPExpReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size,
while (true) {
for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = Evaluate(inputMiniBatches[i]);
std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]);
std::vector<double> error = alg.subtraction(y_hat, outputMiniBatches[i]);
std::vector<real_t> error = alg.subtraction(y_hat, outputMiniBatches[i]);
for (int j = 0; j < k; j++) {
// Calculating the weight gradient
double sum = 0;
real_t sum = 0;
for (int k = 0; k < outputMiniBatches[i].size(); k++) {
sum += error[k] * inputMiniBatches[i][k][j] * std::pow(weights[j], inputMiniBatches[i][k][j] - 1);
}
double w_gradient = sum / outputMiniBatches[i].size();
real_t w_gradient = sum / outputMiniBatches[i].size();
// Calculating the initial gradient
double sum2 = 0;
real_t sum2 = 0;
for (int k = 0; k < outputMiniBatches[i].size(); k++) {
sum2 += error[k] * std::pow(weights[j], inputMiniBatches[i][k][j]);
}
double i_gradient = sum2 / outputMiniBatches[i].size();
real_t i_gradient = sum2 / outputMiniBatches[i].size();
// Weight/initial updation
weights[j] -= learning_rate * w_gradient;
@ -173,11 +173,11 @@ void MLPPExpReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size,
weights = regularization.regWeights(weights, lambda, alpha, reg);
// Calculating the bias gradient
double sum = 0;
real_t sum = 0;
for (int j = 0; j < outputMiniBatches[i].size(); j++) {
sum += (y_hat[j] - outputMiniBatches[i][j]);
}
double b_gradient = sum / outputMiniBatches[i].size();
real_t b_gradient = sum / outputMiniBatches[i].size();
y_hat = Evaluate(inputMiniBatches[i]);
if (UI) {
@ -193,7 +193,7 @@ void MLPPExpReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size,
forwardPass();
}
double MLPPExpReg::score() {
real_t MLPPExpReg::score() {
MLPPUtilities util;
return util.performance(y_hat, outputSet);
}
@ -203,14 +203,14 @@ void MLPPExpReg::save(std::string fileName) {
util.saveParameters(fileName, weights, initial, bias);
}
double MLPPExpReg::Cost(std::vector<double> y_hat, std::vector<double> y) {
real_t MLPPExpReg::Cost(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPReg regularization;
class MLPPCost cost;
return cost.MSE(y_hat, y) + regularization.regTerm(weights, lambda, alpha, reg);
}
std::vector<double> MLPPExpReg::Evaluate(std::vector<std::vector<double>> X) {
std::vector<double> y_hat;
std::vector<real_t> MLPPExpReg::Evaluate(std::vector<std::vector<real_t>> X) {
std::vector<real_t> y_hat;
y_hat.resize(X.size());
for (int i = 0; i < X.size(); i++) {
y_hat[i] = 0;
@ -222,8 +222,8 @@ std::vector<double> MLPPExpReg::Evaluate(std::vector<std::vector<double>> X) {
return y_hat;
}
double MLPPExpReg::Evaluate(std::vector<double> x) {
double y_hat = 0;
real_t MLPPExpReg::Evaluate(std::vector<real_t> x) {
real_t y_hat = 0;
for (int i = 0; i < x.size(); i++) {
y_hat += initial[i] * std::pow(weights[i], x[i]);
}

38
mlpp/exp_reg/exp_reg.h
View File

@ -8,42 +8,44 @@
// Created by Marc Melikyan on 10/2/20.
//
#include "core/math/math_defs.h"
#include <string>
#include <vector>
class MLPPExpReg {
public:
MLPPExpReg(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, std::string reg = "None", double lambda = 0.5, double alpha = 0.5);
std::vector<double> modelSetTest(std::vector<std::vector<double>> X);
double modelTest(std::vector<double> x);
void gradientDescent(double learning_rate, int max_epoch, bool UI = 1);
void SGD(double learning_rate, int max_epoch, bool UI = 1);
void MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
double score();
MLPPExpReg(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
std::vector<real_t> modelSetTest(std::vector<std::vector<real_t>> X);
real_t modelTest(std::vector<real_t> x);
void gradientDescent(real_t learning_rate, int max_epoch, bool UI = 1);
void SGD(real_t learning_rate, int max_epoch, bool UI = 1);
void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
real_t score();
void save(std::string fileName);
private:
double Cost(std::vector<double> y_hat, std::vector<double> y);
real_t Cost(std::vector<real_t> y_hat, std::vector<real_t> y);
std::vector<double> Evaluate(std::vector<std::vector<double>> X);
double Evaluate(std::vector<double> x);
std::vector<real_t> Evaluate(std::vector<std::vector<real_t>> X);
real_t Evaluate(std::vector<real_t> x);
void forwardPass();
std::vector<std::vector<double>> inputSet;
std::vector<double> outputSet;
std::vector<double> y_hat;
std::vector<double> weights;
std::vector<double> initial;
double bias;
std::vector<std::vector<real_t>> inputSet;
std::vector<real_t> outputSet;
std::vector<real_t> y_hat;
std::vector<real_t> weights;
std::vector<real_t> initial;
real_t bias;
int n;
int k;
// Regularization Params
std::string reg;
double lambda;
double alpha; /* This is the controlling param for Elastic Net*/
real_t lambda;
real_t alpha; /* This is the controlling param for Elastic Net*/
};

60
mlpp/gan/gan.cpp
View File

@ -15,7 +15,7 @@
#include <iostream>
MLPPGAN::MLPPGAN(double k, std::vector<std::vector<double>> outputSet) :
MLPPGAN::MLPPGAN(real_t k, std::vector<std::vector<real_t>> outputSet) :
outputSet(outputSet), n(outputSet.size()), k(k) {
}
@ -23,15 +23,15 @@ MLPPGAN::~MLPPGAN() {
delete outputLayer;
}
std::vector<std::vector<double>> MLPPGAN::generateExample(int n) {
std::vector<std::vector<real_t>> MLPPGAN::generateExample(int n) {
MLPPLinAlg alg;
return modelSetTestGenerator(alg.gaussianNoise(n, k));
}
void MLPPGAN::gradientDescent(double learning_rate, int max_epoch, bool UI) {
void MLPPGAN::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
class MLPPCost cost;
MLPPLinAlg alg;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
forwardPass();
@ -40,13 +40,13 @@ void MLPPGAN::gradientDescent(double learning_rate, int max_epoch, bool UI) {
// Training of the discriminator.
std::vector<std::vector<double>> generatorInputSet = alg.gaussianNoise(n, k);
std::vector<std::vector<double>> discriminatorInputSet = modelSetTestGenerator(generatorInputSet);
std::vector<std::vector<real_t>> generatorInputSet = alg.gaussianNoise(n, k);
std::vector<std::vector<real_t>> discriminatorInputSet = modelSetTestGenerator(generatorInputSet);
discriminatorInputSet.insert(discriminatorInputSet.end(), outputSet.begin(), outputSet.end()); // Fake + real inputs.
std::vector<double> y_hat = modelSetTestDiscriminator(discriminatorInputSet);
std::vector<double> outputSet = alg.zerovec(n);
std::vector<double> outputSetReal = alg.onevec(n);
std::vector<real_t> y_hat = modelSetTestDiscriminator(discriminatorInputSet);
std::vector<real_t> outputSet = alg.zerovec(n);
std::vector<real_t> outputSetReal = alg.onevec(n);
outputSet.insert(outputSet.end(), outputSetReal.begin(), outputSetReal.end()); // Fake + real output scores.
auto [cumulativeDiscriminatorHiddenLayerWGrad, outputDiscriminatorWGrad] = computeDiscriminatorGradients(y_hat, outputSet);
@ -60,7 +60,7 @@ void MLPPGAN::gradientDescent(double learning_rate, int max_epoch, bool UI) {
y_hat = modelSetTestDiscriminator(discriminatorInputSet);
outputSet = alg.onevec(n);
std::vector<std::vector<std::vector<double>>> cumulativeGeneratorHiddenLayerWGrad = computeGeneratorGradients(y_hat, outputSet);
std::vector<std::vector<std::vector<real_t>>> cumulativeGeneratorHiddenLayerWGrad = computeGeneratorGradients(y_hat, outputSet);
cumulativeGeneratorHiddenLayerWGrad = alg.scalarMultiply(learning_rate / n, cumulativeGeneratorHiddenLayerWGrad);
updateGeneratorParameters(cumulativeGeneratorHiddenLayerWGrad, learning_rate);
@ -76,7 +76,7 @@ void MLPPGAN::gradientDescent(double learning_rate, int max_epoch, bool UI) {
}
}
double MLPPGAN::score() {
real_t MLPPGAN::score() {
MLPPLinAlg alg;
MLPPUtilities util;
forwardPass();
@ -96,7 +96,7 @@ void MLPPGAN::save(std::string fileName) {
}
}
void MLPPGAN::addLayer(int n_hidden, std::string activation, std::string weightInit, std::string reg, double lambda, double alpha) {
void MLPPGAN::addLayer(int n_hidden, std::string activation, std::string weightInit, std::string reg, real_t lambda, real_t alpha) {
MLPPLinAlg alg;
if (network.empty()) {
network.push_back(MLPPHiddenLayer(n_hidden, activation, alg.gaussianNoise(n, k), weightInit, reg, lambda, alpha));
@ -107,7 +107,7 @@ void MLPPGAN::addLayer(int n_hidden, std::string activation, std::string weightI
}
}
void MLPPGAN::addOutputLayer(std::string weightInit, std::string reg, double lambda, double alpha) {
void MLPPGAN::addOutputLayer(std::string weightInit, std::string reg, real_t lambda, real_t alpha) {
MLPPLinAlg alg;
if (!network.empty()) {
outputLayer = new MLPPOutputLayer(network[network.size() - 1].n_hidden, "Sigmoid", "LogLoss", network[network.size() - 1].a, weightInit, reg, lambda, alpha);
@ -116,7 +116,7 @@ void MLPPGAN::addOutputLayer(std::string weightInit, std::string reg, double lam
}
}
std::vector<std::vector<double>> MLPPGAN::modelSetTestGenerator(std::vector<std::vector<double>> X) {
std::vector<std::vector<real_t>> MLPPGAN::modelSetTestGenerator(std::vector<std::vector<real_t>> X) {
if (!network.empty()) {
network[0].input = X;
network[0].forwardPass();
@ -129,7 +129,7 @@ std::vector<std::vector<double>> MLPPGAN::modelSetTestGenerator(std::vector<std:
return network[network.size() / 2].a;
}
std::vector<double> MLPPGAN::modelSetTestDiscriminator(std::vector<std::vector<double>> X) {
std::vector<real_t> MLPPGAN::modelSetTestDiscriminator(std::vector<std::vector<real_t>> X) {
if (!network.empty()) {
for (int i = network.size() / 2 + 1; i < network.size(); i++) {
if (i == network.size() / 2 + 1) {
@ -145,10 +145,10 @@ std::vector<double> MLPPGAN::modelSetTestDiscriminator(std::vector<std::vector<d
return outputLayer->a;
}
double MLPPGAN::Cost(std::vector<double> y_hat, std::vector<double> y) {
real_t MLPPGAN::Cost(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPReg regularization;
class MLPPCost cost;
double totalRegTerm = 0;
real_t totalRegTerm = 0;
auto cost_function = outputLayer->cost_map[outputLayer->cost];
if (!network.empty()) {
@ -177,7 +177,7 @@ void MLPPGAN::forwardPass() {
y_hat = outputLayer->a;
}
void MLPPGAN::updateDiscriminatorParameters(std::vector<std::vector<std::vector<double>>> hiddenLayerUpdations, std::vector<double> outputLayerUpdation, double learning_rate) {
void MLPPGAN::updateDiscriminatorParameters(std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations, std::vector<real_t> outputLayerUpdation, real_t learning_rate) {
MLPPLinAlg alg;
outputLayer->weights = alg.subtraction(outputLayer->weights, outputLayerUpdation);
@ -194,7 +194,7 @@ void MLPPGAN::updateDiscriminatorParameters(std::vector<std::vector<std::vector<
}
}
void MLPPGAN::updateGeneratorParameters(std::vector<std::vector<std::vector<double>>> hiddenLayerUpdations, double learning_rate) {
void MLPPGAN::updateGeneratorParameters(std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations, real_t learning_rate) {
MLPPLinAlg alg;
if (!network.empty()) {
@ -207,25 +207,25 @@ void MLPPGAN::updateGeneratorParameters(std::vector<std::vector<std::vector<doub
}
}
std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<double>> MLPPGAN::computeDiscriminatorGradients(std::vector<double> y_hat, std::vector<double> outputSet) {
std::tuple<std::vector<std::vector<std::vector<real_t>>>, std::vector<real_t>> MLPPGAN::computeDiscriminatorGradients(std::vector<real_t> y_hat, std::vector<real_t> outputSet) {
class MLPPCost cost;
MLPPActivation avn;
MLPPLinAlg alg;
MLPPReg regularization;
std::vector<std::vector<std::vector<double>>> cumulativeHiddenLayerWGrad; // Tensor containing ALL hidden grads.
std::vector<std::vector<std::vector<real_t>>> cumulativeHiddenLayerWGrad; // Tensor containing ALL hidden grads.
auto costDeriv = outputLayer->costDeriv_map[outputLayer->cost];
auto outputAvn = outputLayer->activation_map[outputLayer->activation];
outputLayer->delta = alg.hadamard_product((cost.*costDeriv)(y_hat, outputSet), (avn.*outputAvn)(outputLayer->z, 1));
std::vector<double> outputWGrad = alg.mat_vec_mult(alg.transpose(outputLayer->input), outputLayer->delta);
std::vector<real_t> outputWGrad = alg.mat_vec_mult(alg.transpose(outputLayer->input), outputLayer->delta);
outputWGrad = alg.addition(outputWGrad, regularization.regDerivTerm(outputLayer->weights, outputLayer->lambda, outputLayer->alpha, outputLayer->reg));
if (!network.empty()) {
auto hiddenLayerAvn = network[network.size() - 1].activation_map[network[network.size() - 1].activation];
network[network.size() - 1].delta = alg.hadamard_product(alg.outerProduct(outputLayer->delta, outputLayer->weights), (avn.*hiddenLayerAvn)(network[network.size() - 1].z, 1));
std::vector<std::vector<double>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[network.size() - 1].input), network[network.size() - 1].delta);
std::vector<std::vector<real_t>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[network.size() - 1].input), network[network.size() - 1].delta);
cumulativeHiddenLayerWGrad.push_back(alg.addition(hiddenLayerWGrad, regularization.regDerivTerm(network[network.size() - 1].weights, network[network.size() - 1].lambda, network[network.size() - 1].alpha, network[network.size() - 1].reg))); // Adding to our cumulative hidden layer grads. Maintain reg terms as well.
@ -235,7 +235,7 @@ std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<double>> M
for (int i = network.size() - 2; i > network.size() / 2; i--) {
auto hiddenLayerAvn = network[i].activation_map[network[i].activation];
network[i].delta = alg.hadamard_product(alg.matmult(network[i + 1].delta, alg.transpose(network[i + 1].weights)), (avn.*hiddenLayerAvn)(network[i].z, 1));
std::vector<std::vector<double>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[i].input), network[i].delta);
std::vector<std::vector<real_t>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[i].input), network[i].delta);
cumulativeHiddenLayerWGrad.push_back(alg.addition(hiddenLayerWGrad, regularization.regDerivTerm(network[i].weights, network[i].lambda, network[i].alpha, network[i].reg))); // Adding to our cumulative hidden layer grads. Maintain reg terms as well.
}
@ -243,36 +243,36 @@ std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<double>> M
return { cumulativeHiddenLayerWGrad, outputWGrad };
}
std::vector<std::vector<std::vector<double>>> MLPPGAN::computeGeneratorGradients(std::vector<double> y_hat, std::vector<double> outputSet) {
std::vector<std::vector<std::vector<real_t>>> MLPPGAN::computeGeneratorGradients(std::vector<real_t> y_hat, std::vector<real_t> outputSet) {
class MLPPCost cost;
MLPPActivation avn;
MLPPLinAlg alg;
MLPPReg regularization;
std::vector<std::vector<std::vector<double>>> cumulativeHiddenLayerWGrad; // Tensor containing ALL hidden grads.
std::vector<std::vector<std::vector<real_t>>> cumulativeHiddenLayerWGrad; // Tensor containing ALL hidden grads.
auto costDeriv = outputLayer->costDeriv_map[outputLayer->cost];
auto outputAvn = outputLayer->activation_map[outputLayer->activation];
outputLayer->delta = alg.hadamard_product((cost.*costDeriv)(y_hat, outputSet), (avn.*outputAvn)(outputLayer->z, 1));
std::vector<double> outputWGrad = alg.mat_vec_mult(alg.transpose(outputLayer->input), outputLayer->delta);
std::vector<real_t> outputWGrad = alg.mat_vec_mult(alg.transpose(outputLayer->input), outputLayer->delta);
outputWGrad = alg.addition(outputWGrad, regularization.regDerivTerm(outputLayer->weights, outputLayer->lambda, outputLayer->alpha, outputLayer->reg));
if (!network.empty()) {
auto hiddenLayerAvn = network[network.size() - 1].activation_map[network[network.size() - 1].activation];
network[network.size() - 1].delta = alg.hadamard_product(alg.outerProduct(outputLayer->delta, outputLayer->weights), (avn.*hiddenLayerAvn)(network[network.size() - 1].z, 1));
std::vector<std::vector<double>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[network.size() - 1].input), network[network.size() - 1].delta);
std::vector<std::vector<real_t>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[network.size() - 1].input), network[network.size() - 1].delta);
cumulativeHiddenLayerWGrad.push_back(alg.addition(hiddenLayerWGrad, regularization.regDerivTerm(network[network.size() - 1].weights, network[network.size() - 1].lambda, network[network.size() - 1].alpha, network[network.size() - 1].reg))); // Adding to our cumulative hidden layer grads. Maintain reg terms as well.
for (int i = network.size() - 2; i >= 0; i--) {
auto hiddenLayerAvn = network[i].activation_map[network[i].activation];
network[i].delta = alg.hadamard_product(alg.matmult(network[i + 1].delta, alg.transpose(network[i + 1].weights)), (avn.*hiddenLayerAvn)(network[i].z, 1));
std::vector<std::vector<double>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[i].input), network[i].delta);
std::vector<std::vector<real_t>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[i].input), network[i].delta);
cumulativeHiddenLayerWGrad.push_back(alg.addition(hiddenLayerWGrad, regularization.regDerivTerm(network[i].weights, network[i].lambda, network[i].alpha, network[i].reg))); // Adding to our cumulative hidden layer grads. Maintain reg terms as well.
}
}
return cumulativeHiddenLayerWGrad;
}
void MLPPGAN::UI(int epoch, double cost_prev, std::vector<double> y_hat, std::vector<double> outputSet) {
void MLPPGAN::UI(int epoch, real_t cost_prev, std::vector<real_t> y_hat, std::vector<real_t> outputSet) {
MLPPUtilities::CostInfo(epoch, cost_prev, Cost(y_hat, outputSet));
std::cout << "Layer " << network.size() + 1 << ": " << std::endl;
MLPPUtilities::UI(outputLayer->weights, outputLayer->bias);

34
mlpp/gan/gan.h
View File

@ -8,6 +8,8 @@
// Created by Marc Melikyan on 11/4/20.
//
#include "core/math/math_defs.h"
#include "../hidden_layer/hidden_layer.h"
#include "../output_layer/output_layer.h"
@ -19,32 +21,32 @@
class MLPPGAN {
public:
MLPPGAN(double k, std::vector<std::vector<double>> outputSet);
MLPPGAN(real_t k, std::vector<std::vector<real_t>> outputSet);
~MLPPGAN();
std::vector<std::vector<double>> generateExample(int n);
void gradientDescent(double learning_rate, int max_epoch, bool UI = 1);
double score();
std::vector<std::vector<real_t>> generateExample(int n);
void gradientDescent(real_t learning_rate, int max_epoch, bool UI = 1);
real_t score();
void save(std::string fileName);
void addLayer(int n_hidden, std::string activation, std::string weightInit = "Default", std::string reg = "None", double lambda = 0.5, double alpha = 0.5);
void addOutputLayer(std::string weightInit = "Default", std::string reg = "None", double lambda = 0.5, double alpha = 0.5);
void addLayer(int n_hidden, std::string activation, std::string weightInit = "Default", std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
void addOutputLayer(std::string weightInit = "Default", std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
private:
std::vector<std::vector<double>> modelSetTestGenerator(std::vector<std::vector<double>> X); // Evaluator for the generator of the gan.
std::vector<double> modelSetTestDiscriminator(std::vector<std::vector<double>> X); // Evaluator for the discriminator of the gan.
std::vector<std::vector<real_t>> modelSetTestGenerator(std::vector<std::vector<real_t>> X); // Evaluator for the generator of the gan.
std::vector<real_t> modelSetTestDiscriminator(std::vector<std::vector<real_t>> X); // Evaluator for the discriminator of the gan.
double Cost(std::vector<double> y_hat, std::vector<double> y);
real_t Cost(std::vector<real_t> y_hat, std::vector<real_t> y);
void forwardPass();
void updateDiscriminatorParameters(std::vector<std::vector<std::vector<double>>> hiddenLayerUpdations, std::vector<double> outputLayerUpdation, double learning_rate);
void updateGeneratorParameters(std::vector<std::vector<std::vector<double>>> hiddenLayerUpdations, double learning_rate);
std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<double>> computeDiscriminatorGradients(std::vector<double> y_hat, std::vector<double> outputSet);
std::vector<std::vector<std::vector<double>>> computeGeneratorGradients(std::vector<double> y_hat, std::vector<double> outputSet);
void updateDiscriminatorParameters(std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations, std::vector<real_t> outputLayerUpdation, real_t learning_rate);
void updateGeneratorParameters(std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations, real_t learning_rate);
std::tuple<std::vector<std::vector<std::vector<real_t>>>, std::vector<real_t>> computeDiscriminatorGradients(std::vector<real_t> y_hat, std::vector<real_t> outputSet);
std::vector<std::vector<std::vector<real_t>>> computeGeneratorGradients(std::vector<real_t> y_hat, std::vector<real_t> outputSet);
void UI(int epoch, double cost_prev, std::vector<double> y_hat, std::vector<double> outputSet);
void UI(int epoch, real_t cost_prev, std::vector<real_t> y_hat, std::vector<real_t> outputSet);
std::vector<std::vector<double>> outputSet;
std::vector<double> y_hat;
std::vector<std::vector<real_t>> outputSet;
std::vector<real_t> y_hat;
std::vector<MLPPHiddenLayer> network;
MLPPOutputLayer *outputLayer;

10
mlpp/gauss_markov_checker/gauss_markov_checker.cpp
View File

@ -9,7 +9,7 @@
#include <iostream>
void MLPPGaussMarkovChecker::checkGMConditions(std::vector<double> eps) {
void MLPPGaussMarkovChecker::checkGMConditions(std::vector<real_t> eps) {
bool condition1 = arithmeticMean(eps);
bool condition2 = homoscedasticity(eps);
bool condition3 = exogeneity(eps);
@ -21,7 +21,7 @@ void MLPPGaussMarkovChecker::checkGMConditions(std::vector<double> eps) {
}
}
bool MLPPGaussMarkovChecker::arithmeticMean(std::vector<double> eps) {
bool MLPPGaussMarkovChecker::arithmeticMean(std::vector<real_t> eps) {
MLPPStat stat;
if (stat.mean(eps) == 0) {
return 1;
@ -30,9 +30,9 @@ bool MLPPGaussMarkovChecker::arithmeticMean(std::vector<double> eps) {
}
}
bool MLPPGaussMarkovChecker::homoscedasticity(std::vector<double> eps) {
bool MLPPGaussMarkovChecker::homoscedasticity(std::vector<real_t> eps) {
MLPPStat stat;
double currentVar = (eps[0] - stat.mean(eps)) * (eps[0] - stat.mean(eps)) / eps.size();
real_t currentVar = (eps[0] - stat.mean(eps)) * (eps[0] - stat.mean(eps)) / eps.size();
for (int i = 0; i < eps.size(); i++) {
if (currentVar != (eps[i] - stat.mean(eps)) * (eps[i] - stat.mean(eps)) / eps.size()) {
return 0;
@ -41,7 +41,7 @@ bool MLPPGaussMarkovChecker::homoscedasticity(std::vector<double> eps) {
return 1;
}
bool MLPPGaussMarkovChecker::exogeneity(std::vector<double> eps) {
bool MLPPGaussMarkovChecker::exogeneity(std::vector<real_t> eps) {
MLPPStat stat;
for (int i = 0; i < eps.size(); i++) {
for (int j = 0; j < eps.size(); j++) {

10
mlpp/gauss_markov_checker/gauss_markov_checker.h
View File

@ -8,18 +8,20 @@
// Created by Marc Melikyan on 11/13/20.
//
#include "core/math/math_defs.h"
#include <string>
#include <vector>
class MLPPGaussMarkovChecker {
public:
void checkGMConditions(std::vector<double> eps);
void checkGMConditions(std::vector<real_t> eps);
// Independent, 3 Gauss-Markov Conditions
bool arithmeticMean(std::vector<double> eps); // 1) Arithmetic Mean of 0.
bool homoscedasticity(std::vector<double> eps); // 2) Homoscedasticity
bool exogeneity(std::vector<double> eps); // 3) Cov of any 2 non-equal eps values = 0.
bool arithmeticMean(std::vector<real_t> eps); // 1) Arithmetic Mean of 0.
bool homoscedasticity(std::vector<real_t> eps); // 2) Homoscedasticity
bool exogeneity(std::vector<real_t> eps); // 3) Cov of any 2 non-equal eps values = 0.
private:
};

28
mlpp/gaussian_nb/gaussian_nb.cpp
View File

@ -14,35 +14,35 @@
#include <random>
MLPPGaussianNB::MLPPGaussianNB(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, int class_num) :
MLPPGaussianNB::MLPPGaussianNB(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, int class_num) :
inputSet(inputSet), outputSet(outputSet), class_num(class_num) {
y_hat.resize(outputSet.size());
Evaluate();
MLPPLinAlg alg;
}
std::vector<double> MLPPGaussianNB::modelSetTest(std::vector<std::vector<double>> X) {
std::vector<double> y_hat;
std::vector<real_t> MLPPGaussianNB::modelSetTest(std::vector<std::vector<real_t>> X) {
std::vector<real_t> y_hat;
for (int i = 0; i < X.size(); i++) {
y_hat.push_back(modelTest(X[i]));
}
return y_hat;
}
double MLPPGaussianNB::modelTest(std::vector<double> x) {
real_t MLPPGaussianNB::modelTest(std::vector<real_t> x) {
MLPPStat stat;
MLPPLinAlg alg;
double score[class_num];
double y_hat_i = 1;
real_t score[class_num];
real_t y_hat_i = 1;
for (int i = class_num - 1; i >= 0; i--) {
y_hat_i += std::log(priors[i] * (1 / sqrt(2 * M_PI * sigma[i] * sigma[i])) * exp(-(x[i] * mu[i]) * (x[i] * mu[i]) / (2 * sigma[i] * sigma[i])));
score[i] = exp(y_hat_i);
}
return std::distance(score, std::max_element(score, score + sizeof(score) / sizeof(double)));
return std::distance(score, std::max_element(score, score + sizeof(score) / sizeof(real_t)));
}
double MLPPGaussianNB::score() {
real_t MLPPGaussianNB::score() {
MLPPUtilities util;
return util.performance(y_hat, outputSet);
}
@ -55,7 +55,7 @@ void MLPPGaussianNB::Evaluate() {
mu.resize(class_num);
sigma.resize(class_num);
for (int i = class_num - 1; i >= 0; i--) {
std::vector<double> set;
std::vector<real_t> set;
for (int j = 0; j < inputSet.size(); j++) {
for (int k = 0; k < inputSet[j].size(); k++) {
if (outputSet[j] == i) {
@ -72,11 +72,11 @@ void MLPPGaussianNB::Evaluate() {
for (int i = 0; i < outputSet.size(); i++) {
priors[int(outputSet[i])]++;
}
priors = alg.scalarMultiply(double(1) / double(outputSet.size()), priors);
priors = alg.scalarMultiply(real_t(1) / real_t(outputSet.size()), priors);
for (int i = 0; i < outputSet.size(); i++) {
double score[class_num];
double y_hat_i = 1;
real_t score[class_num];
real_t y_hat_i = 1;
for (int j = class_num - 1; j >= 0; j--) {
for (int k = 0; k < inputSet[i].size(); k++) {
y_hat_i += std::log(priors[j] * (1 / sqrt(2 * M_PI * sigma[j] * sigma[j])) * exp(-(inputSet[i][k] * mu[j]) * (inputSet[i][k] * mu[j]) / (2 * sigma[j] * sigma[j])));
@ -84,7 +84,7 @@ void MLPPGaussianNB::Evaluate() {
score[j] = exp(y_hat_i);
std::cout << score[j] << std::endl;
}
y_hat[i] = std::distance(score, std::max_element(score, score + sizeof(score) / sizeof(double)));
std::cout << std::distance(score, std::max_element(score, score + sizeof(score) / sizeof(double))) << std::endl;
y_hat[i] = std::distance(score, std::max_element(score, score + sizeof(score) / sizeof(real_t)));
std::cout << std::distance(score, std::max_element(score, score + sizeof(score) / sizeof(real_t))) << std::endl;
}
}

22
mlpp/gaussian_nb/gaussian_nb.h
View File

@ -8,29 +8,31 @@
// Created by Marc Melikyan on 1/17/21.
//
#include "core/math/math_defs.h"
#include <vector>
class MLPPGaussianNB {
public:
MLPPGaussianNB(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, int class_num);
std::vector<double> modelSetTest(std::vector<std::vector<double>> X);
double modelTest(std::vector<double> x);
double score();
MLPPGaussianNB(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, int class_num);
std::vector<real_t> modelSetTest(std::vector<std::vector<real_t>> X);
real_t modelTest(std::vector<real_t> x);
real_t score();
private:
void Evaluate();
int class_num;
std::vector<double> priors;
std::vector<double> mu;
std::vector<double> sigma;
std::vector<real_t> priors;
std::vector<real_t> mu;
std::vector<real_t> sigma;
std::vector<std::vector<double>> inputSet;
std::vector<double> outputSet;
std::vector<std::vector<real_t>> inputSet;
std::vector<real_t> outputSet;
std::vector<double> y_hat;
std::vector<real_t> y_hat;
};
#endif /* GaussianNB_hpp */

4
mlpp/hidden_layer/hidden_layer.cpp
View File

@ -13,7 +13,7 @@
#include <random>
MLPPHiddenLayer::MLPPHiddenLayer(int n_hidden, std::string activation, std::vector<std::vector<double>> input, std::string weightInit, std::string reg, double lambda, double alpha) :
MLPPHiddenLayer::MLPPHiddenLayer(int n_hidden, std::string activation, std::vector<std::vector<real_t>> input, std::string weightInit, std::string reg, real_t lambda, real_t alpha) :
n_hidden(n_hidden), activation(activation), input(input), weightInit(weightInit), reg(reg), lambda(lambda), alpha(alpha) {
weights = MLPPUtilities::weightInitialization(input[0].size(), n_hidden, weightInit);
bias = MLPPUtilities::biasInitialization(n_hidden);
@ -104,7 +104,7 @@ void MLPPHiddenLayer::forwardPass() {
a = (avn.*activation_map[activation])(z, 0);
}
void MLPPHiddenLayer::Test(std::vector<double> x) {
void MLPPHiddenLayer::Test(std::vector<real_t> x) {
MLPPLinAlg alg;
MLPPActivation avn;
z_test = alg.addition(alg.mat_vec_mult(alg.transpose(weights), x), bias);

30
mlpp/hidden_layer/hidden_layer.h
View File

@ -8,6 +8,8 @@
// Created by Marc Melikyan on 11/4/20.
//
#include "core/math/math_defs.h"
#include "../activation/activation.h"
#include <map>
@ -17,36 +19,36 @@
class MLPPHiddenLayer {
public:
MLPPHiddenLayer(int n_hidden, std::string activation, std::vector<std::vector<double>> input, std::string weightInit, std::string reg, double lambda, double alpha);
MLPPHiddenLayer(int n_hidden, std::string activation, std::vector<std::vector<real_t>> input, std::string weightInit, std::string reg, real_t lambda, real_t alpha);
int n_hidden;
std::string activation;
std::vector<std::vector<double>> input;
std::vector<std::vector<real_t>> input;
std::vector<std::vector<double>> weights;
std::vector<double> bias;
std::vector<std::vector<real_t>> weights;
std::vector<real_t> bias;
std::vector<std::vector<double>> z;
std::vector<std::vector<double>> a;
std::vector<std::vector<real_t>> z;
std::vector<std::vector<real_t>> a;
std::map<std::string, std::vector<std::vector<double>> (MLPPActivation::*)(std::vector<std::vector<double>>, bool)> activation_map;
std::map<std::string, std::vector<double> (MLPPActivation::*)(std::vector<double>, bool)> activationTest_map;
std::map<std::string, std::vector<std::vector<real_t>> (MLPPActivation::*)(std::vector<std::vector<real_t>>, bool)> activation_map;
std::map<std::string, std::vector<real_t> (MLPPActivation::*)(std::vector<real_t>, bool)> activationTest_map;
std::vector<double> z_test;
std::vector<double> a_test;
std::vector<real_t> z_test;
std::vector<real_t> a_test;
std::vector<std::vector<double>> delta;
std::vector<std::vector<real_t>> delta;
// Regularization Params
std::string reg;
double lambda; /* Regularization Parameter */
double alpha; /* This is the controlling param for Elastic Net*/
real_t lambda; /* Regularization Parameter */
real_t alpha; /* This is the controlling param for Elastic Net*/
std::string weightInit;
void forwardPass();
void Test(std::vector<double> x);
void Test(std::vector<real_t> x);
};

6
mlpp/hypothesis_testing/hypothesis_testing.cpp
View File

@ -8,9 +8,9 @@
std::tuple<bool, double> MLPPHypothesisTesting::chiSquareTest(std::vector<double> observed, std::vector<double> expected) {
double df = observed.size() - 1; // These are our degrees of freedom
double sum = 0;
std::tuple<bool, real_t> MLPPHypothesisTesting::chiSquareTest(std::vector<real_t> observed, std::vector<real_t> expected) {
real_t df = observed.size() - 1; // These are our degrees of freedom
real_t sum = 0;
for (int i = 0; i < observed.size(); i++) {
sum += (observed[i] - expected[i]) * (observed[i] - expected[i]) / expected[i];
}

4
mlpp/hypothesis_testing/hypothesis_testing.h
View File

@ -8,13 +8,15 @@
// Created by Marc Melikyan on 3/10/21.
//
#include "core/math/math_defs.h"
#include <tuple>
#include <vector>
class MLPPHypothesisTesting {
public:
std::tuple<bool, double> chiSquareTest(std::vector<double> observed, std::vector<double> expected);
std::tuple<bool, real_t> chiSquareTest(std::vector<real_t> observed, std::vector<real_t> expected);
private:
};

48
mlpp/kmeans/kmeans.cpp
View File

@ -13,7 +13,7 @@
#include <random>
MLPPKMeans::MLPPKMeans(std::vector<std::vector<double>> inputSet, int k, std::string init_type) :
MLPPKMeans::MLPPKMeans(std::vector<std::vector<real_t>> inputSet, int k, std::string init_type) :
inputSet(inputSet), k(k), init_type(init_type) {
if (init_type == "KMeans++") {
kmeansppInitialization(k);
@ -22,11 +22,11 @@ MLPPKMeans::MLPPKMeans(std::vector<std::vector<double>> inputSet, int k, std::st
}
}
std::vector<std::vector<double>> MLPPKMeans::modelSetTest(std::vector<std::vector<double>> X) {
std::vector<std::vector<real_t>> MLPPKMeans::modelSetTest(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg;
std::vector<std::vector<double>> closestCentroids;
std::vector<std::vector<real_t>> closestCentroids;
for (int i = 0; i < inputSet.size(); i++) {
std::vector<double> closestCentroid = mu[0];
std::vector<real_t> closestCentroid = mu[0];
for (int j = 0; j < r[0].size(); j++) {
bool isCentroidCloser = alg.euclideanDistance(X[i], mu[j]) < alg.euclideanDistance(X[i], closestCentroid);
if (isCentroidCloser) {
@ -38,9 +38,9 @@ std::vector<std::vector<double>> MLPPKMeans::modelSetTest(std::vector<std::vecto
return closestCentroids;
}
std::vector<double> MLPPKMeans::modelTest(std::vector<double> x) {
std::vector<real_t> MLPPKMeans::modelTest(std::vector<real_t> x) {
MLPPLinAlg alg;
std::vector<double> closestCentroid = mu[0];
std::vector<real_t> closestCentroid = mu[0];
for (int j = 0; j < mu.size(); j++) {
if (alg.euclideanDistance(x, mu[j]) < alg.euclideanDistance(x, closestCentroid)) {
closestCentroid = mu[j];
@ -50,7 +50,7 @@ std::vector<double> MLPPKMeans::modelTest(std::vector<double> x) {
}
void MLPPKMeans::train(int epoch_num, bool UI) {
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
Evaluate();
@ -80,17 +80,17 @@ void MLPPKMeans::train(int epoch_num, bool UI) {
}
}
double MLPPKMeans::score() {
real_t MLPPKMeans::score() {
return Cost();
}
std::vector<double> MLPPKMeans::silhouette_scores() {
std::vector<real_t> MLPPKMeans::silhouette_scores() {
MLPPLinAlg alg;
std::vector<std::vector<double>> closestCentroids = modelSetTest(inputSet);
std::vector<double> silhouette_scores;
std::vector<std::vector<real_t>> closestCentroids = modelSetTest(inputSet);
std::vector<real_t> silhouette_scores;
for (int i = 0; i < inputSet.size(); i++) {
// COMPUTING a[i]
double a = 0;
real_t a = 0;
for (int j = 0; j < inputSet.size(); j++) {
if (i != j && r[i] == r[j]) {
a += alg.euclideanDistance(inputSet[i], inputSet[j]);
@ -100,15 +100,15 @@ std::vector<double> MLPPKMeans::silhouette_scores() {
a /= closestCentroids[i].size() - 1;
// COMPUTING b[i]
double b = INT_MAX;
real_t b = INT_MAX;
for (int j = 0; j < mu.size(); j++) {
if (closestCentroids[i] != mu[j]) {
double sum = 0;
real_t sum = 0;
for (int k = 0; k < inputSet.size(); k++) {
sum += alg.euclideanDistance(inputSet[i], inputSet[k]);
}
// NORMALIZE b[i]
double k_clusterSize = 0;
real_t k_clusterSize = 0;
for (int k = 0; k < closestCentroids.size(); k++) {
if (closestCentroids[k] == mu[j]) {
k_clusterSize++;
@ -144,7 +144,7 @@ void MLPPKMeans::Evaluate() {
}
for (int i = 0; i < r.size(); i++) {
std::vector<double> closestCentroid = mu[0];
std::vector<real_t> closestCentroid = mu[0];
for (int j = 0; j < r[0].size(); j++) {
bool isCentroidCloser = alg.euclideanDistance(inputSet[i], mu[j]) < alg.euclideanDistance(inputSet[i], closestCentroid);
if (isCentroidCloser) {
@ -165,21 +165,21 @@ void MLPPKMeans::Evaluate() {
void MLPPKMeans::computeMu() {
MLPPLinAlg alg;
for (int i = 0; i < mu.size(); i++) {
std::vector<double> num;
std::vector<real_t> num;
num.resize(r.size());
for (int i = 0; i < num.size(); i++) {
num[i] = 0;
}
double den = 0;
real_t den = 0;
for (int j = 0; j < r.size(); j++) {
num = alg.addition(num, alg.scalarMultiply(r[j][i], inputSet[j]));
}
for (int j = 0; j < r.size(); j++) {
den += r[j][i];
}
mu[i] = alg.scalarMultiply(double(1) / double(den), num);
mu[i] = alg.scalarMultiply(real_t(1) / real_t(den), num);
}
}
@ -204,12 +204,12 @@ void MLPPKMeans::kmeansppInitialization(int k) {
mu.push_back(inputSet[distribution(generator)]);
for (int i = 0; i < k - 1; i++) {
std::vector<double> farthestCentroid;
std::vector<real_t> farthestCentroid;
for (int j = 0; j < inputSet.size(); j++) {
double max_dist = 0;
real_t max_dist = 0;
/* SUM ALL THE SQUARED DISTANCES, CHOOSE THE ONE THAT'S FARTHEST
AS TO SPREAD OUT THE CLUSTER CENTROIDS. */
double sum = 0;
real_t sum = 0;
for (int k = 0; k < mu.size(); k++) {
sum += alg.euclideanDistance(inputSet[j], mu[k]);
}
@ -222,9 +222,9 @@ void MLPPKMeans::kmeansppInitialization(int k) {
}
}
double MLPPKMeans::Cost() {
real_t MLPPKMeans::Cost() {
MLPPLinAlg alg;
double sum = 0;
real_t sum = 0;
for (int i = 0; i < r.size(); i++) {
for (int j = 0; j < r[0].size(); j++) {
sum += r[i][j] * alg.norm_sq(alg.subtraction(inputSet[i], mu[j]));

24
mlpp/kmeans/kmeans.h
View File

@ -8,18 +8,20 @@
// Created by Marc Melikyan on 10/2/20.
//
#include "core/math/math_defs.h"
#include <string>
#include <vector>
class MLPPKMeans {
public:
MLPPKMeans(std::vector<std::vector<double>> inputSet, int k, std::string init_type = "Default");
std::vector<std::vector<double>> modelSetTest(std::vector<std::vector<double>> X);
std::vector<double> modelTest(std::vector<double> x);
MLPPKMeans(std::vector<std::vector<real_t>> inputSet, int k, std::string init_type = "Default");
std::vector<std::vector<real_t>> modelSetTest(std::vector<std::vector<real_t>> X);
std::vector<real_t> modelTest(std::vector<real_t> x);
void train(int epoch_num, bool UI = 1);
double score();
std::vector<double> silhouette_scores();
real_t score();
std::vector<real_t> silhouette_scores();
private:
void Evaluate();
@ -27,15 +29,15 @@ private:
void centroidInitialization(int k);
void kmeansppInitialization(int k);
double Cost();
real_t Cost();
std::vector<std::vector<double>> inputSet;
std::vector<std::vector<double>> mu;
std::vector<std::vector<double>> r;
std::vector<std::vector<real_t>> inputSet;
std::vector<std::vector<real_t>> mu;
std::vector<std::vector<real_t>> r;
double euclideanDistance(std::vector<double> A, std::vector<double> B);
real_t euclideanDistance(std::vector<real_t> A, std::vector<real_t> B);
double accuracy_threshold;
real_t accuracy_threshold;
int k;
std::string init_type;

18
mlpp/knn/knn.cpp
View File

@ -13,28 +13,28 @@
#include <map>
MLPPKNN::MLPPKNN(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, int k) :
MLPPKNN::MLPPKNN(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, int k) :
inputSet(inputSet), outputSet(outputSet), k(k) {
}
std::vector<double> MLPPKNN::modelSetTest(std::vector<std::vector<double>> X) {
std::vector<double> y_hat;
std::vector<real_t> MLPPKNN::modelSetTest(std::vector<std::vector<real_t>> X) {
std::vector<real_t> y_hat;
for (int i = 0; i < X.size(); i++) {
y_hat.push_back(modelTest(X[i]));
}
return y_hat;
}
int MLPPKNN::modelTest(std::vector<double> x) {
int MLPPKNN::modelTest(std::vector<real_t> x) {
return determineClass(nearestNeighbors(x));
}
double MLPPKNN::score() {
real_t MLPPKNN::score() {
MLPPUtilities util;
return util.performance(modelSetTest(inputSet), outputSet);
}
int MLPPKNN::determineClass(std::vector<double> knn) {
int MLPPKNN::determineClass(std::vector<real_t> knn) {
std::map<int, int> class_nums;
for (int i = 0; i < outputSet.size(); i++) {
class_nums[outputSet[i]] = 0;
@ -62,12 +62,12 @@ int MLPPKNN::determineClass(std::vector<double> knn) {
return final_class;
}
std::vector<double> MLPPKNN::nearestNeighbors(std::vector<double> x) {
std::vector<real_t> MLPPKNN::nearestNeighbors(std::vector<real_t> x) {
MLPPLinAlg alg;
// The nearest neighbors
std::vector<double> knn;
std::vector<real_t> knn;
std::vector<std::vector<double>> inputUseSet = inputSet;
std::vector<std::vector<real_t>> inputUseSet = inputSet;
//Perfom this loop unless and until all k nearest neighbors are found, appended, and returned
for (int i = 0; i < k; i++) {
int neighbor = 0;

18
mlpp/knn/knn.h
View File

@ -8,24 +8,26 @@
// Created by Marc Melikyan on 10/2/20.
//
#include "core/math/math_defs.h"
#include <vector>
class MLPPKNN {
public:
MLPPKNN(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, int k);
std::vector<double> modelSetTest(std::vector<std::vector<double>> X);
int modelTest(std::vector<double> x);
double score();
MLPPKNN(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, int k);
std::vector<real_t> modelSetTest(std::vector<std::vector<real_t>> X);
int modelTest(std::vector<real_t> x);
real_t score();
private:
// Private Model Functions
std::vector<double> nearestNeighbors(std::vector<double> x);
int determineClass(std::vector<double> knn);
std::vector<real_t> nearestNeighbors(std::vector<real_t> x);
int determineClass(std::vector<real_t> knn);
// Model Inputs and Parameters
std::vector<std::vector<double>> inputSet;
std::vector<double> outputSet;
std::vector<std::vector<real_t>> inputSet;
std::vector<real_t> outputSet;
int k;
};

442
mlpp/lin_alg/lin_alg.cpp
View File

File diff suppressed because it is too large Load Diff

236
mlpp/lin_alg/lin_alg.h
View File

@ -8,6 +8,8 @@
// Created by Marc Melikyan on 1/8/21.
//
#include "core/math/math_defs.h"
#include <tuple>
#include <vector>
@ -15,246 +17,246 @@ class MLPPLinAlg {
public:
// MATRIX FUNCTIONS
std::vector<std::vector<double>> gramMatrix(std::vector<std::vector<double>> A);
std::vector<std::vector<real_t>> gramMatrix(std::vector<std::vector<real_t>> A);
bool linearIndependenceChecker(std::vector<std::vector<double>> A);
bool linearIndependenceChecker(std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> gaussianNoise(int n, int m);
std::vector<std::vector<real_t>> gaussianNoise(int n, int m);
std::vector<std::vector<double>> addition(std::vector<std::vector<double>> A, std::vector<std::vector<double>> B);
std::vector<std::vector<real_t>> addition(std::vector<std::vector<real_t>> A, std::vector<std::vector<real_t>> B);
std::vector<std::vector<double>> subtraction(std::vector<std::vector<double>> A, std::vector<std::vector<double>> B);
std::vector<std::vector<real_t>> subtraction(std::vector<std::vector<real_t>> A, std::vector<std::vector<real_t>> B);
std::vector<std::vector<double>> matmult(std::vector<std::vector<double>> A, std::vector<std::vector<double>> B);
std::vector<std::vector<real_t>> matmult(std::vector<std::vector<real_t>> A, std::vector<std::vector<real_t>> B);
std::vector<std::vector<double>> hadamard_product(std::vector<std::vector<double>> A, std::vector<std::vector<double>> B);
std::vector<std::vector<real_t>> hadamard_product(std::vector<std::vector<real_t>> A, std::vector<std::vector<real_t>> B);
std::vector<std::vector<double>> kronecker_product(std::vector<std::vector<double>> A, std::vector<std::vector<double>> B);
std::vector<std::vector<real_t>> kronecker_product(std::vector<std::vector<real_t>> A, std::vector<std::vector<real_t>> B);
std::vector<std::vector<double>> elementWiseDivision(std::vector<std::vector<double>> A, std::vector<std::vector<double>> B);
std::vector<std::vector<real_t>> elementWiseDivision(std::vector<std::vector<real_t>> A, std::vector<std::vector<real_t>> B);
std::vector<std::vector<double>> transpose(std::vector<std::vector<double>> A);
std::vector<std::vector<real_t>> transpose(std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> scalarMultiply(double scalar, std::vector<std::vector<double>> A);
std::vector<std::vector<real_t>> scalarMultiply(real_t scalar, std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> scalarAdd(double scalar, std::vector<std::vector<double>> A);
std::vector<std::vector<real_t>> scalarAdd(real_t scalar, std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> log(std::vector<std::vector<double>> A);
std::vector<std::vector<real_t>> log(std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> log10(std::vector<std::vector<double>> A);
std::vector<std::vector<real_t>> log10(std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> exp(std::vector<std::vector<double>> A);
std::vector<std::vector<real_t>> exp(std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> erf(std::vector<std::vector<double>> A);
std::vector<std::vector<real_t>> erf(std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> exponentiate(std::vector<std::vector<double>> A, double p);
std::vector<std::vector<real_t>> exponentiate(std::vector<std::vector<real_t>> A, real_t p);
std::vector<std::vector<double>> sqrt(std::vector<std::vector<double>> A);
std::vector<std::vector<real_t>> sqrt(std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> cbrt(std::vector<std::vector<double>> A);
std::vector<std::vector<real_t>> cbrt(std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> matrixPower(std::vector<std::vector<double>> A, int n);
std::vector<std::vector<real_t>> matrixPower(std::vector<std::vector<real_t>> A, int n);
std::vector<std::vector<double>> abs(std::vector<std::vector<double>> A);
std::vector<std::vector<real_t>> abs(std::vector<std::vector<real_t>> A);
double det(std::vector<std::vector<double>> A, int d);
real_t det(std::vector<std::vector<real_t>> A, int d);
double trace(std::vector<std::vector<double>> A);
real_t trace(std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> cofactor(std::vector<std::vector<double>> A, int n, int i, int j);
std::vector<std::vector<real_t>> cofactor(std::vector<std::vector<real_t>> A, int n, int i, int j);
std::vector<std::vector<double>> adjoint(std::vector<std::vector<double>> A);
std::vector<std::vector<real_t>> adjoint(std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> inverse(std::vector<std::vector<double>> A);
std::vector<std::vector<real_t>> inverse(std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> pinverse(std::vector<std::vector<double>> A);
std::vector<std::vector<real_t>> pinverse(std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> zeromat(int n, int m);
std::vector<std::vector<real_t>> zeromat(int n, int m);
std::vector<std::vector<double>> onemat(int n, int m);
std::vector<std::vector<real_t>> onemat(int n, int m);
std::vector<std::vector<double>> full(int n, int m, int k);
std::vector<std::vector<real_t>> full(int n, int m, int k);
std::vector<std::vector<double>> sin(std::vector<std::vector<double>> A);
std::vector<std::vector<real_t>> sin(std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> cos(std::vector<std::vector<double>> A);
std::vector<std::vector<real_t>> cos(std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> rotate(std::vector<std::vector<double>> A, double theta, int axis = -1);
std::vector<std::vector<real_t>> rotate(std::vector<std::vector<real_t>> A, real_t theta, int axis = -1);
std::vector<std::vector<double>> max(std::vector<std::vector<double>> A, std::vector<std::vector<double>> B);
std::vector<std::vector<real_t>> max(std::vector<std::vector<real_t>> A, std::vector<std::vector<real_t>> B);
double max(std::vector<std::vector<double>> A);
real_t max(std::vector<std::vector<real_t>> A);
double min(std::vector<std::vector<double>> A);
real_t min(std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> round(std::vector<std::vector<double>> A);
std::vector<std::vector<real_t>> round(std::vector<std::vector<real_t>> A);
double norm_2(std::vector<std::vector<double>> A);
real_t norm_2(std::vector<std::vector<real_t>> A);
std::vector<std::vector<double>> identity(double d);
std::vector<std::vector<real_t>> identity(real_t d);
std::vector<std::vector<double>> cov(std::vector<std::vector<double>> A);
std::vector<std::vector<real_t>> cov(std::vector<std::vector<real_t>> A);
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> eig(std::vector<std::vector<double>> A);
std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> eig(std::vector<std::vector<real_t>> A);
struct EigenResult {
std::vector<std::vector<double>> eigen_vectors;
std::vector<std::vector<double>> eigen_values;
std::vector<std::vector<real_t>> eigen_vectors;
std::vector<std::vector<real_t>> eigen_values;
};
EigenResult eigen(std::vector<std::vector<double>> A);
EigenResult eigen(std::vector<std::vector<real_t>> A);
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>, std::vector<std::vector<double>>> SVD(std::vector<std::vector<double>> A);
std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> SVD(std::vector<std::vector<real_t>> A);
struct SDVResult {
std::vector<std::vector<double>> U;
std::vector<std::vector<double>> S;
std::vector<std::vector<double>> Vt;
std::vector<std::vector<real_t>> U;
std::vector<std::vector<real_t>> S;
std::vector<std::vector<real_t>> Vt;
};
SDVResult svd(std::vector<std::vector<double>> A);
SDVResult svd(std::vector<std::vector<real_t>> A);
std::vector<double> vectorProjection(std::vector<double> a, std::vector<double> b);
std::vector<real_t> vectorProjection(std::vector<real_t> a, std::vector<real_t> b);
std::vector<std::vector<double>> gramSchmidtProcess(std::vector<std::vector<double>> A);
std::vector<std::vector<real_t>> gramSchmidtProcess(std::vector<std::vector<real_t>> A);
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> QRD(std::vector<std::vector<double>> A);
std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> QRD(std::vector<std::vector<real_t>> A);
struct QRDResult {
std::vector<std::vector<double>> Q;
std::vector<std::vector<double>> R;
std::vector<std::vector<real_t>> Q;
std::vector<std::vector<real_t>> R;
};
QRDResult qrd(std::vector<std::vector<double>> A);
QRDResult qrd(std::vector<std::vector<real_t>> A);
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> chol(std::vector<std::vector<double>> A);
std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> chol(std::vector<std::vector<real_t>> A);
struct CholeskyResult {
std::vector<std::vector<double>> L;
std::vector<std::vector<double>> Lt;
std::vector<std::vector<real_t>> L;
std::vector<std::vector<real_t>> Lt;
};
CholeskyResult cholesky(std::vector<std::vector<double>> A);
CholeskyResult cholesky(std::vector<std::vector<real_t>> A);
double sum_elements(std::vector<std::vector<double>> A);
real_t sum_elements(std::vector<std::vector<real_t>> A);
std::vector<double> flatten(std::vector<std::vector<double>> A);
std::vector<real_t> flatten(std::vector<std::vector<real_t>> A);
std::vector<double> solve(std::vector<std::vector<double>> A, std::vector<double> b);
std::vector<real_t> solve(std::vector<std::vector<real_t>> A, std::vector<real_t> b);
bool positiveDefiniteChecker(std::vector<std::vector<double>> A);
bool positiveDefiniteChecker(std::vector<std::vector<real_t>> A);
bool negativeDefiniteChecker(std::vector<std::vector<double>> A);
bool negativeDefiniteChecker(std::vector<std::vector<real_t>> A);
bool zeroEigenvalue(std::vector<std::vector<double>> A);
bool zeroEigenvalue(std::vector<std::vector<real_t>> A);
void printMatrix(std::vector<std::vector<double>> A);
void printMatrix(std::vector<std::vector<real_t>> A);
// VECTOR FUNCTIONS
std::vector<std::vector<double>> outerProduct(std::vector<double> a, std::vector<double> b); // This multiplies a, bT
std::vector<std::vector<real_t>> outerProduct(std::vector<real_t> a, std::vector<real_t> b); // This multiplies a, bT
std::vector<double> hadamard_product(std::vector<double> a, std::vector<double> b);
std::vector<real_t> hadamard_product(std::vector<real_t> a, std::vector<real_t> b);
std::vector<double> elementWiseDivision(std::vector<double> a, std::vector<double> b);
std::vector<real_t> elementWiseDivision(std::vector<real_t> a, std::vector<real_t> b);
std::vector<double> scalarMultiply(double scalar, std::vector<double> a);
std::vector<real_t> scalarMultiply(real_t scalar, std::vector<real_t> a);
std::vector<double> scalarAdd(double scalar, std::vector<double> a);
std::vector<real_t> scalarAdd(real_t scalar, std::vector<real_t> a);
std::vector<double> addition(std::vector<double> a, std::vector<double> b);
std::vector<real_t> addition(std::vector<real_t> a, std::vector<real_t> b);
std::vector<double> subtraction(std::vector<double> a, std::vector<double> b);
std::vector<real_t> subtraction(std::vector<real_t> a, std::vector<real_t> b);
std::vector<double> subtractMatrixRows(std::vector<double> a, std::vector<std::vector<double>> B);
std::vector<real_t> subtractMatrixRows(std::vector<real_t> a, std::vector<std::vector<real_t>> B);
std::vector<double> log(std::vector<double> a);
std::vector<real_t> log(std::vector<real_t> a);
std::vector<double> log10(std::vector<double> a);
std::vector<real_t> log10(std::vector<real_t> a);
std::vector<double> exp(std::vector<double> a);
std::vector<real_t> exp(std::vector<real_t> a);
std::vector<double> erf(std::vector<double> a);
std::vector<real_t> erf(std::vector<real_t> a);
std::vector<double> exponentiate(std::vector<double> a, double p);
std::vector<real_t> exponentiate(std::vector<real_t> a, real_t p);
std::vector<double> sqrt(std::vector<double> a);
std::vector<real_t> sqrt(std::vector<real_t> a);
std::vector<double> cbrt(std::vector<double> a);
std::vector<real_t> cbrt(std::vector<real_t> a);
double dot(std::vector<double> a, std::vector<double> b);
real_t dot(std::vector<real_t> a, std::vector<real_t> b);
std::vector<double> cross(std::vector<double> a, std::vector<double> b);
std::vector<real_t> cross(std::vector<real_t> a, std::vector<real_t> b);
std::vector<double> abs(std::vector<double> a);
std::vector<real_t> abs(std::vector<real_t> a);
std::vector<double> zerovec(int n);
std::vector<real_t> zerovec(int n);
std::vector<double> onevec(int n);
std::vector<real_t> onevec(int n);
std::vector<std::vector<double>> diag(std::vector<double> a);
std::vector<std::vector<real_t>> diag(std::vector<real_t> a);
std::vector<double> full(int n, int k);
std::vector<real_t> full(int n, int k);
std::vector<double> sin(std::vector<double> a);
std::vector<real_t> sin(std::vector<real_t> a);
std::vector<double> cos(std::vector<double> a);
std::vector<real_t> cos(std::vector<real_t> a);
std::vector<double> max(std::vector<double> a, std::vector<double> b);
std::vector<real_t> max(std::vector<real_t> a, std::vector<real_t> b);
double max(std::vector<double> a);
real_t max(std::vector<real_t> a);
double min(std::vector<double> a);
real_t min(std::vector<real_t> a);
std::vector<double> round(std::vector<double> a);
std::vector<real_t> round(std::vector<real_t> a);
double euclideanDistance(std::vector<double> a, std::vector<double> b);
real_t euclideanDistance(std::vector<real_t> a, std::vector<real_t> b);
double norm_2(std::vector<double> a);
real_t norm_2(std::vector<real_t> a);
double norm_sq(std::vector<double> a);
real_t norm_sq(std::vector<real_t> a);
double sum_elements(std::vector<double> a);
real_t sum_elements(std::vector<real_t> a);
double cosineSimilarity(std::vector<double> a, std::vector<double> b);
real_t cosineSimilarity(std::vector<real_t> a, std::vector<real_t> b);
void printVector(std::vector<double> a);
void printVector(std::vector<real_t> a);
// MATRIX-VECTOR FUNCTIONS
std::vector<std::vector<double>> mat_vec_add(std::vector<std::vector<double>> A, std::vector<double> b);
std::vector<std::vector<real_t>> mat_vec_add(std::vector<std::vector<real_t>> A, std::vector<real_t> b);
std::vector<double> mat_vec_mult(std::vector<std::vector<double>> A, std::vector<double> b);
std::vector<real_t> mat_vec_mult(std::vector<std::vector<real_t>> A, std::vector<real_t> b);
// TENSOR FUNCTIONS
std::vector<std::vector<std::vector<double>>> addition(std::vector<std::vector<std::vector<double>>> A, std::vector<std::vector<std::vector<double>>> B);
std::vector<std::vector<std::vector<real_t>>> addition(std::vector<std::vector<std::vector<real_t>>> A, std::vector<std::vector<std::vector<real_t>>> B);
std::vector<std::vector<std::vector<double>>> elementWiseDivision(std::vector<std::vector<std::vector<double>>> A, std::vector<std::vector<std::vector<double>>> B);
std::vector<std::vector<std::vector<real_t>>> elementWiseDivision(std::vector<std::vector<std::vector<real_t>>> A, std::vector<std::vector<std::vector<real_t>>> B);
std::vector<std::vector<std::vector<double>>> sqrt(std::vector<std::vector<std::vector<double>>> A);
std::vector<std::vector<std::vector<real_t>>> sqrt(std::vector<std::vector<std::vector<real_t>>> A);
std::vector<std::vector<std::vector<double>>> exponentiate(std::vector<std::vector<std::vector<double>>> A, double p);
std::vector<std::vector<std::vector<real_t>>> exponentiate(std::vector<std::vector<std::vector<real_t>>> A, real_t p);
std::vector<std::vector<double>> tensor_vec_mult(std::vector<std::vector<std::vector<double>>> A, std::vector<double> b);
std::vector<std::vector<real_t>> tensor_vec_mult(std::vector<std::vector<std::vector<real_t>>> A, std::vector<real_t> b);
std::vector<double> flatten(std::vector<std::vector<std::vector<double>>> A);
std::vector<real_t> flatten(std::vector<std::vector<std::vector<real_t>>> A);
void printTensor(std::vector<std::vector<std::vector<double>>> A);
void printTensor(std::vector<std::vector<std::vector<real_t>>> A);
std::vector<std::vector<std::vector<double>>> scalarMultiply(double scalar, std::vector<std::vector<std::vector<double>>> A);
std::vector<std::vector<std::vector<real_t>>> scalarMultiply(real_t scalar, std::vector<std::vector<std::vector<real_t>>> A);
std::vector<std::vector<std::vector<double>>> scalarAdd(double scalar, std::vector<std::vector<std::vector<double>>> A);
std::vector<std::vector<std::vector<real_t>>> scalarAdd(real_t scalar, std::vector<std::vector<std::vector<real_t>>> A);
std::vector<std::vector<std::vector<double>>> resize(std::vector<std::vector<std::vector<double>>> A, std::vector<std::vector<std::vector<double>>> B);
std::vector<std::vector<std::vector<real_t>>> resize(std::vector<std::vector<std::vector<real_t>>> A, std::vector<std::vector<std::vector<real_t>>> B);
std::vector<std::vector<std::vector<double>>> hadamard_product(std::vector<std::vector<std::vector<double>>> A, std::vector<std::vector<std::vector<double>>> B);
std::vector<std::vector<std::vector<real_t>>> hadamard_product(std::vector<std::vector<std::vector<real_t>>> A, std::vector<std::vector<std::vector<real_t>>> B);
std::vector<std::vector<std::vector<double>>> max(std::vector<std::vector<std::vector<double>>> A, std::vector<std::vector<std::vector<double>>> B);
std::vector<std::vector<std::vector<real_t>>> max(std::vector<std::vector<std::vector<real_t>>> A, std::vector<std::vector<std::vector<real_t>>> B);
std::vector<std::vector<std::vector<double>>> abs(std::vector<std::vector<std::vector<double>>> A);
std::vector<std::vector<std::vector<real_t>>> abs(std::vector<std::vector<std::vector<real_t>>> A);
double norm_2(std::vector<std::vector<std::vector<double>>> A);
real_t norm_2(std::vector<std::vector<std::vector<real_t>>> A);
std::vector<std::vector<std::vector<double>>> vector_wise_tensor_product(std::vector<std::vector<std::vector<double>>> A, std::vector<std::vector<double>> B);
std::vector<std::vector<std::vector<real_t>>> vector_wise_tensor_product(std::vector<std::vector<std::vector<real_t>>> A, std::vector<std::vector<real_t>> B);
private:
};

84
mlpp/lin_alg/mlpp_matrix.h
View File

@ -1,6 +1,8 @@
#ifndef MLPP_MATRIX_H
#define MLPP_MATRIX_H
#include "core/math/math_defs.h"
#include "core/containers/pool_vector.h"
#include "core/containers/sort_array.h"
#include "core/containers/vector.h"
@ -19,15 +21,15 @@ class MLPPMatrix : public Reference {
GDCLASS(MLPPMatrix, Reference);
public:
double *ptr() {
real_t *ptr() {
return _data;
}
const double *ptr() const {
const real_t *ptr() const {
return _data;
}
_FORCE_INLINE_ void add_row(const Vector<double> &p_row) {
_FORCE_INLINE_ void add_row(const Vector<real_t> &p_row) {
if (_size.x == 0) {
_size.x = p_row.size();
}
@ -38,10 +40,10 @@ public:
++_size.y;
_data = (double *)memrealloc(_data, data_size() * sizeof(double));
_data = (real_t *)memrealloc(_data, data_size() * sizeof(real_t));
CRASH_COND_MSG(!_data, "Out of memory");
const double *row_arr = p_row.ptr();
const real_t *row_arr = p_row.ptr();
for (int i = 0; i < p_row.size(); ++i) {
_data[ci + i] = row_arr[i];
@ -59,7 +61,7 @@ public:
++_size.y;
_data = (double *)memrealloc(_data, data_size() * sizeof(double));
_data = (real_t *)memrealloc(_data, data_size() * sizeof(real_t));
CRASH_COND_MSG(!_data, "Out of memory");
PoolRealArray::Read rread = p_row.read();
@ -70,7 +72,7 @@ public:
}
}
void remove_row(double p_index) {
void remove_row(real_t p_index) {
ERR_FAIL_INDEX(p_index, _size.y);
--_size.y;
@ -87,7 +89,7 @@ public:
_data[i] = _data[i + _size.x];
}
_data = (double *)memrealloc(_data, data_size() * sizeof(double));
_data = (real_t *)memrealloc(_data, data_size() * sizeof(real_t));
CRASH_COND_MSG(!_data, "Out of memory");
}
@ -113,7 +115,7 @@ public:
_data[i] = _data[ds + i];
}
_data = (double *)memrealloc(_data, data_size() * sizeof(double));
_data = (real_t *)memrealloc(_data, data_size() * sizeof(real_t));
CRASH_COND_MSG(!_data, "Out of memory");
}
@ -154,26 +156,26 @@ public:
return;
}
_data = (double *)memrealloc(_data, ds * sizeof(double));
_data = (real_t *)memrealloc(_data, ds * sizeof(real_t));
CRASH_COND_MSG(!_data, "Out of memory");
}
_FORCE_INLINE_ const double &operator[](int p_index) const {
_FORCE_INLINE_ const real_t &operator[](int p_index) const {
CRASH_BAD_INDEX(p_index, data_size());
return _data[p_index];
}
_FORCE_INLINE_ double &operator[](int p_index) {
_FORCE_INLINE_ real_t &operator[](int p_index) {
CRASH_BAD_INDEX(p_index, data_size());
return _data[p_index];
}
_FORCE_INLINE_ double get_element(int p_index_x, int p_index_y) const {
_FORCE_INLINE_ real_t get_element(int p_index_x, int p_index_y) const {
ERR_FAIL_INDEX_V(p_index_x, _size.x, 0);
ERR_FAIL_INDEX_V(p_index_y, _size.y, 0);
return _data[p_index_x * p_index_y];
}
_FORCE_INLINE_ double get_element(int p_index_x, int p_index_y) {
_FORCE_INLINE_ real_t get_element(int p_index_x, int p_index_y) {
ERR_FAIL_INDEX_V(p_index_x, _size.x, 0);
ERR_FAIL_INDEX_V(p_index_y, _size.y, 0);
@ -187,7 +189,7 @@ public:
return static_cast<real_t>(_data[p_index_x * p_index_y]);
}
_FORCE_INLINE_ void set_element(int p_index_x, int p_index_y, double p_val) {
_FORCE_INLINE_ void set_element(int p_index_x, int p_index_y, real_t p_val) {
ERR_FAIL_INDEX(p_index_x, _size.x);
ERR_FAIL_INDEX(p_index_y, _size.y);
@ -201,13 +203,13 @@ public:
_data[p_index_x * p_index_y] = p_val;
}
_FORCE_INLINE_ void set_row_vector(int p_index_y, const Vector<double> &p_row) {
_FORCE_INLINE_ void set_row_vector(int p_index_y, const Vector<real_t> &p_row) {
ERR_FAIL_COND(p_row.size() != _size.x);
ERR_FAIL_INDEX(p_index_y, _size.y);
int ind_start = p_index_y * _size.x;
const double *row_ptr = p_row.ptr();
const real_t *row_ptr = p_row.ptr();
for (int i = 0; i < _size.x; ++i) {
_data[ind_start + i] = row_ptr[i];
@ -228,18 +230,18 @@ public:
}
}
void fill(double p_val) {
void fill(real_t p_val) {
int ds = data_size();
for (int i = 0; i < ds; i++) {
_data[i] = p_val;
}
}
Vector<double> to_flat_vector() const {
Vector<double> ret;
Vector<real_t> to_flat_vector() const {
Vector<real_t> ret;
ret.resize(data_size());
double *w = ret.ptrw();
memcpy(w, _data, sizeof(double) * data_size());
real_t *w = ret.ptrw();
memcpy(w, _data, sizeof(real_t) * data_size());
return ret;
}
@ -259,9 +261,9 @@ public:
Vector<uint8_t> to_flat_byte_array() const {
Vector<uint8_t> ret;
ret.resize(data_size() * sizeof(double));
ret.resize(data_size() * sizeof(real_t));
uint8_t *w = ret.ptrw();
memcpy(w, _data, sizeof(double) * data_size());
memcpy(w, _data, sizeof(real_t) * data_size());
return ret;
}
@ -307,7 +309,7 @@ public:
int start_index = i * _size.x;
const double *from_ptr = r->ptr();
const real_t *from_ptr = r->ptr();
for (int j = 0; j < _size.x; j++) {
_data[start_index + j] = from_ptr[j];
}
@ -342,14 +344,14 @@ public:
int start_index = i * _size.x;
const double *from_ptr = r->ptr();
const real_t *from_ptr = r->ptr();
for (int j = 0; j < _size.x; j++) {
_data[start_index + j] = from_ptr[j];
}
}
}
_FORCE_INLINE_ void set_from_vectors(const Vector<Vector<double>> &p_from) {
_FORCE_INLINE_ void set_from_vectors(const Vector<Vector<real_t>> &p_from) {
if (p_from.size() == 0) {
reset();
return;
@ -363,13 +365,13 @@ public:
}
for (int i = 0; i < p_from.size(); ++i) {
const Vector<double> &r = p_from[i];
const Vector<real_t> &r = p_from[i];
ERR_CONTINUE(r.size() != _size.x);
int start_index = i * _size.x;
const double *from_ptr = r.ptr();
const real_t *from_ptr = r.ptr();
for (int j = 0; j < _size.x; j++) {
_data[start_index + j] = from_ptr[j];
}
@ -420,7 +422,7 @@ public:
}
}
MLPPMatrix(const Vector<Vector<double>> &p_from) {
MLPPMatrix(const Vector<Vector<real_t>> &p_from) {
_data = NULL;
set_from_vectors(p_from);
@ -439,15 +441,15 @@ public:
}
// TODO: These are temporary
std::vector<double> to_flat_std_vector() const {
std::vector<double> ret;
std::vector<real_t> to_flat_std_vector() const {
std::vector<real_t> ret;
ret.resize(data_size());
double *w = &ret[0];
memcpy(w, _data, sizeof(double) * data_size());
real_t *w = &ret[0];
memcpy(w, _data, sizeof(real_t) * data_size());
return ret;
}
_FORCE_INLINE_ void set_from_std_vectors(const std::vector<std::vector<double>> &p_from) {
_FORCE_INLINE_ void set_from_std_vectors(const std::vector<std::vector<real_t>> &p_from) {
if (p_from.size() == 0) {
reset();
return;
@ -461,33 +463,33 @@ public:
}
for (uint32_t i = 0; i < p_from.size(); ++i) {
const std::vector<double> &r = p_from[i];
const std::vector<real_t> &r = p_from[i];
ERR_CONTINUE(r.size() != static_cast<uint32_t>(_size.x));
int start_index = i * _size.x;
const double *from_ptr = &r[0];
const real_t *from_ptr = &r[0];
for (int j = 0; j < _size.x; j++) {
_data[start_index + j] = from_ptr[j];
}
}
}
_FORCE_INLINE_ void set_row_std_vector(int p_index_y, const std::vector<double> &p_row) {
_FORCE_INLINE_ void set_row_std_vector(int p_index_y, const std::vector<real_t> &p_row) {
ERR_FAIL_COND(p_row.size() != static_cast<uint32_t>(_size.x));
ERR_FAIL_INDEX(p_index_y, _size.y);
int ind_start = p_index_y * _size.x;
const double *row_ptr = &p_row[0];
const real_t *row_ptr = &p_row[0];
for (int i = 0; i < _size.x; ++i) {
_data[ind_start + i] = row_ptr[i];
}
}
MLPPMatrix(const std::vector<std::vector<double>> &p_from) {
MLPPMatrix(const std::vector<std::vector<real_t>> &p_from) {
_data = NULL;
set_from_std_vectors(p_from);
@ -498,7 +500,7 @@ protected:
protected:
Size2i _size;
double *_data;
real_t *_data;
};
#endif

74
mlpp/lin_alg/mlpp_vector.h
View File

@ -1,6 +1,8 @@
#ifndef MLPP_VECTOR_H
#define MLPP_VECTOR_H
#include "core/math/math_defs.h"
#include "core/containers/pool_vector.h"
#include "core/containers/sort_array.h"
#include "core/containers/vector.h"
@ -16,24 +18,24 @@ class MLPPVector : public Reference {
GDCLASS(MLPPVector, Reference);
public:
double *ptr() {
real_t *ptr() {
return _data;
}
const double *ptr() const {
const real_t *ptr() const {
return _data;
}
_FORCE_INLINE_ void push_back(double p_elem) {
_FORCE_INLINE_ void push_back(real_t p_elem) {
++_size;
_data = (double *)memrealloc(_data, _size * sizeof(double));
_data = (real_t *)memrealloc(_data, _size * sizeof(real_t));
CRASH_COND_MSG(!_data, "Out of memory");
_data[_size - 1] = p_elem;
}
void remove(double p_index) {
void remove(real_t p_index) {
ERR_FAIL_INDEX(p_index, _size);
--_size;
@ -42,7 +44,7 @@ public:
_data[i] = _data[i + 1];
}
_data = (double *)memrealloc(_data, _size * sizeof(double));
_data = (real_t *)memrealloc(_data, _size * sizeof(real_t));
CRASH_COND_MSG(!_data, "Out of memory");
}
@ -56,18 +58,18 @@ public:
_data[p_index] = _data[_size];
}
_data = (double *)memrealloc(_data, _size * sizeof(double));
_data = (real_t *)memrealloc(_data, _size * sizeof(real_t));
CRASH_COND_MSG(!_data, "Out of memory");
}
void erase(const double &p_val) {
void erase(const real_t &p_val) {
int idx = find(p_val);
if (idx >= 0) {
remove(idx);
}
}
int erase_multiple_unordered(const double &p_val) {
int erase_multiple_unordered(const real_t &p_val) {
int from = 0;
int count = 0;
while (true) {
@ -111,24 +113,24 @@ public:
return;
}
_data = (double *)memrealloc(_data, _size * sizeof(double));
_data = (real_t *)memrealloc(_data, _size * sizeof(real_t));
CRASH_COND_MSG(!_data, "Out of memory");
}
_FORCE_INLINE_ const double &operator[](int p_index) const {
_FORCE_INLINE_ const real_t &operator[](int p_index) const {
CRASH_BAD_INDEX(p_index, _size);
return _data[p_index];
}
_FORCE_INLINE_ double &operator[](int p_index) {
_FORCE_INLINE_ real_t &operator[](int p_index) {
CRASH_BAD_INDEX(p_index, _size);
return _data[p_index];
}
_FORCE_INLINE_ double get_element(int p_index) const {
_FORCE_INLINE_ real_t get_element(int p_index) const {
ERR_FAIL_INDEX_V(p_index, _size, 0);
return _data[p_index];
}
_FORCE_INLINE_ double get_element(int p_index) {
_FORCE_INLINE_ real_t get_element(int p_index) {
ERR_FAIL_INDEX_V(p_index, _size, 0);
return _data[p_index];
}
@ -138,7 +140,7 @@ public:
return static_cast<real_t>(_data[p_index]);
}
_FORCE_INLINE_ void set_element(int p_index, double p_val) {
_FORCE_INLINE_ void set_element(int p_index, real_t p_val) {
ERR_FAIL_INDEX(p_index, _size);
_data[p_index] = p_val;
}
@ -148,13 +150,13 @@ public:
_data[p_index] = p_val;
}
void fill(double p_val) {
void fill(real_t p_val) {
for (int i = 0; i < _size; i++) {
_data[i] = p_val;
}
}
void insert(int p_pos, double p_val) {
void insert(int p_pos, real_t p_val) {
ERR_FAIL_INDEX(p_pos, _size + 1);
if (p_pos == _size) {
push_back(p_val);
@ -167,7 +169,7 @@ public:
}
}
int find(const double &p_val, int p_from = 0) const {
int find(const real_t &p_val, int p_from = 0) const {
for (int i = p_from; i < _size; i++) {
if (_data[i] == p_val) {
return i;
@ -183,15 +185,15 @@ public:
return;
}
SortArray<double, C> sorter;
SortArray<real_t, C> sorter;
sorter.sort(_data, len);
}
void sort() {
sort_custom<_DefaultComparator<double>>();
sort_custom<_DefaultComparator<real_t>>();
}
void ordered_insert(double p_val) {
void ordered_insert(real_t p_val) {
int i;
for (i = 0; i < _size; i++) {
if (p_val < _data[i]) {
@ -201,11 +203,11 @@ public:
insert(i, p_val);
}
Vector<double> to_vector() const {
Vector<double> ret;
Vector<real_t> to_vector() const {
Vector<real_t> ret;
ret.resize(size());
double *w = ret.ptrw();
memcpy(w, _data, sizeof(double) * _size);
real_t *w = ret.ptrw();
memcpy(w, _data, sizeof(real_t) * _size);
return ret;
}
@ -225,9 +227,9 @@ public:
Vector<uint8_t> to_byte_array() const {
Vector<uint8_t> ret;
ret.resize(_size * sizeof(double));
ret.resize(_size * sizeof(real_t));
uint8_t *w = ret.ptrw();
memcpy(w, _data, sizeof(double) * _size);
memcpy(w, _data, sizeof(real_t) * _size);
return ret;
}
@ -255,7 +257,7 @@ public:
}
}
_FORCE_INLINE_ void set_from_vector(const Vector<double> &p_from) {
_FORCE_INLINE_ void set_from_vector(const Vector<real_t> &p_from) {
resize(p_from.size());
for (int i = 0; i < _size; i++) {
_data[i] = p_from[i];
@ -286,7 +288,7 @@ public:
}
}
MLPPVector(const Vector<double> &p_from) {
MLPPVector(const Vector<real_t> &p_from) {
_size = 0;
_data = NULL;
@ -314,22 +316,22 @@ public:
}
// TODO: These are temporary
std::vector<double> to_std_vector() const {
std::vector<double> ret;
std::vector<real_t> to_std_vector() const {
std::vector<real_t> ret;
ret.resize(size());
double *w = &ret[0];
memcpy(w, _data, sizeof(double) * _size);
real_t *w = &ret[0];
memcpy(w, _data, sizeof(real_t) * _size);
return ret;
}
_FORCE_INLINE_ void set_from_std_vector(const std::vector<double> &p_from) {
_FORCE_INLINE_ void set_from_std_vector(const std::vector<real_t> &p_from) {
resize(p_from.size());
for (int i = 0; i < _size; i++) {
_data[i] = p_from[i];
}
}
MLPPVector(const std::vector<double> &p_from) {
MLPPVector(const std::vector<real_t> &p_from) {
_size = 0;
_data = NULL;
@ -344,7 +346,7 @@ protected:
protected:
int _size;
double *_data;
real_t *_data;
};
#endif

182
mlpp/lin_reg/lin_reg.cpp
View File

@ -15,7 +15,7 @@
#include <iostream>
#include <random>
MLPPLinReg::MLPPLinReg(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, std::string reg, double lambda, double alpha) :
MLPPLinReg::MLPPLinReg(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, std::string reg, real_t lambda, real_t alpha) :
inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) {
y_hat.resize(n);
@ -23,28 +23,28 @@ MLPPLinReg::MLPPLinReg(std::vector<std::vector<double>> inputSet, std::vector<do
bias = MLPPUtilities::biasInitialization();
}
std::vector<double> MLPPLinReg::modelSetTest(std::vector<std::vector<double>> X) {
std::vector<real_t> MLPPLinReg::modelSetTest(std::vector<std::vector<real_t>> X) {
return Evaluate(X);
}
double MLPPLinReg::modelTest(std::vector<double> x) {
real_t MLPPLinReg::modelTest(std::vector<real_t> x) {
return Evaluate(x);
}
void MLPPLinReg::NewtonRaphson(double learning_rate, int max_epoch, bool UI) {
void MLPPLinReg::NewtonRaphson(real_t learning_rate, int max_epoch, bool UI) {
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
forwardPass();
while (true) {
cost_prev = Cost(y_hat, outputSet);
std::vector<double> error = alg.subtraction(y_hat, outputSet);
std::vector<real_t> error = alg.subtraction(y_hat, outputSet);
// Calculating the weight gradients (2nd derivative)
std::vector<double> first_derivative = alg.mat_vec_mult(alg.transpose(inputSet), error);
std::vector<std::vector<double>> second_derivative = alg.matmult(alg.transpose(inputSet), inputSet);
std::vector<real_t> first_derivative = alg.mat_vec_mult(alg.transpose(inputSet), error);
std::vector<std::vector<real_t>> second_derivative = alg.matmult(alg.transpose(inputSet), inputSet);
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(alg.inverse(second_derivative)), first_derivative)));
weights = regularization.regWeights(weights, lambda, alpha, reg);
@ -63,17 +63,17 @@ void MLPPLinReg::NewtonRaphson(double learning_rate, int max_epoch, bool UI) {
}
}
void MLPPLinReg::gradientDescent(double learning_rate, int max_epoch, bool UI) {
void MLPPLinReg::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
forwardPass();
while (true) {
cost_prev = Cost(y_hat, outputSet);
std::vector<double> error = alg.subtraction(y_hat, outputSet);
std::vector<real_t> error = alg.subtraction(y_hat, outputSet);
// Calculating the weight gradients
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputSet), error)));
@ -94,10 +94,10 @@ void MLPPLinReg::gradientDescent(double learning_rate, int max_epoch, bool UI) {
}
}
void MLPPLinReg::SGD(double learning_rate, int max_epoch, bool UI) {
void MLPPLinReg::SGD(real_t learning_rate, int max_epoch, bool UI) {
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
while (true) {
@ -106,10 +106,10 @@ void MLPPLinReg::SGD(double learning_rate, int max_epoch, bool UI) {
std::uniform_int_distribution<int> distribution(0, int(n - 1));
int outputIndex = distribution(generator);
double y_hat = Evaluate(inputSet[outputIndex]);
real_t y_hat = Evaluate(inputSet[outputIndex]);
cost_prev = Cost({ y_hat }, { outputSet[outputIndex] });
double error = y_hat - outputSet[outputIndex];
real_t error = y_hat - outputSet[outputIndex];
// Weight updation
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate * error, inputSet[outputIndex]));
@ -133,10 +133,10 @@ void MLPPLinReg::SGD(double learning_rate, int max_epoch, bool UI) {
forwardPass();
}
void MLPPLinReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI) {
void MLPPLinReg::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI) {
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
// Creating the mini-batches
@ -145,10 +145,10 @@ void MLPPLinReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size,
while (true) {
for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = Evaluate(inputMiniBatches[i]);
std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]);
std::vector<double> error = alg.subtraction(y_hat, outputMiniBatches[i]);
std::vector<real_t> error = alg.subtraction(y_hat, outputMiniBatches[i]);
// Calculating the weight gradients
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error)));
@ -171,10 +171,10 @@ void MLPPLinReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size,
forwardPass();
}
void MLPPLinReg::Momentum(double learning_rate, int max_epoch, int mini_batch_size, double gamma, bool UI) {
void MLPPLinReg::Momentum(real_t learning_rate, int max_epoch, int mini_batch_size, real_t gamma, bool UI) {
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
// Creating the mini-batches
@ -182,18 +182,18 @@ void MLPPLinReg::Momentum(double learning_rate, int max_epoch, int mini_batch_si
auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch);
// Initializing necessary components for Momentum.
std::vector<double> v = alg.zerovec(weights.size());
std::vector<real_t> v = alg.zerovec(weights.size());
while (true) {
for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = Evaluate(inputMiniBatches[i]);
std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]);
std::vector<double> error = alg.subtraction(y_hat, outputMiniBatches[i]);
std::vector<real_t> error = alg.subtraction(y_hat, outputMiniBatches[i]);
// Calculating the weight gradients
std::vector<double> gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error));
std::vector<double> RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg);
std::vector<double> weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final
std::vector<real_t> gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error));
std::vector<real_t> RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg);
std::vector<real_t> weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final
v = alg.addition(alg.scalarMultiply(gamma, v), alg.scalarMultiply(learning_rate, weight_grad));
@ -216,10 +216,10 @@ void MLPPLinReg::Momentum(double learning_rate, int max_epoch, int mini_batch_si
forwardPass();
}
void MLPPLinReg::NAG(double learning_rate, int max_epoch, int mini_batch_size, double gamma, bool UI) {
void MLPPLinReg::NAG(real_t learning_rate, int max_epoch, int mini_batch_size, real_t gamma, bool UI) {
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
// Creating the mini-batches
@ -227,20 +227,20 @@ void MLPPLinReg::NAG(double learning_rate, int max_epoch, int mini_batch_size, d
auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch);
// Initializing necessary components for Momentum.
std::vector<double> v = alg.zerovec(weights.size());
std::vector<real_t> v = alg.zerovec(weights.size());
while (true) {
for (int i = 0; i < n_mini_batch; i++) {
weights = alg.subtraction(weights, alg.scalarMultiply(gamma, v)); // "Aposterori" calculation
std::vector<double> y_hat = Evaluate(inputMiniBatches[i]);
std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]);
std::vector<double> error = alg.subtraction(y_hat, outputMiniBatches[i]);
std::vector<real_t> error = alg.subtraction(y_hat, outputMiniBatches[i]);
// Calculating the weight gradients
std::vector<double> gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error));
std::vector<double> RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg);
std::vector<double> weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final
std::vector<real_t> gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error));
std::vector<real_t> RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg);
std::vector<real_t> weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final
v = alg.addition(alg.scalarMultiply(gamma, v), alg.scalarMultiply(learning_rate, weight_grad));
@ -263,10 +263,10 @@ void MLPPLinReg::NAG(double learning_rate, int max_epoch, int mini_batch_size, d
forwardPass();
}
void MLPPLinReg::Adagrad(double learning_rate, int max_epoch, int mini_batch_size, double e, bool UI) {
void MLPPLinReg::Adagrad(real_t learning_rate, int max_epoch, int mini_batch_size, real_t e, bool UI) {
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
// Creating the mini-batches
@ -274,18 +274,18 @@ void MLPPLinReg::Adagrad(double learning_rate, int max_epoch, int mini_batch_siz
auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch);
// Initializing necessary components for Adagrad.
std::vector<double> v = alg.zerovec(weights.size());
std::vector<real_t> v = alg.zerovec(weights.size());
while (true) {
for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = Evaluate(inputMiniBatches[i]);
std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]);
std::vector<double> error = alg.subtraction(y_hat, outputMiniBatches[i]);
std::vector<real_t> error = alg.subtraction(y_hat, outputMiniBatches[i]);
// Calculating the weight gradients
std::vector<double> gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error));
std::vector<double> RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg);
std::vector<double> weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final
std::vector<real_t> gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error));
std::vector<real_t> RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg);
std::vector<real_t> weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final
v = alg.hadamard_product(weight_grad, weight_grad);
@ -308,11 +308,11 @@ void MLPPLinReg::Adagrad(double learning_rate, int max_epoch, int mini_batch_siz
forwardPass();
}
void MLPPLinReg::Adadelta(double learning_rate, int max_epoch, int mini_batch_size, double b1, double e, bool UI) {
void MLPPLinReg::Adadelta(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t e, bool UI) {
// Adagrad upgrade. Momentum is applied.
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
// Creating the mini-batches
@ -320,18 +320,18 @@ void MLPPLinReg::Adadelta(double learning_rate, int max_epoch, int mini_batch_si
auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch);
// Initializing necessary components for Adagrad.
std::vector<double> v = alg.zerovec(weights.size());
std::vector<real_t> v = alg.zerovec(weights.size());
while (true) {
for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = Evaluate(inputMiniBatches[i]);
std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]);
std::vector<double> error = alg.subtraction(y_hat, outputMiniBatches[i]);
std::vector<real_t> error = alg.subtraction(y_hat, outputMiniBatches[i]);
// Calculating the weight gradients
std::vector<double> gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error));
std::vector<double> RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg);
std::vector<double> weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final
std::vector<real_t> gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error));
std::vector<real_t> RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg);
std::vector<real_t> weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final
v = alg.addition(alg.scalarMultiply(b1, v), alg.scalarMultiply(1 - b1, alg.hadamard_product(weight_grad, weight_grad)));
@ -354,10 +354,10 @@ void MLPPLinReg::Adadelta(double learning_rate, int max_epoch, int mini_batch_si
forwardPass();
}
void MLPPLinReg::Adam(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI) {
void MLPPLinReg::Adam(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t b2, real_t e, bool UI) {
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
// Creating the mini-batches
@ -365,26 +365,26 @@ void MLPPLinReg::Adam(double learning_rate, int max_epoch, int mini_batch_size,
auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch);
// Initializing necessary components for Adam.
std::vector<double> m = alg.zerovec(weights.size());
std::vector<real_t> m = alg.zerovec(weights.size());
std::vector<double> v = alg.zerovec(weights.size());
std::vector<real_t> v = alg.zerovec(weights.size());
while (true) {
for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = Evaluate(inputMiniBatches[i]);
std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]);
std::vector<double> error = alg.subtraction(y_hat, outputMiniBatches[i]);
std::vector<real_t> error = alg.subtraction(y_hat, outputMiniBatches[i]);
// Calculating the weight gradients
std::vector<double> gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error));
std::vector<double> RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg);
std::vector<double> weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final
std::vector<real_t> gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error));
std::vector<real_t> RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg);
std::vector<real_t> weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final
m = alg.addition(alg.scalarMultiply(b1, m), alg.scalarMultiply(1 - b1, weight_grad));
v = alg.addition(alg.scalarMultiply(b2, v), alg.scalarMultiply(1 - b2, alg.exponentiate(weight_grad, 2)));
std::vector<double> m_hat = alg.scalarMultiply(1 / (1 - pow(b1, epoch)), m);
std::vector<double> v_hat = alg.scalarMultiply(1 / (1 - pow(b2, epoch)), v);
std::vector<real_t> m_hat = alg.scalarMultiply(1 / (1 - pow(b1, epoch)), m);
std::vector<real_t> v_hat = alg.scalarMultiply(1 / (1 - pow(b2, epoch)), v);
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate, alg.elementWiseDivision(m_hat, alg.scalarAdd(e, alg.sqrt(v_hat)))));
@ -405,35 +405,35 @@ void MLPPLinReg::Adam(double learning_rate, int max_epoch, int mini_batch_size,
forwardPass();
}
void MLPPLinReg::Adamax(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI) {
void MLPPLinReg::Adamax(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t b2, real_t e, bool UI) {
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
// Creating the mini-batches
int n_mini_batch = n / mini_batch_size;
auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch);
std::vector<double> m = alg.zerovec(weights.size());
std::vector<real_t> m = alg.zerovec(weights.size());
std::vector<double> u = alg.zerovec(weights.size());
std::vector<real_t> u = alg.zerovec(weights.size());
while (true) {
for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = Evaluate(inputMiniBatches[i]);
std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]);
std::vector<double> error = alg.subtraction(y_hat, outputMiniBatches[i]);
std::vector<real_t> error = alg.subtraction(y_hat, outputMiniBatches[i]);
// Calculating the weight gradients
std::vector<double> gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error));
std::vector<double> RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg);
std::vector<double> weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final
std::vector<real_t> gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error));
std::vector<real_t> RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg);
std::vector<real_t> weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final
m = alg.addition(alg.scalarMultiply(b1, m), alg.scalarMultiply(1 - b1, weight_grad));
u = alg.max(alg.scalarMultiply(b2, u), alg.abs(weight_grad));
std::vector<double> m_hat = alg.scalarMultiply(1 / (1 - pow(b1, epoch)), m);
std::vector<real_t> m_hat = alg.scalarMultiply(1 / (1 - pow(b1, epoch)), m);
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate, alg.elementWiseDivision(m_hat, u)));
@ -454,10 +454,10 @@ void MLPPLinReg::Adamax(double learning_rate, int max_epoch, int mini_batch_size
forwardPass();
}
void MLPPLinReg::Nadam(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI) {
void MLPPLinReg::Nadam(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t b2, real_t e, bool UI) {
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
// Creating the mini-batches
@ -465,27 +465,27 @@ void MLPPLinReg::Nadam(double learning_rate, int max_epoch, int mini_batch_size,
auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch);
// Initializing necessary components for Adam.
std::vector<double> m = alg.zerovec(weights.size());
std::vector<double> v = alg.zerovec(weights.size());
std::vector<double> m_final = alg.zerovec(weights.size());
std::vector<real_t> m = alg.zerovec(weights.size());
std::vector<real_t> v = alg.zerovec(weights.size());
std::vector<real_t> m_final = alg.zerovec(weights.size());
while (true) {
for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = Evaluate(inputMiniBatches[i]);
std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]);
std::vector<double> error = alg.subtraction(y_hat, outputMiniBatches[i]);
std::vector<real_t> error = alg.subtraction(y_hat, outputMiniBatches[i]);
// Calculating the weight gradients
std::vector<double> gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error));
std::vector<double> RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg);
std::vector<double> weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final
std::vector<real_t> gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error));
std::vector<real_t> RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg);
std::vector<real_t> weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final
m = alg.addition(alg.scalarMultiply(b1, m), alg.scalarMultiply(1 - b1, weight_grad));
v = alg.addition(alg.scalarMultiply(b2, v), alg.scalarMultiply(1 - b2, alg.exponentiate(weight_grad, 2)));
m_final = alg.addition(alg.scalarMultiply(b1, m), alg.scalarMultiply((1 - b1) / (1 - pow(b1, epoch)), weight_grad));
std::vector<double> m_hat = alg.scalarMultiply(1 / (1 - pow(b1, epoch)), m);
std::vector<double> v_hat = alg.scalarMultiply(1 / (1 - pow(b2, epoch)), v);
std::vector<real_t> m_hat = alg.scalarMultiply(1 / (1 - pow(b1, epoch)), m);
std::vector<real_t> v_hat = alg.scalarMultiply(1 / (1 - pow(b2, epoch)), v);
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate, alg.elementWiseDivision(m_final, alg.scalarAdd(e, alg.sqrt(v_hat)))));
@ -509,8 +509,8 @@ void MLPPLinReg::Nadam(double learning_rate, int max_epoch, int mini_batch_size,
void MLPPLinReg::normalEquation() {
MLPPLinAlg alg;
MLPPStat stat;
std::vector<double> x_means;
std::vector<std::vector<double>> inputSetT = alg.transpose(inputSet);
std::vector<real_t> x_means;
std::vector<std::vector<real_t>> inputSetT = alg.transpose(inputSet);
x_means.resize(inputSetT.size());
for (int i = 0; i < inputSetT.size(); i++) {
@ -518,7 +518,7 @@ void MLPPLinReg::normalEquation() {
}
//try {
std::vector<double> temp;
std::vector<real_t> temp;
temp.resize(k);
temp = alg.mat_vec_mult(alg.inverse(alg.matmult(alg.transpose(inputSet), inputSet)), alg.mat_vec_mult(alg.transpose(inputSet), outputSet));
if (std::isnan(temp[0])) {
@ -542,7 +542,7 @@ void MLPPLinReg::normalEquation() {
//}
}
double MLPPLinReg::score() {
real_t MLPPLinReg::score() {
MLPPUtilities util;
return util.performance(y_hat, outputSet);
}
@ -552,18 +552,18 @@ void MLPPLinReg::save(std::string fileName) {
util.saveParameters(fileName, weights, bias);
}
double MLPPLinReg::Cost(std::vector<double> y_hat, std::vector<double> y) {
real_t MLPPLinReg::Cost(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPReg regularization;
class MLPPCost cost;
return cost.MSE(y_hat, y) + regularization.regTerm(weights, lambda, alpha, reg);
}
std::vector<double> MLPPLinReg::Evaluate(std::vector<std::vector<double>> X) {
std::vector<real_t> MLPPLinReg::Evaluate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg;
return alg.scalarAdd(bias, alg.mat_vec_mult(X, weights));
}
double MLPPLinReg::Evaluate(std::vector<double> x) {
real_t MLPPLinReg::Evaluate(std::vector<real_t> x) {
MLPPLinAlg alg;
return alg.dot(weights, x) + bias;
}

48
mlpp/lin_reg/lin_reg.h
View File

@ -8,43 +8,45 @@
// Created by Marc Melikyan on 10/2/20.
//
#include "core/math/math_defs.h"
#include <string>
#include <vector>
class MLPPLinReg {
public:
MLPPLinReg(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, std::string reg = "None", double lambda = 0.5, double alpha = 0.5);
std::vector<double> modelSetTest(std::vector<std::vector<double>> X);
double modelTest(std::vector<double> x);
void NewtonRaphson(double learning_rate, int max_epoch, bool UI);
void gradientDescent(double learning_rate, int max_epoch, bool UI = 1);
void SGD(double learning_rate, int max_epoch, bool UI = 1);
MLPPLinReg(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
std::vector<real_t> modelSetTest(std::vector<std::vector<real_t>> X);
real_t modelTest(std::vector<real_t> x);
void NewtonRaphson(real_t learning_rate, int max_epoch, bool UI);
void gradientDescent(real_t learning_rate, int max_epoch, bool UI = 1);
void SGD(real_t learning_rate, int max_epoch, bool UI = 1);
void Momentum(double learning_rate, int max_epoch, int mini_batch_size, double gamma, bool UI = 1);
void NAG(double learning_rate, int max_epoch, int mini_batch_size, double gamma, bool UI = 1);
void Adagrad(double learning_rate, int max_epoch, int mini_batch_size, double e, bool UI = 1);
void Adadelta(double learning_rate, int max_epoch, int mini_batch_size, double b1, double e, bool UI = 1);
void Adam(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI = 1);
void Adamax(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI = 1);
void Nadam(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI = 1);
void Momentum(real_t learning_rate, int max_epoch, int mini_batch_size, real_t gamma, bool UI = 1);
void NAG(real_t learning_rate, int max_epoch, int mini_batch_size, real_t gamma, bool UI = 1);
void Adagrad(real_t learning_rate, int max_epoch, int mini_batch_size, real_t e, bool UI = 1);
void Adadelta(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t e, bool UI = 1);
void Adam(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t b2, real_t e, bool UI = 1);
void Adamax(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t b2, real_t e, bool UI = 1);
void Nadam(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t b2, real_t e, bool UI = 1);
void MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
void normalEquation();
double score();
real_t score();
void save(std::string fileName);
private:
double Cost(std::vector<double> y_hat, std::vector<double> y);
real_t Cost(std::vector<real_t> y_hat, std::vector<real_t> y);
std::vector<double> Evaluate(std::vector<std::vector<double>> X);
double Evaluate(std::vector<double> x);
std::vector<real_t> Evaluate(std::vector<std::vector<real_t>> X);
real_t Evaluate(std::vector<real_t> x);
void forwardPass();
std::vector<std::vector<double>> inputSet;
std::vector<double> outputSet;
std::vector<double> y_hat;
std::vector<double> weights;
double bias;
std::vector<std::vector<real_t>> inputSet;
std::vector<real_t> outputSet;
std::vector<real_t> y_hat;
std::vector<real_t> weights;
real_t bias;
int n;
int k;

42
mlpp/log_reg/log_reg.cpp
View File

@ -15,32 +15,32 @@
#include <random>
MLPPLogReg::MLPPLogReg(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, std::string reg, double lambda, double alpha) :
MLPPLogReg::MLPPLogReg(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, std::string reg, real_t lambda, real_t alpha) :
inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) {
y_hat.resize(n);
weights = MLPPUtilities::weightInitialization(k);
bias = MLPPUtilities::biasInitialization();
}
std::vector<double> MLPPLogReg::modelSetTest(std::vector<std::vector<double>> X) {
std::vector<real_t> MLPPLogReg::modelSetTest(std::vector<std::vector<real_t>> X) {
return Evaluate(X);
}
double MLPPLogReg::modelTest(std::vector<double> x) {
real_t MLPPLogReg::modelTest(std::vector<real_t> x) {
return Evaluate(x);
}
void MLPPLogReg::gradientDescent(double learning_rate, int max_epoch, bool UI) {
void MLPPLogReg::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
forwardPass();
while (true) {
cost_prev = Cost(y_hat, outputSet);
std::vector<double> error = alg.subtraction(y_hat, outputSet);
std::vector<real_t> error = alg.subtraction(y_hat, outputSet);
// Calculating the weight gradients
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputSet), error)));
@ -62,17 +62,17 @@ void MLPPLogReg::gradientDescent(double learning_rate, int max_epoch, bool UI) {
}
}
void MLPPLogReg::MLE(double learning_rate, int max_epoch, bool UI) {
void MLPPLogReg::MLE(real_t learning_rate, int max_epoch, bool UI) {
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
forwardPass();
while (true) {
cost_prev = Cost(y_hat, outputSet);
std::vector<double> error = alg.subtraction(outputSet, y_hat);
std::vector<real_t> error = alg.subtraction(outputSet, y_hat);
// Calculating the weight gradients
weights = alg.addition(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputSet), error)));
@ -93,10 +93,10 @@ void MLPPLogReg::MLE(double learning_rate, int max_epoch, bool UI) {
}
}
void MLPPLogReg::SGD(double learning_rate, int max_epoch, bool UI) {
void MLPPLogReg::SGD(real_t learning_rate, int max_epoch, bool UI) {
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
while (true) {
@ -105,10 +105,10 @@ void MLPPLogReg::SGD(double learning_rate, int max_epoch, bool UI) {
std::uniform_int_distribution<int> distribution(0, int(n - 1));
int outputIndex = distribution(generator);
double y_hat = Evaluate(inputSet[outputIndex]);
real_t y_hat = Evaluate(inputSet[outputIndex]);
cost_prev = Cost({ y_hat }, { outputSet[outputIndex] });
double error = y_hat - outputSet[outputIndex];
real_t error = y_hat - outputSet[outputIndex];
// Weight updation
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate * error, inputSet[outputIndex]));
@ -132,10 +132,10 @@ void MLPPLogReg::SGD(double learning_rate, int max_epoch, bool UI) {
forwardPass();
}
void MLPPLogReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI) {
void MLPPLogReg::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI) {
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
// Creating the mini-batches
@ -144,10 +144,10 @@ void MLPPLogReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size,
while (true) {
for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = Evaluate(inputMiniBatches[i]);
std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]);
std::vector<double> error = alg.subtraction(y_hat, outputMiniBatches[i]);
std::vector<real_t> error = alg.subtraction(y_hat, outputMiniBatches[i]);
// Calculating the weight gradients
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error)));
@ -170,7 +170,7 @@ void MLPPLogReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size,
forwardPass();
}
double MLPPLogReg::score() {
real_t MLPPLogReg::score() {
MLPPUtilities util;
return util.performance(y_hat, outputSet);
}
@ -180,19 +180,19 @@ void MLPPLogReg::save(std::string fileName) {
util.saveParameters(fileName, weights, bias);
}
double MLPPLogReg::Cost(std::vector<double> y_hat, std::vector<double> y) {
real_t MLPPLogReg::Cost(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPReg regularization;
class MLPPCost cost;
return cost.LogLoss(y_hat, y) + regularization.regTerm(weights, lambda, alpha, reg);
}
std::vector<double> MLPPLogReg::Evaluate(std::vector<std::vector<double>> X) {
std::vector<real_t> MLPPLogReg::Evaluate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg;
MLPPActivation avn;
return avn.sigmoid(alg.scalarAdd(bias, alg.mat_vec_mult(X, weights)));
}
double MLPPLogReg::Evaluate(std::vector<double> x) {
real_t MLPPLogReg::Evaluate(std::vector<real_t> x) {
MLPPLinAlg alg;
MLPPActivation avn;
return avn.sigmoid(alg.dot(weights, x) + bias);

40
mlpp/log_reg/log_reg.h
View File

@ -8,6 +8,8 @@
// Created by Marc Melikyan on 10/2/20.
//
#include "core/math/math_defs.h"
#include <string>
#include <vector>
@ -15,37 +17,37 @@
class MLPPLogReg {
public:
MLPPLogReg(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, std::string reg = "None", double lambda = 0.5, double alpha = 0.5);
std::vector<double> modelSetTest(std::vector<std::vector<double>> X);
double modelTest(std::vector<double> x);
void gradientDescent(double learning_rate, int max_epoch, bool UI = 1);
void MLE(double learning_rate, int max_epoch, bool UI = 1);
void SGD(double learning_rate, int max_epoch, bool UI = 1);
void MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
double score();
MLPPLogReg(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
std::vector<real_t> modelSetTest(std::vector<std::vector<real_t>> X);
real_t modelTest(std::vector<real_t> x);
void gradientDescent(real_t learning_rate, int max_epoch, bool UI = 1);
void MLE(real_t learning_rate, int max_epoch, bool UI = 1);
void SGD(real_t learning_rate, int max_epoch, bool UI = 1);
void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
real_t score();
void save(std::string fileName);
private:
double Cost(std::vector<double> y_hat, std::vector<double> y);
real_t Cost(std::vector<real_t> y_hat, std::vector<real_t> y);
std::vector<double> Evaluate(std::vector<std::vector<double>> X);
double Evaluate(std::vector<double> x);
std::vector<real_t> Evaluate(std::vector<std::vector<real_t>> X);
real_t Evaluate(std::vector<real_t> x);
void forwardPass();
std::vector<std::vector<double>> inputSet;
std::vector<double> outputSet;
std::vector<double> y_hat;
std::vector<double> weights;
double bias;
std::vector<std::vector<real_t>> inputSet;
std::vector<real_t> outputSet;
std::vector<real_t> y_hat;
std::vector<real_t> weights;
real_t bias;
int n;
int k;
double learning_rate;
real_t learning_rate;
// Regularization Params
std::string reg;
double lambda; /* Regularization Parameter */
double alpha; /* This is the controlling param for Elastic Net*/
real_t lambda; /* Regularization Parameter */
real_t alpha; /* This is the controlling param for Elastic Net*/
};

26
mlpp/mann/mann.cpp
View File

@ -14,7 +14,7 @@
#include <iostream>
MLPPMANN::MLPPMANN(std::vector<std::vector<double>> inputSet, std::vector<std::vector<double>> outputSet) :
MLPPMANN::MLPPMANN(std::vector<std::vector<real_t>> inputSet, std::vector<std::vector<real_t>> outputSet) :
inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), n_output(outputSet[0].size()) {
}
@ -22,7 +22,7 @@ MLPPMANN::~MLPPMANN() {
delete outputLayer;
}
std::vector<std::vector<double>> MLPPMANN::modelSetTest(std::vector<std::vector<double>> X) {
std::vector<std::vector<real_t>> MLPPMANN::modelSetTest(std::vector<std::vector<real_t>> X) {
if (!network.empty()) {
network[0].input = X;
network[0].forwardPass();
@ -39,7 +39,7 @@ std::vector<std::vector<double>> MLPPMANN::modelSetTest(std::vector<std::vector<
return outputLayer->a;
}
std::vector<double> MLPPMANN::modelTest(std::vector<double> x) {
std::vector<real_t> MLPPMANN::modelTest(std::vector<real_t> x) {
if (!network.empty()) {
network[0].Test(x);
for (int i = 1; i < network.size(); i++) {
@ -52,13 +52,13 @@ std::vector<double> MLPPMANN::modelTest(std::vector<double> x) {
return outputLayer->a_test;
}
void MLPPMANN::gradientDescent(double learning_rate, int max_epoch, bool UI) {
void MLPPMANN::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
class MLPPCost cost;
MLPPActivation avn;
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
forwardPass();
@ -73,7 +73,7 @@ void MLPPMANN::gradientDescent(double learning_rate, int max_epoch, bool UI) {
outputLayer->delta = alg.hadamard_product((cost.*costDeriv)(y_hat, outputSet), (avn.*outputAvn)(outputLayer->z, 1));
}
std::vector<std::vector<double>> outputWGrad = alg.matmult(alg.transpose(outputLayer->input), outputLayer->delta);
std::vector<std::vector<real_t>> outputWGrad = alg.matmult(alg.transpose(outputLayer->input), outputLayer->delta);
outputLayer->weights = alg.subtraction(outputLayer->weights, alg.scalarMultiply(learning_rate / n, outputWGrad));
outputLayer->weights = regularization.regWeights(outputLayer->weights, outputLayer->lambda, outputLayer->alpha, outputLayer->reg);
@ -82,7 +82,7 @@ void MLPPMANN::gradientDescent(double learning_rate, int max_epoch, bool UI) {
if (!network.empty()) {
auto hiddenLayerAvn = network[network.size() - 1].activation_map[network[network.size() - 1].activation];
network[network.size() - 1].delta = alg.hadamard_product(alg.matmult(outputLayer->delta, alg.transpose(outputLayer->weights)), (avn.*hiddenLayerAvn)(network[network.size() - 1].z, 1));
std::vector<std::vector<double>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[network.size() - 1].input), network[network.size() - 1].delta);
std::vector<std::vector<real_t>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[network.size() - 1].input), network[network.size() - 1].delta);
network[network.size() - 1].weights = alg.subtraction(network[network.size() - 1].weights, alg.scalarMultiply(learning_rate / n, hiddenLayerWGrad));
network[network.size() - 1].weights = regularization.regWeights(network[network.size() - 1].weights, network[network.size() - 1].lambda, network[network.size() - 1].alpha, network[network.size() - 1].reg);
@ -91,7 +91,7 @@ void MLPPMANN::gradientDescent(double learning_rate, int max_epoch, bool UI) {
for (int i = network.size() - 2; i >= 0; i--) {
auto hiddenLayerAvn = network[i].activation_map[network[i].activation];
network[i].delta = alg.hadamard_product(alg.matmult(network[i + 1].delta, network[i + 1].weights), (avn.*hiddenLayerAvn)(network[i].z, 1));
std::vector<std::vector<double>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[i].input), network[i].delta);
std::vector<std::vector<real_t>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[i].input), network[i].delta);
network[i].weights = alg.subtraction(network[i].weights, alg.scalarMultiply(learning_rate / n, hiddenLayerWGrad));
network[i].weights = regularization.regWeights(network[i].weights, network[i].lambda, network[i].alpha, network[i].reg);
network[i].bias = alg.subtractMatrixRows(network[i].bias, alg.scalarMultiply(learning_rate / n, network[i].delta));
@ -120,7 +120,7 @@ void MLPPMANN::gradientDescent(double learning_rate, int max_epoch, bool UI) {
}
}
double MLPPMANN::score() {
real_t MLPPMANN::score() {
MLPPUtilities util;
forwardPass();
return util.performance(y_hat, outputSet);
@ -139,7 +139,7 @@ void MLPPMANN::save(std::string fileName) {
}
}
void MLPPMANN::addLayer(int n_hidden, std::string activation, std::string weightInit, std::string reg, double lambda, double alpha) {
void MLPPMANN::addLayer(int n_hidden, std::string activation, std::string weightInit, std::string reg, real_t lambda, real_t alpha) {
if (network.empty()) {
network.push_back(MLPPHiddenLayer(n_hidden, activation, inputSet, weightInit, reg, lambda, alpha));
network[0].forwardPass();
@ -149,7 +149,7 @@ void MLPPMANN::addLayer(int n_hidden, std::string activation, std::string weight
}
}
void MLPPMANN::addOutputLayer(std::string activation, std::string loss, std::string weightInit, std::string reg, double lambda, double alpha) {
void MLPPMANN::addOutputLayer(std::string activation, std::string loss, std::string weightInit, std::string reg, real_t lambda, real_t alpha) {
if (!network.empty()) {
outputLayer = new MLPPMultiOutputLayer(n_output, network[0].n_hidden, activation, loss, network[network.size() - 1].a, weightInit, reg, lambda, alpha);
} else {
@ -157,10 +157,10 @@ void MLPPMANN::addOutputLayer(std::string activation, std::string loss, std::str
}
}
double MLPPMANN::Cost(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) {
real_t MLPPMANN::Cost(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
MLPPReg regularization;
class MLPPCost cost;
double totalRegTerm = 0;
real_t totalRegTerm = 0;
auto cost_function = outputLayer->cost_map[outputLayer->cost];
if (!network.empty()) {

24
mlpp/mann/mann.h
View File

@ -8,6 +8,8 @@
// Created by Marc Melikyan on 11/4/20.
//
#include "core/math/math_defs.h"
#include "../hidden_layer/hidden_layer.h"
#include "../multi_output_layer/multi_output_layer.h"
@ -18,24 +20,24 @@
class MLPPMANN {
public:
MLPPMANN(std::vector<std::vector<double>> inputSet, std::vector<std::vector<double>> outputSet);
MLPPMANN(std::vector<std::vector<real_t>> inputSet, std::vector<std::vector<real_t>> outputSet);
~MLPPMANN();
std::vector<std::vector<double>> modelSetTest(std::vector<std::vector<double>> X);
std::vector<double> modelTest(std::vector<double> x);
void gradientDescent(double learning_rate, int max_epoch, bool UI = 1);
double score();
std::vector<std::vector<real_t>> modelSetTest(std::vector<std::vector<real_t>> X);
std::vector<real_t> modelTest(std::vector<real_t> x);
void gradientDescent(real_t learning_rate, int max_epoch, bool UI = 1);
real_t score();
void save(std::string fileName);
void addLayer(int n_hidden, std::string activation, std::string weightInit = "Default", std::string reg = "None", double lambda = 0.5, double alpha = 0.5);
void addOutputLayer(std::string activation, std::string loss, std::string weightInit = "Default", std::string reg = "None", double lambda = 0.5, double alpha = 0.5);
void addLayer(int n_hidden, std::string activation, std::string weightInit = "Default", std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
void addOutputLayer(std::string activation, std::string loss, std::string weightInit = "Default", std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
private:
double Cost(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y);
real_t Cost(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
void forwardPass();
std::vector<std::vector<double>> inputSet;
std::vector<std::vector<double>> outputSet;
std::vector<std::vector<double>> y_hat;
std::vector<std::vector<real_t>> inputSet;
std::vector<std::vector<real_t>> outputSet;
std::vector<std::vector<real_t>> y_hat;
std::vector<MLPPHiddenLayer> network;
MLPPMultiOutputLayer *outputLayer;

82
mlpp/mlp/mlp.cpp
View File

@ -16,7 +16,7 @@
#include <random>
MLPPMLP::MLPPMLP(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, int n_hidden, std::string reg, double lambda, double alpha) :
MLPPMLP::MLPPMLP(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, int n_hidden, std::string reg, real_t lambda, real_t alpha) :
inputSet(inputSet), outputSet(outputSet), n_hidden(n_hidden), n(inputSet.size()), k(inputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) {
MLPPActivation avn;
y_hat.resize(n);
@ -27,19 +27,19 @@ MLPPMLP::MLPPMLP(std::vector<std::vector<double>> inputSet, std::vector<double>
bias2 = MLPPUtilities::biasInitialization();
}
std::vector<double> MLPPMLP::modelSetTest(std::vector<std::vector<double>> X) {
std::vector<real_t> MLPPMLP::modelSetTest(std::vector<std::vector<real_t>> X) {
return Evaluate(X);
}
double MLPPMLP::modelTest(std::vector<double> x) {
real_t MLPPMLP::modelTest(std::vector<real_t> x) {
return Evaluate(x);
}
void MLPPMLP::gradientDescent(double learning_rate, int max_epoch, bool UI) {
void MLPPMLP::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
MLPPActivation avn;
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
forwardPass();
@ -47,11 +47,11 @@ void MLPPMLP::gradientDescent(double learning_rate, int max_epoch, bool UI) {
cost_prev = Cost(y_hat, outputSet);
// Calculating the errors
std::vector<double> error = alg.subtraction(y_hat, outputSet);
std::vector<real_t> error = alg.subtraction(y_hat, outputSet);
// Calculating the weight/bias gradients for layer 2
std::vector<double> D2_1 = alg.mat_vec_mult(alg.transpose(a2), error);
std::vector<real_t> D2_1 = alg.mat_vec_mult(alg.transpose(a2), error);
// weights and bias updation for layer 2
weights2 = alg.subtraction(weights2, alg.scalarMultiply(learning_rate / n, D2_1));
@ -61,14 +61,14 @@ void MLPPMLP::gradientDescent(double learning_rate, int max_epoch, bool UI) {
// Calculating the weight/bias for layer 1
std::vector<std::vector<double>> D1_1;
std::vector<std::vector<real_t>> D1_1;
D1_1.resize(n);
D1_1 = alg.outerProduct(error, weights2);
std::vector<std::vector<double>> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1));
std::vector<std::vector<real_t>> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1));
std::vector<std::vector<double>> D1_3 = alg.matmult(alg.transpose(inputSet), D1_2);
std::vector<std::vector<real_t>> D1_3 = alg.matmult(alg.transpose(inputSet), D1_2);
// weight an bias updation for layer 1
weights1 = alg.subtraction(weights1, alg.scalarMultiply(learning_rate / n, D1_3));
@ -94,11 +94,11 @@ void MLPPMLP::gradientDescent(double learning_rate, int max_epoch, bool UI) {
}
}
void MLPPMLP::SGD(double learning_rate, int max_epoch, bool UI) {
void MLPPMLP::SGD(real_t learning_rate, int max_epoch, bool UI) {
MLPPActivation avn;
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
while (true) {
@ -107,13 +107,13 @@ void MLPPMLP::SGD(double learning_rate, int max_epoch, bool UI) {
std::uniform_int_distribution<int> distribution(0, int(n - 1));
int outputIndex = distribution(generator);
double y_hat = Evaluate(inputSet[outputIndex]);
real_t y_hat = Evaluate(inputSet[outputIndex]);
auto [z2, a2] = propagate(inputSet[outputIndex]);
cost_prev = Cost({ y_hat }, { outputSet[outputIndex] });
double error = y_hat - outputSet[outputIndex];
real_t error = y_hat - outputSet[outputIndex];
// Weight updation for layer 2
std::vector<double> D2_1 = alg.scalarMultiply(error, a2);
std::vector<real_t> D2_1 = alg.scalarMultiply(error, a2);
weights2 = alg.subtraction(weights2, alg.scalarMultiply(learning_rate, D2_1));
weights2 = regularization.regWeights(weights2, lambda, alpha, reg);
@ -121,9 +121,9 @@ void MLPPMLP::SGD(double learning_rate, int max_epoch, bool UI) {
bias2 -= learning_rate * error;
// Weight updation for layer 1
std::vector<double> D1_1 = alg.scalarMultiply(error, weights2);
std::vector<double> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1));
std::vector<std::vector<double>> D1_3 = alg.outerProduct(inputSet[outputIndex], D1_2);
std::vector<real_t> D1_1 = alg.scalarMultiply(error, weights2);
std::vector<real_t> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1));
std::vector<std::vector<real_t>> D1_3 = alg.outerProduct(inputSet[outputIndex], D1_2);
weights1 = alg.subtraction(weights1, alg.scalarMultiply(learning_rate, D1_3));
weights1 = regularization.regWeights(weights1, lambda, alpha, reg);
@ -148,11 +148,11 @@ void MLPPMLP::SGD(double learning_rate, int max_epoch, bool UI) {
forwardPass();
}
void MLPPMLP::MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI) {
void MLPPMLP::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI) {
MLPPActivation avn;
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
// Creating the mini-batches
@ -161,34 +161,34 @@ void MLPPMLP::MBGD(double learning_rate, int max_epoch, int mini_batch_size, boo
while (true) {
for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = Evaluate(inputMiniBatches[i]);
std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]);
auto [z2, a2] = propagate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]);
// Calculating the errors
std::vector<double> error = alg.subtraction(y_hat, outputMiniBatches[i]);
std::vector<real_t> error = alg.subtraction(y_hat, outputMiniBatches[i]);
// Calculating the weight/bias gradients for layer 2
std::vector<double> D2_1 = alg.mat_vec_mult(alg.transpose(a2), error);
std::vector<real_t> D2_1 = alg.mat_vec_mult(alg.transpose(a2), error);
// weights and bias updation for layser 2
weights2 = alg.subtraction(weights2, alg.scalarMultiply(learning_rate / outputMiniBatches[i].size(), D2_1));
weights2 = regularization.regWeights(weights2, lambda, alpha, reg);
// Calculating the bias gradients for layer 2
double b_gradient = alg.sum_elements(error);
real_t b_gradient = alg.sum_elements(error);
// Bias Updation for layer 2
bias2 -= learning_rate * alg.sum_elements(error) / outputMiniBatches[i].size();
//Calculating the weight/bias for layer 1
std::vector<std::vector<double>> D1_1 = alg.outerProduct(error, weights2);
std::vector<std::vector<real_t>> D1_1 = alg.outerProduct(error, weights2);
std::vector<std::vector<double>> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1));
std::vector<std::vector<real_t>> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1));
std::vector<std::vector<double>> D1_3 = alg.matmult(alg.transpose(inputMiniBatches[i]), D1_2);
std::vector<std::vector<real_t>> D1_3 = alg.matmult(alg.transpose(inputMiniBatches[i]), D1_2);
// weight an bias updation for layer 1
weights1 = alg.subtraction(weights1, alg.scalarMultiply(learning_rate / outputMiniBatches[i].size(), D1_3));
@ -214,7 +214,7 @@ void MLPPMLP::MBGD(double learning_rate, int max_epoch, int mini_batch_size, boo
forwardPass();
}
double MLPPMLP::score() {
real_t MLPPMLP::score() {
MLPPUtilities util;
return util.performance(y_hat, outputSet);
}
@ -225,41 +225,41 @@ void MLPPMLP::save(std::string fileName) {
util.saveParameters(fileName, weights2, bias2, 1, 2);
}
double MLPPMLP::Cost(std::vector<double> y_hat, std::vector<double> y) {
real_t MLPPMLP::Cost(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPReg regularization;
class MLPPCost cost;
return cost.LogLoss(y_hat, y) + regularization.regTerm(weights2, lambda, alpha, reg) + regularization.regTerm(weights1, lambda, alpha, reg);
}
std::vector<double> MLPPMLP::Evaluate(std::vector<std::vector<double>> X) {
std::vector<real_t> MLPPMLP::Evaluate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg;
MLPPActivation avn;
std::vector<std::vector<double>> z2 = alg.mat_vec_add(alg.matmult(X, weights1), bias1);
std::vector<std::vector<double>> a2 = avn.sigmoid(z2);
std::vector<std::vector<real_t>> z2 = alg.mat_vec_add(alg.matmult(X, weights1), bias1);
std::vector<std::vector<real_t>> a2 = avn.sigmoid(z2);
return avn.sigmoid(alg.scalarAdd(bias2, alg.mat_vec_mult(a2, weights2)));
}
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> MLPPMLP::propagate(std::vector<std::vector<double>> X) {
std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> MLPPMLP::propagate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg;
MLPPActivation avn;
std::vector<std::vector<double>> z2 = alg.mat_vec_add(alg.matmult(X, weights1), bias1);
std::vector<std::vector<double>> a2 = avn.sigmoid(z2);
std::vector<std::vector<real_t>> z2 = alg.mat_vec_add(alg.matmult(X, weights1), bias1);
std::vector<std::vector<real_t>> a2 = avn.sigmoid(z2);
return { z2, a2 };
}
double MLPPMLP::Evaluate(std::vector<double> x) {
real_t MLPPMLP::Evaluate(std::vector<real_t> x) {
MLPPLinAlg alg;
MLPPActivation avn;
std::vector<double> z2 = alg.addition(alg.mat_vec_mult(alg.transpose(weights1), x), bias1);
std::vector<double> a2 = avn.sigmoid(z2);
std::vector<real_t> z2 = alg.addition(alg.mat_vec_mult(alg.transpose(weights1), x), bias1);
std::vector<real_t> a2 = avn.sigmoid(z2);
return avn.sigmoid(alg.dot(weights2, a2) + bias2);
}
std::tuple<std::vector<double>, std::vector<double>> MLPPMLP::propagate(std::vector<double> x) {
std::tuple<std::vector<real_t>, std::vector<real_t>> MLPPMLP::propagate(std::vector<real_t> x) {
MLPPLinAlg alg;
MLPPActivation avn;
std::vector<double> z2 = alg.addition(alg.mat_vec_mult(alg.transpose(weights1), x), bias1);
std::vector<double> a2 = avn.sigmoid(z2);
std::vector<real_t> z2 = alg.addition(alg.mat_vec_mult(alg.transpose(weights1), x), bias1);
std::vector<real_t> a2 = avn.sigmoid(z2);
return { z2, a2 };
}

48
mlpp/mlp/mlp.h
View File

@ -8,6 +8,8 @@
// Created by Marc Melikyan on 11/4/20.
//
#include "core/math/math_defs.h"
#include <map>
#include <string>
#include <vector>
@ -16,36 +18,36 @@
class MLPPMLP {
public:
MLPPMLP(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, int n_hidden, std::string reg = "None", double lambda = 0.5, double alpha = 0.5);
std::vector<double> modelSetTest(std::vector<std::vector<double>> X);
double modelTest(std::vector<double> x);
void gradientDescent(double learning_rate, int max_epoch, bool UI = 1);
void SGD(double learning_rate, int max_epoch, bool UI = 1);
void MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
double score();
MLPPMLP(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, int n_hidden, std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
std::vector<real_t> modelSetTest(std::vector<std::vector<real_t>> X);
real_t modelTest(std::vector<real_t> x);
void gradientDescent(real_t learning_rate, int max_epoch, bool UI = 1);
void SGD(real_t learning_rate, int max_epoch, bool UI = 1);
void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
real_t score();
void save(std::string fileName);
private:
double Cost(std::vector<double> y_hat, std::vector<double> y);
real_t Cost(std::vector<real_t> y_hat, std::vector<real_t> y);
std::vector<double> Evaluate(std::vector<std::vector<double>> X);
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> propagate(std::vector<std::vector<double>> X);
double Evaluate(std::vector<double> x);
std::tuple<std::vector<double>, std::vector<double>> propagate(std::vector<double> x);
std::vector<real_t> Evaluate(std::vector<std::vector<real_t>> X);
std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> propagate(std::vector<std::vector<real_t>> X);
real_t Evaluate(std::vector<real_t> x);
std::tuple<std::vector<real_t>, std::vector<real_t>> propagate(std::vector<real_t> x);
void forwardPass();
std::vector<std::vector<double>> inputSet;
std::vector<double> outputSet;
std::vector<double> y_hat;
std::vector<std::vector<real_t>> inputSet;
std::vector<real_t> outputSet;
std::vector<real_t> y_hat;
std::vector<std::vector<double>> weights1;
std::vector<double> weights2;
std::vector<std::vector<real_t>> weights1;
std::vector<real_t> weights2;
std::vector<double> bias1;
double bias2;
std::vector<real_t> bias1;
real_t bias2;
std::vector<std::vector<double>> z2;
std::vector<std::vector<double>> a2;
std::vector<std::vector<real_t>> z2;
std::vector<std::vector<real_t>> a2;
int n;
int k;
@ -53,8 +55,8 @@ private:
// Regularization Params
std::string reg;
double lambda; /* Regularization Parameter */
double alpha; /* This is the controlling param for Elastic Net*/
real_t lambda; /* Regularization Parameter */
real_t alpha; /* This is the controlling param for Elastic Net*/
};

4
mlpp/multi_output_layer/multi_output_layer.cpp
View File

@ -12,7 +12,7 @@
#include <random>
MLPPMultiOutputLayer::MLPPMultiOutputLayer(int n_output, int n_hidden, std::string activation, std::string cost, std::vector<std::vector<double>> input, std::string weightInit, std::string reg, double lambda, double alpha) :
MLPPMultiOutputLayer::MLPPMultiOutputLayer(int n_output, int n_hidden, std::string activation, std::string cost, std::vector<std::vector<real_t>> input, std::string weightInit, std::string reg, real_t lambda, real_t alpha) :
n_output(n_output), n_hidden(n_hidden), activation(activation), cost(cost), input(input), weightInit(weightInit), reg(reg), lambda(lambda), alpha(alpha) {
weights = MLPPUtilities::weightInitialization(n_hidden, n_output, weightInit);
bias = MLPPUtilities::biasInitialization(n_output);
@ -123,7 +123,7 @@ void MLPPMultiOutputLayer::forwardPass() {
a = (avn.*activation_map[activation])(z, 0);
}
void MLPPMultiOutputLayer::Test(std::vector<double> x) {
void MLPPMultiOutputLayer::Test(std::vector<real_t> x) {
MLPPLinAlg alg;
MLPPActivation avn;
z_test = alg.addition(alg.mat_vec_mult(alg.transpose(weights), x), bias);

34
mlpp/multi_output_layer/multi_output_layer.h
View File

@ -8,6 +8,8 @@
// Created by Marc Melikyan on 11/4/20.
//
#include "core/math/math_defs.h"
#include "../activation/activation.h"
#include "../cost/cost.h"
@ -18,40 +20,40 @@
class MLPPMultiOutputLayer {
public:
MLPPMultiOutputLayer(int n_output, int n_hidden, std::string activation, std::string cost, std::vector<std::vector<double>> input, std::string weightInit, std::string reg, double lambda, double alpha);
MLPPMultiOutputLayer(int n_output, int n_hidden, std::string activation, std::string cost, std::vector<std::vector<real_t>> input, std::string weightInit, std::string reg, real_t lambda, real_t alpha);
int n_output;
int n_hidden;
std::string activation;
std::string cost;
std::vector<std::vector<double>> input;
std::vector<std::vector<real_t>> input;
std::vector<std::vector<double>> weights;
std::vector<double> bias;
std::vector<std::vector<real_t>> weights;
std::vector<real_t> bias;
std::vector<std::vector<double>> z;
std::vector<std::vector<double>> a;
std::vector<std::vector<real_t>> z;
std::vector<std::vector<real_t>> a;
std::map<std::string, std::vector<std::vector<double>> (MLPPActivation::*)(std::vector<std::vector<double>>, bool)> activation_map;
std::map<std::string, std::vector<double> (MLPPActivation::*)(std::vector<double>, bool)> activationTest_map;
std::map<std::string, double (MLPPCost::*)(std::vector<std::vector<double>>, std::vector<std::vector<double>>)> cost_map;
std::map<std::string, std::vector<std::vector<double>> (MLPPCost::*)(std::vector<std::vector<double>>, std::vector<std::vector<double>>)> costDeriv_map;
std::map<std::string, std::vector<std::vector<real_t>> (MLPPActivation::*)(std::vector<std::vector<real_t>>, bool)> activation_map;
std::map<std::string, std::vector<real_t> (MLPPActivation::*)(std::vector<real_t>, bool)> activationTest_map;
std::map<std::string, real_t (MLPPCost::*)(std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>)> cost_map;
std::map<std::string, std::vector<std::vector<real_t>> (MLPPCost::*)(std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>)> costDeriv_map;
std::vector<double> z_test;
std::vector<double> a_test;
std::vector<real_t> z_test;
std::vector<real_t> a_test;
std::vector<std::vector<double>> delta;
std::vector<std::vector<real_t>> delta;
// Regularization Params
std::string reg;
double lambda; /* Regularization Parameter */
double alpha; /* This is the controlling param for Elastic Net*/
real_t lambda; /* Regularization Parameter */
real_t alpha; /* This is the controlling param for Elastic Net*/
std::string weightInit;
void forwardPass();
void Test(std::vector<double> x);
void Test(std::vector<real_t> x);
};

20
mlpp/multinomial_nb/multinomial_nb.cpp
View File

@ -13,22 +13,22 @@
#include <random>
MLPPMultinomialNB::MLPPMultinomialNB(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, int class_num) :
MLPPMultinomialNB::MLPPMultinomialNB(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, int class_num) :
inputSet(inputSet), outputSet(outputSet), class_num(class_num) {
y_hat.resize(outputSet.size());
Evaluate();
}
std::vector<double> MLPPMultinomialNB::modelSetTest(std::vector<std::vector<double>> X) {
std::vector<double> y_hat;
std::vector<real_t> MLPPMultinomialNB::modelSetTest(std::vector<std::vector<real_t>> X) {
std::vector<real_t> y_hat;
for (int i = 0; i < X.size(); i++) {
y_hat.push_back(modelTest(X[i]));
}
return y_hat;
}
double MLPPMultinomialNB::modelTest(std::vector<double> x) {
double score[class_num];
real_t MLPPMultinomialNB::modelTest(std::vector<real_t> x) {
real_t score[class_num];
computeTheta();
for (int j = 0; j < x.size(); j++) {
@ -45,10 +45,10 @@ double MLPPMultinomialNB::modelTest(std::vector<double> x) {
score[i] += std::log(priors[i]);
}
return std::distance(score, std::max_element(score, score + sizeof(score) / sizeof(double)));
return std::distance(score, std::max_element(score, score + sizeof(score) / sizeof(real_t)));
}
double MLPPMultinomialNB::score() {
real_t MLPPMultinomialNB::score() {
MLPPUtilities util;
return util.performance(y_hat, outputSet);
}
@ -81,14 +81,14 @@ void MLPPMultinomialNB::Evaluate() {
MLPPLinAlg alg;
for (int i = 0; i < outputSet.size(); i++) {
// Pr(B | A) * Pr(A)
double score[class_num];
real_t score[class_num];
// Easy computation of priors, i.e. Pr(C_k)
priors.resize(class_num);
for (int i = 0; i < outputSet.size(); i++) {
priors[int(outputSet[i])]++;
}
priors = alg.scalarMultiply(double(1) / double(outputSet.size()), priors);
priors = alg.scalarMultiply(real_t(1) / real_t(outputSet.size()), priors);
// Evaluating Theta...
computeTheta();
@ -113,6 +113,6 @@ void MLPPMultinomialNB::Evaluate() {
}
// Assigning the traning example's y_hat to a class
y_hat[i] = std::distance(score, std::max_element(score, score + sizeof(score) / sizeof(double)));
y_hat[i] = std::distance(score, std::max_element(score, score + sizeof(score) / sizeof(real_t)));
}
}

22
mlpp/multinomial_nb/multinomial_nb.h
View File

@ -8,32 +8,34 @@
// Created by Marc Melikyan on 1/17/21.
//
#include "core/math/math_defs.h"
#include <map>
#include <vector>
class MLPPMultinomialNB {
public:
MLPPMultinomialNB(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, int class_num);
std::vector<double> modelSetTest(std::vector<std::vector<double>> X);
double modelTest(std::vector<double> x);
double score();
MLPPMultinomialNB(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, int class_num);
std::vector<real_t> modelSetTest(std::vector<std::vector<real_t>> X);
real_t modelTest(std::vector<real_t> x);
real_t score();
private:
void computeTheta();
void Evaluate();
// Model Params
std::vector<double> priors;
std::vector<real_t> priors;
std::vector<std::map<double, int>> theta;
std::vector<double> vocab;
std::vector<std::map<real_t, int>> theta;
std::vector<real_t> vocab;
int class_num;
// Datasets
std::vector<std::vector<double>> inputSet;
std::vector<double> outputSet;
std::vector<double> y_hat;
std::vector<std::vector<real_t>> inputSet;
std::vector<real_t> outputSet;
std::vector<real_t> y_hat;
};
#endif /* MultinomialNB_hpp */

140
mlpp/numerical_analysis/numerical_analysis.cpp
View File

@ -14,127 +14,127 @@
double MLPPNumericalAnalysis::numDiff(double (*function)(double), double x) {
double eps = 1e-10;
real_t MLPPNumericalAnalysis::numDiff(real_t (*function)(real_t), real_t x) {
real_t eps = 1e-10;
return (function(x + eps) - function(x)) / eps; // This is just the formal def. of the derivative.
}
double MLPPNumericalAnalysis::numDiff_2(double (*function)(double), double x) {
double eps = 1e-5;
real_t MLPPNumericalAnalysis::numDiff_2(real_t (*function)(real_t), real_t x) {
real_t eps = 1e-5;
return (function(x + 2 * eps) - 2 * function(x + eps) + function(x)) / (eps * eps);
}
double MLPPNumericalAnalysis::numDiff_3(double (*function)(double), double x) {
double eps = 1e-5;
double t1 = function(x + 3 * eps) - 2 * function(x + 2 * eps) + function(x + eps);
double t2 = function(x + 2 * eps) - 2 * function(x + eps) + function(x);
real_t MLPPNumericalAnalysis::numDiff_3(real_t (*function)(real_t), real_t x) {
real_t eps = 1e-5;
real_t t1 = function(x + 3 * eps) - 2 * function(x + 2 * eps) + function(x + eps);
real_t t2 = function(x + 2 * eps) - 2 * function(x + eps) + function(x);
return (t1 - t2) / (eps * eps * eps);
}
double MLPPNumericalAnalysis::constantApproximation(double (*function)(double), double c) {
real_t MLPPNumericalAnalysis::constantApproximation(real_t (*function)(real_t), real_t c) {
return function(c);
}
double MLPPNumericalAnalysis::linearApproximation(double (*function)(double), double c, double x) {
real_t MLPPNumericalAnalysis::linearApproximation(real_t (*function)(real_t), real_t c, real_t x) {
return constantApproximation(function, c) + numDiff(function, c) * (x - c);
}
double MLPPNumericalAnalysis::quadraticApproximation(double (*function)(double), double c, double x) {
real_t MLPPNumericalAnalysis::quadraticApproximation(real_t (*function)(real_t), real_t c, real_t x) {
return linearApproximation(function, c, x) + 0.5 * numDiff_2(function, c) * (x - c) * (x - c);
}
double MLPPNumericalAnalysis::cubicApproximation(double (*function)(double), double c, double x) {
real_t MLPPNumericalAnalysis::cubicApproximation(real_t (*function)(real_t), real_t c, real_t x) {
return quadraticApproximation(function, c, x) + (1 / 6) * numDiff_3(function, c) * (x - c) * (x - c) * (x - c);
}
double MLPPNumericalAnalysis::numDiff(double (*function)(std::vector<double>), std::vector<double> x, int axis) {
real_t MLPPNumericalAnalysis::numDiff(real_t (*function)(std::vector<real_t>), std::vector<real_t> x, int axis) {
// For multivariable function analysis.
// This will be used for calculating Jacobian vectors.
// Diffrentiate with respect to indicated axis. (0, 1, 2 ...)
double eps = 1e-10;
std::vector<double> x_eps = x;
real_t eps = 1e-10;
std::vector<real_t> x_eps = x;
x_eps[axis] += eps;
return (function(x_eps) - function(x)) / eps;
}
double MLPPNumericalAnalysis::numDiff_2(double (*function)(std::vector<double>), std::vector<double> x, int axis1, int axis2) {
real_t MLPPNumericalAnalysis::numDiff_2(real_t (*function)(std::vector<real_t>), std::vector<real_t> x, int axis1, int axis2) {
//For Hessians.
double eps = 1e-5;
real_t eps = 1e-5;
std::vector<double> x_pp = x;
std::vector<real_t> x_pp = x;
x_pp[axis1] += eps;
x_pp[axis2] += eps;
std::vector<double> x_np = x;
std::vector<real_t> x_np = x;
x_np[axis2] += eps;
std::vector<double> x_pn = x;
std::vector<real_t> x_pn = x;
x_pn[axis1] += eps;
return (function(x_pp) - function(x_np) - function(x_pn) + function(x)) / (eps * eps);
}
double MLPPNumericalAnalysis::numDiff_3(double (*function)(std::vector<double>), std::vector<double> x, int axis1, int axis2, int axis3) {
real_t MLPPNumericalAnalysis::numDiff_3(real_t (*function)(std::vector<real_t>), std::vector<real_t> x, int axis1, int axis2, int axis3) {
// For third order derivative tensors.
// NOTE: Approximations do not appear to be accurate for sinusodial functions...
// Should revisit this later.
double eps = INT_MAX;
real_t eps = INT_MAX;
std::vector<double> x_ppp = x;
std::vector<real_t> x_ppp = x;
x_ppp[axis1] += eps;
x_ppp[axis2] += eps;
x_ppp[axis3] += eps;
std::vector<double> x_npp = x;
std::vector<real_t> x_npp = x;
x_npp[axis2] += eps;
x_npp[axis3] += eps;
std::vector<double> x_pnp = x;
std::vector<real_t> x_pnp = x;
x_pnp[axis1] += eps;
x_pnp[axis3] += eps;
std::vector<double> x_nnp = x;
std::vector<real_t> x_nnp = x;
x_nnp[axis3] += eps;
std::vector<double> x_ppn = x;
std::vector<real_t> x_ppn = x;
x_ppn[axis1] += eps;
x_ppn[axis2] += eps;
std::vector<double> x_npn = x;
std::vector<real_t> x_npn = x;
x_npn[axis2] += eps;
std::vector<double> x_pnn = x;
std::vector<real_t> x_pnn = x;
x_pnn[axis1] += eps;
double thirdAxis = function(x_ppp) - function(x_npp) - function(x_pnp) + function(x_nnp);
double noThirdAxis = function(x_ppn) - function(x_npn) - function(x_pnn) + function(x);
real_t thirdAxis = function(x_ppp) - function(x_npp) - function(x_pnp) + function(x_nnp);
real_t noThirdAxis = function(x_ppn) - function(x_npn) - function(x_pnn) + function(x);
return (thirdAxis - noThirdAxis) / (eps * eps * eps);
}
double MLPPNumericalAnalysis::newtonRaphsonMethod(double (*function)(double), double x_0, double epoch_num) {
double x = x_0;
real_t MLPPNumericalAnalysis::newtonRaphsonMethod(real_t (*function)(real_t), real_t x_0, real_t epoch_num) {
real_t x = x_0;
for (int i = 0; i < epoch_num; i++) {
x -= function(x) / numDiff(function, x);
}
return x;
}
double MLPPNumericalAnalysis::halleyMethod(double (*function)(double), double x_0, double epoch_num) {
double x = x_0;
real_t MLPPNumericalAnalysis::halleyMethod(real_t (*function)(real_t), real_t x_0, real_t epoch_num) {
real_t x = x_0;
for (int i = 0; i < epoch_num; i++) {
x -= ((2 * function(x) * numDiff(function, x)) / (2 * numDiff(function, x) * numDiff(function, x) - function(x) * numDiff_2(function, x)));
}
return x;
}
double MLPPNumericalAnalysis::invQuadraticInterpolation(double (*function)(double), std::vector<double> x_0, double epoch_num) {
double x = 0;
std::vector<double> currentThree = x_0;
real_t MLPPNumericalAnalysis::invQuadraticInterpolation(real_t (*function)(real_t), std::vector<real_t> x_0, real_t epoch_num) {
real_t x = 0;
std::vector<real_t> currentThree = x_0;
for (int i = 0; i < epoch_num; i++) {
double t1 = ((function(currentThree[1]) * function(currentThree[2])) / ((function(currentThree[0]) - function(currentThree[1])) * (function(currentThree[0]) - function(currentThree[2])))) * currentThree[0];
double t2 = ((function(currentThree[0]) * function(currentThree[2])) / ((function(currentThree[1]) - function(currentThree[0])) * (function(currentThree[1]) - function(currentThree[2])))) * currentThree[1];
double t3 = ((function(currentThree[0]) * function(currentThree[1])) / ((function(currentThree[2]) - function(currentThree[0])) * (function(currentThree[2]) - function(currentThree[1])))) * currentThree[2];
real_t t1 = ((function(currentThree[1]) * function(currentThree[2])) / ((function(currentThree[0]) - function(currentThree[1])) * (function(currentThree[0]) - function(currentThree[2])))) * currentThree[0];
real_t t2 = ((function(currentThree[0]) * function(currentThree[2])) / ((function(currentThree[1]) - function(currentThree[0])) * (function(currentThree[1]) - function(currentThree[2])))) * currentThree[1];
real_t t3 = ((function(currentThree[0]) * function(currentThree[1])) / ((function(currentThree[2]) - function(currentThree[0])) * (function(currentThree[2]) - function(currentThree[1])))) * currentThree[2];
x = t1 + t2 + t3;
currentThree.erase(currentThree.begin());
@ -143,10 +143,10 @@ double MLPPNumericalAnalysis::invQuadraticInterpolation(double (*function)(doubl
return x;
}
double MLPPNumericalAnalysis::eulerianMethod(double (*derivative)(double), std::vector<double> q_0, double p, double h) {
double max_epoch = (p - q_0[0]) / h;
double x = q_0[0];
double y = q_0[1];
real_t MLPPNumericalAnalysis::eulerianMethod(real_t (*derivative)(real_t), std::vector<real_t> q_0, real_t p, real_t h) {
real_t max_epoch = (p - q_0[0]) / h;
real_t x = q_0[0];
real_t y = q_0[1];
for (int i = 0; i < max_epoch; i++) {
y = y + h * derivative(x);
x += h;
@ -154,10 +154,10 @@ double MLPPNumericalAnalysis::eulerianMethod(double (*derivative)(double), std::
return y;
}
double MLPPNumericalAnalysis::eulerianMethod(double (*derivative)(std::vector<double>), std::vector<double> q_0, double p, double h) {
double max_epoch = (p - q_0[0]) / h;
double x = q_0[0];
double y = q_0[1];
real_t MLPPNumericalAnalysis::eulerianMethod(real_t (*derivative)(std::vector<real_t>), std::vector<real_t> q_0, real_t p, real_t h) {
real_t max_epoch = (p - q_0[0]) / h;
real_t x = q_0[0];
real_t y = q_0[1];
for (int i = 0; i < max_epoch; i++) {
y = y + h * derivative({ x, y });
x += h;
@ -165,7 +165,7 @@ double MLPPNumericalAnalysis::eulerianMethod(double (*derivative)(std::vector<do
return y;
}
double MLPPNumericalAnalysis::growthMethod(double C, double k, double t) {
real_t MLPPNumericalAnalysis::growthMethod(real_t C, real_t k, real_t t) {
/*
dP/dt = kP
dP/P = kdt
@ -177,20 +177,20 @@ double MLPPNumericalAnalysis::growthMethod(double C, double k, double t) {
P = C * e^(kt)
*/
// auto growthFunction = [&C, &k](double t) { return C * exp(k * t); };
// auto growthFunction = [&C, &k](real_t t) { return C * exp(k * t); };
return C * std::exp(k * t);
}
std::vector<double> MLPPNumericalAnalysis::jacobian(double (*function)(std::vector<double>), std::vector<double> x) {
std::vector<double> jacobian;
std::vector<real_t> MLPPNumericalAnalysis::jacobian(real_t (*function)(std::vector<real_t>), std::vector<real_t> x) {
std::vector<real_t> jacobian;
jacobian.resize(x.size());
for (int i = 0; i < jacobian.size(); i++) {
jacobian[i] = numDiff(function, x, i); // Derivative w.r.t axis i evaluated at x. For all x_i.
}
return jacobian;
}
std::vector<std::vector<double>> MLPPNumericalAnalysis::hessian(double (*function)(std::vector<double>), std::vector<double> x) {
std::vector<std::vector<double>> hessian;
std::vector<std::vector<real_t>> MLPPNumericalAnalysis::hessian(real_t (*function)(std::vector<real_t>), std::vector<real_t> x) {
std::vector<std::vector<real_t>> hessian;
hessian.resize(x.size());
for (int i = 0; i < hessian.size(); i++) {
hessian[i].resize(x.size());
@ -203,8 +203,8 @@ std::vector<std::vector<double>> MLPPNumericalAnalysis::hessian(double (*functio
return hessian;
}
std::vector<std::vector<std::vector<double>>> MLPPNumericalAnalysis::thirdOrderTensor(double (*function)(std::vector<double>), std::vector<double> x) {
std::vector<std::vector<std::vector<double>>> tensor;
std::vector<std::vector<std::vector<real_t>>> MLPPNumericalAnalysis::thirdOrderTensor(real_t (*function)(std::vector<real_t>), std::vector<real_t> x) {
std::vector<std::vector<std::vector<real_t>>> tensor;
tensor.resize(x.size());
for (int i = 0; i < tensor.size(); i++) {
tensor[i].resize(x.size());
@ -221,21 +221,21 @@ std::vector<std::vector<std::vector<double>>> MLPPNumericalAnalysis::thirdOrderT
return tensor;
}
double MLPPNumericalAnalysis::constantApproximation(double (*function)(std::vector<double>), std::vector<double> c) {
real_t MLPPNumericalAnalysis::constantApproximation(real_t (*function)(std::vector<real_t>), std::vector<real_t> c) {
return function(c);
}
double MLPPNumericalAnalysis::linearApproximation(double (*function)(std::vector<double>), std::vector<double> c, std::vector<double> x) {
real_t MLPPNumericalAnalysis::linearApproximation(real_t (*function)(std::vector<real_t>), std::vector<real_t> c, std::vector<real_t> x) {
MLPPLinAlg alg;
return constantApproximation(function, c) + alg.matmult(alg.transpose({ jacobian(function, c) }), { alg.subtraction(x, c) })[0][0];
}
double MLPPNumericalAnalysis::quadraticApproximation(double (*function)(std::vector<double>), std::vector<double> c, std::vector<double> x) {
real_t MLPPNumericalAnalysis::quadraticApproximation(real_t (*function)(std::vector<real_t>), std::vector<real_t> c, std::vector<real_t> x) {
MLPPLinAlg alg;
return linearApproximation(function, c, x) + 0.5 * alg.matmult({ (alg.subtraction(x, c)) }, alg.matmult(hessian(function, c), alg.transpose({ alg.subtraction(x, c) })))[0][0];
}
double MLPPNumericalAnalysis::cubicApproximation(double (*function)(std::vector<double>), std::vector<double> c, std::vector<double> x) {
real_t MLPPNumericalAnalysis::cubicApproximation(real_t (*function)(std::vector<real_t>), std::vector<real_t> c, std::vector<real_t> x) {
/*
Not completely sure as the literature seldom discusses the third order taylor approximation,
in particular for multivariate cases, but ostensibly, the matrix/tensor/vector multiplies
@ -246,32 +246,32 @@ double MLPPNumericalAnalysis::cubicApproximation(double (*function)(std::vector<
Result is a scalar.
*/
MLPPLinAlg alg;
std::vector<std::vector<double>> resultMat = alg.tensor_vec_mult(thirdOrderTensor(function, c), alg.subtraction(x, c));
double resultScalar = alg.matmult({ (alg.subtraction(x, c)) }, alg.matmult(resultMat, alg.transpose({ alg.subtraction(x, c) })))[0][0];
std::vector<std::vector<real_t>> resultMat = alg.tensor_vec_mult(thirdOrderTensor(function, c), alg.subtraction(x, c));
real_t resultScalar = alg.matmult({ (alg.subtraction(x, c)) }, alg.matmult(resultMat, alg.transpose({ alg.subtraction(x, c) })))[0][0];
return quadraticApproximation(function, c, x) + (1 / 6) * resultScalar;
}
double MLPPNumericalAnalysis::laplacian(double (*function)(std::vector<double>), std::vector<double> x) {
real_t MLPPNumericalAnalysis::laplacian(real_t (*function)(std::vector<real_t>), std::vector<real_t> x) {
MLPPLinAlg alg;
std::vector<std::vector<double>> hessian_matrix = hessian(function, x);
double laplacian = 0;
std::vector<std::vector<real_t>> hessian_matrix = hessian(function, x);
real_t laplacian = 0;
for (int i = 0; i < hessian_matrix.size(); i++) {
laplacian += hessian_matrix[i][i]; // homogenous 2nd derivs w.r.t i, then i
}
return laplacian;
}
std::string MLPPNumericalAnalysis::secondPartialDerivativeTest(double (*function)(std::vector<double>), std::vector<double> x) {
std::string MLPPNumericalAnalysis::secondPartialDerivativeTest(real_t (*function)(std::vector<real_t>), std::vector<real_t> x) {
MLPPLinAlg alg;
std::vector<std::vector<double>> hessianMatrix = hessian(function, x);
std::vector<std::vector<real_t>> hessianMatrix = hessian(function, x);
/*
The reason we do this is because the 2nd partial derivative test is less conclusive for functions of variables greater than
2, and the calculations specific to the bivariate case are less computationally intensive.
*/
if (x.size() == 2) {
double det = alg.det(hessianMatrix, hessianMatrix.size());
double secondDerivative = numDiff_2(function, x, 0, 0);
real_t det = alg.det(hessianMatrix, hessianMatrix.size());
real_t secondDerivative = numDiff_2(function, x, 0, 0);
if (secondDerivative > 0 && det > 0) {
return "min";
} else if (secondDerivative < 0 && det > 0) {

52
mlpp/numerical_analysis/numerical_analysis.h
View File

@ -7,6 +7,8 @@
//
//
#include "core/math/math_defs.h"
#include <string>
#include <vector>
@ -17,40 +19,40 @@ public:
as an analytical method for calculating derivatives will most likely be used in
the future.
*/
double numDiff(double (*function)(double), double x);
double numDiff_2(double (*function)(double), double x);
double numDiff_3(double (*function)(double), double x);
real_t numDiff(real_t (*function)(real_t), real_t x);
real_t numDiff_2(real_t (*function)(real_t), real_t x);
real_t numDiff_3(real_t (*function)(real_t), real_t x);
double constantApproximation(double (*function)(double), double c);
double linearApproximation(double (*function)(double), double c, double x);
double quadraticApproximation(double (*function)(double), double c, double x);
double cubicApproximation(double (*function)(double), double c, double x);
real_t constantApproximation(real_t (*function)(real_t), real_t c);
real_t linearApproximation(real_t (*function)(real_t), real_t c, real_t x);
real_t quadraticApproximation(real_t (*function)(real_t), real_t c, real_t x);
real_t cubicApproximation(real_t (*function)(real_t), real_t c, real_t x);
double numDiff(double (*function)(std::vector<double>), std::vector<double> x, int axis);
double numDiff_2(double (*function)(std::vector<double>), std::vector<double> x, int axis1, int axis2);
double numDiff_3(double (*function)(std::vector<double>), std::vector<double> x, int axis1, int axis2, int axis3);
real_t numDiff(real_t (*function)(std::vector<real_t>), std::vector<real_t> x, int axis);
real_t numDiff_2(real_t (*function)(std::vector<real_t>), std::vector<real_t> x, int axis1, int axis2);
real_t numDiff_3(real_t (*function)(std::vector<real_t>), std::vector<real_t> x, int axis1, int axis2, int axis3);
double newtonRaphsonMethod(double (*function)(double), double x_0, double epoch_num);
double halleyMethod(double (*function)(double), double x_0, double epoch_num);
double invQuadraticInterpolation(double (*function)(double), std::vector<double> x_0, double epoch_num);
real_t newtonRaphsonMethod(real_t (*function)(real_t), real_t x_0, real_t epoch_num);
real_t halleyMethod(real_t (*function)(real_t), real_t x_0, real_t epoch_num);
real_t invQuadraticInterpolation(real_t (*function)(real_t), std::vector<real_t> x_0, real_t epoch_num);
double eulerianMethod(double (*derivative)(double), std::vector<double> q_0, double p, double h); // Euler's method for solving diffrential equations.
double eulerianMethod(double (*derivative)(std::vector<double>), std::vector<double> q_0, double p, double h); // Euler's method for solving diffrential equations.
real_t eulerianMethod(real_t (*derivative)(real_t), std::vector<real_t> q_0, real_t p, real_t h); // Euler's method for solving diffrential equations.
real_t eulerianMethod(real_t (*derivative)(std::vector<real_t>), std::vector<real_t> q_0, real_t p, real_t h); // Euler's method for solving diffrential equations.
double growthMethod(double C, double k, double t); // General growth-based diffrential equations can be solved by seperation of variables.
real_t growthMethod(real_t C, real_t k, real_t t); // General growth-based diffrential equations can be solved by seperation of variables.
std::vector<double> jacobian(double (*function)(std::vector<double>), std::vector<double> x); // Indeed, for functions with scalar outputs the Jacobians will be vectors.
std::vector<std::vector<double>> hessian(double (*function)(std::vector<double>), std::vector<double> x);
std::vector<std::vector<std::vector<double>>> thirdOrderTensor(double (*function)(std::vector<double>), std::vector<double> x);
std::vector<real_t> jacobian(real_t (*function)(std::vector<real_t>), std::vector<real_t> x); // Indeed, for functions with scalar outputs the Jacobians will be vectors.
std::vector<std::vector<real_t>> hessian(real_t (*function)(std::vector<real_t>), std::vector<real_t> x);
std::vector<std::vector<std::vector<real_t>>> thirdOrderTensor(real_t (*function)(std::vector<real_t>), std::vector<real_t> x);
double constantApproximation(double (*function)(std::vector<double>), std::vector<double> c);
double linearApproximation(double (*function)(std::vector<double>), std::vector<double> c, std::vector<double> x);
double quadraticApproximation(double (*function)(std::vector<double>), std::vector<double> c, std::vector<double> x);
double cubicApproximation(double (*function)(std::vector<double>), std::vector<double> c, std::vector<double> x);
real_t constantApproximation(real_t (*function)(std::vector<real_t>), std::vector<real_t> c);
real_t linearApproximation(real_t (*function)(std::vector<real_t>), std::vector<real_t> c, std::vector<real_t> x);
real_t quadraticApproximation(real_t (*function)(std::vector<real_t>), std::vector<real_t> c, std::vector<real_t> x);
real_t cubicApproximation(real_t (*function)(std::vector<real_t>), std::vector<real_t> c, std::vector<real_t> x);
double laplacian(double (*function)(std::vector<double>), std::vector<double> x); // laplacian
real_t laplacian(real_t (*function)(std::vector<real_t>), std::vector<real_t> x); // laplacian
std::string secondPartialDerivativeTest(double (*function)(std::vector<double>), std::vector<double> x);
std::string secondPartialDerivativeTest(real_t (*function)(std::vector<real_t>), std::vector<real_t> x);
};

12
mlpp/outlier_finder/outlier_finder.cpp
View File

@ -13,13 +13,13 @@ MLPPOutlierFinder::MLPPOutlierFinder(int threshold) :
threshold(threshold) {
}
std::vector<std::vector<double>> MLPPOutlierFinder::modelSetTest(std::vector<std::vector<double>> inputSet) {
std::vector<std::vector<real_t>> MLPPOutlierFinder::modelSetTest(std::vector<std::vector<real_t>> inputSet) {
MLPPStat stat;
std::vector<std::vector<double>> outliers;
std::vector<std::vector<real_t>> outliers;
outliers.resize(inputSet.size());
for (int i = 0; i < inputSet.size(); i++) {
for (int j = 0; j < inputSet[i].size(); j++) {
double z = (inputSet[i][j] - stat.mean(inputSet[i])) / stat.standardDeviation(inputSet[i]);
real_t z = (inputSet[i][j] - stat.mean(inputSet[i])) / stat.standardDeviation(inputSet[i]);
if (abs(z) > threshold) {
outliers[i].push_back(inputSet[i][j]);
}
@ -28,11 +28,11 @@ std::vector<std::vector<double>> MLPPOutlierFinder::modelSetTest(std::vector<std
return outliers;
}
std::vector<double> MLPPOutlierFinder::modelTest(std::vector<double> inputSet) {
std::vector<real_t> MLPPOutlierFinder::modelTest(std::vector<real_t> inputSet) {
MLPPStat stat;
std::vector<double> outliers;
std::vector<real_t> outliers;
for (int i = 0; i < inputSet.size(); i++) {
double z = (inputSet[i] - stat.mean(inputSet)) / stat.standardDeviation(inputSet);
real_t z = (inputSet[i] - stat.mean(inputSet)) / stat.standardDeviation(inputSet);
if (abs(z) > threshold) {
outliers.push_back(inputSet[i]);
}

6
mlpp/outlier_finder/outlier_finder.h
View File

@ -8,6 +8,8 @@
// Created by Marc Melikyan on 11/13/20.
//
#include "core/math/math_defs.h"
#include <vector>
@ -16,8 +18,8 @@ public:
// Cnstr
MLPPOutlierFinder(int threshold);
std::vector<std::vector<double>> modelSetTest(std::vector<std::vector<double>> inputSet);
std::vector<double> modelTest(std::vector<double> inputSet);
std::vector<std::vector<real_t>> modelSetTest(std::vector<std::vector<real_t>> inputSet);
std::vector<real_t> modelTest(std::vector<real_t> inputSet);
// Variables required
int threshold;

4
mlpp/output_layer/output_layer.cpp
View File

@ -12,7 +12,7 @@
#include <random>
MLPPOutputLayer::MLPPOutputLayer(int n_hidden, std::string activation, std::string cost, std::vector<std::vector<double>> input, std::string weightInit, std::string reg, double lambda, double alpha) :
MLPPOutputLayer::MLPPOutputLayer(int n_hidden, std::string activation, std::string cost, std::vector<std::vector<real_t>> input, std::string weightInit, std::string reg, real_t lambda, real_t alpha) :
n_hidden(n_hidden), activation(activation), cost(cost), input(input), weightInit(weightInit), reg(reg), lambda(lambda), alpha(alpha) {
weights = MLPPUtilities::weightInitialization(n_hidden, weightInit);
bias = MLPPUtilities::biasInitialization();
@ -120,7 +120,7 @@ void MLPPOutputLayer::forwardPass() {
a = (avn.*activation_map[activation])(z, 0);
}
void MLPPOutputLayer::Test(std::vector<double> x) {
void MLPPOutputLayer::Test(std::vector<real_t> x) {
MLPPLinAlg alg;
MLPPActivation avn;
z_test = alg.dot(weights, x) + bias;

34
mlpp/output_layer/output_layer.h
View File

@ -8,6 +8,8 @@
// Created by Marc Melikyan on 11/4/20.
//
#include "core/math/math_defs.h"
#include "../activation/activation.h"
#include "../cost/cost.h"
@ -18,39 +20,39 @@
class MLPPOutputLayer {
public:
MLPPOutputLayer(int n_hidden, std::string activation, std::string cost, std::vector<std::vector<double>> input, std::string weightInit, std::string reg, double lambda, double alpha);
MLPPOutputLayer(int n_hidden, std::string activation, std::string cost, std::vector<std::vector<real_t>> input, std::string weightInit, std::string reg, real_t lambda, real_t alpha);
int n_hidden;
std::string activation;
std::string cost;
std::vector<std::vector<double>> input;
std::vector<std::vector<real_t>> input;
std::vector<double> weights;
double bias;
std::vector<real_t> weights;
real_t bias;
std::vector<double> z;
std::vector<double> a;
std::vector<real_t> z;
std::vector<real_t> a;
std::map<std::string, std::vector<double> (MLPPActivation::*)(std::vector<double>, bool)> activation_map;
std::map<std::string, double (MLPPActivation::*)(double, bool)> activationTest_map;
std::map<std::string, double (MLPPCost::*)(std::vector<double>, std::vector<double>)> cost_map;
std::map<std::string, std::vector<double> (MLPPCost::*)(std::vector<double>, std::vector<double>)> costDeriv_map;
std::map<std::string, std::vector<real_t> (MLPPActivation::*)(std::vector<real_t>, bool)> activation_map;
std::map<std::string, real_t (MLPPActivation::*)(real_t, bool)> activationTest_map;
std::map<std::string, real_t (MLPPCost::*)(std::vector<real_t>, std::vector<real_t>)> cost_map;
std::map<std::string, std::vector<real_t> (MLPPCost::*)(std::vector<real_t>, std::vector<real_t>)> costDeriv_map;
double z_test;
double a_test;
real_t z_test;
real_t a_test;
std::vector<double> delta;
std::vector<real_t> delta;
// Regularization Params
std::string reg;
double lambda; /* Regularization Parameter */
double alpha; /* This is the controlling param for Elastic Net*/
real_t lambda; /* Regularization Parameter */
real_t alpha; /* This is the controlling param for Elastic Net*/
std::string weightInit;
void forwardPass();
void Test(std::vector<double> x);
void Test(std::vector<real_t> x);
};

10
mlpp/pca/pca.cpp
View File

@ -13,11 +13,11 @@
MLPPPCA::MLPPPCA(std::vector<std::vector<double>> inputSet, int k) :
MLPPPCA::MLPPPCA(std::vector<std::vector<real_t>> inputSet, int k) :
inputSet(inputSet), k(k) {
}
std::vector<std::vector<double>> MLPPPCA::principalComponents() {
std::vector<std::vector<real_t>> MLPPPCA::principalComponents() {
MLPPLinAlg alg;
MLPPData data;
@ -33,10 +33,10 @@ std::vector<std::vector<double>> MLPPPCA::principalComponents() {
return Z;
}
// Simply tells us the percentage of variance maintained.
double MLPPPCA::score() {
real_t MLPPPCA::score() {
MLPPLinAlg alg;
std::vector<std::vector<double>> X_approx = alg.matmult(U_reduce, Z);
double num, den = 0;
std::vector<std::vector<real_t>> X_approx = alg.matmult(U_reduce, Z);
real_t num, den = 0;
for (int i = 0; i < X_normalized.size(); i++) {
num += alg.norm_sq(alg.subtraction(X_normalized[i], X_approx[i]));
}

16
mlpp/pca/pca.h
View File

@ -8,20 +8,22 @@
// Created by Marc Melikyan on 10/2/20.
//
#include "core/math/math_defs.h"
#include <vector>
class MLPPPCA {
public:
MLPPPCA(std::vector<std::vector<double>> inputSet, int k);
std::vector<std::vector<double>> principalComponents();
double score();
MLPPPCA(std::vector<std::vector<real_t>> inputSet, int k);
std::vector<std::vector<real_t>> principalComponents();
real_t score();
private:
std::vector<std::vector<double>> inputSet;
std::vector<std::vector<double>> X_normalized;
std::vector<std::vector<double>> U_reduce;
std::vector<std::vector<double>> Z;
std::vector<std::vector<real_t>> inputSet;
std::vector<std::vector<real_t>> X_normalized;
std::vector<std::vector<real_t>> U_reduce;
std::vector<std::vector<real_t>> Z;
int k;
};

56
mlpp/probit_reg/probit_reg.cpp
View File

@ -15,33 +15,33 @@
#include <random>
MLPPProbitReg::MLPPProbitReg(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, std::string reg, double lambda, double alpha) :
MLPPProbitReg::MLPPProbitReg(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, std::string reg, real_t lambda, real_t alpha) :
inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) {
y_hat.resize(n);
weights = MLPPUtilities::weightInitialization(k);
bias = MLPPUtilities::biasInitialization();
}
std::vector<double> MLPPProbitReg::modelSetTest(std::vector<std::vector<double>> X) {
std::vector<real_t> MLPPProbitReg::modelSetTest(std::vector<std::vector<real_t>> X) {
return Evaluate(X);
}
double MLPPProbitReg::modelTest(std::vector<double> x) {
real_t MLPPProbitReg::modelTest(std::vector<real_t> x) {
return Evaluate(x);
}
void MLPPProbitReg::gradientDescent(double learning_rate, int max_epoch, bool UI) {
void MLPPProbitReg::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
MLPPActivation avn;
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
forwardPass();
while (true) {
cost_prev = Cost(y_hat, outputSet);
std::vector<double> error = alg.subtraction(y_hat, outputSet);
std::vector<real_t> error = alg.subtraction(y_hat, outputSet);
// Calculating the weight gradients
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputSet), alg.hadamard_product(error, avn.gaussianCDF(z, 1)))));
@ -63,18 +63,18 @@ void MLPPProbitReg::gradientDescent(double learning_rate, int max_epoch, bool UI
}
}
void MLPPProbitReg::MLE(double learning_rate, int max_epoch, bool UI) {
void MLPPProbitReg::MLE(real_t learning_rate, int max_epoch, bool UI) {
MLPPActivation avn;
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
forwardPass();
while (true) {
cost_prev = Cost(y_hat, outputSet);
std::vector<double> error = alg.subtraction(outputSet, y_hat);
std::vector<real_t> error = alg.subtraction(outputSet, y_hat);
// Calculating the weight gradients
weights = alg.addition(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputSet), alg.hadamard_product(error, avn.gaussianCDF(z, 1)))));
@ -96,12 +96,12 @@ void MLPPProbitReg::MLE(double learning_rate, int max_epoch, bool UI) {
}
}
void MLPPProbitReg::SGD(double learning_rate, int max_epoch, bool UI) {
void MLPPProbitReg::SGD(real_t learning_rate, int max_epoch, bool UI) {
// NOTE: ∂y_hat/∂z is sparse
MLPPActivation avn;
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
while (true) {
@ -110,11 +110,11 @@ void MLPPProbitReg::SGD(double learning_rate, int max_epoch, bool UI) {
std::uniform_int_distribution<int> distribution(0, int(n - 1));
int outputIndex = distribution(generator);
double y_hat = Evaluate(inputSet[outputIndex]);
double z = propagate(inputSet[outputIndex]);
real_t y_hat = Evaluate(inputSet[outputIndex]);
real_t z = propagate(inputSet[outputIndex]);
cost_prev = Cost({ y_hat }, { outputSet[outputIndex] });
double error = y_hat - outputSet[outputIndex];
real_t error = y_hat - outputSet[outputIndex];
// Weight Updation
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate * error * ((1 / sqrt(2 * M_PI)) * exp(-z * z / 2)), inputSet[outputIndex]));
@ -138,11 +138,11 @@ void MLPPProbitReg::SGD(double learning_rate, int max_epoch, bool UI) {
forwardPass();
}
void MLPPProbitReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI) {
void MLPPProbitReg::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI) {
MLPPActivation avn;
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
// Creating the mini-batches
@ -151,8 +151,8 @@ void MLPPProbitReg::MBGD(double learning_rate, int max_epoch, int mini_batch_siz
// Creating the mini-batches
for (int i = 0; i < n_mini_batch; i++) {
std::vector<std::vector<double>> currentInputSet;
std::vector<double> currentOutputSet;
std::vector<std::vector<real_t>> currentInputSet;
std::vector<real_t> currentOutputSet;
for (int j = 0; j < n / n_mini_batch; j++) {
currentInputSet.push_back(inputSet[n / n_mini_batch * i + j]);
currentOutputSet.push_back(outputSet[n / n_mini_batch * i + j]);
@ -161,7 +161,7 @@ void MLPPProbitReg::MBGD(double learning_rate, int max_epoch, int mini_batch_siz
outputMiniBatches.push_back(currentOutputSet);
}
if (double(n) / double(n_mini_batch) - int(n / n_mini_batch) != 0) {
if (real_t(n) / real_t(n_mini_batch) - int(n / n_mini_batch) != 0) {
for (int i = 0; i < n - n / n_mini_batch * n_mini_batch; i++) {
inputMiniBatches[n_mini_batch - 1].push_back(inputSet[n / n_mini_batch * n_mini_batch + i]);
outputMiniBatches[n_mini_batch - 1].push_back(outputSet[n / n_mini_batch * n_mini_batch + i]);
@ -170,11 +170,11 @@ void MLPPProbitReg::MBGD(double learning_rate, int max_epoch, int mini_batch_siz
while (true) {
for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = Evaluate(inputMiniBatches[i]);
std::vector<double> z = propagate(inputMiniBatches[i]);
std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]);
std::vector<real_t> z = propagate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]);
std::vector<double> error = alg.subtraction(y_hat, outputMiniBatches[i]);
std::vector<real_t> error = alg.subtraction(y_hat, outputMiniBatches[i]);
// Calculating the weight gradients
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / outputMiniBatches.size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), alg.hadamard_product(error, avn.gaussianCDF(z, 1)))));
@ -197,7 +197,7 @@ void MLPPProbitReg::MBGD(double learning_rate, int max_epoch, int mini_batch_siz
forwardPass();
}
double MLPPProbitReg::score() {
real_t MLPPProbitReg::score() {
MLPPUtilities util;
return util.performance(y_hat, outputSet);
}
@ -207,30 +207,30 @@ void MLPPProbitReg::save(std::string fileName) {
util.saveParameters(fileName, weights, bias);
}
double MLPPProbitReg::Cost(std::vector<double> y_hat, std::vector<double> y) {
real_t MLPPProbitReg::Cost(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPReg regularization;
class MLPPCost cost;
return cost.MSE(y_hat, y) + regularization.regTerm(weights, lambda, alpha, reg);
}
std::vector<double> MLPPProbitReg::Evaluate(std::vector<std::vector<double>> X) {
std::vector<real_t> MLPPProbitReg::Evaluate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg;
MLPPActivation avn;
return avn.gaussianCDF(alg.scalarAdd(bias, alg.mat_vec_mult(X, weights)));
}
std::vector<double> MLPPProbitReg::propagate(std::vector<std::vector<double>> X) {
std::vector<real_t> MLPPProbitReg::propagate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg;
return alg.scalarAdd(bias, alg.mat_vec_mult(X, weights));
}
double MLPPProbitReg::Evaluate(std::vector<double> x) {
real_t MLPPProbitReg::Evaluate(std::vector<real_t> x) {
MLPPLinAlg alg;
MLPPActivation avn;
return avn.gaussianCDF(alg.dot(weights, x) + bias);
}
double MLPPProbitReg::propagate(std::vector<double> x) {
real_t MLPPProbitReg::propagate(std::vector<real_t> x) {
MLPPLinAlg alg;
return alg.dot(weights, x) + bias;
}

44
mlpp/probit_reg/probit_reg.h
View File

@ -8,6 +8,8 @@
// Created by Marc Melikyan on 10/2/20.
//
#include "core/math/math_defs.h"
#include <string>
#include <vector>
@ -15,39 +17,39 @@
class MLPPProbitReg {
public:
MLPPProbitReg(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, std::string reg = "None", double lambda = 0.5, double alpha = 0.5);
std::vector<double> modelSetTest(std::vector<std::vector<double>> X);
double modelTest(std::vector<double> x);
void gradientDescent(double learning_rate, int max_epoch = 0, bool UI = 1);
void MLE(double learning_rate, int max_epoch = 0, bool UI = 1);
void SGD(double learning_rate, int max_epoch = 0, bool UI = 1);
void MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
double score();
MLPPProbitReg(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
std::vector<real_t> modelSetTest(std::vector<std::vector<real_t>> X);
real_t modelTest(std::vector<real_t> x);
void gradientDescent(real_t learning_rate, int max_epoch = 0, bool UI = 1);
void MLE(real_t learning_rate, int max_epoch = 0, bool UI = 1);
void SGD(real_t learning_rate, int max_epoch = 0, bool UI = 1);
void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
real_t score();
void save(std::string fileName);
private:
double Cost(std::vector<double> y_hat, std::vector<double> y);
real_t Cost(std::vector<real_t> y_hat, std::vector<real_t> y);
std::vector<double> Evaluate(std::vector<std::vector<double>> X);
std::vector<double> propagate(std::vector<std::vector<double>> X);
double Evaluate(std::vector<double> x);
double propagate(std::vector<double> x);
std::vector<real_t> Evaluate(std::vector<std::vector<real_t>> X);
std::vector<real_t> propagate(std::vector<std::vector<real_t>> X);
real_t Evaluate(std::vector<real_t> x);
real_t propagate(std::vector<real_t> x);
void forwardPass();
std::vector<std::vector<double>> inputSet;
std::vector<double> outputSet;
std::vector<double> z;
std::vector<double> y_hat;
std::vector<double> weights;
double bias;
std::vector<std::vector<real_t>> inputSet;
std::vector<real_t> outputSet;
std::vector<real_t> z;
std::vector<real_t> y_hat;
std::vector<real_t> weights;
real_t bias;
int n;
int k;
// Regularization Params
std::string reg;
double lambda;
double alpha; /* This is the controlling param for Elastic Net*/
real_t lambda;
real_t alpha; /* This is the controlling param for Elastic Net*/
};

32
mlpp/regularization/reg.cpp
View File

@ -12,21 +12,21 @@
double MLPPReg::regTerm(std::vector<double> weights, double lambda, double alpha, std::string reg) {
real_t MLPPReg::regTerm(std::vector<real_t> weights, real_t lambda, real_t alpha, std::string reg) {
if (reg == "Ridge") {
double reg = 0;
real_t reg = 0;
for (int i = 0; i < weights.size(); i++) {
reg += weights[i] * weights[i];
}
return reg * lambda / 2;
} else if (reg == "Lasso") {
double reg = 0;
real_t reg = 0;
for (int i = 0; i < weights.size(); i++) {
reg += abs(weights[i]);
}
return reg * lambda;
} else if (reg == "ElasticNet") {
double reg = 0;
real_t reg = 0;
for (int i = 0; i < weights.size(); i++) {
reg += alpha * abs(weights[i]); // Lasso Reg
reg += ((1 - alpha) / 2) * weights[i] * weights[i]; // Ridge Reg
@ -36,9 +36,9 @@ double MLPPReg::regTerm(std::vector<double> weights, double lambda, double alpha
return 0;
}
double MLPPReg::regTerm(std::vector<std::vector<double>> weights, double lambda, double alpha, std::string reg) {
real_t MLPPReg::regTerm(std::vector<std::vector<real_t>> weights, real_t lambda, real_t alpha, std::string reg) {
if (reg == "Ridge") {
double reg = 0;
real_t reg = 0;
for (int i = 0; i < weights.size(); i++) {
for (int j = 0; j < weights[i].size(); j++) {
reg += weights[i][j] * weights[i][j];
@ -46,7 +46,7 @@ double MLPPReg::regTerm(std::vector<std::vector<double>> weights, double lambda,
}
return reg * lambda / 2;
} else if (reg == "Lasso") {
double reg = 0;
real_t reg = 0;
for (int i = 0; i < weights.size(); i++) {
for (int j = 0; j < weights[i].size(); j++) {
reg += abs(weights[i][j]);
@ -54,7 +54,7 @@ double MLPPReg::regTerm(std::vector<std::vector<double>> weights, double lambda,
}
return reg * lambda;
} else if (reg == "ElasticNet") {
double reg = 0;
real_t reg = 0;
for (int i = 0; i < weights.size(); i++) {
for (int j = 0; j < weights[i].size(); j++) {
reg += alpha * abs(weights[i][j]); // Lasso Reg
@ -66,7 +66,7 @@ double MLPPReg::regTerm(std::vector<std::vector<double>> weights, double lambda,
return 0;
}
std::vector<double> MLPPReg::regWeights(std::vector<double> weights, double lambda, double alpha, std::string reg) {
std::vector<real_t> MLPPReg::regWeights(std::vector<real_t> weights, real_t lambda, real_t alpha, std::string reg) {
MLPPLinAlg alg;
if (reg == "WeightClipping") {
return regDerivTerm(weights, lambda, alpha, reg);
@ -78,7 +78,7 @@ std::vector<double> MLPPReg::regWeights(std::vector<double> weights, double lamb
// return weights;
}
std::vector<std::vector<double>> MLPPReg::regWeights(std::vector<std::vector<double>> weights, double lambda, double alpha, std::string reg) {
std::vector<std::vector<real_t>> MLPPReg::regWeights(std::vector<std::vector<real_t>> weights, real_t lambda, real_t alpha, std::string reg) {
MLPPLinAlg alg;
if (reg == "WeightClipping") {
return regDerivTerm(weights, lambda, alpha, reg);
@ -92,8 +92,8 @@ std::vector<std::vector<double>> MLPPReg::regWeights(std::vector<std::vector<dou
// return weights;
}
std::vector<double> MLPPReg::regDerivTerm(std::vector<double> weights, double lambda, double alpha, std::string reg) {
std::vector<double> regDeriv;
std::vector<real_t> MLPPReg::regDerivTerm(std::vector<real_t> weights, real_t lambda, real_t alpha, std::string reg) {
std::vector<real_t> regDeriv;
regDeriv.resize(weights.size());
for (int i = 0; i < regDeriv.size(); i++) {
@ -102,8 +102,8 @@ std::vector<double> MLPPReg::regDerivTerm(std::vector<double> weights, double la
return regDeriv;
}
std::vector<std::vector<double>> MLPPReg::regDerivTerm(std::vector<std::vector<double>> weights, double lambda, double alpha, std::string reg) {
std::vector<std::vector<double>> regDeriv;
std::vector<std::vector<real_t>> MLPPReg::regDerivTerm(std::vector<std::vector<real_t>> weights, real_t lambda, real_t alpha, std::string reg) {
std::vector<std::vector<real_t>> regDeriv;
regDeriv.resize(weights.size());
for (int i = 0; i < regDeriv.size(); i++) {
regDeriv[i].resize(weights[0].size());
@ -117,7 +117,7 @@ std::vector<std::vector<double>> MLPPReg::regDerivTerm(std::vector<std::vector<d
return regDeriv;
}
double MLPPReg::regDerivTerm(std::vector<double> weights, double lambda, double alpha, std::string reg, int j) {
real_t MLPPReg::regDerivTerm(std::vector<real_t> weights, real_t lambda, real_t alpha, std::string reg, int j) {
MLPPActivation act;
if (reg == "Ridge") {
return lambda * weights[j];
@ -140,7 +140,7 @@ double MLPPReg::regDerivTerm(std::vector<double> weights, double lambda, double
}
}
double MLPPReg::regDerivTerm(std::vector<std::vector<double>> weights, double lambda, double alpha, std::string reg, int i, int j) {
real_t MLPPReg::regDerivTerm(std::vector<std::vector<real_t>> weights, real_t lambda, real_t alpha, std::string reg, int i, int j) {
MLPPActivation act;
if (reg == "Ridge") {
return lambda * weights[i][j];

18
mlpp/regularization/reg.h
View File

@ -9,24 +9,26 @@
// Created by Marc Melikyan on 1/16/21.
//
#include "core/math/math_defs.h"
#include <vector>
#include <string>
class MLPPReg {
public:
double regTerm(std::vector<double> weights, double lambda, double alpha, std::string reg);
double regTerm(std::vector<std::vector<double>> weights, double lambda, double alpha, std::string reg);
real_t regTerm(std::vector<real_t> weights, real_t lambda, real_t alpha, std::string reg);
real_t regTerm(std::vector<std::vector<real_t>> weights, real_t lambda, real_t alpha, std::string reg);
std::vector<double> regWeights(std::vector<double> weights, double lambda, double alpha, std::string reg);
std::vector<std::vector<double>> regWeights(std::vector<std::vector<double>> weights, double lambda, double alpha, std::string reg);
std::vector<real_t> regWeights(std::vector<real_t> weights, real_t lambda, real_t alpha, std::string reg);
std::vector<std::vector<real_t>> regWeights(std::vector<std::vector<real_t>> weights, real_t lambda, real_t alpha, std::string reg);
std::vector<double> regDerivTerm(std::vector<double> weights, double lambda, double alpha, std::string reg);
std::vector<std::vector<double>> regDerivTerm(std::vector<std::vector<double>>, double lambda, double alpha, std::string reg);
std::vector<real_t> regDerivTerm(std::vector<real_t> weights, real_t lambda, real_t alpha, std::string reg);
std::vector<std::vector<real_t>> regDerivTerm(std::vector<std::vector<real_t>>, real_t lambda, real_t alpha, std::string reg);
private:
double regDerivTerm(std::vector<double> weights, double lambda, double alpha, std::string reg, int j);
double regDerivTerm(std::vector<std::vector<double>> weights, double lambda, double alpha, std::string reg, int i, int j);
real_t regDerivTerm(std::vector<real_t> weights, real_t lambda, real_t alpha, std::string reg, int j);
real_t regDerivTerm(std::vector<std::vector<real_t>> weights, real_t lambda, real_t alpha, std::string reg, int i, int j);
};

88
mlpp/softmax_net/softmax_net.cpp
View File

@ -16,7 +16,7 @@
#include <random>
MLPPSoftmaxNet::MLPPSoftmaxNet(std::vector<std::vector<double>> inputSet, std::vector<std::vector<double>> outputSet, int n_hidden, std::string reg, double lambda, double alpha) :
MLPPSoftmaxNet::MLPPSoftmaxNet(std::vector<std::vector<real_t>> inputSet, std::vector<std::vector<real_t>> outputSet, int n_hidden, std::string reg, real_t lambda, real_t alpha) :
inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), n_hidden(n_hidden), n_class(outputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) {
y_hat.resize(n);
@ -26,19 +26,19 @@ MLPPSoftmaxNet::MLPPSoftmaxNet(std::vector<std::vector<double>> inputSet, std::v
bias2 = MLPPUtilities::biasInitialization(n_class);
}
std::vector<double> MLPPSoftmaxNet::modelTest(std::vector<double> x) {
std::vector<real_t> MLPPSoftmaxNet::modelTest(std::vector<real_t> x) {
return Evaluate(x);
}
std::vector<std::vector<double>> MLPPSoftmaxNet::modelSetTest(std::vector<std::vector<double>> X) {
std::vector<std::vector<real_t>> MLPPSoftmaxNet::modelSetTest(std::vector<std::vector<real_t>> X) {
return Evaluate(X);
}
void MLPPSoftmaxNet::gradientDescent(double learning_rate, int max_epoch, bool UI) {
void MLPPSoftmaxNet::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
MLPPActivation avn;
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
forwardPass();
@ -46,11 +46,11 @@ void MLPPSoftmaxNet::gradientDescent(double learning_rate, int max_epoch, bool U
cost_prev = Cost(y_hat, outputSet);
// Calculating the errors
std::vector<std::vector<double>> error = alg.subtraction(y_hat, outputSet);
std::vector<std::vector<real_t>> error = alg.subtraction(y_hat, outputSet);
// Calculating the weight/bias gradients for layer 2
std::vector<std::vector<double>> D2_1 = alg.matmult(alg.transpose(a2), error);
std::vector<std::vector<real_t>> D2_1 = alg.matmult(alg.transpose(a2), error);
// weights and bias updation for layer 2
weights2 = alg.subtraction(weights2, alg.scalarMultiply(learning_rate, D2_1));
@ -60,11 +60,11 @@ void MLPPSoftmaxNet::gradientDescent(double learning_rate, int max_epoch, bool U
//Calculating the weight/bias for layer 1
std::vector<std::vector<double>> D1_1 = alg.matmult(error, alg.transpose(weights2));
std::vector<std::vector<real_t>> D1_1 = alg.matmult(error, alg.transpose(weights2));
std::vector<std::vector<double>> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1));
std::vector<std::vector<real_t>> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1));
std::vector<std::vector<double>> D1_3 = alg.matmult(alg.transpose(inputSet), D1_2);
std::vector<std::vector<real_t>> D1_3 = alg.matmult(alg.transpose(inputSet), D1_2);
// weight an bias updation for layer 1
weights1 = alg.subtraction(weights1, alg.scalarMultiply(learning_rate, D1_3));
@ -90,11 +90,11 @@ void MLPPSoftmaxNet::gradientDescent(double learning_rate, int max_epoch, bool U
}
}
void MLPPSoftmaxNet::SGD(double learning_rate, int max_epoch, bool UI) {
void MLPPSoftmaxNet::SGD(real_t learning_rate, int max_epoch, bool UI) {
MLPPActivation avn;
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
while (true) {
@ -103,13 +103,13 @@ void MLPPSoftmaxNet::SGD(double learning_rate, int max_epoch, bool UI) {
std::uniform_int_distribution<int> distribution(0, int(n - 1));
int outputIndex = distribution(generator);
std::vector<double> y_hat = Evaluate(inputSet[outputIndex]);
std::vector<real_t> y_hat = Evaluate(inputSet[outputIndex]);
auto [z2, a2] = propagate(inputSet[outputIndex]);
cost_prev = Cost({ y_hat }, { outputSet[outputIndex] });
std::vector<double> error = alg.subtraction(y_hat, outputSet[outputIndex]);
std::vector<real_t> error = alg.subtraction(y_hat, outputSet[outputIndex]);
// Weight updation for layer 2
std::vector<std::vector<double>> D2_1 = alg.outerProduct(error, a2);
std::vector<std::vector<real_t>> D2_1 = alg.outerProduct(error, a2);
weights2 = alg.subtraction(weights2, alg.scalarMultiply(learning_rate, alg.transpose(D2_1)));
weights2 = regularization.regWeights(weights2, lambda, alpha, reg);
@ -117,9 +117,9 @@ void MLPPSoftmaxNet::SGD(double learning_rate, int max_epoch, bool UI) {
bias2 = alg.subtraction(bias2, alg.scalarMultiply(learning_rate, error));
// Weight updation for layer 1
std::vector<double> D1_1 = alg.mat_vec_mult(weights2, error);
std::vector<double> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1));
std::vector<std::vector<double>> D1_3 = alg.outerProduct(inputSet[outputIndex], D1_2);
std::vector<real_t> D1_1 = alg.mat_vec_mult(weights2, error);
std::vector<real_t> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1));
std::vector<std::vector<real_t>> D1_3 = alg.outerProduct(inputSet[outputIndex], D1_2);
weights1 = alg.subtraction(weights1, alg.scalarMultiply(learning_rate, D1_3));
weights1 = regularization.regWeights(weights1, lambda, alpha, reg);
@ -144,11 +144,11 @@ void MLPPSoftmaxNet::SGD(double learning_rate, int max_epoch, bool UI) {
forwardPass();
}
void MLPPSoftmaxNet::MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI) {
void MLPPSoftmaxNet::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI) {
MLPPActivation avn;
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
// Creating the mini-batches
@ -157,8 +157,8 @@ void MLPPSoftmaxNet::MBGD(double learning_rate, int max_epoch, int mini_batch_si
// Creating the mini-batches
for (int i = 0; i < n_mini_batch; i++) {
std::vector<std::vector<double>> currentInputSet;
std::vector<std::vector<double>> currentOutputSet;
std::vector<std::vector<real_t>> currentInputSet;
std::vector<std::vector<real_t>> currentOutputSet;
for (int j = 0; j < n / n_mini_batch; j++) {
currentInputSet.push_back(inputSet[n / n_mini_batch * i + j]);
currentOutputSet.push_back(outputSet[n / n_mini_batch * i + j]);
@ -167,7 +167,7 @@ void MLPPSoftmaxNet::MBGD(double learning_rate, int max_epoch, int mini_batch_si
outputMiniBatches.push_back(currentOutputSet);
}
if (double(n) / double(n_mini_batch) - int(n / n_mini_batch) != 0) {
if (real_t(n) / real_t(n_mini_batch) - int(n / n_mini_batch) != 0) {
for (int i = 0; i < n - n / n_mini_batch * n_mini_batch; i++) {
inputMiniBatches[n_mini_batch - 1].push_back(inputSet[n / n_mini_batch * n_mini_batch + i]);
outputMiniBatches[n_mini_batch - 1].push_back(outputSet[n / n_mini_batch * n_mini_batch + i]);
@ -176,16 +176,16 @@ void MLPPSoftmaxNet::MBGD(double learning_rate, int max_epoch, int mini_batch_si
while (true) {
for (int i = 0; i < n_mini_batch; i++) {
std::vector<std::vector<double>> y_hat = Evaluate(inputMiniBatches[i]);
std::vector<std::vector<real_t>> y_hat = Evaluate(inputMiniBatches[i]);
auto [z2, a2] = propagate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]);
// Calculating the errors
std::vector<std::vector<double>> error = alg.subtraction(y_hat, outputMiniBatches[i]);
std::vector<std::vector<real_t>> error = alg.subtraction(y_hat, outputMiniBatches[i]);
// Calculating the weight/bias gradients for layer 2
std::vector<std::vector<double>> D2_1 = alg.matmult(alg.transpose(a2), error);
std::vector<std::vector<real_t>> D2_1 = alg.matmult(alg.transpose(a2), error);
// weights and bias updation for layser 2
weights2 = alg.subtraction(weights2, alg.scalarMultiply(learning_rate, D2_1));
@ -196,11 +196,11 @@ void MLPPSoftmaxNet::MBGD(double learning_rate, int max_epoch, int mini_batch_si
//Calculating the weight/bias for layer 1
std::vector<std::vector<double>> D1_1 = alg.matmult(error, alg.transpose(weights2));
std::vector<std::vector<real_t>> D1_1 = alg.matmult(error, alg.transpose(weights2));
std::vector<std::vector<double>> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1));
std::vector<std::vector<real_t>> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1));
std::vector<std::vector<double>> D1_3 = alg.matmult(alg.transpose(inputMiniBatches[i]), D1_2);
std::vector<std::vector<real_t>> D1_3 = alg.matmult(alg.transpose(inputMiniBatches[i]), D1_2);
// weight an bias updation for layer 1
weights1 = alg.subtraction(weights1, alg.scalarMultiply(learning_rate, D1_3));
@ -226,7 +226,7 @@ void MLPPSoftmaxNet::MBGD(double learning_rate, int max_epoch, int mini_batch_si
forwardPass();
}
double MLPPSoftmaxNet::score() {
real_t MLPPSoftmaxNet::score() {
MLPPUtilities util;
return util.performance(y_hat, outputSet);
}
@ -239,46 +239,46 @@ void MLPPSoftmaxNet::save(std::string fileName) {
MLPPLinAlg alg;
}
std::vector<std::vector<double>> MLPPSoftmaxNet::getEmbeddings() {
std::vector<std::vector<real_t>> MLPPSoftmaxNet::getEmbeddings() {
return weights1;
}
double MLPPSoftmaxNet::Cost(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) {
real_t MLPPSoftmaxNet::Cost(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
MLPPReg regularization;
MLPPData data;
class MLPPCost cost;
return cost.CrossEntropy(y_hat, y) + regularization.regTerm(weights1, lambda, alpha, reg) + regularization.regTerm(weights2, lambda, alpha, reg);
}
std::vector<std::vector<double>> MLPPSoftmaxNet::Evaluate(std::vector<std::vector<double>> X) {
std::vector<std::vector<real_t>> MLPPSoftmaxNet::Evaluate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg;
MLPPActivation avn;
std::vector<std::vector<double>> z2 = alg.mat_vec_add(alg.matmult(X, weights1), bias1);
std::vector<std::vector<double>> a2 = avn.sigmoid(z2);
std::vector<std::vector<real_t>> z2 = alg.mat_vec_add(alg.matmult(X, weights1), bias1);
std::vector<std::vector<real_t>> a2 = avn.sigmoid(z2);
return avn.adjSoftmax(alg.mat_vec_add(alg.matmult(a2, weights2), bias2));
}
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> MLPPSoftmaxNet::propagate(std::vector<std::vector<double>> X) {
std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> MLPPSoftmaxNet::propagate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg;
MLPPActivation avn;
std::vector<std::vector<double>> z2 = alg.mat_vec_add(alg.matmult(X, weights1), bias1);
std::vector<std::vector<double>> a2 = avn.sigmoid(z2);
std::vector<std::vector<real_t>> z2 = alg.mat_vec_add(alg.matmult(X, weights1), bias1);
std::vector<std::vector<real_t>> a2 = avn.sigmoid(z2);
return { z2, a2 };
}
std::vector<double> MLPPSoftmaxNet::Evaluate(std::vector<double> x) {
std::vector<real_t> MLPPSoftmaxNet::Evaluate(std::vector<real_t> x) {
MLPPLinAlg alg;
MLPPActivation avn;
std::vector<double> z2 = alg.addition(alg.mat_vec_mult(alg.transpose(weights1), x), bias1);
std::vector<double> a2 = avn.sigmoid(z2);
std::vector<real_t> z2 = alg.addition(alg.mat_vec_mult(alg.transpose(weights1), x), bias1);
std::vector<real_t> a2 = avn.sigmoid(z2);
return avn.adjSoftmax(alg.addition(alg.mat_vec_mult(alg.transpose(weights2), a2), bias2));
}
std::tuple<std::vector<double>, std::vector<double>> MLPPSoftmaxNet::propagate(std::vector<double> x) {
std::tuple<std::vector<real_t>, std::vector<real_t>> MLPPSoftmaxNet::propagate(std::vector<real_t> x) {
MLPPLinAlg alg;
MLPPActivation avn;
std::vector<double> z2 = alg.addition(alg.mat_vec_mult(alg.transpose(weights1), x), bias1);
std::vector<double> a2 = avn.sigmoid(z2);
std::vector<real_t> z2 = alg.addition(alg.mat_vec_mult(alg.transpose(weights1), x), bias1);
std::vector<real_t> a2 = avn.sigmoid(z2);
return { z2, a2 };
}

50
mlpp/softmax_net/softmax_net.h
View File

@ -8,6 +8,8 @@
// Created by Marc Melikyan on 10/2/20.
//
#include "core/math/math_defs.h"
#include <string>
#include <vector>
@ -15,37 +17,37 @@
class MLPPSoftmaxNet {
public:
MLPPSoftmaxNet(std::vector<std::vector<double>> inputSet, std::vector<std::vector<double>> outputSet, int n_hidden, std::string reg = "None", double lambda = 0.5, double alpha = 0.5);
std::vector<double> modelTest(std::vector<double> x);
std::vector<std::vector<double>> modelSetTest(std::vector<std::vector<double>> X);
void gradientDescent(double learning_rate, int max_epoch, bool UI = 1);
void SGD(double learning_rate, int max_epoch, bool UI = 1);
void MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
double score();
MLPPSoftmaxNet(std::vector<std::vector<real_t>> inputSet, std::vector<std::vector<real_t>> outputSet, int n_hidden, std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
std::vector<real_t> modelTest(std::vector<real_t> x);
std::vector<std::vector<real_t>> modelSetTest(std::vector<std::vector<real_t>> X);
void gradientDescent(real_t learning_rate, int max_epoch, bool UI = 1);
void SGD(real_t learning_rate, int max_epoch, bool UI = 1);
void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
real_t score();
void save(std::string fileName);
std::vector<std::vector<double>> getEmbeddings(); // This class is used (mostly) for word2Vec. This function returns our embeddings.
std::vector<std::vector<real_t>> getEmbeddings(); // This class is used (mostly) for word2Vec. This function returns our embeddings.
private:
double Cost(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y);
real_t Cost(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
std::vector<std::vector<double>> Evaluate(std::vector<std::vector<double>> X);
std::tuple<std::vector<std::vector<double>>, std::vector<std::vector<double>>> propagate(std::vector<std::vector<double>> X);
std::vector<double> Evaluate(std::vector<double> x);
std::tuple<std::vector<double>, std::vector<double>> propagate(std::vector<double> x);
std::vector<std::vector<real_t>> Evaluate(std::vector<std::vector<real_t>> X);
std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> propagate(std::vector<std::vector<real_t>> X);
std::vector<real_t> Evaluate(std::vector<real_t> x);
std::tuple<std::vector<real_t>, std::vector<real_t>> propagate(std::vector<real_t> x);
void forwardPass();
std::vector<std::vector<double>> inputSet;
std::vector<std::vector<double>> outputSet;
std::vector<std::vector<double>> y_hat;
std::vector<std::vector<real_t>> inputSet;
std::vector<std::vector<real_t>> outputSet;
std::vector<std::vector<real_t>> y_hat;
std::vector<std::vector<double>> weights1;
std::vector<std::vector<double>> weights2;
std::vector<std::vector<real_t>> weights1;
std::vector<std::vector<real_t>> weights2;
std::vector<double> bias1;
std::vector<double> bias2;
std::vector<real_t> bias1;
std::vector<real_t> bias2;
std::vector<std::vector<double>> z2;
std::vector<std::vector<double>> a2;
std::vector<std::vector<real_t>> z2;
std::vector<std::vector<real_t>> a2;
int n;
int k;
@ -54,8 +56,8 @@ private:
// Regularization Params
std::string reg;
double lambda;
double alpha; /* This is the controlling param for Elastic Net*/
real_t lambda;
real_t alpha; /* This is the controlling param for Elastic Net*/
};

46
mlpp/softmax_reg/softmax_reg.cpp
View File

@ -15,41 +15,41 @@
#include <random>
MLPPSoftmaxReg::MLPPSoftmaxReg(std::vector<std::vector<double>> inputSet, std::vector<std::vector<double>> outputSet, std::string reg, double lambda, double alpha) :
MLPPSoftmaxReg::MLPPSoftmaxReg(std::vector<std::vector<real_t>> inputSet, std::vector<std::vector<real_t>> outputSet, std::string reg, real_t lambda, real_t alpha) :
inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), n_class(outputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) {
y_hat.resize(n);
weights = MLPPUtilities::weightInitialization(k, n_class);
bias = MLPPUtilities::biasInitialization(n_class);
}
std::vector<double> MLPPSoftmaxReg::modelTest(std::vector<double> x) {
std::vector<real_t> MLPPSoftmaxReg::modelTest(std::vector<real_t> x) {
return Evaluate(x);
}
std::vector<std::vector<double>> MLPPSoftmaxReg::modelSetTest(std::vector<std::vector<double>> X) {
std::vector<std::vector<real_t>> MLPPSoftmaxReg::modelSetTest(std::vector<std::vector<real_t>> X) {
return Evaluate(X);
}
void MLPPSoftmaxReg::gradientDescent(double learning_rate, int max_epoch, bool UI) {
void MLPPSoftmaxReg::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
forwardPass();
while (true) {
cost_prev = Cost(y_hat, outputSet);
std::vector<std::vector<double>> error = alg.subtraction(y_hat, outputSet);
std::vector<std::vector<real_t>> error = alg.subtraction(y_hat, outputSet);
//Calculating the weight gradients
std::vector<std::vector<double>> w_gradient = alg.matmult(alg.transpose(inputSet), error);
std::vector<std::vector<real_t>> w_gradient = alg.matmult(alg.transpose(inputSet), error);
//Weight updation
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate, w_gradient));
weights = regularization.regWeights(weights, lambda, alpha, reg);
// Calculating the bias gradients
//double b_gradient = alg.sum_elements(error);
//real_t b_gradient = alg.sum_elements(error);
// Bias Updation
bias = alg.subtractMatrixRows(bias, alg.scalarMultiply(learning_rate, error));
@ -69,30 +69,30 @@ void MLPPSoftmaxReg::gradientDescent(double learning_rate, int max_epoch, bool U
}
}
void MLPPSoftmaxReg::SGD(double learning_rate, int max_epoch, bool UI) {
void MLPPSoftmaxReg::SGD(real_t learning_rate, int max_epoch, bool UI) {
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
while (true) {
std::random_device rd;
std::default_random_engine generator(rd());
std::uniform_int_distribution<int> distribution(0, int(n - 1));
double outputIndex = distribution(generator);
real_t outputIndex = distribution(generator);
std::vector<double> y_hat = Evaluate(inputSet[outputIndex]);
std::vector<real_t> y_hat = Evaluate(inputSet[outputIndex]);
cost_prev = Cost({ y_hat }, { outputSet[outputIndex] });
// Calculating the weight gradients
std::vector<std::vector<double>> w_gradient = alg.outerProduct(inputSet[outputIndex], alg.subtraction(y_hat, outputSet[outputIndex]));
std::vector<std::vector<real_t>> w_gradient = alg.outerProduct(inputSet[outputIndex], alg.subtraction(y_hat, outputSet[outputIndex]));
// Weight Updation
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate, w_gradient));
weights = regularization.regWeights(weights, lambda, alpha, reg);
// Calculating the bias gradients
std::vector<double> b_gradient = alg.subtraction(y_hat, outputSet[outputIndex]);
std::vector<real_t> b_gradient = alg.subtraction(y_hat, outputSet[outputIndex]);
// Bias updation
bias = alg.subtraction(bias, alg.scalarMultiply(learning_rate, b_gradient));
@ -112,10 +112,10 @@ void MLPPSoftmaxReg::SGD(double learning_rate, int max_epoch, bool UI) {
forwardPass();
}
void MLPPSoftmaxReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI) {
void MLPPSoftmaxReg::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI) {
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
// Creating the mini-batches
@ -124,13 +124,13 @@ void MLPPSoftmaxReg::MBGD(double learning_rate, int max_epoch, int mini_batch_si
while (true) {
for (int i = 0; i < n_mini_batch; i++) {
std::vector<std::vector<double>> y_hat = Evaluate(inputMiniBatches[i]);
std::vector<std::vector<real_t>> y_hat = Evaluate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]);
std::vector<std::vector<double>> error = alg.subtraction(y_hat, outputMiniBatches[i]);
std::vector<std::vector<real_t>> error = alg.subtraction(y_hat, outputMiniBatches[i]);
// Calculating the weight gradients
std::vector<std::vector<double>> w_gradient = alg.matmult(alg.transpose(inputMiniBatches[i]), error);
std::vector<std::vector<real_t>> w_gradient = alg.matmult(alg.transpose(inputMiniBatches[i]), error);
//Weight updation
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate, w_gradient));
@ -153,7 +153,7 @@ void MLPPSoftmaxReg::MBGD(double learning_rate, int max_epoch, int mini_batch_si
forwardPass();
}
double MLPPSoftmaxReg::score() {
real_t MLPPSoftmaxReg::score() {
MLPPUtilities util;
return util.performance(y_hat, outputSet);
}
@ -163,19 +163,19 @@ void MLPPSoftmaxReg::save(std::string fileName) {
util.saveParameters(fileName, weights, bias);
}
double MLPPSoftmaxReg::Cost(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) {
real_t MLPPSoftmaxReg::Cost(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
MLPPReg regularization;
class MLPPCost cost;
return cost.CrossEntropy(y_hat, y) + regularization.regTerm(weights, lambda, alpha, reg);
}
std::vector<double> MLPPSoftmaxReg::Evaluate(std::vector<double> x) {
std::vector<real_t> MLPPSoftmaxReg::Evaluate(std::vector<real_t> x) {
MLPPLinAlg alg;
MLPPActivation avn;
return avn.softmax(alg.addition(bias, alg.mat_vec_mult(alg.transpose(weights), x)));
}
std::vector<std::vector<double>> MLPPSoftmaxReg::Evaluate(std::vector<std::vector<double>> X) {
std::vector<std::vector<real_t>> MLPPSoftmaxReg::Evaluate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg;
MLPPActivation avn;

36
mlpp/softmax_reg/softmax_reg.h
View File

@ -8,6 +8,8 @@
// Created by Marc Melikyan on 10/2/20.
//
#include "core/math/math_defs.h"
#include <string>
#include <vector>
@ -15,27 +17,27 @@
class MLPPSoftmaxReg {
public:
MLPPSoftmaxReg(std::vector<std::vector<double>> inputSet, std::vector<std::vector<double>> outputSet, std::string reg = "None", double lambda = 0.5, double alpha = 0.5);
std::vector<double> modelTest(std::vector<double> x);
std::vector<std::vector<double>> modelSetTest(std::vector<std::vector<double>> X);
void gradientDescent(double learning_rate, int max_epoch, bool UI = 1);
void SGD(double learning_rate, int max_epoch, bool UI = 1);
void MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
double score();
MLPPSoftmaxReg(std::vector<std::vector<real_t>> inputSet, std::vector<std::vector<real_t>> outputSet, std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
std::vector<real_t> modelTest(std::vector<real_t> x);
std::vector<std::vector<real_t>> modelSetTest(std::vector<std::vector<real_t>> X);
void gradientDescent(real_t learning_rate, int max_epoch, bool UI = 1);
void SGD(real_t learning_rate, int max_epoch, bool UI = 1);
void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
real_t score();
void save(std::string fileName);
private:
double Cost(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y);
real_t Cost(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
std::vector<std::vector<double>> Evaluate(std::vector<std::vector<double>> X);
std::vector<double> Evaluate(std::vector<double> x);
std::vector<std::vector<real_t>> Evaluate(std::vector<std::vector<real_t>> X);
std::vector<real_t> Evaluate(std::vector<real_t> x);
void forwardPass();
std::vector<std::vector<double>> inputSet;
std::vector<std::vector<double>> outputSet;
std::vector<std::vector<double>> y_hat;
std::vector<std::vector<double>> weights;
std::vector<double> bias;
std::vector<std::vector<real_t>> inputSet;
std::vector<std::vector<real_t>> outputSet;
std::vector<std::vector<real_t>> y_hat;
std::vector<std::vector<real_t>> weights;
std::vector<real_t> bias;
int n;
int k;
@ -43,8 +45,8 @@ private:
// Regularization Params
std::string reg;
double lambda;
double alpha; /* This is the controlling param for Elastic Net*/
real_t lambda;
real_t alpha; /* This is the controlling param for Elastic Net*/
};

94
mlpp/stat/stat.cpp
View File

@ -15,24 +15,24 @@
#include <iostream>
double MLPPStat::b0Estimation(const std::vector<double> &x, const std::vector<double> &y) {
real_t MLPPStat::b0Estimation(const std::vector<real_t> &x, const std::vector<real_t> &y) {
return mean(y) - b1Estimation(x, y) * mean(x);
}
double MLPPStat::b1Estimation(const std::vector<double> &x, const std::vector<double> &y) {
real_t MLPPStat::b1Estimation(const std::vector<real_t> &x, const std::vector<real_t> &y) {
return covariance(x, y) / variance(x);
}
double MLPPStat::mean(const std::vector<double> &x) {
double sum = 0;
real_t MLPPStat::mean(const std::vector<real_t> &x) {
real_t sum = 0;
for (int i = 0; i < x.size(); i++) {
sum += x[i];
}
return sum / x.size();
}
double MLPPStat::median(std::vector<double> x) {
double center = double(x.size()) / double(2);
real_t MLPPStat::median(std::vector<real_t> x) {
real_t center = real_t(x.size()) / real_t(2);
sort(x.begin(), x.end());
if (x.size() % 2 == 0) {
return mean({ x[center - 1], x[center] });
@ -41,18 +41,18 @@ double MLPPStat::median(std::vector<double> x) {
}
}
std::vector<double> MLPPStat::mode(const std::vector<double> &x) {
std::vector<real_t> MLPPStat::mode(const std::vector<real_t> &x) {
MLPPData data;
std::vector<double> x_set = data.vecToSet(x);
std::map<double, int> element_num;
std::vector<real_t> x_set = data.vecToSet(x);
std::map<real_t, int> element_num;
for (int i = 0; i < x_set.size(); i++) {
element_num[x[i]] = 0;
}
for (int i = 0; i < x.size(); i++) {
element_num[x[i]]++;
}
std::vector<double> modes;
double max_num = element_num[x_set[0]];
std::vector<real_t> modes;
real_t max_num = element_num[x_set[0]];
for (int i = 0; i < x_set.size(); i++) {
if (element_num[x_set[i]] > max_num) {
max_num = element_num[x_set[i]];
@ -65,59 +65,59 @@ std::vector<double> MLPPStat::mode(const std::vector<double> &x) {
return modes;
}
double MLPPStat::range(const std::vector<double> &x) {
real_t MLPPStat::range(const std::vector<real_t> &x) {
MLPPLinAlg alg;
return alg.max(x) - alg.min(x);
}
double MLPPStat::midrange(const std::vector<double> &x) {
real_t MLPPStat::midrange(const std::vector<real_t> &x) {
return range(x) / 2;
}
double MLPPStat::absAvgDeviation(const std::vector<double> &x) {
double sum = 0;
real_t MLPPStat::absAvgDeviation(const std::vector<real_t> &x) {
real_t sum = 0;
for (int i = 0; i < x.size(); i++) {
sum += std::abs(x[i] - mean(x));
}
return sum / x.size();
}
double MLPPStat::standardDeviation(const std::vector<double> &x) {
real_t MLPPStat::standardDeviation(const std::vector<real_t> &x) {
return std::sqrt(variance(x));
}
double MLPPStat::variance(const std::vector<double> &x) {
double sum = 0;
real_t MLPPStat::variance(const std::vector<real_t> &x) {
real_t sum = 0;
for (int i = 0; i < x.size(); i++) {
sum += (x[i] - mean(x)) * (x[i] - mean(x));
}
return sum / (x.size() - 1);
}
double MLPPStat::covariance(const std::vector<double> &x, const std::vector<double> &y) {
double sum = 0;
real_t MLPPStat::covariance(const std::vector<real_t> &x, const std::vector<real_t> &y) {
real_t sum = 0;
for (int i = 0; i < x.size(); i++) {
sum += (x[i] - mean(x)) * (y[i] - mean(y));
}
return sum / (x.size() - 1);
}
double MLPPStat::correlation(const std::vector<double> &x, const std::vector<double> &y) {
real_t MLPPStat::correlation(const std::vector<real_t> &x, const std::vector<real_t> &y) {
return covariance(x, y) / (standardDeviation(x) * standardDeviation(y));
}
double MLPPStat::R2(const std::vector<double> &x, const std::vector<double> &y) {
real_t MLPPStat::R2(const std::vector<real_t> &x, const std::vector<real_t> &y) {
return correlation(x, y) * correlation(x, y);
}
double MLPPStat::chebyshevIneq(const double k) {
real_t MLPPStat::chebyshevIneq(const real_t k) {
// X may or may not belong to a Gaussian Distribution
return 1 - 1 / (k * k);
}
double MLPPStat::weightedMean(const std::vector<double> &x, const std::vector<double> &weights) {
double sum = 0;
double weights_sum = 0;
real_t MLPPStat::weightedMean(const std::vector<real_t> &x, const std::vector<real_t> &weights) {
real_t sum = 0;
real_t weights_sum = 0;
for (int i = 0; i < x.size(); i++) {
sum += x[i] * weights[i];
weights_sum += weights[i];
@ -125,41 +125,41 @@ double MLPPStat::weightedMean(const std::vector<double> &x, const std::vector<do
return sum / weights_sum;
}
double MLPPStat::geometricMean(const std::vector<double> &x) {
double product = 1;
real_t MLPPStat::geometricMean(const std::vector<real_t> &x) {
real_t product = 1;
for (int i = 0; i < x.size(); i++) {
product *= x[i];
}
return std::pow(product, 1.0 / x.size());
}
double MLPPStat::harmonicMean(const std::vector<double> &x) {
double sum = 0;
real_t MLPPStat::harmonicMean(const std::vector<real_t> &x) {
real_t sum = 0;
for (int i = 0; i < x.size(); i++) {
sum += 1 / x[i];
}
return x.size() / sum;
}
double MLPPStat::RMS(const std::vector<double> &x) {
double sum = 0;
real_t MLPPStat::RMS(const std::vector<real_t> &x) {
real_t sum = 0;
for (int i = 0; i < x.size(); i++) {
sum += x[i] * x[i];
}
return sqrt(sum / x.size());
}
double MLPPStat::powerMean(const std::vector<double> &x, const double p) {
double sum = 0;
real_t MLPPStat::powerMean(const std::vector<real_t> &x, const real_t p) {
real_t sum = 0;
for (int i = 0; i < x.size(); i++) {
sum += std::pow(x[i], p);
}
return std::pow(sum / x.size(), 1 / p);
}
double MLPPStat::lehmerMean(const std::vector<double> &x, const double p) {
double num = 0;
double den = 0;
real_t MLPPStat::lehmerMean(const std::vector<real_t> &x, const real_t p) {
real_t num = 0;
real_t den = 0;
for (int i = 0; i < x.size(); i++) {
num += std::pow(x[i], p);
den += std::pow(x[i], p - 1);
@ -167,9 +167,9 @@ double MLPPStat::lehmerMean(const std::vector<double> &x, const double p) {
return num / den;
}
double MLPPStat::weightedLehmerMean(const std::vector<double> &x, const std::vector<double> &weights, const double p) {
double num = 0;
double den = 0;
real_t MLPPStat::weightedLehmerMean(const std::vector<real_t> &x, const std::vector<real_t> &weights, const real_t p) {
real_t num = 0;
real_t den = 0;
for (int i = 0; i < x.size(); i++) {
num += weights[i] * std::pow(x[i], p);
den += weights[i] * std::pow(x[i], p - 1);
@ -177,38 +177,38 @@ double MLPPStat::weightedLehmerMean(const std::vector<double> &x, const std::vec
return num / den;
}
double MLPPStat::heronianMean(const double A, const double B) {
real_t MLPPStat::heronianMean(const real_t A, const real_t B) {
return (A + sqrt(A * B) + B) / 3;
}
double MLPPStat::contraHarmonicMean(const std::vector<double> &x) {
real_t MLPPStat::contraHarmonicMean(const std::vector<real_t> &x) {
return lehmerMean(x, 2);
}
double MLPPStat::heinzMean(const double A, const double B, const double x) {
real_t MLPPStat::heinzMean(const real_t A, const real_t B, const real_t x) {
return (std::pow(A, x) * std::pow(B, 1 - x) + std::pow(A, 1 - x) * std::pow(B, x)) / 2;
}
double MLPPStat::neumanSandorMean(const double a, const double b) {
real_t MLPPStat::neumanSandorMean(const real_t a, const real_t b) {
MLPPActivation avn;
return (a - b) / 2 * avn.arsinh((a - b) / (a + b));
}
double MLPPStat::stolarskyMean(const double x, const double y, const double p) {
real_t MLPPStat::stolarskyMean(const real_t x, const real_t y, const real_t p) {
if (x == y) {
return x;
}
return std::pow((std::pow(x, p) - std::pow(y, p)) / (p * (x - y)), 1 / (p - 1));
}
double MLPPStat::identricMean(const double x, const double y) {
real_t MLPPStat::identricMean(const real_t x, const real_t y) {
if (x == y) {
return x;
}
return (1 / M_E) * std::pow(std::pow(x, x) / std::pow(y, y), 1 / (x - y));
}
double MLPPStat::logMean(const double x, const double y) {
real_t MLPPStat::logMean(const real_t x, const real_t y) {
if (x == y) {
return x;
}

58
mlpp/stat/stat.h
View File

@ -8,44 +8,46 @@
// Created by Marc Melikyan on 9/29/20.
//
#include "core/math/math_defs.h"
#include <vector>
class MLPPStat {
public:
// These functions are for univariate lin reg module- not for users.
double b0Estimation(const std::vector<double> &x, const std::vector<double> &y);
double b1Estimation(const std::vector<double> &x, const std::vector<double> &y);
real_t b0Estimation(const std::vector<real_t> &x, const std::vector<real_t> &y);
real_t b1Estimation(const std::vector<real_t> &x, const std::vector<real_t> &y);
// Statistical Functions
double mean(const std::vector<double> &x);
double median(std::vector<double> x);
std::vector<double> mode(const std::vector<double> &x);
double range(const std::vector<double> &x);
double midrange(const std::vector<double> &x);
double absAvgDeviation(const std::vector<double> &x);
double standardDeviation(const std::vector<double> &x);
double variance(const std::vector<double> &x);
double covariance(const std::vector<double> &x, const std::vector<double> &y);
double correlation(const std::vector<double> &x, const std::vector<double> &y);
double R2(const std::vector<double> &x, const std::vector<double> &y);
double chebyshevIneq(const double k);
real_t mean(const std::vector<real_t> &x);
real_t median(std::vector<real_t> x);
std::vector<real_t> mode(const std::vector<real_t> &x);
real_t range(const std::vector<real_t> &x);
real_t midrange(const std::vector<real_t> &x);
real_t absAvgDeviation(const std::vector<real_t> &x);
real_t standardDeviation(const std::vector<real_t> &x);
real_t variance(const std::vector<real_t> &x);
real_t covariance(const std::vector<real_t> &x, const std::vector<real_t> &y);
real_t correlation(const std::vector<real_t> &x, const std::vector<real_t> &y);
real_t R2(const std::vector<real_t> &x, const std::vector<real_t> &y);
real_t chebyshevIneq(const real_t k);
// Extras
double weightedMean(const std::vector<double> &x, const std::vector<double> &weights);
double geometricMean(const std::vector<double> &x);
double harmonicMean(const std::vector<double> &x);
double RMS(const std::vector<double> &x);
double powerMean(const std::vector<double> &x, const double p);
double lehmerMean(const std::vector<double> &x, const double p);
double weightedLehmerMean(const std::vector<double> &x, const std::vector<double> &weights, const double p);
double contraHarmonicMean(const std::vector<double> &x);
double heronianMean(const double A, const double B);
double heinzMean(const double A, const double B, const double x);
double neumanSandorMean(const double a, const double b);
double stolarskyMean(const double x, const double y, const double p);
double identricMean(const double x, const double y);
double logMean(const double x, const double y);
real_t weightedMean(const std::vector<real_t> &x, const std::vector<real_t> &weights);
real_t geometricMean(const std::vector<real_t> &x);
real_t harmonicMean(const std::vector<real_t> &x);
real_t RMS(const std::vector<real_t> &x);
real_t powerMean(const std::vector<real_t> &x, const real_t p);
real_t lehmerMean(const std::vector<real_t> &x, const real_t p);
real_t weightedLehmerMean(const std::vector<real_t> &x, const std::vector<real_t> &weights, const real_t p);
real_t contraHarmonicMean(const std::vector<real_t> &x);
real_t heronianMean(const real_t A, const real_t B);
real_t heinzMean(const real_t A, const real_t B, const real_t x);
real_t neumanSandorMean(const real_t a, const real_t b);
real_t stolarskyMean(const real_t x, const real_t y, const real_t p);
real_t identricMean(const real_t x, const real_t y);
real_t logMean(const real_t x, const real_t y);
};

40
mlpp/svc/svc.cpp
View File

@ -15,27 +15,27 @@
#include <random>
MLPPSVC::MLPPSVC(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, double C) :
MLPPSVC::MLPPSVC(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, real_t C) :
inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), C(C) {
y_hat.resize(n);
weights = MLPPUtilities::weightInitialization(k);
bias = MLPPUtilities::biasInitialization();
}
std::vector<double> MLPPSVC::modelSetTest(std::vector<std::vector<double>> X) {
std::vector<real_t> MLPPSVC::modelSetTest(std::vector<std::vector<real_t>> X) {
return Evaluate(X);
}
double MLPPSVC::modelTest(std::vector<double> x) {
real_t MLPPSVC::modelTest(std::vector<real_t> x) {
return Evaluate(x);
}
void MLPPSVC::gradientDescent(double learning_rate, int max_epoch, bool UI) {
void MLPPSVC::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
class MLPPCost cost;
MLPPActivation avn;
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
forwardPass();
@ -63,13 +63,13 @@ void MLPPSVC::gradientDescent(double learning_rate, int max_epoch, bool UI) {
}
}
void MLPPSVC::SGD(double learning_rate, int max_epoch, bool UI) {
void MLPPSVC::SGD(real_t learning_rate, int max_epoch, bool UI) {
class MLPPCost cost;
MLPPActivation avn;
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
while (true) {
@ -78,11 +78,11 @@ void MLPPSVC::SGD(double learning_rate, int max_epoch, bool UI) {
std::uniform_int_distribution<int> distribution(0, int(n - 1));
int outputIndex = distribution(generator);
double y_hat = Evaluate(inputSet[outputIndex]);
double z = propagate(inputSet[outputIndex]);
real_t y_hat = Evaluate(inputSet[outputIndex]);
real_t z = propagate(inputSet[outputIndex]);
cost_prev = Cost({ z }, { outputSet[outputIndex] }, weights, C);
double costDeriv = cost.HingeLossDeriv(std::vector<double>({ z }), std::vector<double>({ outputSet[outputIndex] }), C)[0]; // Explicit conversion to avoid ambiguity with overloaded function. Error occured on Ubuntu.
real_t costDeriv = cost.HingeLossDeriv(std::vector<real_t>({ z }), std::vector<real_t>({ outputSet[outputIndex] }), C)[0]; // Explicit conversion to avoid ambiguity with overloaded function. Error occured on Ubuntu.
// Weight Updation
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate * costDeriv, inputSet[outputIndex]));
@ -106,12 +106,12 @@ void MLPPSVC::SGD(double learning_rate, int max_epoch, bool UI) {
forwardPass();
}
void MLPPSVC::MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI) {
void MLPPSVC::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI) {
class MLPPCost cost;
MLPPActivation avn;
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
// Creating the mini-batches
@ -120,8 +120,8 @@ void MLPPSVC::MBGD(double learning_rate, int max_epoch, int mini_batch_size, boo
while (true) {
for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = Evaluate(inputMiniBatches[i]);
std::vector<double> z = propagate(inputMiniBatches[i]);
std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]);
std::vector<real_t> z = propagate(inputMiniBatches[i]);
cost_prev = Cost(z, outputMiniBatches[i], weights, C);
// Calculating the weight gradients
@ -148,7 +148,7 @@ void MLPPSVC::MBGD(double learning_rate, int max_epoch, int mini_batch_size, boo
forwardPass();
}
double MLPPSVC::score() {
real_t MLPPSVC::score() {
MLPPUtilities util;
return util.performance(y_hat, outputSet);
}
@ -158,30 +158,30 @@ void MLPPSVC::save(std::string fileName) {
util.saveParameters(fileName, weights, bias);
}
double MLPPSVC::Cost(std::vector<double> z, std::vector<double> y, std::vector<double> weights, double C) {
real_t MLPPSVC::Cost(std::vector<real_t> z, std::vector<real_t> y, std::vector<real_t> weights, real_t C) {
class MLPPCost cost;
return cost.HingeLoss(z, y, weights, C);
}
std::vector<double> MLPPSVC::Evaluate(std::vector<std::vector<double>> X) {
std::vector<real_t> MLPPSVC::Evaluate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg;
MLPPActivation avn;
return avn.sign(alg.scalarAdd(bias, alg.mat_vec_mult(X, weights)));
}
std::vector<double> MLPPSVC::propagate(std::vector<std::vector<double>> X) {
std::vector<real_t> MLPPSVC::propagate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg;
MLPPActivation avn;
return alg.scalarAdd(bias, alg.mat_vec_mult(X, weights));
}
double MLPPSVC::Evaluate(std::vector<double> x) {
real_t MLPPSVC::Evaluate(std::vector<real_t> x) {
MLPPLinAlg alg;
MLPPActivation avn;
return avn.sign(alg.dot(weights, x) + bias);
}
double MLPPSVC::propagate(std::vector<double> x) {
real_t MLPPSVC::propagate(std::vector<real_t> x) {
MLPPLinAlg alg;
MLPPActivation avn;
return alg.dot(weights, x) + bias;

42
mlpp/svc/svc.h
View File

@ -11,6 +11,8 @@
// https://towardsdatascience.com/svm-implementation-from-scratch-python-2db2fc52e5c2
// Illustratd a practical definition of the Hinge Loss function and its gradient when optimizing with SGD.
#include "core/math/math_defs.h"
#include <string>
#include <vector>
@ -18,37 +20,37 @@
class MLPPSVC {
public:
MLPPSVC(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, double C);
std::vector<double> modelSetTest(std::vector<std::vector<double>> X);
double modelTest(std::vector<double> x);
void gradientDescent(double learning_rate, int max_epoch, bool UI = 1);
void SGD(double learning_rate, int max_epoch, bool UI = 1);
void MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
double score();
MLPPSVC(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, real_t C);
std::vector<real_t> modelSetTest(std::vector<std::vector<real_t>> X);
real_t modelTest(std::vector<real_t> x);
void gradientDescent(real_t learning_rate, int max_epoch, bool UI = 1);
void SGD(real_t learning_rate, int max_epoch, bool UI = 1);
void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
real_t score();
void save(std::string fileName);
private:
double Cost(std::vector<double> y_hat, std::vector<double> y, std::vector<double> weights, double C);
real_t Cost(std::vector<real_t> y_hat, std::vector<real_t> y, std::vector<real_t> weights, real_t C);
std::vector<double> Evaluate(std::vector<std::vector<double>> X);
std::vector<double> propagate(std::vector<std::vector<double>> X);
double Evaluate(std::vector<double> x);
double propagate(std::vector<double> x);
std::vector<real_t> Evaluate(std::vector<std::vector<real_t>> X);
std::vector<real_t> propagate(std::vector<std::vector<real_t>> X);
real_t Evaluate(std::vector<real_t> x);
real_t propagate(std::vector<real_t> x);
void forwardPass();
std::vector<std::vector<double>> inputSet;
std::vector<double> outputSet;
std::vector<double> z;
std::vector<double> y_hat;
std::vector<double> weights;
double bias;
std::vector<std::vector<real_t>> inputSet;
std::vector<real_t> outputSet;
std::vector<real_t> z;
std::vector<real_t> y_hat;
std::vector<real_t> weights;
real_t bias;
double C;
real_t C;
int n;
int k;
// UI Portion
void UI(int epoch, double cost_prev);
void UI(int epoch, real_t cost_prev);
};

42
mlpp/tanh_reg/tanh_reg.cpp
View File

@ -15,33 +15,33 @@
#include <random>
MLPPTanhReg::MLPPTanhReg(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, std::string reg, double lambda, double alpha) :
MLPPTanhReg::MLPPTanhReg(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, std::string reg, real_t lambda, real_t alpha) :
inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) {
y_hat.resize(n);
weights = MLPPUtilities::weightInitialization(k);
bias = MLPPUtilities::biasInitialization();
}
std::vector<double> MLPPTanhReg::modelSetTest(std::vector<std::vector<double>> X) {
std::vector<real_t> MLPPTanhReg::modelSetTest(std::vector<std::vector<real_t>> X) {
return Evaluate(X);
}
double MLPPTanhReg::modelTest(std::vector<double> x) {
real_t MLPPTanhReg::modelTest(std::vector<real_t> x) {
return Evaluate(x);
}
void MLPPTanhReg::gradientDescent(double learning_rate, int max_epoch, bool UI) {
void MLPPTanhReg::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
MLPPActivation avn;
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
forwardPass();
while (true) {
cost_prev = Cost(y_hat, outputSet);
std::vector<double> error = alg.subtraction(y_hat, outputSet);
std::vector<real_t> error = alg.subtraction(y_hat, outputSet);
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputSet), alg.hadamard_product(error, avn.tanh(z, 1)))));
weights = regularization.regWeights(weights, lambda, alpha, reg);
@ -64,10 +64,10 @@ void MLPPTanhReg::gradientDescent(double learning_rate, int max_epoch, bool UI)
}
}
void MLPPTanhReg::SGD(double learning_rate, int max_epoch, bool UI) {
void MLPPTanhReg::SGD(real_t learning_rate, int max_epoch, bool UI) {
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
while (true) {
@ -76,10 +76,10 @@ void MLPPTanhReg::SGD(double learning_rate, int max_epoch, bool UI) {
std::uniform_int_distribution<int> distribution(0, int(n - 1));
int outputIndex = distribution(generator);
double y_hat = Evaluate(inputSet[outputIndex]);
real_t y_hat = Evaluate(inputSet[outputIndex]);
cost_prev = Cost({ y_hat }, { outputSet[outputIndex] });
double error = y_hat - outputSet[outputIndex];
real_t error = y_hat - outputSet[outputIndex];
// Weight Updation
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate * error * (1 - y_hat * y_hat), inputSet[outputIndex]));
@ -103,11 +103,11 @@ void MLPPTanhReg::SGD(double learning_rate, int max_epoch, bool UI) {
forwardPass();
}
void MLPPTanhReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI) {
void MLPPTanhReg::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI) {
MLPPActivation avn;
MLPPLinAlg alg;
MLPPReg regularization;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
// Creating the mini-batches
@ -116,11 +116,11 @@ void MLPPTanhReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size,
while (true) {
for (int i = 0; i < n_mini_batch; i++) {
std::vector<double> y_hat = Evaluate(inputMiniBatches[i]);
std::vector<double> z = propagate(inputMiniBatches[i]);
std::vector<real_t> y_hat = Evaluate(inputMiniBatches[i]);
std::vector<real_t> z = propagate(inputMiniBatches[i]);
cost_prev = Cost(y_hat, outputMiniBatches[i]);
std::vector<double> error = alg.subtraction(y_hat, outputMiniBatches[i]);
std::vector<real_t> error = alg.subtraction(y_hat, outputMiniBatches[i]);
// Calculating the weight gradients
weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate / n, alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), alg.hadamard_product(error, avn.tanh(z, 1)))));
@ -146,7 +146,7 @@ void MLPPTanhReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size,
forwardPass();
}
double MLPPTanhReg::score() {
real_t MLPPTanhReg::score() {
MLPPUtilities util;
return util.performance(y_hat, outputSet);
}
@ -156,30 +156,30 @@ void MLPPTanhReg::save(std::string fileName) {
util.saveParameters(fileName, weights, bias);
}
double MLPPTanhReg::Cost(std::vector<double> y_hat, std::vector<double> y) {
real_t MLPPTanhReg::Cost(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPReg regularization;
class MLPPCost cost;
return cost.MSE(y_hat, y) + regularization.regTerm(weights, lambda, alpha, reg);
}
std::vector<double> MLPPTanhReg::Evaluate(std::vector<std::vector<double>> X) {
std::vector<real_t> MLPPTanhReg::Evaluate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg;
MLPPActivation avn;
return avn.tanh(alg.scalarAdd(bias, alg.mat_vec_mult(X, weights)));
}
std::vector<double> MLPPTanhReg::propagate(std::vector<std::vector<double>> X) {
std::vector<real_t> MLPPTanhReg::propagate(std::vector<std::vector<real_t>> X) {
MLPPLinAlg alg;
return alg.scalarAdd(bias, alg.mat_vec_mult(X, weights));
}
double MLPPTanhReg::Evaluate(std::vector<double> x) {
real_t MLPPTanhReg::Evaluate(std::vector<real_t> x) {
MLPPLinAlg alg;
MLPPActivation avn;
return avn.tanh(alg.dot(weights, x) + bias);
}
double MLPPTanhReg::propagate(std::vector<double> x) {
real_t MLPPTanhReg::propagate(std::vector<real_t> x) {
MLPPLinAlg alg;
return alg.dot(weights, x) + bias;
}

44
mlpp/tanh_reg/tanh_reg.h
View File

@ -8,6 +8,8 @@
// Created by Marc Melikyan on 10/2/20.
//
#include "core/math/math_defs.h"
#include <string>
#include <vector>
@ -15,41 +17,41 @@
class MLPPTanhReg {
public:
MLPPTanhReg(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, std::string reg = "None", double lambda = 0.5, double alpha = 0.5);
std::vector<double> modelSetTest(std::vector<std::vector<double>> X);
double modelTest(std::vector<double> x);
void gradientDescent(double learning_rate, int max_epoch, bool UI = 1);
void SGD(double learning_rate, int max_epoch, bool UI = 1);
void MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
double score();
MLPPTanhReg(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
std::vector<real_t> modelSetTest(std::vector<std::vector<real_t>> X);
real_t modelTest(std::vector<real_t> x);
void gradientDescent(real_t learning_rate, int max_epoch, bool UI = 1);
void SGD(real_t learning_rate, int max_epoch, bool UI = 1);
void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = 1);
real_t score();
void save(std::string fileName);
private:
double Cost(std::vector<double> y_hat, std::vector<double> y);
real_t Cost(std::vector<real_t> y_hat, std::vector<real_t> y);
std::vector<double> Evaluate(std::vector<std::vector<double>> X);
std::vector<double> propagate(std::vector<std::vector<double>> X);
double Evaluate(std::vector<double> x);
double propagate(std::vector<double> x);
std::vector<real_t> Evaluate(std::vector<std::vector<real_t>> X);
std::vector<real_t> propagate(std::vector<std::vector<real_t>> X);
real_t Evaluate(std::vector<real_t> x);
real_t propagate(std::vector<real_t> x);
void forwardPass();
std::vector<std::vector<double>> inputSet;
std::vector<double> outputSet;
std::vector<double> z;
std::vector<double> y_hat;
std::vector<double> weights;
double bias;
std::vector<std::vector<real_t>> inputSet;
std::vector<real_t> outputSet;
std::vector<real_t> z;
std::vector<real_t> y_hat;
std::vector<real_t> weights;
real_t bias;
int n;
int k;
// UI Portion
void UI(int epoch, double cost_prev);
void UI(int epoch, real_t cost_prev);
// Regularization Params
std::string reg;
double lambda;
double alpha; /* This is the controlling param for Elastic Net*/
real_t lambda;
real_t alpha; /* This is the controlling param for Elastic Net*/
};

14
mlpp/transforms/transforms.cpp
View File

@ -14,11 +14,11 @@
// DCT ii.
// https://www.mathworks.com/help/images/discrete-cosine-transform.html
std::vector<std::vector<double>> MLPPTransforms::discreteCosineTransform(std::vector<std::vector<double>> A) {
std::vector<std::vector<real_t>> MLPPTransforms::discreteCosineTransform(std::vector<std::vector<real_t>> A) {
MLPPLinAlg alg;
A = alg.scalarAdd(-128, A); // Center around 0.
std::vector<std::vector<double>> B;
std::vector<std::vector<real_t>> B;
B.resize(A.size());
for (int i = 0; i < B.size(); i++) {
B[i].resize(A[i].size());
@ -28,18 +28,18 @@ std::vector<std::vector<double>> MLPPTransforms::discreteCosineTransform(std::ve
for (int i = 0; i < B.size(); i++) {
for (int j = 0; j < B[i].size(); j++) {
double sum = 0;
double alphaI;
real_t sum = 0;
real_t alphaI;
if (i == 0) {
alphaI = 1 / std::sqrt(M);
} else {
alphaI = std::sqrt(double(2) / double(M));
alphaI = std::sqrt(real_t(2) / real_t(M));
}
double alphaJ;
real_t alphaJ;
if (j == 0) {
alphaJ = 1 / std::sqrt(M);
} else {
alphaJ = std::sqrt(double(2) / double(M));
alphaJ = std::sqrt(real_t(2) / real_t(M));
}
for (int k = 0; k < B.size(); k++) {

4
mlpp/transforms/transforms.h
View File

@ -7,13 +7,15 @@
//
//
#include "core/math/math_defs.h"
#include <string>
#include <vector>
class MLPPTransforms {
public:
std::vector<std::vector<double>> discreteCosineTransform(std::vector<std::vector<double>> A);
std::vector<std::vector<real_t>> discreteCosineTransform(std::vector<std::vector<real_t>> A);
};

6
mlpp/uni_lin_reg/uni_lin_reg.cpp
View File

@ -16,19 +16,19 @@
// ŷ = b0 + b1x1
MLPPUniLinReg::MLPPUniLinReg(std::vector<double> x, std::vector<double> y) :
MLPPUniLinReg::MLPPUniLinReg(std::vector<real_t> x, std::vector<real_t> y) :
inputSet(x), outputSet(y) {
MLPPStat estimator;
b1 = estimator.b1Estimation(inputSet, outputSet);
b0 = estimator.b0Estimation(inputSet, outputSet);
}
std::vector<double> MLPPUniLinReg::modelSetTest(std::vector<double> x) {
std::vector<real_t> MLPPUniLinReg::modelSetTest(std::vector<real_t> x) {
MLPPLinAlg alg;
return alg.scalarAdd(b0, alg.scalarMultiply(b1, x));
}
double MLPPUniLinReg::modelTest(double input) {
real_t MLPPUniLinReg::modelTest(real_t input) {
return b0 + b1 * input;
}

16
mlpp/uni_lin_reg/uni_lin_reg.h
View File

@ -8,21 +8,23 @@
// Created by Marc Melikyan on 9/29/20.
//
#include "core/math/math_defs.h"
#include <vector>
class MLPPUniLinReg {
public:
MLPPUniLinReg(std::vector<double> x, std::vector<double> y);
std::vector<double> modelSetTest(std::vector<double> x);
double modelTest(double x);
MLPPUniLinReg(std::vector<real_t> x, std::vector<real_t> y);
std::vector<real_t> modelSetTest(std::vector<real_t> x);
real_t modelTest(real_t x);
private:
std::vector<double> inputSet;
std::vector<double> outputSet;
std::vector<real_t> inputSet;
std::vector<real_t> outputSet;
double b0;
double b1;
real_t b0;
real_t b1;
};

116
mlpp/utilities/utilities.cpp
View File

@ -12,81 +12,81 @@
std::vector<double> MLPPUtilities::weightInitialization(int n, std::string type) {
std::vector<real_t> MLPPUtilities::weightInitialization(int n, std::string type) {
std::random_device rd;
std::default_random_engine generator(rd());
std::vector<double> weights;
std::vector<real_t> weights;
for (int i = 0; i < n; i++) {
if (type == "XavierNormal") {
std::normal_distribution<double> distribution(0, sqrt(2 / (n + 1)));
std::normal_distribution<real_t> distribution(0, sqrt(2 / (n + 1)));
weights.push_back(distribution(generator));
} else if (type == "XavierUniform") {
std::uniform_real_distribution<double> distribution(-sqrt(6 / (n + 1)), sqrt(6 / (n + 1)));
std::uniform_real_distribution<real_t> distribution(-sqrt(6 / (n + 1)), sqrt(6 / (n + 1)));
weights.push_back(distribution(generator));
} else if (type == "HeNormal") {
std::normal_distribution<double> distribution(0, sqrt(2 / n));
std::normal_distribution<real_t> distribution(0, sqrt(2 / n));
weights.push_back(distribution(generator));
} else if (type == "HeUniform") {
std::uniform_real_distribution<double> distribution(-sqrt(6 / n), sqrt(6 / n));
std::uniform_real_distribution<real_t> distribution(-sqrt(6 / n), sqrt(6 / n));
weights.push_back(distribution(generator));
} else if (type == "LeCunNormal") {
std::normal_distribution<double> distribution(0, sqrt(1 / n));
std::normal_distribution<real_t> distribution(0, sqrt(1 / n));
weights.push_back(distribution(generator));
} else if (type == "LeCunUniform") {
std::uniform_real_distribution<double> distribution(-sqrt(3 / n), sqrt(3 / n));
std::uniform_real_distribution<real_t> distribution(-sqrt(3 / n), sqrt(3 / n));
weights.push_back(distribution(generator));
} else if (type == "Uniform") {
std::uniform_real_distribution<double> distribution(-1 / sqrt(n), 1 / sqrt(n));
std::uniform_real_distribution<real_t> distribution(-1 / sqrt(n), 1 / sqrt(n));
weights.push_back(distribution(generator));
} else {
std::uniform_real_distribution<double> distribution(0, 1);
std::uniform_real_distribution<real_t> distribution(0, 1);
weights.push_back(distribution(generator));
}
}
return weights;
}
double MLPPUtilities::biasInitialization() {
real_t MLPPUtilities::biasInitialization() {
std::random_device rd;
std::default_random_engine generator(rd());
std::uniform_real_distribution<double> distribution(0, 1);
std::uniform_real_distribution<real_t> distribution(0, 1);
return distribution(generator);
}
std::vector<std::vector<double>> MLPPUtilities::weightInitialization(int n, int m, std::string type) {
std::vector<std::vector<real_t>> MLPPUtilities::weightInitialization(int n, int m, std::string type) {
std::random_device rd;
std::default_random_engine generator(rd());
std::vector<std::vector<double>> weights;
std::vector<std::vector<real_t>> weights;
weights.resize(n);
for (int i = 0; i < n; i++) {
for (int j = 0; j < m; j++) {
if (type == "XavierNormal") {
std::normal_distribution<double> distribution(0, sqrt(2 / (n + m)));
std::normal_distribution<real_t> distribution(0, sqrt(2 / (n + m)));
weights[i].push_back(distribution(generator));
} else if (type == "XavierUniform") {
std::uniform_real_distribution<double> distribution(-sqrt(6 / (n + m)), sqrt(6 / (n + m)));
std::uniform_real_distribution<real_t> distribution(-sqrt(6 / (n + m)), sqrt(6 / (n + m)));
weights[i].push_back(distribution(generator));
} else if (type == "HeNormal") {
std::normal_distribution<double> distribution(0, sqrt(2 / n));
std::normal_distribution<real_t> distribution(0, sqrt(2 / n));
weights[i].push_back(distribution(generator));
} else if (type == "HeUniform") {
std::uniform_real_distribution<double> distribution(-sqrt(6 / n), sqrt(6 / n));
std::uniform_real_distribution<real_t> distribution(-sqrt(6 / n), sqrt(6 / n));
weights[i].push_back(distribution(generator));
} else if (type == "LeCunNormal") {
std::normal_distribution<double> distribution(0, sqrt(1 / n));
std::normal_distribution<real_t> distribution(0, sqrt(1 / n));
weights[i].push_back(distribution(generator));
} else if (type == "LeCunUniform") {
std::uniform_real_distribution<double> distribution(-sqrt(3 / n), sqrt(3 / n));
std::uniform_real_distribution<real_t> distribution(-sqrt(3 / n), sqrt(3 / n));
weights[i].push_back(distribution(generator));
} else if (type == "Uniform") {
std::uniform_real_distribution<double> distribution(-1 / sqrt(n), 1 / sqrt(n));
std::uniform_real_distribution<real_t> distribution(-1 / sqrt(n), 1 / sqrt(n));
weights[i].push_back(distribution(generator));
} else {
std::uniform_real_distribution<double> distribution(0, 1);
std::uniform_real_distribution<real_t> distribution(0, 1);
weights[i].push_back(distribution(generator));
}
}
@ -94,11 +94,11 @@ std::vector<std::vector<double>> MLPPUtilities::weightInitialization(int n, int
return weights;
}
std::vector<double> MLPPUtilities::biasInitialization(int n) {
std::vector<double> bias;
std::vector<real_t> MLPPUtilities::biasInitialization(int n) {
std::vector<real_t> bias;
std::random_device rd;
std::default_random_engine generator(rd());
std::uniform_real_distribution<double> distribution(0, 1);
std::uniform_real_distribution<real_t> distribution(0, 1);
for (int i = 0; i < n; i++) {
bias.push_back(distribution(generator));
@ -106,8 +106,8 @@ std::vector<double> MLPPUtilities::biasInitialization(int n) {
return bias;
}
double MLPPUtilities::performance(std::vector<double> y_hat, std::vector<double> outputSet) {
double correct = 0;
real_t MLPPUtilities::performance(std::vector<real_t> y_hat, std::vector<real_t> outputSet) {
real_t correct = 0;
for (int i = 0; i < y_hat.size(); i++) {
if (std::round(y_hat[i]) == outputSet[i]) {
correct++;
@ -116,8 +116,8 @@ double MLPPUtilities::performance(std::vector<double> y_hat, std::vector<double>
return correct / y_hat.size();
}
double MLPPUtilities::performance(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y) {
double correct = 0;
real_t MLPPUtilities::performance(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
real_t correct = 0;
for (int i = 0; i < y_hat.size(); i++) {
int sub_correct = 0;
for (int j = 0; j < y_hat[i].size(); j++) {
@ -132,7 +132,7 @@ double MLPPUtilities::performance(std::vector<std::vector<double>> y_hat, std::v
return correct / y_hat.size();
}
void MLPPUtilities::saveParameters(std::string fileName, std::vector<double> weights, double bias, bool app, int layer) {
void MLPPUtilities::saveParameters(std::string fileName, std::vector<real_t> weights, real_t bias, bool app, int layer) {
std::string layer_info = "";
std::ofstream saveFile;
@ -160,7 +160,7 @@ void MLPPUtilities::saveParameters(std::string fileName, std::vector<double> wei
saveFile.close();
}
void MLPPUtilities::saveParameters(std::string fileName, std::vector<double> weights, std::vector<double> initial, double bias, bool app, int layer) {
void MLPPUtilities::saveParameters(std::string fileName, std::vector<real_t> weights, std::vector<real_t> initial, real_t bias, bool app, int layer) {
std::string layer_info = "";
std::ofstream saveFile;
@ -194,7 +194,7 @@ void MLPPUtilities::saveParameters(std::string fileName, std::vector<double> wei
saveFile.close();
}
void MLPPUtilities::saveParameters(std::string fileName, std::vector<std::vector<double>> weights, std::vector<double> bias, bool app, int layer) {
void MLPPUtilities::saveParameters(std::string fileName, std::vector<std::vector<real_t>> weights, std::vector<real_t> bias, bool app, int layer) {
std::string layer_info = "";
std::ofstream saveFile;
@ -226,7 +226,7 @@ void MLPPUtilities::saveParameters(std::string fileName, std::vector<std::vector
saveFile.close();
}
void MLPPUtilities::UI(std::vector<double> weights, double bias) {
void MLPPUtilities::UI(std::vector<real_t> weights, real_t bias) {
std::cout << "Values of the weight(s):" << std::endl;
for (int i = 0; i < weights.size(); i++) {
std::cout << weights[i] << std::endl;
@ -235,7 +235,7 @@ void MLPPUtilities::UI(std::vector<double> weights, double bias) {
std::cout << bias << std::endl;
}
void MLPPUtilities::UI(std::vector<std::vector<double>> weights, std::vector<double> bias) {
void MLPPUtilities::UI(std::vector<std::vector<real_t>> weights, std::vector<real_t> bias) {
std::cout << "Values of the weight(s):" << std::endl;
for (int i = 0; i < weights.size(); i++) {
for (int j = 0; j < weights[i].size(); j++) {
@ -248,7 +248,7 @@ void MLPPUtilities::UI(std::vector<std::vector<double>> weights, std::vector<dou
}
}
void MLPPUtilities::UI(std::vector<double> weights, std::vector<double> initial, double bias) {
void MLPPUtilities::UI(std::vector<real_t> weights, std::vector<real_t> initial, real_t bias) {
std::cout << "Values of the weight(s):" << std::endl;
for (int i = 0; i < weights.size(); i++) {
std::cout << weights[i] << std::endl;
@ -261,7 +261,7 @@ void MLPPUtilities::UI(std::vector<double> weights, std::vector<double> initial,
std::cout << bias << std::endl;
}
void MLPPUtilities::CostInfo(int epoch, double cost_prev, double Cost) {
void MLPPUtilities::CostInfo(int epoch, real_t cost_prev, real_t Cost) {
std::cout << "-----------------------------------" << std::endl;
std::cout << "This is epoch: " << epoch << std::endl;
std::cout << "The cost function has been minimized by " << cost_prev - Cost << std::endl;
@ -269,21 +269,21 @@ void MLPPUtilities::CostInfo(int epoch, double cost_prev, double Cost) {
std::cout << Cost << std::endl;
}
std::vector<std::vector<std::vector<double>>> MLPPUtilities::createMiniBatches(std::vector<std::vector<double>> inputSet, int n_mini_batch) {
std::vector<std::vector<std::vector<real_t>>> MLPPUtilities::createMiniBatches(std::vector<std::vector<real_t>> inputSet, int n_mini_batch) {
int n = inputSet.size();
std::vector<std::vector<std::vector<double>>> inputMiniBatches;
std::vector<std::vector<std::vector<real_t>>> inputMiniBatches;
// Creating the mini-batches
for (int i = 0; i < n_mini_batch; i++) {
std::vector<std::vector<double>> currentInputSet;
std::vector<std::vector<real_t>> currentInputSet;
for (int j = 0; j < n / n_mini_batch; j++) {
currentInputSet.push_back(inputSet[n / n_mini_batch * i + j]);
}
inputMiniBatches.push_back(currentInputSet);
}
if (double(n) / double(n_mini_batch) - int(n / n_mini_batch) != 0) {
if (real_t(n) / real_t(n_mini_batch) - int(n / n_mini_batch) != 0) {
for (int i = 0; i < n - n / n_mini_batch * n_mini_batch; i++) {
inputMiniBatches[n_mini_batch - 1].push_back(inputSet[n / n_mini_batch * n_mini_batch + i]);
}
@ -291,15 +291,15 @@ std::vector<std::vector<std::vector<double>>> MLPPUtilities::createMiniBatches(s
return inputMiniBatches;
}
std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<std::vector<double>>> MLPPUtilities::createMiniBatches(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, int n_mini_batch) {
std::tuple<std::vector<std::vector<std::vector<real_t>>>, std::vector<std::vector<real_t>>> MLPPUtilities::createMiniBatches(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, int n_mini_batch) {
int n = inputSet.size();
std::vector<std::vector<std::vector<double>>> inputMiniBatches;
std::vector<std::vector<double>> outputMiniBatches;
std::vector<std::vector<std::vector<real_t>>> inputMiniBatches;
std::vector<std::vector<real_t>> outputMiniBatches;
for (int i = 0; i < n_mini_batch; i++) {
std::vector<std::vector<double>> currentInputSet;
std::vector<double> currentOutputSet;
std::vector<std::vector<real_t>> currentInputSet;
std::vector<real_t> currentOutputSet;
for (int j = 0; j < n / n_mini_batch; j++) {
currentInputSet.push_back(inputSet[n / n_mini_batch * i + j]);
currentOutputSet.push_back(outputSet[n / n_mini_batch * i + j]);
@ -308,7 +308,7 @@ std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<std::vecto
outputMiniBatches.push_back(currentOutputSet);
}
if (double(n) / double(n_mini_batch) - int(n / n_mini_batch) != 0) {
if (real_t(n) / real_t(n_mini_batch) - int(n / n_mini_batch) != 0) {
for (int i = 0; i < n - n / n_mini_batch * n_mini_batch; i++) {
inputMiniBatches[n_mini_batch - 1].push_back(inputSet[n / n_mini_batch * n_mini_batch + i]);
outputMiniBatches[n_mini_batch - 1].push_back(outputSet[n / n_mini_batch * n_mini_batch + i]);
@ -317,15 +317,15 @@ std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<std::vecto
return { inputMiniBatches, outputMiniBatches };
}
std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<std::vector<std::vector<double>>>> MLPPUtilities::createMiniBatches(std::vector<std::vector<double>> inputSet, std::vector<std::vector<double>> outputSet, int n_mini_batch) {
std::tuple<std::vector<std::vector<std::vector<real_t>>>, std::vector<std::vector<std::vector<real_t>>>> MLPPUtilities::createMiniBatches(std::vector<std::vector<real_t>> inputSet, std::vector<std::vector<real_t>> outputSet, int n_mini_batch) {
int n = inputSet.size();
std::vector<std::vector<std::vector<double>>> inputMiniBatches;
std::vector<std::vector<std::vector<double>>> outputMiniBatches;
std::vector<std::vector<std::vector<real_t>>> inputMiniBatches;
std::vector<std::vector<std::vector<real_t>>> outputMiniBatches;
for (int i = 0; i < n_mini_batch; i++) {
std::vector<std::vector<double>> currentInputSet;
std::vector<std::vector<double>> currentOutputSet;
std::vector<std::vector<real_t>> currentInputSet;
std::vector<std::vector<real_t>> currentOutputSet;
for (int j = 0; j < n / n_mini_batch; j++) {
currentInputSet.push_back(inputSet[n / n_mini_batch * i + j]);
currentOutputSet.push_back(outputSet[n / n_mini_batch * i + j]);
@ -334,7 +334,7 @@ std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<std::vecto
outputMiniBatches.push_back(currentOutputSet);
}
if (double(n) / double(n_mini_batch) - int(n / n_mini_batch) != 0) {
if (real_t(n) / real_t(n_mini_batch) - int(n / n_mini_batch) != 0) {
for (int i = 0; i < n - n / n_mini_batch * n_mini_batch; i++) {
inputMiniBatches[n_mini_batch - 1].push_back(inputSet[n / n_mini_batch * n_mini_batch + i]);
outputMiniBatches[n_mini_batch - 1].push_back(outputSet[n / n_mini_batch * n_mini_batch + i]);
@ -343,8 +343,8 @@ std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<std::vecto
return { inputMiniBatches, outputMiniBatches };
}
std::tuple<double, double, double, double> MLPPUtilities::TF_PN(std::vector<double> y_hat, std::vector<double> y) {
double TP, FP, TN, FN = 0;
std::tuple<real_t, real_t, real_t, real_t> MLPPUtilities::TF_PN(std::vector<real_t> y_hat, std::vector<real_t> y) {
real_t TP, FP, TN, FN = 0;
for (int i = 0; i < y_hat.size(); i++) {
if (y_hat[i] == y[i]) {
if (y_hat[i] == 1) {
@ -363,20 +363,20 @@ std::tuple<double, double, double, double> MLPPUtilities::TF_PN(std::vector<doub
return { TP, FP, TN, FN };
}
double MLPPUtilities::recall(std::vector<double> y_hat, std::vector<double> y) {
real_t MLPPUtilities::recall(std::vector<real_t> y_hat, std::vector<real_t> y) {
auto [TP, FP, TN, FN] = TF_PN(y_hat, y);
return TP / (TP + FN);
}
double MLPPUtilities::precision(std::vector<double> y_hat, std::vector<double> y) {
real_t MLPPUtilities::precision(std::vector<real_t> y_hat, std::vector<real_t> y) {
auto [TP, FP, TN, FN] = TF_PN(y_hat, y);
return TP / (TP + FP);
}
double MLPPUtilities::accuracy(std::vector<double> y_hat, std::vector<double> y) {
real_t MLPPUtilities::accuracy(std::vector<real_t> y_hat, std::vector<real_t> y) {
auto [TP, FP, TN, FN] = TF_PN(y_hat, y);
return (TP + TN) / (TP + FP + FN + TN);
}
double MLPPUtilities::f1_score(std::vector<double> y_hat, std::vector<double> y) {
real_t MLPPUtilities::f1_score(std::vector<real_t> y_hat, std::vector<real_t> y) {
return 2 * precision(y_hat, y) * recall(y_hat, y) / (precision(y_hat, y) + recall(y_hat, y));
}

45
mlpp/utilities/utilities.h
View File

@ -8,6 +8,9 @@
// Created by Marc Melikyan on 1/16/21.
//
#include "core/math/math_defs.h"
#include <string>
#include <tuple>
#include <vector>
@ -16,37 +19,37 @@
class MLPPUtilities {
public:
// Weight Init
static std::vector<double> weightInitialization(int n, std::string type = "Default");
static double biasInitialization();
static std::vector<real_t> weightInitialization(int n, std::string type = "Default");
static real_t biasInitialization();
static std::vector<std::vector<double>> weightInitialization(int n, int m, std::string type = "Default");
static std::vector<double> biasInitialization(int n);
static std::vector<std::vector<real_t>> weightInitialization(int n, int m, std::string type = "Default");
static std::vector<real_t> biasInitialization(int n);
// Cost/Performance related Functions
double performance(std::vector<double> y_hat, std::vector<double> y);
double performance(std::vector<std::vector<double>> y_hat, std::vector<std::vector<double>> y);
real_t performance(std::vector<real_t> y_hat, std::vector<real_t> y);
real_t performance(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
// Parameter Saving Functions
void saveParameters(std::string fileName, std::vector<double> weights, double bias, bool app = 0, int layer = -1);
void saveParameters(std::string fileName, std::vector<double> weights, std::vector<double> initial, double bias, bool app = 0, int layer = -1);
void saveParameters(std::string fileName, std::vector<std::vector<double>> weights, std::vector<double> bias, bool app = 0, int layer = -1);
void saveParameters(std::string fileName, std::vector<real_t> weights, real_t bias, bool app = 0, int layer = -1);
void saveParameters(std::string fileName, std::vector<real_t> weights, std::vector<real_t> initial, real_t bias, bool app = 0, int layer = -1);
void saveParameters(std::string fileName, std::vector<std::vector<real_t>> weights, std::vector<real_t> bias, bool app = 0, int layer = -1);
// Gradient Descent related
static void UI(std::vector<double> weights, double bias);
static void UI(std::vector<double> weights, std::vector<double> initial, double bias);
static void UI(std::vector<std::vector<double>>, std::vector<double> bias);
static void CostInfo(int epoch, double cost_prev, double Cost);
static void UI(std::vector<real_t> weights, real_t bias);
static void UI(std::vector<real_t> weights, std::vector<real_t> initial, real_t bias);
static void UI(std::vector<std::vector<real_t>>, std::vector<real_t> bias);
static void CostInfo(int epoch, real_t cost_prev, real_t Cost);
static std::vector<std::vector<std::vector<double>>> createMiniBatches(std::vector<std::vector<double>> inputSet, int n_mini_batch);
static std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<std::vector<double>>> createMiniBatches(std::vector<std::vector<double>> inputSet, std::vector<double> outputSet, int n_mini_batch);
static std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<std::vector<std::vector<double>>>> createMiniBatches(std::vector<std::vector<double>> inputSet, std::vector<std::vector<double>> outputSet, int n_mini_batch);
static std::vector<std::vector<std::vector<real_t>>> createMiniBatches(std::vector<std::vector<real_t>> inputSet, int n_mini_batch);
static std::tuple<std::vector<std::vector<std::vector<real_t>>>, std::vector<std::vector<real_t>>> createMiniBatches(std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet, int n_mini_batch);
static std::tuple<std::vector<std::vector<std::vector<real_t>>>, std::vector<std::vector<std::vector<real_t>>>> createMiniBatches(std::vector<std::vector<real_t>> inputSet, std::vector<std::vector<real_t>> outputSet, int n_mini_batch);
// F1 score, Precision/Recall, TP, FP, TN, FN, etc.
std::tuple<double, double, double, double> TF_PN(std::vector<double> y_hat, std::vector<double> y); //TF_PN = "True", "False", "Positive", "Negative"
double recall(std::vector<double> y_hat, std::vector<double> y);
double precision(std::vector<double> y_hat, std::vector<double> y);
double accuracy(std::vector<double> y_hat, std::vector<double> y);
double f1_score(std::vector<double> y_hat, std::vector<double> y);
std::tuple<real_t, real_t, real_t, real_t> TF_PN(std::vector<real_t> y_hat, std::vector<real_t> y); //TF_PN = "True", "False", "Positive", "Negative"
real_t recall(std::vector<real_t> y_hat, std::vector<real_t> y);
real_t precision(std::vector<real_t> y_hat, std::vector<real_t> y);
real_t accuracy(std::vector<real_t> y_hat, std::vector<real_t> y);
real_t f1_score(std::vector<real_t> y_hat, std::vector<real_t> y);
private:
};

60
mlpp/wgan/wgan.cpp
View File

@ -15,7 +15,7 @@
#include <iostream>
MLPPWGAN::MLPPWGAN(double k, std::vector<std::vector<double>> outputSet) :
MLPPWGAN::MLPPWGAN(real_t k, std::vector<std::vector<real_t>> outputSet) :
outputSet(outputSet), n(outputSet.size()), k(k) {
}
@ -23,15 +23,15 @@ MLPPWGAN::~MLPPWGAN() {
delete outputLayer;
}
std::vector<std::vector<double>> MLPPWGAN::generateExample(int n) {
std::vector<std::vector<real_t>> MLPPWGAN::generateExample(int n) {
MLPPLinAlg alg;
return modelSetTestGenerator(alg.gaussianNoise(n, k));
}
void MLPPWGAN::gradientDescent(double learning_rate, int max_epoch, bool UI) {
void MLPPWGAN::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
class MLPPCost cost;
MLPPLinAlg alg;
double cost_prev = 0;
real_t cost_prev = 0;
int epoch = 1;
forwardPass();
@ -40,11 +40,11 @@ void MLPPWGAN::gradientDescent(double learning_rate, int max_epoch, bool UI) {
while (true) {
cost_prev = Cost(y_hat, alg.onevec(n));
std::vector<std::vector<double>> generatorInputSet;
std::vector<std::vector<double>> discriminatorInputSet;
std::vector<std::vector<real_t>> generatorInputSet;
std::vector<std::vector<real_t>> discriminatorInputSet;
std::vector<double> y_hat;
std::vector<double> outputSet;
std::vector<real_t> y_hat;
std::vector<real_t> outputSet;
// Training of the discriminator.
for (int i = 0; i < CRITIC_INTERATIONS; i++) {
@ -54,7 +54,7 @@ void MLPPWGAN::gradientDescent(double learning_rate, int max_epoch, bool UI) {
y_hat = modelSetTestDiscriminator(discriminatorInputSet);
outputSet = alg.scalarMultiply(-1, alg.onevec(n)); // WGAN changes y_i = 1 and y_i = 0 to y_i = 1 and y_i = -1
std::vector<double> outputSetReal = alg.onevec(n);
std::vector<real_t> outputSetReal = alg.onevec(n);
outputSet.insert(outputSet.end(), outputSetReal.begin(), outputSetReal.end()); // Fake + real output scores.
auto [cumulativeDiscriminatorHiddenLayerWGrad, outputDiscriminatorWGrad] = computeDiscriminatorGradients(y_hat, outputSet);
@ -69,7 +69,7 @@ void MLPPWGAN::gradientDescent(double learning_rate, int max_epoch, bool UI) {
y_hat = modelSetTestDiscriminator(discriminatorInputSet);
outputSet = alg.onevec(n);
std::vector<std::vector<std::vector<double>>> cumulativeGeneratorHiddenLayerWGrad = computeGeneratorGradients(y_hat, outputSet);
std::vector<std::vector<std::vector<real_t>>> cumulativeGeneratorHiddenLayerWGrad = computeGeneratorGradients(y_hat, outputSet);
cumulativeGeneratorHiddenLayerWGrad = alg.scalarMultiply(learning_rate / n, cumulativeGeneratorHiddenLayerWGrad);
updateGeneratorParameters(cumulativeGeneratorHiddenLayerWGrad, learning_rate);
@ -85,7 +85,7 @@ void MLPPWGAN::gradientDescent(double learning_rate, int max_epoch, bool UI) {
}
}
double MLPPWGAN::score() {
real_t MLPPWGAN::score() {
MLPPLinAlg alg;
MLPPUtilities util;
forwardPass();
@ -105,7 +105,7 @@ void MLPPWGAN::save(std::string fileName) {
}
}
void MLPPWGAN::addLayer(int n_hidden, std::string activation, std::string weightInit, std::string reg, double lambda, double alpha) {
void MLPPWGAN::addLayer(int n_hidden, std::string activation, std::string weightInit, std::string reg, real_t lambda, real_t alpha) {
MLPPLinAlg alg;
if (network.empty()) {
network.push_back(MLPPHiddenLayer(n_hidden, activation, alg.gaussianNoise(n, k), weightInit, reg, lambda, alpha));
@ -116,7 +116,7 @@ void MLPPWGAN::addLayer(int n_hidden, std::string activation, std::string weight
}
}
void MLPPWGAN::addOutputLayer(std::string weightInit, std::string reg, double lambda, double alpha) {
void MLPPWGAN::addOutputLayer(std::string weightInit, std::string reg, real_t lambda, real_t alpha) {
MLPPLinAlg alg;
if (!network.empty()) {
outputLayer = new MLPPOutputLayer(network[network.size() - 1].n_hidden, "Linear", "WassersteinLoss", network[network.size() - 1].a, weightInit, "WeightClipping", -0.01, 0.01);
@ -125,7 +125,7 @@ void MLPPWGAN::addOutputLayer(std::string weightInit, std::string reg, double la
}
}
std::vector<std::vector<double>> MLPPWGAN::modelSetTestGenerator(std::vector<std::vector<double>> X) {
std::vector<std::vector<real_t>> MLPPWGAN::modelSetTestGenerator(std::vector<std::vector<real_t>> X) {
if (!network.empty()) {
network[0].input = X;
network[0].forwardPass();
@ -138,7 +138,7 @@ std::vector<std::vector<double>> MLPPWGAN::modelSetTestGenerator(std::vector<std
return network[network.size() / 2].a;
}
std::vector<double> MLPPWGAN::modelSetTestDiscriminator(std::vector<std::vector<double>> X) {
std::vector<real_t> MLPPWGAN::modelSetTestDiscriminator(std::vector<std::vector<real_t>> X) {
if (!network.empty()) {
for (int i = network.size() / 2 + 1; i < network.size(); i++) {
if (i == network.size() / 2 + 1) {
@ -154,10 +154,10 @@ std::vector<double> MLPPWGAN::modelSetTestDiscriminator(std::vector<std::vector<
return outputLayer->a;
}
double MLPPWGAN::Cost(std::vector<double> y_hat, std::vector<double> y) {
real_t MLPPWGAN::Cost(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPReg regularization;
class MLPPCost cost;
double totalRegTerm = 0;
real_t totalRegTerm = 0;
auto cost_function = outputLayer->cost_map[outputLayer->cost];
if (!network.empty()) {
@ -186,7 +186,7 @@ void MLPPWGAN::forwardPass() {
y_hat = outputLayer->a;
}
void MLPPWGAN::updateDiscriminatorParameters(std::vector<std::vector<std::vector<double>>> hiddenLayerUpdations, std::vector<double> outputLayerUpdation, double learning_rate) {
void MLPPWGAN::updateDiscriminatorParameters(std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations, std::vector<real_t> outputLayerUpdation, real_t learning_rate) {
MLPPLinAlg alg;
outputLayer->weights = alg.subtraction(outputLayer->weights, outputLayerUpdation);
@ -203,7 +203,7 @@ void MLPPWGAN::updateDiscriminatorParameters(std::vector<std::vector<std::vector
}
}
void MLPPWGAN::updateGeneratorParameters(std::vector<std::vector<std::vector<double>>> hiddenLayerUpdations, double learning_rate) {
void MLPPWGAN::updateGeneratorParameters(std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations, real_t learning_rate) {
MLPPLinAlg alg;
if (!network.empty()) {
@ -216,25 +216,25 @@ void MLPPWGAN::updateGeneratorParameters(std::vector<std::vector<std::vector<dou
}
}
std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<double>> MLPPWGAN::computeDiscriminatorGradients(std::vector<double> y_hat, std::vector<double> outputSet) {
std::tuple<std::vector<std::vector<std::vector<real_t>>>, std::vector<real_t>> MLPPWGAN::computeDiscriminatorGradients(std::vector<real_t> y_hat, std::vector<real_t> outputSet) {
class MLPPCost cost;
MLPPActivation avn;
MLPPLinAlg alg;
MLPPReg regularization;
std::vector<std::vector<std::vector<double>>> cumulativeHiddenLayerWGrad; // Tensor containing ALL hidden grads.
std::vector<std::vector<std::vector<real_t>>> cumulativeHiddenLayerWGrad; // Tensor containing ALL hidden grads.
auto costDeriv = outputLayer->costDeriv_map[outputLayer->cost];
auto outputAvn = outputLayer->activation_map[outputLayer->activation];
outputLayer->delta = alg.hadamard_product((cost.*costDeriv)(y_hat, outputSet), (avn.*outputAvn)(outputLayer->z, 1));
std::vector<double> outputWGrad = alg.mat_vec_mult(alg.transpose(outputLayer->input), outputLayer->delta);
std::vector<real_t> outputWGrad = alg.mat_vec_mult(alg.transpose(outputLayer->input), outputLayer->delta);
outputWGrad = alg.addition(outputWGrad, regularization.regDerivTerm(outputLayer->weights, outputLayer->lambda, outputLayer->alpha, outputLayer->reg));
if (!network.empty()) {
auto hiddenLayerAvn = network[network.size() - 1].activation_map[network[network.size() - 1].activation];
network[network.size() - 1].delta = alg.hadamard_product(alg.outerProduct(outputLayer->delta, outputLayer->weights), (avn.*hiddenLayerAvn)(network[network.size() - 1].z, 1));
std::vector<std::vector<double>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[network.size() - 1].input), network[network.size() - 1].delta);
std::vector<std::vector<real_t>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[network.size() - 1].input), network[network.size() - 1].delta);
cumulativeHiddenLayerWGrad.push_back(alg.addition(hiddenLayerWGrad, regularization.regDerivTerm(network[network.size() - 1].weights, network[network.size() - 1].lambda, network[network.size() - 1].alpha, network[network.size() - 1].reg))); // Adding to our cumulative hidden layer grads. Maintain reg terms as well.
@ -244,7 +244,7 @@ std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<double>> M
for (int i = network.size() - 2; i > network.size() / 2; i--) {
auto hiddenLayerAvn = network[i].activation_map[network[i].activation];
network[i].delta = alg.hadamard_product(alg.matmult(network[i + 1].delta, alg.transpose(network[i + 1].weights)), (avn.*hiddenLayerAvn)(network[i].z, 1));
std::vector<std::vector<double>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[i].input), network[i].delta);
std::vector<std::vector<real_t>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[i].input), network[i].delta);
cumulativeHiddenLayerWGrad.push_back(alg.addition(hiddenLayerWGrad, regularization.regDerivTerm(network[i].weights, network[i].lambda, network[i].alpha, network[i].reg))); // Adding to our cumulative hidden layer grads. Maintain reg terms as well.
}
@ -252,36 +252,36 @@ std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<double>> M
return { cumulativeHiddenLayerWGrad, outputWGrad };
}
std::vector<std::vector<std::vector<double>>> MLPPWGAN::computeGeneratorGradients(std::vector<double> y_hat, std::vector<double> outputSet) {
std::vector<std::vector<std::vector<real_t>>> MLPPWGAN::computeGeneratorGradients(std::vector<real_t> y_hat, std::vector<real_t> outputSet) {
class MLPPCost cost;
MLPPActivation avn;
MLPPLinAlg alg;
MLPPReg regularization;
std::vector<std::vector<std::vector<double>>> cumulativeHiddenLayerWGrad; // Tensor containing ALL hidden grads.
std::vector<std::vector<std::vector<real_t>>> cumulativeHiddenLayerWGrad; // Tensor containing ALL hidden grads.
auto costDeriv = outputLayer->costDeriv_map[outputLayer->cost];
auto outputAvn = outputLayer->activation_map[outputLayer->activation];
outputLayer->delta = alg.hadamard_product((cost.*costDeriv)(y_hat, outputSet), (avn.*outputAvn)(outputLayer->z, 1));
std::vector<double> outputWGrad = alg.mat_vec_mult(alg.transpose(outputLayer->input), outputLayer->delta);
std::vector<real_t> outputWGrad = alg.mat_vec_mult(alg.transpose(outputLayer->input), outputLayer->delta);
outputWGrad = alg.addition(outputWGrad, regularization.regDerivTerm(outputLayer->weights, outputLayer->lambda, outputLayer->alpha, outputLayer->reg));
if (!network.empty()) {
auto hiddenLayerAvn = network[network.size() - 1].activation_map[network[network.size() - 1].activation];
network[network.size() - 1].delta = alg.hadamard_product(alg.outerProduct(outputLayer->delta, outputLayer->weights), (avn.*hiddenLayerAvn)(network[network.size() - 1].z, 1));
std::vector<std::vector<double>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[network.size() - 1].input), network[network.size() - 1].delta);
std::vector<std::vector<real_t>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[network.size() - 1].input), network[network.size() - 1].delta);
cumulativeHiddenLayerWGrad.push_back(alg.addition(hiddenLayerWGrad, regularization.regDerivTerm(network[network.size() - 1].weights, network[network.size() - 1].lambda, network[network.size() - 1].alpha, network[network.size() - 1].reg))); // Adding to our cumulative hidden layer grads. Maintain reg terms as well.
for (int i = network.size() - 2; i >= 0; i--) {
auto hiddenLayerAvn = network[i].activation_map[network[i].activation];
network[i].delta = alg.hadamard_product(alg.matmult(network[i + 1].delta, alg.transpose(network[i + 1].weights)), (avn.*hiddenLayerAvn)(network[i].z, 1));
std::vector<std::vector<double>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[i].input), network[i].delta);
std::vector<std::vector<real_t>> hiddenLayerWGrad = alg.matmult(alg.transpose(network[i].input), network[i].delta);
cumulativeHiddenLayerWGrad.push_back(alg.addition(hiddenLayerWGrad, regularization.regDerivTerm(network[i].weights, network[i].lambda, network[i].alpha, network[i].reg))); // Adding to our cumulative hidden layer grads. Maintain reg terms as well.
}
}
return cumulativeHiddenLayerWGrad;
}
void MLPPWGAN::UI(int epoch, double cost_prev, std::vector<double> y_hat, std::vector<double> outputSet) {
void MLPPWGAN::UI(int epoch, real_t cost_prev, std::vector<real_t> y_hat, std::vector<real_t> outputSet) {
MLPPUtilities::CostInfo(epoch, cost_prev, Cost(y_hat, outputSet));
std::cout << "Layer " << network.size() + 1 << ": " << std::endl;
MLPPUtilities::UI(outputLayer->weights, outputLayer->bias);

32
mlpp/wgan/wgan.h
View File

@ -19,32 +19,32 @@
class MLPPWGAN {
public:
MLPPWGAN(double k, std::vector<std::vector<double>> outputSet);
MLPPWGAN(real_t k, std::vector<std::vector<real_t>> outputSet);
~MLPPWGAN();
std::vector<std::vector<double>> generateExample(int n);
void gradientDescent(double learning_rate, int max_epoch, bool UI = 1);
double score();
std::vector<std::vector<real_t>> generateExample(int n);
void gradientDescent(real_t learning_rate, int max_epoch, bool UI = 1);
real_t score();
void save(std::string fileName);
void addLayer(int n_hidden, std::string activation, std::string weightInit = "Default", std::string reg = "None", double lambda = 0.5, double alpha = 0.5);
void addOutputLayer(std::string weightInit = "Default", std::string reg = "None", double lambda = 0.5, double alpha = 0.5);
void addLayer(int n_hidden, std::string activation, std::string weightInit = "Default", std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
void addOutputLayer(std::string weightInit = "Default", std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
private:
std::vector<std::vector<double>> modelSetTestGenerator(std::vector<std::vector<double>> X); // Evaluator for the generator of the WGAN.
std::vector<double> modelSetTestDiscriminator(std::vector<std::vector<double>> X); // Evaluator for the discriminator of the WGAN.
std::vector<std::vector<real_t>> modelSetTestGenerator(std::vector<std::vector<real_t>> X); // Evaluator for the generator of the WGAN.
std::vector<real_t> modelSetTestDiscriminator(std::vector<std::vector<real_t>> X); // Evaluator for the discriminator of the WGAN.
double Cost(std::vector<double> y_hat, std::vector<double> y);
real_t Cost(std::vector<real_t> y_hat, std::vector<real_t> y);
void forwardPass();
void updateDiscriminatorParameters(std::vector<std::vector<std::vector<double>>> hiddenLayerUpdations, std::vector<double> outputLayerUpdation, double learning_rate);
void updateGeneratorParameters(std::vector<std::vector<std::vector<double>>> hiddenLayerUpdations, double learning_rate);
std::tuple<std::vector<std::vector<std::vector<double>>>, std::vector<double>> computeDiscriminatorGradients(std::vector<double> y_hat, std::vector<double> outputSet);
std::vector<std::vector<std::vector<double>>> computeGeneratorGradients(std::vector<double> y_hat, std::vector<double> outputSet);
void updateDiscriminatorParameters(std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations, std::vector<real_t> outputLayerUpdation, real_t learning_rate);
void updateGeneratorParameters(std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations, real_t learning_rate);
std::tuple<std::vector<std::vector<std::vector<real_t>>>, std::vector<real_t>> computeDiscriminatorGradients(std::vector<real_t> y_hat, std::vector<real_t> outputSet);
std::vector<std::vector<std::vector<real_t>>> computeGeneratorGradients(std::vector<real_t> y_hat, std::vector<real_t> outputSet);
void UI(int epoch, double cost_prev, std::vector<double> y_hat, std::vector<double> outputSet);
void UI(int epoch, real_t cost_prev, std::vector<real_t> y_hat, std::vector<real_t> outputSet);
std::vector<std::vector<double>> outputSet;
std::vector<double> y_hat;
std::vector<std::vector<real_t>> outputSet;
std::vector<real_t> y_hat;
std::vector<MLPPHiddenLayer> network;
MLPPOutputLayer *outputLayer;

165
test/mlpp_tests.cpp
View File

@ -45,11 +45,11 @@
#include "../mlpp/uni_lin_reg/uni_lin_reg.h"
#include "../mlpp/wgan/wgan.h"
Vector<double> dstd_vec_to_vec(const std::vector<double> &in) {
Vector<double> r;
Vector<real_t> dstd_vec_to_vec(const std::vector<real_t> &in) {
Vector<real_t> r;
r.resize(static_cast<int>(in.size()));
double *darr = r.ptrw();
real_t *darr = r.ptrw();
for (uint32_t i = 0; i < in.size(); ++i) {
darr[i] = in[i];
@ -58,8 +58,8 @@ Vector<double> dstd_vec_to_vec(const std::vector<double> &in) {
return r;
}
Vector<Vector<double>> dstd_mat_to_mat(const std::vector<std::vector<double>> &in) {
Vector<Vector<double>> r;
Vector<Vector<real_t>> dstd_mat_to_mat(const std::vector<std::vector<real_t>> &in) {
Vector<Vector<real_t>> r;
for (uint32_t i = 0; i < in.size(); ++i) {
r.push_back(dstd_vec_to_vec(in[i]));
@ -75,9 +75,9 @@ void MLPPTests::test_statistics() {
MLPPConvolutions conv;
// STATISTICS
std::vector<double> x = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
std::vector<double> y = { 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 };
std::vector<double> w = { 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1 };
std::vector<real_t> x = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
std::vector<real_t> y = { 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 };
std::vector<real_t> w = { 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1 };
is_approx_equalsd(stat.mean(x), 5.5, "Arithmetic Mean");
is_approx_equalsd(stat.mean(x), 5.5, "Median");
@ -117,19 +117,19 @@ void MLPPTests::test_statistics() {
void MLPPTests::test_linear_algebra() {
MLPPLinAlg alg;
std::vector<std::vector<double>> square = { { 1, 1 }, { -1, 1 }, { 1, -1 }, { -1, -1 } };
std::vector<std::vector<double>> square_rot_res = { { 1.41421, 1.11022e-16 }, { -1.11022e-16, 1.41421 }, { 1.11022e-16, -1.41421 }, { -1.41421, -1.11022e-16 } };
std::vector<std::vector<real_t>> square = { { 1, 1 }, { -1, 1 }, { 1, -1 }, { -1, -1 } };
std::vector<std::vector<real_t>> square_rot_res = { { 1.41421, 1.11022e-16 }, { -1.11022e-16, 1.41421 }, { 1.11022e-16, -1.41421 }, { -1.41421, -1.11022e-16 } };
is_approx_equals_dmat(dstd_mat_to_mat(alg.rotate(square, M_PI / 4)), dstd_mat_to_mat(square_rot_res), "alg.rotate(square, M_PI / 4)");
std::vector<std::vector<double>> A = {
std::vector<std::vector<real_t>> A = {
{ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 },
{ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 },
};
std::vector<double> a = { 4, 3, 1, 3 };
std::vector<double> b = { 3, 5, 6, 1 };
std::vector<real_t> a = { 4, 3, 1, 3 };
std::vector<real_t> b = { 3, 5, 6, 1 };
std::vector<std::vector<double>> mmtr_res = {
std::vector<std::vector<real_t>> mmtr_res = {
{ 2, 4, 6, 8, 10, 12, 14, 16, 18, 20 },
{ 4, 8, 12, 16, 20, 24, 28, 32, 36, 40 },
{ 6, 12, 18, 24, 30, 36, 42, 48, 54, 60 },
@ -146,14 +146,14 @@ void MLPPTests::test_linear_algebra() {
is_approx_equalsd(alg.dot(a, b), 36, "alg.dot(a, b)");
std::vector<std::vector<double>> had_prod_res = {
std::vector<std::vector<real_t>> had_prod_res = {
{ 1, 4, 9, 16, 25, 36, 49, 64, 81, 100 },
{ 1, 4, 9, 16, 25, 36, 49, 64, 81, 100 }
};
is_approx_equals_dmat(dstd_mat_to_mat(alg.hadamard_product(A, A)), dstd_mat_to_mat(had_prod_res), "alg.hadamard_product(A, A)");
std::vector<std::vector<double>> id_10_res = {
std::vector<std::vector<real_t>> id_10_res = {
{ 1, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
{ 0, 1, 0, 0, 0, 0, 0, 0, 0, 0 },
{ 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 },
@ -177,7 +177,7 @@ void MLPPTests::test_univariate_linear_regression() {
MLPPUniLinReg model(ds->input, ds->output);
std::vector<double> slr_res = {
std::vector<real_t> slr_res = {
24.1095, 28.4829, 29.8082, 26.0974, 27.2902, 61.0851, 30.4709, 25.0372, 25.5673, 35.9046,
54.4587, 18.8083, 23.4468, 18.5432, 19.2059, 21.1938, 23.0492, 18.8083, 25.4348, 35.9046,
37.76, 40.278, 63.8683, 68.5068, 40.4106, 46.772, 32.0612, 23.3143, 44.784, 44.519,
@ -254,8 +254,8 @@ void MLPPTests::test_multivariate_linear_regression_score_sgd_adam(bool ui) {
const int TRIAL_NUM = 1000;
double scoreSGD = 0;
double scoreADAM = 0;
real_t scoreSGD = 0;
real_t scoreADAM = 0;
for (int i = 0; i < TRIAL_NUM; i++) {
MLPPLinReg modelf(alg.transpose(ds->input), ds->output);
modelf.MBGD(0.001, 5, 1, ui);
@ -327,8 +327,8 @@ void MLPPTests::test_c_log_log_regression(bool ui) {
MLPPLinAlg alg;
// CLOGLOG REGRESSION
std::vector<std::vector<double>> inputSet = { { 1, 2, 3, 4, 5, 6, 7, 8 }, { 0, 0, 0, 0, 1, 1, 1, 1 } };
std::vector<double> outputSet = { 0, 0, 0, 0, 1, 1, 1, 1 };
std::vector<std::vector<real_t>> inputSet = { { 1, 2, 3, 4, 5, 6, 7, 8 }, { 0, 0, 0, 0, 1, 1, 1, 1 } };
std::vector<real_t> outputSet = { 0, 0, 0, 0, 1, 1, 1, 1 };
MLPPCLogLogReg model(alg.transpose(inputSet), outputSet);
model.SGD(0.1, 10000, ui);
alg.printVector(model.modelSetTest(alg.transpose(inputSet)));
@ -338,8 +338,8 @@ void MLPPTests::test_exp_reg_regression(bool ui) {
MLPPLinAlg alg;
// EXPREG REGRESSION
std::vector<std::vector<double>> inputSet = { { 0, 1, 2, 3, 4 } };
std::vector<double> outputSet = { 1, 2, 4, 8, 16 };
std::vector<std::vector<real_t>> inputSet = { { 0, 1, 2, 3, 4 } };
std::vector<real_t> outputSet = { 1, 2, 4, 8, 16 };
MLPPExpReg model(alg.transpose(inputSet), outputSet);
model.SGD(0.001, 10000, ui);
alg.printVector(model.modelSetTest(alg.transpose(inputSet)));
@ -349,8 +349,8 @@ void MLPPTests::test_tanh_regression(bool ui) {
MLPPLinAlg alg;
// TANH REGRESSION
std::vector<std::vector<double>> inputSet = { { 4, 3, 0, -3, -4 }, { 0, 0, 0, 1, 1 } };
std::vector<double> outputSet = { 1, 1, 0, -1, -1 };
std::vector<std::vector<real_t>> inputSet = { { 4, 3, 0, -3, -4 }, { 0, 0, 0, 1, 1 } };
std::vector<real_t> outputSet = { 1, 1, 0, -1, -1 };
MLPPTanhReg model(alg.transpose(inputSet), outputSet);
model.SGD(0.1, 10000, ui);
alg.printVector(model.modelSetTest(alg.transpose(inputSet)));
@ -387,9 +387,9 @@ void MLPPTests::test_mlp(bool ui) {
MLPPLinAlg alg;
// MLP
std::vector<std::vector<double>> inputSet = { { 0, 0, 1, 1 }, { 0, 1, 0, 1 } };
std::vector<std::vector<real_t>> inputSet = { { 0, 0, 1, 1 }, { 0, 1, 0, 1 } };
inputSet = alg.transpose(inputSet);
std::vector<double> outputSet = { 0, 1, 1, 0 };
std::vector<real_t> outputSet = { 0, 1, 1, 0 };
MLPPMLP model(inputSet, outputSet, 2);
model.gradientDescent(0.1, 10000, ui);
@ -412,7 +412,7 @@ void MLPPTests::test_autoencoder(bool ui) {
MLPPLinAlg alg;
// AUTOENCODER
std::vector<std::vector<double>> inputSet = { { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, { 3, 5, 9, 12, 15, 18, 21, 24, 27, 30 } };
std::vector<std::vector<real_t>> inputSet = { { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, { 3, 5, 9, 12, 15, 18, 21, 24, 27, 30 } };
MLPPAutoEncoder model(alg.transpose(inputSet), 5);
model.SGD(0.001, 300000, ui);
alg.printMatrix(model.modelSetTest(alg.transpose(inputSet)));
@ -425,8 +425,8 @@ void MLPPTests::test_dynamically_sized_ann(bool ui) {
// Possible Weight Init Methods: Default, Uniform, HeNormal, HeUniform, XavierNormal, XavierUniform
// Possible Activations: Linear, Sigmoid, Swish, Softplus, Softsign, CLogLog, Ar{Sinh, Cosh, Tanh, Csch, Sech, Coth}, GaussianCDF, GELU, UnitStep
// Possible Loss Functions: MSE, RMSE, MBE, LogLoss, CrossEntropy, HingeLoss
std::vector<std::vector<double>> inputSet = { { 0, 0, 1, 1 }, { 0, 1, 0, 1 } };
std::vector<double> outputSet = { 0, 1, 1, 0 };
std::vector<std::vector<real_t>> inputSet = { { 0, 0, 1, 1 }, { 0, 1, 0, 1 } };
std::vector<real_t> outputSet = { 0, 1, 1, 0 };
MLPPANN ann(alg.transpose(inputSet), outputSet);
ann.addLayer(2, "Cosh");
ann.addOutputLayer("Sigmoid", "LogLoss");
@ -448,7 +448,7 @@ void MLPPTests::test_wgan(bool ui) {
//MLPPData data;
//MLPPConvolutions conv;
std::vector<std::vector<double>> outputSet = {
std::vector<std::vector<real_t>> outputSet = {
{ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20 },
{ 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40 }
};
@ -465,8 +465,8 @@ void MLPPTests::test_wgan(bool ui) {
void MLPPTests::test_ann(bool ui) {
MLPPLinAlg alg;
std::vector<std::vector<double>> inputSet = { { 0, 0 }, { 0, 1 }, { 1, 0 }, { 1, 1 } }; // XOR
std::vector<double> outputSet = { 0, 1, 1, 0 };
std::vector<std::vector<real_t>> inputSet = { { 0, 0 }, { 0, 1 }, { 1, 0 }, { 1, 1 } }; // XOR
std::vector<real_t> outputSet = { 0, 1, 1, 0 };
MLPPANN ann(inputSet, outputSet);
ann.addLayer(5, "Sigmoid");
@ -474,7 +474,7 @@ void MLPPTests::test_ann(bool ui) {
ann.addOutputLayer("Sigmoid", "LogLoss");
ann.gradientDescent(1, 20000, ui);
std::vector<double> predictions = ann.modelSetTest(inputSet);
std::vector<real_t> predictions = ann.modelSetTest(inputSet);
alg.printVector(predictions); // Testing out the model's preds for train set.
std::cout << "ACCURACY: " << 100 * ann.score() << "%" << std::endl; // Accuracy.
}
@ -483,8 +483,8 @@ void MLPPTests::test_dynamically_sized_mann(bool ui) {
MLPPData data;
// DYNAMICALLY SIZED MANN (Multidimensional Output ANN)
std::vector<std::vector<double>> inputSet = { { 1, 2, 3 }, { 2, 4, 6 }, { 3, 6, 9 }, { 4, 8, 12 } };
std::vector<std::vector<double>> outputSet = { { 1, 5 }, { 2, 10 }, { 3, 15 }, { 4, 20 } };
std::vector<std::vector<real_t>> inputSet = { { 1, 2, 3 }, { 2, 4, 6 }, { 3, 6, 9 }, { 4, 8, 12 } };
std::vector<std::vector<real_t>> outputSet = { { 1, 5 }, { 2, 10 }, { 3, 15 }, { 4, 20 } };
MLPPMANN mann(inputSet, outputSet);
mann.addOutputLayer("Linear", "MSE");
@ -497,8 +497,8 @@ void MLPPTests::test_train_test_split_mann(bool ui) {
MLPPData data;
// TRAIN TEST SPLIT CHECK
std::vector<std::vector<double>> inputSet1 = { { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, { 3, 5, 9, 12, 15, 18, 21, 24, 27, 30 } };
std::vector<std::vector<double>> outputSet1 = { { 2, 4, 6, 8, 10, 12, 14, 16, 18, 20 } };
std::vector<std::vector<real_t>> inputSet1 = { { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, { 3, 5, 9, 12, 15, 18, 21, 24, 27, 30 } };
std::vector<std::vector<real_t>> outputSet1 = { { 2, 4, 6, 8, 10, 12, 14, 16, 18, 20 } };
Ref<MLPPDataComplex> d;
d.instance();
@ -525,8 +525,8 @@ void MLPPTests::test_naive_bayes() {
MLPPLinAlg alg;
// NAIVE BAYES
std::vector<std::vector<double>> inputSet = { { 1, 1, 1, 1, 1 }, { 0, 0, 1, 1, 1 }, { 0, 0, 1, 0, 1 } };
std::vector<double> outputSet = { 0, 1, 0, 1, 1 };
std::vector<std::vector<real_t>> inputSet = { { 1, 1, 1, 1, 1 }, { 0, 0, 1, 1, 1 }, { 0, 0, 1, 0, 1 } };
std::vector<real_t> outputSet = { 0, 1, 0, 1, 1 };
MLPPMultinomialNB MNB(alg.transpose(inputSet), outputSet, 2);
alg.printVector(MNB.modelSetTest(alg.transpose(inputSet)));
@ -541,7 +541,7 @@ void MLPPTests::test_k_means(bool ui) {
MLPPLinAlg alg;
// KMeans
std::vector<std::vector<double>> inputSet = { { 32, 0, 7 }, { 2, 28, 17 }, { 0, 9, 23 } };
std::vector<std::vector<real_t>> inputSet = { { 32, 0, 7 }, { 2, 28, 17 }, { 0, 9, 23 } };
MLPPKMeans kmeans(inputSet, 3, "KMeans++");
kmeans.train(3, ui);
std::cout << std::endl;
@ -553,8 +553,8 @@ void MLPPTests::test_knn(bool ui) {
MLPPLinAlg alg;
// kNN
std::vector<std::vector<double>> inputSet = { { 1, 2, 3, 4, 5, 6, 7, 8 }, { 0, 0, 0, 0, 1, 1, 1, 1 } };
std::vector<double> outputSet = { 0, 0, 0, 0, 1, 1, 1, 1 };
std::vector<std::vector<real_t>> inputSet = { { 1, 2, 3, 4, 5, 6, 7, 8 }, { 0, 0, 0, 0, 1, 1, 1, 1 } };
std::vector<real_t> outputSet = { 0, 0, 0, 0, 1, 1, 1, 1 };
MLPPKNN knn(alg.transpose(inputSet), outputSet, 8);
alg.printVector(knn.modelSetTest(alg.transpose(inputSet)));
std::cout << "ACCURACY: " << 100 * knn.score() << "%" << std::endl;
@ -566,18 +566,18 @@ void MLPPTests::test_convolution_tensors_etc() {
MLPPConvolutions conv;
// CONVOLUTION, POOLING, ETC..
std::vector<std::vector<double>> input = {
std::vector<std::vector<real_t>> input = {
{ 1 },
};
std::vector<std::vector<std::vector<double>>> tensorSet;
std::vector<std::vector<std::vector<real_t>>> tensorSet;
tensorSet.push_back(input);
tensorSet.push_back(input);
tensorSet.push_back(input);
alg.printTensor(data.rgb2xyz(tensorSet));
std::vector<std::vector<double>> input2 = {
std::vector<std::vector<real_t>> input2 = {
{ 62, 55, 55, 54, 49, 48, 47, 55 },
{ 62, 57, 54, 52, 48, 47, 48, 53 },
{ 61, 60, 52, 49, 48, 47, 49, 54 },
@ -595,19 +595,19 @@ void MLPPTests::test_convolution_tensors_etc() {
alg.printMatrix(conv.convolve(input2, conv.getPrewittVertical(), 1)); // Can use padding
alg.printMatrix(conv.pool(input2, 4, 4, "Max")); // Can use Max, Min, or Average pooling.
std::vector<std::vector<std::vector<double>>> tensorSet2;
std::vector<std::vector<std::vector<real_t>>> tensorSet2;
tensorSet2.push_back(input2);
tensorSet2.push_back(input2);
alg.printVector(conv.globalPool(tensorSet2, "Average")); // Can use Max, Min, or Average global pooling.
std::vector<std::vector<double>> laplacian = { { 1, 1, 1 }, { 1, -4, 1 }, { 1, 1, 1 } };
std::vector<std::vector<real_t>> laplacian = { { 1, 1, 1 }, { 1, -4, 1 }, { 1, 1, 1 } };
alg.printMatrix(conv.convolve(conv.gaussianFilter2D(5, 1), laplacian, 1));
}
void MLPPTests::test_pca_svd_eigenvalues_eigenvectors(bool ui) {
MLPPLinAlg alg;
// PCA, SVD, eigenvalues & eigenvectors
std::vector<std::vector<double>> inputSet = { { 1, 1 }, { 1, 1 } };
std::vector<std::vector<real_t>> inputSet = { { 1, 1 }, { 1, 1 } };
MLPPLinAlg::EigenResult eigen = alg.eigen(inputSet);
@ -675,7 +675,7 @@ void MLPPTests::test_nlp_and_data(bool ui) {
//alg.printMatrix(data.BOW(textArchive, "Default"));
std::cout << std::endl;
std::vector<std::vector<double>> inputSet = { { 1, 2 }, { 2, 3 }, { 3, 4 }, { 4, 5 }, { 5, 6 } };
std::vector<std::vector<real_t>> inputSet = { { 1, 2 }, { 2, 3 }, { 3, 4 }, { 4, 5 }, { 5, 6 } };
std::cout << "Feature Scaling Example:" << std::endl;
alg.printMatrix(data.featureScaling(inputSet));
std::cout << std::endl;
@ -692,7 +692,8 @@ void MLPPTests::test_outlier_finder(bool ui) {
MLPPLinAlg alg;
// Outlier Finder
std::vector<double> inputSet = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 23554332523523 };
//std::vector<real_t> inputSet = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 23554332523523 };
std::vector<real_t> inputSet = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 23554332 };
MLPPOutlierFinder outlierFinder(2); // Any datapoint outside of 2 stds from the mean is marked as an outlier.
alg.printVector(outlierFinder.modelTest(inputSet));
}
@ -702,15 +703,15 @@ void MLPPTests::test_new_math_functions() {
MLPPData data;
// Testing new Functions
double z_s = 0.001;
real_t z_s = 0.001;
std::cout << avn.logit(z_s) << std::endl;
std::cout << avn.logit(z_s, 1) << std::endl;
std::vector<double> z_v = { 0.001 };
std::vector<real_t> z_v = { 0.001 };
alg.printVector(avn.logit(z_v));
alg.printVector(avn.logit(z_v, 1));
std::vector<std::vector<double>> Z_m = { { 0.001 } };
std::vector<std::vector<real_t>> Z_m = { { 0.001 } };
alg.printMatrix(avn.logit(Z_m));
alg.printMatrix(avn.logit(Z_m, 1));
@ -721,18 +722,18 @@ void MLPPTests::test_new_math_functions() {
alg.printMatrix(alg.matrixPower({ { 5, 5 }, { 5, 5 } }, 2));
alg.printVector(alg.solve({ { 1, 1 }, { 1.5, 4.0 } }, { 2200, 5050 }));
std::vector<std::vector<double>> matrixOfCubes = { { 1, 2, 64, 27 } };
std::vector<double> vectorOfCubes = { 1, 2, 64, 27 };
std::vector<std::vector<real_t>> matrixOfCubes = { { 1, 2, 64, 27 } };
std::vector<real_t> vectorOfCubes = { 1, 2, 64, 27 };
alg.printMatrix(alg.cbrt(matrixOfCubes));
alg.printVector(alg.cbrt(vectorOfCubes));
std::cout << alg.max({ { 1, 2, 3, 4, 5 }, { 6, 5, 3, 4, 1 }, { 9, 9, 9, 9, 9 } }) << std::endl;
std::cout << alg.min({ { 1, 2, 3, 4, 5 }, { 6, 5, 3, 4, 1 }, { 9, 9, 9, 9, 9 } }) << std::endl;
//std::vector<double> chicken;
//std::vector<real_t> chicken;
//data.getImage("../../Data/apple.jpeg", chicken);
//alg.printVector(chicken);
std::vector<std::vector<double>> P = { { 12, -51, 4 }, { 6, 167, -68 }, { -4, 24, -41 } };
std::vector<std::vector<real_t>> P = { { 12, -51, 4 }, { 6, 167, -68 }, { -4, 24, -41 } };
alg.printMatrix(P);
alg.printMatrix(alg.gramSchmidtProcess(P));
@ -752,7 +753,7 @@ void MLPPTests::test_positive_definiteness_checker() {
//MLPPConvolutions conv;
// Checking positive-definiteness checker. For Cholesky Decomp.
std::vector<std::vector<double>> A = {
std::vector<std::vector<real_t>> A = {
{ 1, -1, -1, -1 },
{ -1, 2, 2, 2 },
{ -1, 2, 3, 1 },
@ -765,19 +766,19 @@ void MLPPTests::test_positive_definiteness_checker() {
alg.printMatrix(chres.Lt);
}
// double f(double x){
// real_t f(real_t x){
// return x*x*x + 2*x - 2;
// }
double f(double x) {
real_t f(real_t x) {
return sin(x);
}
double f_prime(double x) {
real_t f_prime(real_t x) {
return 2 * x;
}
double f_prime_2var(std::vector<double> x) {
real_t f_prime_2var(std::vector<real_t> x) {
return 2 * x[0] + x[1];
}
/*
@ -787,7 +788,7 @@ double f_prime_2var(std::vector<double> x) {
y''(2) = 12
*/
// double f_mv(std::vector<double> x){
// real_t f_mv(std::vector<real_t> x){
// return x[0] * x[0] + x[0] * x[1] * x[1] + x[1] + 5;
// }
@ -798,7 +799,7 @@ double f_prime_2var(std::vector<double> x) {
∂^2f/∂x∂y = 2
*/
double f_mv(std::vector<double> x) {
real_t f_mv(std::vector<real_t> x) {
return x[0] * x[0] * x[0] + x[0] + x[1] * x[1] * x[1] * x[0] + x[2] * x[2] * x[1];
}
@ -865,7 +866,7 @@ void MLPPTests::test_numerical_analysis() {
std::cout << numAn.laplacian(f_mv, { 1, 1, 1 }) << std::endl;
std::vector<std::vector<std::vector<double>>> tensor;
std::vector<std::vector<std::vector<real_t>>> tensor;
tensor.push_back({ { 1, 2 }, { 1, 2 }, { 1, 2 } });
tensor.push_back({ { 1, 2 }, { 1, 2 }, { 1, 2 } });
@ -877,7 +878,7 @@ void MLPPTests::test_numerical_analysis() {
std::cout << numAn.eulerianMethod(f_prime, { 1, 1 }, 1.5, 0.000001) << std::endl;
std::cout << numAn.eulerianMethod(f_prime_2var, { 2, 3 }, 2.5, 0.00000001) << std::endl;
std::vector<std::vector<double>> A = {
std::vector<std::vector<real_t>> A = {
{ 1, 0, 0, 0 },
{ 0, 0, 0, 0 },
{ 0, 0, 0, 0 },
@ -898,8 +899,8 @@ void MLPPTests::test_numerical_analysis() {
std::cout << std::endl;
} // Harris detector works. Life is good!
std::vector<double> a = { 3, 4, 4 };
std::vector<double> b = { 4, 4, 4 };
std::vector<real_t> a = { 3, 4, 4 };
std::vector<real_t> b = { 4, 4, 4 };
alg.printVector(alg.cross(a, b));
}
void MLPPTests::test_support_vector_classification_kernel(bool ui) {
@ -913,16 +914,16 @@ void MLPPTests::test_support_vector_classification_kernel(bool ui) {
kernelSVM.gradientDescent(0.0001, 20, ui);
std::cout << "SCORE: " << kernelSVM.score() << std::endl;
std::vector<std::vector<double>> linearlyIndependentMat = {
std::vector<std::vector<real_t>> linearlyIndependentMat = {
{ 1, 2, 3, 4 },
{ 234538495, 4444, 6111, 55 }
{ 2345384, 4444, 6111, 55 }
};
std::cout << "True of false: linearly independent?: " << std::boolalpha << alg.linearIndependenceChecker(linearlyIndependentMat) << std::endl;
}
void MLPPTests::test_mlpp_vector() {
std::vector<double> a = { 4, 3, 1, 3 };
std::vector<real_t> a = { 4, 3, 1, 3 };
Ref<MLPPVector> rv;
rv.instance();
@ -939,7 +940,7 @@ void MLPPTests::test_mlpp_vector() {
is_approx_equals_vec(rv, rv2, "re-set_from_std_vectors test.");
}
void MLPPTests::test_mlpp_matrix() {
std::vector<std::vector<double>> A = {
std::vector<std::vector<real_t>> A = {
{ 1, 0, 0, 0 },
{ 0, 1, 0, 0 },
{ 0, 0, 1, 0 },
@ -961,13 +962,13 @@ void MLPPTests::test_mlpp_matrix() {
is_approx_equals_mat(rmat, rmat2, "re-set_from_std_vectors test.");
}
void MLPPTests::is_approx_equalsd(double a, double b, const String &str) {
void MLPPTests::is_approx_equalsd(real_t a, real_t b, const String &str) {
if (!Math::is_equal_approx(a, b)) {
ERR_PRINT("TEST FAILED: " + str + " Got: " + String::num(a) + " Should be: " + String::num(b));
}
}
void MLPPTests::is_approx_equals_dvec(const Vector<double> &a, const Vector<double> &b, const String &str) {
void MLPPTests::is_approx_equals_dvec(const Vector<real_t> &a, const Vector<real_t> &b, const String &str) {
if (a.size() != b.size()) {
goto IAEDVEC_FAILED;
}
@ -1003,7 +1004,7 @@ IAEDVEC_FAILED:
ERR_PRINT(fail_str);
}
String vmat_to_str(const Vector<Vector<double>> &a) {
String vmat_to_str(const Vector<Vector<real_t>> &a) {
String str;
str += "[ \n";
@ -1011,7 +1012,7 @@ String vmat_to_str(const Vector<Vector<double>> &a) {
for (int i = 0; i < a.size(); ++i) {
str += " [ ";
const Vector<double> &aa = a[i];
const Vector<real_t> &aa = a[i];
for (int j = 0; j < aa.size(); ++j) {
str += String::num(aa[j]);
@ -1026,14 +1027,14 @@ String vmat_to_str(const Vector<Vector<double>> &a) {
return str;
}
void MLPPTests::is_approx_equals_dmat(const Vector<Vector<double>> &a, const Vector<Vector<double>> &b, const String &str) {
void MLPPTests::is_approx_equals_dmat(const Vector<Vector<real_t>> &a, const Vector<Vector<real_t>> &b, const String &str) {
if (a.size() != b.size()) {
goto IAEDMAT_FAILED;
}
for (int i = 0; i < a.size(); ++i) {
const Vector<double> &aa = a[i];
const Vector<double> &bb = b[i];
const Vector<real_t> &aa = a[i];
const Vector<real_t> &bb = b[i];
if (aa.size() != bb.size()) {
goto IAEDMAT_FAILED;
@ -1066,8 +1067,8 @@ void MLPPTests::is_approx_equals_mat(Ref<MLPPMatrix> a, Ref<MLPPMatrix> b, const
int ds = a->data_size();
const double *aa = a->ptr();
const double *bb = b->ptr();
const real_t *aa = a->ptr();
const real_t *bb = b->ptr();
if (a->size() != b->size()) {
goto IAEMAT_FAILED;

8
test/mlpp_tests.h
View File

@ -4,6 +4,8 @@
// TODO port this class to use the test module once it's working
// Also don't forget to remove it's bindings
#include "core/math/math_defs.h"
#include "core/containers/vector.h"
#include "core/object/reference.h"
@ -64,9 +66,9 @@ public:
void test_mlpp_vector();
void test_mlpp_matrix();
void is_approx_equalsd(double a, double b, const String &str);
void is_approx_equals_dvec(const Vector<double> &a, const Vector<double> &b, const String &str);
void is_approx_equals_dmat(const Vector<Vector<double>> &a, const Vector<Vector<double>> &b, const String &str);
void is_approx_equalsd(real_t a, real_t b, const String &str);
void is_approx_equals_dvec(const Vector<real_t> &a, const Vector<real_t> &b, const String &str);
void is_approx_equals_dmat(const Vector<Vector<real_t>> &a, const Vector<Vector<real_t>> &b, const String &str);
void is_approx_equals_mat(Ref<MLPPMatrix> a, Ref<MLPPMatrix> b, const String &str);
void is_approx_equals_vec(Ref<MLPPVector> a, Ref<MLPPVector> b, const String &str);