Also split layers and old layers into different files.

SCsub

@@ -51,6 +51,9 @@ sources = [
     "mlpp/wgan/wgan.cpp",
 
     "mlpp/wgan/wgan_old.cpp",
+    "mlpp/output_layer/output_layer_old.cpp",
+    "mlpp/multi_output_layer/multi_output_layer_old.cpp",
+    "mlpp/hidden_layer/hidden_layer_old.cpp",
 
     "test/mlpp_tests.cpp",
 ]

mlpp/ann/ann.h

@@ -12,6 +12,9 @@
 #include "../hidden_layer/hidden_layer.h"
 #include "../output_layer/output_layer.h"
 
+#include "../hidden_layer/hidden_layer_old.h"
+#include "../output_layer/output_layer_old.h"
+
 #include <string>
 #include <tuple>
 #include <vector>

mlpp/gan/gan.h

@@ -13,6 +13,9 @@
 #include "../hidden_layer/hidden_layer.h"
 #include "../output_layer/output_layer.h"
 
+#include "../hidden_layer/hidden_layer_old.h"
+#include "../output_layer/output_layer_old.h"
+
 #include <string>
 #include <tuple>
 #include <vector>

mlpp/hidden_layer/hidden_layer.cpp

@@ -291,109 +291,3 @@ void MLPPHiddenLayer::_bind_methods() {
 	ClassDB::bind_method(D_METHOD("forward_pass"), &MLPPHiddenLayer::forward_pass);
 	ClassDB::bind_method(D_METHOD("test", "x"), &MLPPHiddenLayer::test);
 }
-
-MLPPOldHiddenLayer::MLPPOldHiddenLayer(int p_n_hidden, std::string p_activation, std::vector<std::vector<real_t>> p_input, std::string p_weightInit, std::string p_reg, real_t p_lambda, real_t p_alpha) {
-	n_hidden = p_n_hidden;
-	activation = p_activation;
-	input = p_input;
-	weightInit = p_weightInit;
-	reg = p_reg;
-	lambda = p_lambda;
-	alpha = p_alpha;
-
-	weights = MLPPUtilities::weightInitialization(input[0].size(), n_hidden, weightInit);
-	bias = MLPPUtilities::biasInitialization(n_hidden);
-
-	activation_map["Linear"] = &MLPPActivation::linear;
-	activationTest_map["Linear"] = &MLPPActivation::linear;
-
-	activation_map["Sigmoid"] = &MLPPActivation::sigmoid;
-	activationTest_map["Sigmoid"] = &MLPPActivation::sigmoid;
-
-	activation_map["Swish"] = &MLPPActivation::swish;
-	activationTest_map["Swish"] = &MLPPActivation::swish;
-
-	activation_map["Mish"] = &MLPPActivation::mish;
-	activationTest_map["Mish"] = &MLPPActivation::mish;
-
-	activation_map["SinC"] = &MLPPActivation::sinc;
-	activationTest_map["SinC"] = &MLPPActivation::sinc;
-
-	activation_map["Softplus"] = &MLPPActivation::softplus;
-	activationTest_map["Softplus"] = &MLPPActivation::softplus;
-
-	activation_map["Softsign"] = &MLPPActivation::softsign;
-	activationTest_map["Softsign"] = &MLPPActivation::softsign;
-
-	activation_map["CLogLog"] = &MLPPActivation::cloglog;
-	activationTest_map["CLogLog"] = &MLPPActivation::cloglog;
-
-	activation_map["Logit"] = &MLPPActivation::logit;
-	activationTest_map["Logit"] = &MLPPActivation::logit;
-
-	activation_map["GaussianCDF"] = &MLPPActivation::gaussianCDF;
-	activationTest_map["GaussianCDF"] = &MLPPActivation::gaussianCDF;
-
-	activation_map["RELU"] = &MLPPActivation::RELU;
-	activationTest_map["RELU"] = &MLPPActivation::RELU;
-
-	activation_map["GELU"] = &MLPPActivation::GELU;
-	activationTest_map["GELU"] = &MLPPActivation::GELU;
-
-	activation_map["Sign"] = &MLPPActivation::sign;
-	activationTest_map["Sign"] = &MLPPActivation::sign;
-
-	activation_map["UnitStep"] = &MLPPActivation::unitStep;
-	activationTest_map["UnitStep"] = &MLPPActivation::unitStep;
-
-	activation_map["Sinh"] = &MLPPActivation::sinh;
-	activationTest_map["Sinh"] = &MLPPActivation::sinh;
-
-	activation_map["Cosh"] = &MLPPActivation::cosh;
-	activationTest_map["Cosh"] = &MLPPActivation::cosh;
-
-	activation_map["Tanh"] = &MLPPActivation::tanh;
-	activationTest_map["Tanh"] = &MLPPActivation::tanh;
-
-	activation_map["Csch"] = &MLPPActivation::csch;
-	activationTest_map["Csch"] = &MLPPActivation::csch;
-
-	activation_map["Sech"] = &MLPPActivation::sech;
-	activationTest_map["Sech"] = &MLPPActivation::sech;
-
-	activation_map["Coth"] = &MLPPActivation::coth;
-	activationTest_map["Coth"] = &MLPPActivation::coth;
-
-	activation_map["Arsinh"] = &MLPPActivation::arsinh;
-	activationTest_map["Arsinh"] = &MLPPActivation::arsinh;
-
-	activation_map["Arcosh"] = &MLPPActivation::arcosh;
-	activationTest_map["Arcosh"] = &MLPPActivation::arcosh;
-
-	activation_map["Artanh"] = &MLPPActivation::artanh;
-	activationTest_map["Artanh"] = &MLPPActivation::artanh;
-
-	activation_map["Arcsch"] = &MLPPActivation::arcsch;
-	activationTest_map["Arcsch"] = &MLPPActivation::arcsch;
-
-	activation_map["Arsech"] = &MLPPActivation::arsech;
-	activationTest_map["Arsech"] = &MLPPActivation::arsech;
-
-	activation_map["Arcoth"] = &MLPPActivation::arcoth;
-	activationTest_map["Arcoth"] = &MLPPActivation::arcoth;
-}
-
-void MLPPOldHiddenLayer::forwardPass() {
-	MLPPLinAlg alg;
-	MLPPActivation avn;
-
-	z = alg.mat_vec_add(alg.matmult(input, weights), bias);
-	a = (avn.*activation_map[activation])(z, false);
-}
-
-void MLPPOldHiddenLayer::Test(std::vector<real_t> x) {
-	MLPPLinAlg alg;
-	MLPPActivation avn;
-	z_test = alg.addition(alg.mat_vec_mult(alg.transpose(weights), x), bias);
-	a_test = (avn.*activationTest_map[activation])(z_test, false);
-}

mlpp/hidden_layer/hidden_layer.h

@@ -110,38 +110,4 @@ protected:
 	bool _initialized;
 };
 
-class MLPPOldHiddenLayer {
-public:
-	MLPPOldHiddenLayer(int n_hidden, std::string activation, std::vector<std::vector<real_t>> input, std::string weightInit, std::string reg, real_t lambda, real_t alpha);
-
-	int n_hidden;
-	std::string activation;
-
-	std::vector<std::vector<real_t>> input;
-
-	std::vector<std::vector<real_t>> weights;
-	std::vector<real_t> bias;
-
-	std::vector<std::vector<real_t>> z;
-	std::vector<std::vector<real_t>> a;
-
-	std::map<std::string, std::vector<std::vector<real_t>> (MLPPActivation::*)(std::vector<std::vector<real_t>>, bool)> activation_map;
-	std::map<std::string, std::vector<real_t> (MLPPActivation::*)(std::vector<real_t>, bool)> activationTest_map;
-
-	std::vector<real_t> z_test;
-	std::vector<real_t> a_test;
-
-	std::vector<std::vector<real_t>> delta;
-
-	// Regularization Params
-	std::string reg;
-	real_t lambda; /* Regularization Parameter */
-	real_t alpha; /* This is the controlling param for Elastic Net*/
-
-	std::string weightInit;
-
-	void forwardPass();
-	void Test(std::vector<real_t> x);
-};
-
 #endif /* HiddenLayer_hpp */

mlpp/hidden_layer/hidden_layer_old.cpp

@@ -0,0 +1,118 @@
+//
+//  HiddenLayer.cpp
+//
+//  Created by Marc Melikyan on 11/4/20.
+//
+
+#include "hidden_layer_old.h"
+#include "../activation/activation.h"
+#include "../lin_alg/lin_alg.h"
+
+#include <iostream>
+#include <random>
+
+MLPPOldHiddenLayer::MLPPOldHiddenLayer(int p_n_hidden, std::string p_activation, std::vector<std::vector<real_t>> p_input, std::string p_weightInit, std::string p_reg, real_t p_lambda, real_t p_alpha) {
+	n_hidden = p_n_hidden;
+	activation = p_activation;
+	input = p_input;
+	weightInit = p_weightInit;
+	reg = p_reg;
+	lambda = p_lambda;
+	alpha = p_alpha;
+
+	weights = MLPPUtilities::weightInitialization(input[0].size(), n_hidden, weightInit);
+	bias = MLPPUtilities::biasInitialization(n_hidden);
+
+	activation_map["Linear"] = &MLPPActivation::linear;
+	activationTest_map["Linear"] = &MLPPActivation::linear;
+
+	activation_map["Sigmoid"] = &MLPPActivation::sigmoid;
+	activationTest_map["Sigmoid"] = &MLPPActivation::sigmoid;
+
+	activation_map["Swish"] = &MLPPActivation::swish;
+	activationTest_map["Swish"] = &MLPPActivation::swish;
+
+	activation_map["Mish"] = &MLPPActivation::mish;
+	activationTest_map["Mish"] = &MLPPActivation::mish;
+
+	activation_map["SinC"] = &MLPPActivation::sinc;
+	activationTest_map["SinC"] = &MLPPActivation::sinc;
+
+	activation_map["Softplus"] = &MLPPActivation::softplus;
+	activationTest_map["Softplus"] = &MLPPActivation::softplus;
+
+	activation_map["Softsign"] = &MLPPActivation::softsign;
+	activationTest_map["Softsign"] = &MLPPActivation::softsign;
+
+	activation_map["CLogLog"] = &MLPPActivation::cloglog;
+	activationTest_map["CLogLog"] = &MLPPActivation::cloglog;
+
+	activation_map["Logit"] = &MLPPActivation::logit;
+	activationTest_map["Logit"] = &MLPPActivation::logit;
+
+	activation_map["GaussianCDF"] = &MLPPActivation::gaussianCDF;
+	activationTest_map["GaussianCDF"] = &MLPPActivation::gaussianCDF;
+
+	activation_map["RELU"] = &MLPPActivation::RELU;
+	activationTest_map["RELU"] = &MLPPActivation::RELU;
+
+	activation_map["GELU"] = &MLPPActivation::GELU;
+	activationTest_map["GELU"] = &MLPPActivation::GELU;
+
+	activation_map["Sign"] = &MLPPActivation::sign;
+	activationTest_map["Sign"] = &MLPPActivation::sign;
+
+	activation_map["UnitStep"] = &MLPPActivation::unitStep;
+	activationTest_map["UnitStep"] = &MLPPActivation::unitStep;
+
+	activation_map["Sinh"] = &MLPPActivation::sinh;
+	activationTest_map["Sinh"] = &MLPPActivation::sinh;
+
+	activation_map["Cosh"] = &MLPPActivation::cosh;
+	activationTest_map["Cosh"] = &MLPPActivation::cosh;
+
+	activation_map["Tanh"] = &MLPPActivation::tanh;
+	activationTest_map["Tanh"] = &MLPPActivation::tanh;
+
+	activation_map["Csch"] = &MLPPActivation::csch;
+	activationTest_map["Csch"] = &MLPPActivation::csch;
+
+	activation_map["Sech"] = &MLPPActivation::sech;
+	activationTest_map["Sech"] = &MLPPActivation::sech;
+
+	activation_map["Coth"] = &MLPPActivation::coth;
+	activationTest_map["Coth"] = &MLPPActivation::coth;
+
+	activation_map["Arsinh"] = &MLPPActivation::arsinh;
+	activationTest_map["Arsinh"] = &MLPPActivation::arsinh;
+
+	activation_map["Arcosh"] = &MLPPActivation::arcosh;
+	activationTest_map["Arcosh"] = &MLPPActivation::arcosh;
+
+	activation_map["Artanh"] = &MLPPActivation::artanh;
+	activationTest_map["Artanh"] = &MLPPActivation::artanh;
+
+	activation_map["Arcsch"] = &MLPPActivation::arcsch;
+	activationTest_map["Arcsch"] = &MLPPActivation::arcsch;
+
+	activation_map["Arsech"] = &MLPPActivation::arsech;
+	activationTest_map["Arsech"] = &MLPPActivation::arsech;
+
+	activation_map["Arcoth"] = &MLPPActivation::arcoth;
+	activationTest_map["Arcoth"] = &MLPPActivation::arcoth;
+}
+
+void MLPPOldHiddenLayer::forwardPass() {
+	MLPPLinAlg alg;
+	MLPPActivation avn;
+
+	z = alg.mat_vec_add(alg.matmult(input, weights), bias);
+	a = (avn.*activation_map[activation])(z, false);
+}
+
+void MLPPOldHiddenLayer::Test(std::vector<real_t> x) {
+	MLPPLinAlg alg;
+	MLPPActivation avn;
+	z_test = alg.addition(alg.mat_vec_mult(alg.transpose(weights), x), bias);
+	a_test = (avn.*activationTest_map[activation])(z_test, false);
+}

mlpp/hidden_layer/hidden_layer_old.h

@@ -0,0 +1,61 @@
+
+#ifndef MLPP_HIDDEN_LAYER_OLD_H
+#define MLPP_HIDDEN_LAYER_OLD_H
+
+//
+//  HiddenLayer.hpp
+//
+//  Created by Marc Melikyan on 11/4/20.
+//
+
+#include "core/math/math_defs.h"
+#include "core/string/ustring.h"
+
+#include "core/object/reference.h"
+
+#include "../activation/activation.h"
+#include "../regularization/reg.h"
+#include "../utilities/utilities.h"
+
+#include "../lin_alg/mlpp_matrix.h"
+#include "../lin_alg/mlpp_vector.h"
+
+#include <map>
+#include <string>
+#include <vector>
+
+class MLPPOldHiddenLayer {
+public:
+	MLPPOldHiddenLayer(int n_hidden, std::string activation, std::vector<std::vector<real_t>> input, std::string weightInit, std::string reg, real_t lambda, real_t alpha);
+
+	int n_hidden;
+	std::string activation;
+
+	std::vector<std::vector<real_t>> input;
+
+	std::vector<std::vector<real_t>> weights;
+	std::vector<real_t> bias;
+
+	std::vector<std::vector<real_t>> z;
+	std::vector<std::vector<real_t>> a;
+
+	std::map<std::string, std::vector<std::vector<real_t>> (MLPPActivation::*)(std::vector<std::vector<real_t>>, bool)> activation_map;
+	std::map<std::string, std::vector<real_t> (MLPPActivation::*)(std::vector<real_t>, bool)> activationTest_map;
+
+	std::vector<real_t> z_test;
+	std::vector<real_t> a_test;
+
+	std::vector<std::vector<real_t>> delta;
+
+	// Regularization Params
+	std::string reg;
+	real_t lambda; /* Regularization Parameter */
+	real_t alpha; /* This is the controlling param for Elastic Net*/
+
+	std::string weightInit;
+
+	void forwardPass();
+	void Test(std::vector<real_t> x);
+};
+
+#endif /* HiddenLayer_hpp */

mlpp/mann/mann.h

@@ -13,11 +13,12 @@
 #include "../hidden_layer/hidden_layer.h"
 #include "../multi_output_layer/multi_output_layer.h"
 
+#include "../hidden_layer/hidden_layer_old.h"
+#include "../multi_output_layer/multi_output_layer_old.h"
+
 #include <string>
 #include <vector>
 
-
-
 class MLPPMANN {
 public:
 	MLPPMANN(std::vector<std::vector<real_t>> inputSet, std::vector<std::vector<real_t>> outputSet);
@@ -47,5 +48,4 @@ private:
 	int n_output;
 };
 
-
 #endif /* MANN_hpp */

mlpp/multi_output_layer/multi_output_layer.cpp

@@ -8,9 +8,6 @@
 #include "../lin_alg/lin_alg.h"
 #include "../utilities/utilities.h"
 
-#include <iostream>
-#include <random>
-
 int MLPPMultiOutputLayer::get_n_output() {
 	return n_output;
 }
@@ -265,130 +262,3 @@ void MLPPMultiOutputLayer::_bind_methods() {
 	ClassDB::bind_method(D_METHOD("forward_pass"), &MLPPMultiOutputLayer::forward_pass);
 	ClassDB::bind_method(D_METHOD("test", "x"), &MLPPMultiOutputLayer::test);
 }
-
-MLPPOldMultiOutputLayer::MLPPOldMultiOutputLayer(int p_n_output, int p_n_hidden, std::string p_activation, std::string p_cost, std::vector<std::vector<real_t>> p_input, std::string p_weightInit, std::string p_reg, real_t p_lambda, real_t p_alpha) {
-	n_output = p_n_output;
-	n_hidden = p_n_hidden;
-	activation = p_activation;
-	cost = p_cost;
-	input = p_input;
-	weightInit = p_weightInit;
-	reg = p_reg;
-	lambda = p_lambda;
-	alpha = p_alpha;
-
-	weights = MLPPUtilities::weightInitialization(n_hidden, n_output, weightInit);
-	bias = MLPPUtilities::biasInitialization(n_output);
-
-	activation_map["Linear"] = &MLPPActivation::linear;
-	activationTest_map["Linear"] = &MLPPActivation::linear;
-
-	activation_map["Sigmoid"] = &MLPPActivation::sigmoid;
-	activationTest_map["Sigmoid"] = &MLPPActivation::sigmoid;
-
-	activation_map["Softmax"] = &MLPPActivation::softmax;
-	activationTest_map["Softmax"] = &MLPPActivation::softmax;
-
-	activation_map["Swish"] = &MLPPActivation::swish;
-	activationTest_map["Swish"] = &MLPPActivation::swish;
-
-	activation_map["Mish"] = &MLPPActivation::mish;
-	activationTest_map["Mish"] = &MLPPActivation::mish;
-
-	activation_map["SinC"] = &MLPPActivation::sinc;
-	activationTest_map["SinC"] = &MLPPActivation::sinc;
-
-	activation_map["Softplus"] = &MLPPActivation::softplus;
-	activationTest_map["Softplus"] = &MLPPActivation::softplus;
-
-	activation_map["Softsign"] = &MLPPActivation::softsign;
-	activationTest_map["Softsign"] = &MLPPActivation::softsign;
-
-	activation_map["CLogLog"] = &MLPPActivation::cloglog;
-	activationTest_map["CLogLog"] = &MLPPActivation::cloglog;
-
-	activation_map["Logit"] = &MLPPActivation::logit;
-	activationTest_map["Logit"] = &MLPPActivation::logit;
-
-	activation_map["GaussianCDF"] = &MLPPActivation::gaussianCDF;
-	activationTest_map["GaussianCDF"] = &MLPPActivation::gaussianCDF;
-
-	activation_map["RELU"] = &MLPPActivation::RELU;
-	activationTest_map["RELU"] = &MLPPActivation::RELU;
-
-	activation_map["GELU"] = &MLPPActivation::GELU;
-	activationTest_map["GELU"] = &MLPPActivation::GELU;
-
-	activation_map["Sign"] = &MLPPActivation::sign;
-	activationTest_map["Sign"] = &MLPPActivation::sign;
-
-	activation_map["UnitStep"] = &MLPPActivation::unitStep;
-	activationTest_map["UnitStep"] = &MLPPActivation::unitStep;
-
-	activation_map["Sinh"] = &MLPPActivation::sinh;
-	activationTest_map["Sinh"] = &MLPPActivation::sinh;
-
-	activation_map["Cosh"] = &MLPPActivation::cosh;
-	activationTest_map["Cosh"] = &MLPPActivation::cosh;
-
-	activation_map["Tanh"] = &MLPPActivation::tanh;
-	activationTest_map["Tanh"] = &MLPPActivation::tanh;
-
-	activation_map["Csch"] = &MLPPActivation::csch;
-	activationTest_map["Csch"] = &MLPPActivation::csch;
-
-	activation_map["Sech"] = &MLPPActivation::sech;
-	activationTest_map["Sech"] = &MLPPActivation::sech;
-
-	activation_map["Coth"] = &MLPPActivation::coth;
-	activationTest_map["Coth"] = &MLPPActivation::coth;
-
-	activation_map["Arsinh"] = &MLPPActivation::arsinh;
-	activationTest_map["Arsinh"] = &MLPPActivation::arsinh;
-
-	activation_map["Arcosh"] = &MLPPActivation::arcosh;
-	activationTest_map["Arcosh"] = &MLPPActivation::arcosh;
-
-	activation_map["Artanh"] = &MLPPActivation::artanh;
-	activationTest_map["Artanh"] = &MLPPActivation::artanh;
-
-	activation_map["Arcsch"] = &MLPPActivation::arcsch;
-	activationTest_map["Arcsch"] = &MLPPActivation::arcsch;
-
-	activation_map["Arsech"] = &MLPPActivation::arsech;
-	activationTest_map["Arsech"] = &MLPPActivation::arsech;
-
-	activation_map["Arcoth"] = &MLPPActivation::arcoth;
-	activationTest_map["Arcoth"] = &MLPPActivation::arcoth;
-
-	costDeriv_map["MSE"] = &MLPPCost::MSEDeriv;
-	cost_map["MSE"] = &MLPPCost::MSE;
-	costDeriv_map["RMSE"] = &MLPPCost::RMSEDeriv;
-	cost_map["RMSE"] = &MLPPCost::RMSE;
-	costDeriv_map["MAE"] = &MLPPCost::MAEDeriv;
-	cost_map["MAE"] = &MLPPCost::MAE;
-	costDeriv_map["MBE"] = &MLPPCost::MBEDeriv;
-	cost_map["MBE"] = &MLPPCost::MBE;
-	costDeriv_map["LogLoss"] = &MLPPCost::LogLossDeriv;
-	cost_map["LogLoss"] = &MLPPCost::LogLoss;
-	costDeriv_map["CrossEntropy"] = &MLPPCost::CrossEntropyDeriv;
-	cost_map["CrossEntropy"] = &MLPPCost::CrossEntropy;
-	costDeriv_map["HingeLoss"] = &MLPPCost::HingeLossDeriv;
-	cost_map["HingeLoss"] = &MLPPCost::HingeLoss;
-	costDeriv_map["WassersteinLoss"] = &MLPPCost::HingeLossDeriv;
-	cost_map["WassersteinLoss"] = &MLPPCost::HingeLoss;
-}
-
-void MLPPOldMultiOutputLayer::forwardPass() {
-	MLPPLinAlg alg;
-	MLPPActivation avn;
-	z = alg.mat_vec_add(alg.matmult(input, weights), bias);
-	a = (avn.*activation_map[activation])(z, false);
-}
-
-void MLPPOldMultiOutputLayer::Test(std::vector<real_t> x) {
-	MLPPLinAlg alg;
-	MLPPActivation avn;
-	z_test = alg.addition(alg.mat_vec_mult(alg.transpose(weights), x), bias);
-	a_test = (avn.*activationTest_map[activation])(z_test, false);
-}

mlpp/multi_output_layer/multi_output_layer.h

@@ -21,10 +21,6 @@
 #include "../lin_alg/mlpp_matrix.h"
 #include "../lin_alg/mlpp_vector.h"
 
-#include <map>
-#include <string>
-#include <vector>
-
 class MLPPMultiOutputLayer : public Reference {
 	GDCLASS(MLPPMultiOutputLayer, Reference);
 
@@ -114,42 +110,4 @@ protected:
 	MLPPUtilities::WeightDistributionType weight_init;
 };
 
-class MLPPOldMultiOutputLayer {
-public:
-	MLPPOldMultiOutputLayer(int n_output, int n_hidden, std::string activation, std::string cost, std::vector<std::vector<real_t>> input, std::string weightInit, std::string reg, real_t lambda, real_t alpha);
-
-	int n_output;
-	int n_hidden;
-	std::string activation;
-	std::string cost;
-
-	std::vector<std::vector<real_t>> input;
-
-	std::vector<std::vector<real_t>> weights;
-	std::vector<real_t> bias;
-
-	std::vector<std::vector<real_t>> z;
-	std::vector<std::vector<real_t>> a;
-
-	std::map<std::string, std::vector<std::vector<real_t>> (MLPPActivation::*)(std::vector<std::vector<real_t>>, bool)> activation_map;
-	std::map<std::string, std::vector<real_t> (MLPPActivation::*)(std::vector<real_t>, bool)> activationTest_map;
-	std::map<std::string, real_t (MLPPCost::*)(std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>)> cost_map;
-	std::map<std::string, std::vector<std::vector<real_t>> (MLPPCost::*)(std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>)> costDeriv_map;
-
-	std::vector<real_t> z_test;
-	std::vector<real_t> a_test;
-
-	std::vector<std::vector<real_t>> delta;
-
-	// Regularization Params
-	std::string reg;
-	real_t lambda; /* Regularization Parameter */
-	real_t alpha; /* This is the controlling param for Elastic Net*/
-
-	std::string weightInit;
-
-	void forwardPass();
-	void Test(std::vector<real_t> x);
-};
-
 #endif /* MultiOutputLayer_hpp */

mlpp/multi_output_layer/multi_output_layer_old.cpp

@@ -0,0 +1,139 @@
+//
+//  MultiOutputLayer.cpp
+//
+//  Created by Marc Melikyan on 11/4/20.
+//
+
+#include "multi_output_layer_old.h"
+#include "../lin_alg/lin_alg.h"
+#include "../utilities/utilities.h"
+
+#include <iostream>
+#include <random>
+
+MLPPOldMultiOutputLayer::MLPPOldMultiOutputLayer(int p_n_output, int p_n_hidden, std::string p_activation, std::string p_cost, std::vector<std::vector<real_t>> p_input, std::string p_weightInit, std::string p_reg, real_t p_lambda, real_t p_alpha) {
+	n_output = p_n_output;
+	n_hidden = p_n_hidden;
+	activation = p_activation;
+	cost = p_cost;
+	input = p_input;
+	weightInit = p_weightInit;
+	reg = p_reg;
+	lambda = p_lambda;
+	alpha = p_alpha;
+
+	weights = MLPPUtilities::weightInitialization(n_hidden, n_output, weightInit);
+	bias = MLPPUtilities::biasInitialization(n_output);
+
+	activation_map["Linear"] = &MLPPActivation::linear;
+	activationTest_map["Linear"] = &MLPPActivation::linear;
+
+	activation_map["Sigmoid"] = &MLPPActivation::sigmoid;
+	activationTest_map["Sigmoid"] = &MLPPActivation::sigmoid;
+
+	activation_map["Softmax"] = &MLPPActivation::softmax;
+	activationTest_map["Softmax"] = &MLPPActivation::softmax;
+
+	activation_map["Swish"] = &MLPPActivation::swish;
+	activationTest_map["Swish"] = &MLPPActivation::swish;
+
+	activation_map["Mish"] = &MLPPActivation::mish;
+	activationTest_map["Mish"] = &MLPPActivation::mish;
+
+	activation_map["SinC"] = &MLPPActivation::sinc;
+	activationTest_map["SinC"] = &MLPPActivation::sinc;
+
+	activation_map["Softplus"] = &MLPPActivation::softplus;
+	activationTest_map["Softplus"] = &MLPPActivation::softplus;
+
+	activation_map["Softsign"] = &MLPPActivation::softsign;
+	activationTest_map["Softsign"] = &MLPPActivation::softsign;
+
+	activation_map["CLogLog"] = &MLPPActivation::cloglog;
+	activationTest_map["CLogLog"] = &MLPPActivation::cloglog;
+
+	activation_map["Logit"] = &MLPPActivation::logit;
+	activationTest_map["Logit"] = &MLPPActivation::logit;
+
+	activation_map["GaussianCDF"] = &MLPPActivation::gaussianCDF;
+	activationTest_map["GaussianCDF"] = &MLPPActivation::gaussianCDF;
+
+	activation_map["RELU"] = &MLPPActivation::RELU;
+	activationTest_map["RELU"] = &MLPPActivation::RELU;
+
+	activation_map["GELU"] = &MLPPActivation::GELU;
+	activationTest_map["GELU"] = &MLPPActivation::GELU;
+
+	activation_map["Sign"] = &MLPPActivation::sign;
+	activationTest_map["Sign"] = &MLPPActivation::sign;
+
+	activation_map["UnitStep"] = &MLPPActivation::unitStep;
+	activationTest_map["UnitStep"] = &MLPPActivation::unitStep;
+
+	activation_map["Sinh"] = &MLPPActivation::sinh;
+	activationTest_map["Sinh"] = &MLPPActivation::sinh;
+
+	activation_map["Cosh"] = &MLPPActivation::cosh;
+	activationTest_map["Cosh"] = &MLPPActivation::cosh;
+
+	activation_map["Tanh"] = &MLPPActivation::tanh;
+	activationTest_map["Tanh"] = &MLPPActivation::tanh;
+
+	activation_map["Csch"] = &MLPPActivation::csch;
+	activationTest_map["Csch"] = &MLPPActivation::csch;
+
+	activation_map["Sech"] = &MLPPActivation::sech;
+	activationTest_map["Sech"] = &MLPPActivation::sech;
+
+	activation_map["Coth"] = &MLPPActivation::coth;
+	activationTest_map["Coth"] = &MLPPActivation::coth;
+
+	activation_map["Arsinh"] = &MLPPActivation::arsinh;
+	activationTest_map["Arsinh"] = &MLPPActivation::arsinh;
+
+	activation_map["Arcosh"] = &MLPPActivation::arcosh;
+	activationTest_map["Arcosh"] = &MLPPActivation::arcosh;
+
+	activation_map["Artanh"] = &MLPPActivation::artanh;
+	activationTest_map["Artanh"] = &MLPPActivation::artanh;
+
+	activation_map["Arcsch"] = &MLPPActivation::arcsch;
+	activationTest_map["Arcsch"] = &MLPPActivation::arcsch;
+
+	activation_map["Arsech"] = &MLPPActivation::arsech;
+	activationTest_map["Arsech"] = &MLPPActivation::arsech;
+
+	activation_map["Arcoth"] = &MLPPActivation::arcoth;
+	activationTest_map["Arcoth"] = &MLPPActivation::arcoth;
+
+	costDeriv_map["MSE"] = &MLPPCost::MSEDeriv;
+	cost_map["MSE"] = &MLPPCost::MSE;
+	costDeriv_map["RMSE"] = &MLPPCost::RMSEDeriv;
+	cost_map["RMSE"] = &MLPPCost::RMSE;
+	costDeriv_map["MAE"] = &MLPPCost::MAEDeriv;
+	cost_map["MAE"] = &MLPPCost::MAE;
+	costDeriv_map["MBE"] = &MLPPCost::MBEDeriv;
+	cost_map["MBE"] = &MLPPCost::MBE;
+	costDeriv_map["LogLoss"] = &MLPPCost::LogLossDeriv;
+	cost_map["LogLoss"] = &MLPPCost::LogLoss;
+	costDeriv_map["CrossEntropy"] = &MLPPCost::CrossEntropyDeriv;
+	cost_map["CrossEntropy"] = &MLPPCost::CrossEntropy;
+	costDeriv_map["HingeLoss"] = &MLPPCost::HingeLossDeriv;
+	cost_map["HingeLoss"] = &MLPPCost::HingeLoss;
+	costDeriv_map["WassersteinLoss"] = &MLPPCost::HingeLossDeriv;
+	cost_map["WassersteinLoss"] = &MLPPCost::HingeLoss;
+}
+
+void MLPPOldMultiOutputLayer::forwardPass() {
+	MLPPLinAlg alg;
+	MLPPActivation avn;
+	z = alg.mat_vec_add(alg.matmult(input, weights), bias);
+	a = (avn.*activation_map[activation])(z, false);
+}
+
+void MLPPOldMultiOutputLayer::Test(std::vector<real_t> x) {
+	MLPPLinAlg alg;
+	MLPPActivation avn;
+	z_test = alg.addition(alg.mat_vec_mult(alg.transpose(weights), x), bias);
+	a_test = (avn.*activationTest_map[activation])(z_test, false);
+}

mlpp/multi_output_layer/multi_output_layer_old.h

@@ -0,0 +1,66 @@
+
+#ifndef MLPP_MULTI_OUTPUT_LAYER_OLD_H
+#define MLPP_MULTI_OUTPUT_LAYER_OLD_H
+
+//
+//  MultiOutputLayer.hpp
+//
+//  Created by Marc Melikyan on 11/4/20.
+//
+
+#include "core/math/math_defs.h"
+#include "core/string/ustring.h"
+
+#include "core/object/reference.h"
+
+#include "../activation/activation.h"
+#include "../cost/cost.h"
+#include "../regularization/reg.h"
+#include "../utilities/utilities.h"
+
+#include "../lin_alg/mlpp_matrix.h"
+#include "../lin_alg/mlpp_vector.h"
+
+#include <map>
+#include <string>
+#include <vector>
+
+class MLPPOldMultiOutputLayer {
+public:
+	MLPPOldMultiOutputLayer(int n_output, int n_hidden, std::string activation, std::string cost, std::vector<std::vector<real_t>> input, std::string weightInit, std::string reg, real_t lambda, real_t alpha);
+
+	int n_output;
+	int n_hidden;
+	std::string activation;
+	std::string cost;
+
+	std::vector<std::vector<real_t>> input;
+
+	std::vector<std::vector<real_t>> weights;
+	std::vector<real_t> bias;
+
+	std::vector<std::vector<real_t>> z;
+	std::vector<std::vector<real_t>> a;
+
+	std::map<std::string, std::vector<std::vector<real_t>> (MLPPActivation::*)(std::vector<std::vector<real_t>>, bool)> activation_map;
+	std::map<std::string, std::vector<real_t> (MLPPActivation::*)(std::vector<real_t>, bool)> activationTest_map;
+	std::map<std::string, real_t (MLPPCost::*)(std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>)> cost_map;
+	std::map<std::string, std::vector<std::vector<real_t>> (MLPPCost::*)(std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>)> costDeriv_map;
+
+	std::vector<real_t> z_test;
+	std::vector<real_t> a_test;
+
+	std::vector<std::vector<real_t>> delta;
+
+	// Regularization Params
+	std::string reg;
+	real_t lambda; /* Regularization Parameter */
+	real_t alpha; /* This is the controlling param for Elastic Net*/
+
+	std::string weightInit;
+
+	void forwardPass();
+	void Test(std::vector<real_t> x);
+};
+
+#endif /* MultiOutputLayer_hpp */

mlpp/output_layer/output_layer.cpp

@@ -8,9 +8,6 @@
 #include "../lin_alg/lin_alg.h"
 #include "../utilities/utilities.h"
 
-#include <iostream>
-#include <random>
-
 int MLPPOutputLayer::get_n_hidden() {
 	return n_hidden;
 }
@@ -300,126 +297,3 @@ void MLPPOutputLayer::_bind_methods() {
 	ClassDB::bind_method(D_METHOD("forward_pass"), &MLPPOutputLayer::forward_pass);
 	ClassDB::bind_method(D_METHOD("test", "x"), &MLPPOutputLayer::test);
 }
-
-MLPPOldOutputLayer::MLPPOldOutputLayer(int p_n_hidden, std::string p_activation, std::string p_cost, std::vector<std::vector<real_t>> p_input, std::string p_weightInit, std::string p_reg, real_t p_lambda, real_t p_alpha) {
-	n_hidden = p_n_hidden;
-	activation = p_activation;
-	cost = p_cost;
-	input = p_input;
-	weightInit = p_weightInit;
-	reg = p_reg;
-	lambda = p_lambda;
-	alpha = p_alpha;
-
-	weights = MLPPUtilities::weightInitialization(n_hidden, weightInit);
-	bias = MLPPUtilities::biasInitialization();
-
-	activation_map["Linear"] = &MLPPActivation::linear;
-	activationTest_map["Linear"] = &MLPPActivation::linear;
-
-	activation_map["Sigmoid"] = &MLPPActivation::sigmoid;
-	activationTest_map["Sigmoid"] = &MLPPActivation::sigmoid;
-
-	activation_map["Swish"] = &MLPPActivation::swish;
-	activationTest_map["Swish"] = &MLPPActivation::swish;
-
-	activation_map["Mish"] = &MLPPActivation::mish;
-	activationTest_map["Mish"] = &MLPPActivation::mish;
-
-	activation_map["SinC"] = &MLPPActivation::sinc;
-	activationTest_map["SinC"] = &MLPPActivation::sinc;
-
-	activation_map["Softplus"] = &MLPPActivation::softplus;
-	activationTest_map["Softplus"] = &MLPPActivation::softplus;
-
-	activation_map["Softsign"] = &MLPPActivation::softsign;
-	activationTest_map["Softsign"] = &MLPPActivation::softsign;
-
-	activation_map["CLogLog"] = &MLPPActivation::cloglog;
-	activationTest_map["CLogLog"] = &MLPPActivation::cloglog;
-
-	activation_map["Logit"] = &MLPPActivation::logit;
-	activationTest_map["Logit"] = &MLPPActivation::logit;
-
-	activation_map["GaussianCDF"] = &MLPPActivation::gaussianCDF;
-	activationTest_map["GaussianCDF"] = &MLPPActivation::gaussianCDF;
-
-	activation_map["RELU"] = &MLPPActivation::RELU;
-	activationTest_map["RELU"] = &MLPPActivation::RELU;
-
-	activation_map["GELU"] = &MLPPActivation::GELU;
-	activationTest_map["GELU"] = &MLPPActivation::GELU;
-
-	activation_map["Sign"] = &MLPPActivation::sign;
-	activationTest_map["Sign"] = &MLPPActivation::sign;
-
-	activation_map["UnitStep"] = &MLPPActivation::unitStep;
-	activationTest_map["UnitStep"] = &MLPPActivation::unitStep;
-
-	activation_map["Sinh"] = &MLPPActivation::sinh;
-	activationTest_map["Sinh"] = &MLPPActivation::sinh;
-
-	activation_map["Cosh"] = &MLPPActivation::cosh;
-	activationTest_map["Cosh"] = &MLPPActivation::cosh;
-
-	activation_map["Tanh"] = &MLPPActivation::tanh;
-	activationTest_map["Tanh"] = &MLPPActivation::tanh;
-
-	activation_map["Csch"] = &MLPPActivation::csch;
-	activationTest_map["Csch"] = &MLPPActivation::csch;
-
-	activation_map["Sech"] = &MLPPActivation::sech;
-	activationTest_map["Sech"] = &MLPPActivation::sech;
-
-	activation_map["Coth"] = &MLPPActivation::coth;
-	activationTest_map["Coth"] = &MLPPActivation::coth;
-
-	activation_map["Arsinh"] = &MLPPActivation::arsinh;
-	activationTest_map["Arsinh"] = &MLPPActivation::arsinh;
-
-	activation_map["Arcosh"] = &MLPPActivation::arcosh;
-	activationTest_map["Arcosh"] = &MLPPActivation::arcosh;
-
-	activation_map["Artanh"] = &MLPPActivation::artanh;
-	activationTest_map["Artanh"] = &MLPPActivation::artanh;
-
-	activation_map["Arcsch"] = &MLPPActivation::arcsch;
-	activationTest_map["Arcsch"] = &MLPPActivation::arcsch;
-
-	activation_map["Arsech"] = &MLPPActivation::arsech;
-	activationTest_map["Arsech"] = &MLPPActivation::arsech;
-
-	activation_map["Arcoth"] = &MLPPActivation::arcoth;
-	activationTest_map["Arcoth"] = &MLPPActivation::arcoth;
-
-	costDeriv_map["MSE"] = &MLPPCost::MSEDeriv;
-	cost_map["MSE"] = &MLPPCost::MSE;
-	costDeriv_map["RMSE"] = &MLPPCost::RMSEDeriv;
-	cost_map["RMSE"] = &MLPPCost::RMSE;
-	costDeriv_map["MAE"] = &MLPPCost::MAEDeriv;
-	cost_map["MAE"] = &MLPPCost::MAE;
-	costDeriv_map["MBE"] = &MLPPCost::MBEDeriv;
-	cost_map["MBE"] = &MLPPCost::MBE;
-	costDeriv_map["LogLoss"] = &MLPPCost::LogLossDeriv;
-	cost_map["LogLoss"] = &MLPPCost::LogLoss;
-	costDeriv_map["CrossEntropy"] = &MLPPCost::CrossEntropyDeriv;
-	cost_map["CrossEntropy"] = &MLPPCost::CrossEntropy;
-	costDeriv_map["HingeLoss"] = &MLPPCost::HingeLossDeriv;
-	cost_map["HingeLoss"] = &MLPPCost::HingeLoss;
-	costDeriv_map["WassersteinLoss"] = &MLPPCost::HingeLossDeriv;
-	cost_map["WassersteinLoss"] = &MLPPCost::HingeLoss;
-}
-
-void MLPPOldOutputLayer::forwardPass() {
-	MLPPLinAlg alg;
-	MLPPActivation avn;
-	z = alg.scalarAdd(bias, alg.mat_vec_mult(input, weights));
-	a = (avn.*activation_map[activation])(z, false);
-}
-
-void MLPPOldOutputLayer::Test(std::vector<real_t> x) {
-	MLPPLinAlg alg;
-	MLPPActivation avn;
-	z_test = alg.dot(weights, x) + bias;
-	a_test = (avn.*activationTest_map[activation])(z_test, false);
-}

mlpp/output_layer/output_layer.h

@@ -20,11 +20,6 @@
 
 #include "../lin_alg/mlpp_matrix.h"
 #include "../lin_alg/mlpp_vector.h"
-
-#include <map>
-#include <string>
-#include <vector>
-
 class MLPPOutputLayer : public Reference {
 	GDCLASS(MLPPOutputLayer, Reference);
 
@@ -115,41 +110,4 @@ protected:
 	bool _initialized;
 };
 
-class MLPPOldOutputLayer {
-public:
-	MLPPOldOutputLayer(int n_hidden, std::string activation, std::string cost, std::vector<std::vector<real_t>> input, std::string weightInit, std::string reg, real_t lambda, real_t alpha);
-
-	int n_hidden;
-	std::string activation;
-	std::string cost;
-
-	std::vector<std::vector<real_t>> input;
-
-	std::vector<real_t> weights;
-	real_t bias;
-
-	std::vector<real_t> z;
-	std::vector<real_t> a;
-
-	std::map<std::string, std::vector<real_t> (MLPPActivation::*)(std::vector<real_t>, bool)> activation_map;
-	std::map<std::string, real_t (MLPPActivation::*)(real_t, bool)> activationTest_map;
-	std::map<std::string, real_t (MLPPCost::*)(std::vector<real_t>, std::vector<real_t>)> cost_map;
-	std::map<std::string, std::vector<real_t> (MLPPCost::*)(std::vector<real_t>, std::vector<real_t>)> costDeriv_map;
-
-	real_t z_test;
-	real_t a_test;
-
-	std::vector<real_t> delta;
-
-	// Regularization Params
-	std::string reg;
-	real_t lambda; /* Regularization Parameter */
-	real_t alpha; /* This is the controlling param for Elastic Net*/
-
-	std::string weightInit;
-
-	void forwardPass();
-	void Test(std::vector<real_t> x);
-};
-
 #endif /* OutputLayer_hpp */

mlpp/output_layer/output_layer_old.cpp

@@ -0,0 +1,135 @@
+//
+//  OutputLayer.cpp
+//
+//  Created by Marc Melikyan on 11/4/20.
+//
+
+#include "output_layer_old.h"
+#include "../lin_alg/lin_alg.h"
+#include "../utilities/utilities.h"
+
+#include <iostream>
+#include <random>
+
+MLPPOldOutputLayer::MLPPOldOutputLayer(int p_n_hidden, std::string p_activation, std::string p_cost, std::vector<std::vector<real_t>> p_input, std::string p_weightInit, std::string p_reg, real_t p_lambda, real_t p_alpha) {
+	n_hidden = p_n_hidden;
+	activation = p_activation;
+	cost = p_cost;
+	input = p_input;
+	weightInit = p_weightInit;
+	reg = p_reg;
+	lambda = p_lambda;
+	alpha = p_alpha;
+
+	weights = MLPPUtilities::weightInitialization(n_hidden, weightInit);
+	bias = MLPPUtilities::biasInitialization();
+
+	activation_map["Linear"] = &MLPPActivation::linear;
+	activationTest_map["Linear"] = &MLPPActivation::linear;
+
+	activation_map["Sigmoid"] = &MLPPActivation::sigmoid;
+	activationTest_map["Sigmoid"] = &MLPPActivation::sigmoid;
+
+	activation_map["Swish"] = &MLPPActivation::swish;
+	activationTest_map["Swish"] = &MLPPActivation::swish;
+
+	activation_map["Mish"] = &MLPPActivation::mish;
+	activationTest_map["Mish"] = &MLPPActivation::mish;
+
+	activation_map["SinC"] = &MLPPActivation::sinc;
+	activationTest_map["SinC"] = &MLPPActivation::sinc;
+
+	activation_map["Softplus"] = &MLPPActivation::softplus;
+	activationTest_map["Softplus"] = &MLPPActivation::softplus;
+
+	activation_map["Softsign"] = &MLPPActivation::softsign;
+	activationTest_map["Softsign"] = &MLPPActivation::softsign;
+
+	activation_map["CLogLog"] = &MLPPActivation::cloglog;
+	activationTest_map["CLogLog"] = &MLPPActivation::cloglog;
+
+	activation_map["Logit"] = &MLPPActivation::logit;
+	activationTest_map["Logit"] = &MLPPActivation::logit;
+
+	activation_map["GaussianCDF"] = &MLPPActivation::gaussianCDF;
+	activationTest_map["GaussianCDF"] = &MLPPActivation::gaussianCDF;
+
+	activation_map["RELU"] = &MLPPActivation::RELU;
+	activationTest_map["RELU"] = &MLPPActivation::RELU;
+
+	activation_map["GELU"] = &MLPPActivation::GELU;
+	activationTest_map["GELU"] = &MLPPActivation::GELU;
+
+	activation_map["Sign"] = &MLPPActivation::sign;
+	activationTest_map["Sign"] = &MLPPActivation::sign;
+
+	activation_map["UnitStep"] = &MLPPActivation::unitStep;
+	activationTest_map["UnitStep"] = &MLPPActivation::unitStep;
+
+	activation_map["Sinh"] = &MLPPActivation::sinh;
+	activationTest_map["Sinh"] = &MLPPActivation::sinh;
+
+	activation_map["Cosh"] = &MLPPActivation::cosh;
+	activationTest_map["Cosh"] = &MLPPActivation::cosh;
+
+	activation_map["Tanh"] = &MLPPActivation::tanh;
+	activationTest_map["Tanh"] = &MLPPActivation::tanh;
+
+	activation_map["Csch"] = &MLPPActivation::csch;
+	activationTest_map["Csch"] = &MLPPActivation::csch;
+
+	activation_map["Sech"] = &MLPPActivation::sech;
+	activationTest_map["Sech"] = &MLPPActivation::sech;
+
+	activation_map["Coth"] = &MLPPActivation::coth;
+	activationTest_map["Coth"] = &MLPPActivation::coth;
+
+	activation_map["Arsinh"] = &MLPPActivation::arsinh;
+	activationTest_map["Arsinh"] = &MLPPActivation::arsinh;
+
+	activation_map["Arcosh"] = &MLPPActivation::arcosh;
+	activationTest_map["Arcosh"] = &MLPPActivation::arcosh;
+
+	activation_map["Artanh"] = &MLPPActivation::artanh;
+	activationTest_map["Artanh"] = &MLPPActivation::artanh;
+
+	activation_map["Arcsch"] = &MLPPActivation::arcsch;
+	activationTest_map["Arcsch"] = &MLPPActivation::arcsch;
+
+	activation_map["Arsech"] = &MLPPActivation::arsech;
+	activationTest_map["Arsech"] = &MLPPActivation::arsech;
+
+	activation_map["Arcoth"] = &MLPPActivation::arcoth;
+	activationTest_map["Arcoth"] = &MLPPActivation::arcoth;
+
+	costDeriv_map["MSE"] = &MLPPCost::MSEDeriv;
+	cost_map["MSE"] = &MLPPCost::MSE;
+	costDeriv_map["RMSE"] = &MLPPCost::RMSEDeriv;
+	cost_map["RMSE"] = &MLPPCost::RMSE;
+	costDeriv_map["MAE"] = &MLPPCost::MAEDeriv;
+	cost_map["MAE"] = &MLPPCost::MAE;
+	costDeriv_map["MBE"] = &MLPPCost::MBEDeriv;
+	cost_map["MBE"] = &MLPPCost::MBE;
+	costDeriv_map["LogLoss"] = &MLPPCost::LogLossDeriv;
+	cost_map["LogLoss"] = &MLPPCost::LogLoss;
+	costDeriv_map["CrossEntropy"] = &MLPPCost::CrossEntropyDeriv;
+	cost_map["CrossEntropy"] = &MLPPCost::CrossEntropy;
+	costDeriv_map["HingeLoss"] = &MLPPCost::HingeLossDeriv;
+	cost_map["HingeLoss"] = &MLPPCost::HingeLoss;
+	costDeriv_map["WassersteinLoss"] = &MLPPCost::HingeLossDeriv;
+	cost_map["WassersteinLoss"] = &MLPPCost::HingeLoss;
+}
+
+void MLPPOldOutputLayer::forwardPass() {
+	MLPPLinAlg alg;
+	MLPPActivation avn;
+	z = alg.scalarAdd(bias, alg.mat_vec_mult(input, weights));
+	a = (avn.*activation_map[activation])(z, false);
+}
+
+void MLPPOldOutputLayer::Test(std::vector<real_t> x) {
+	MLPPLinAlg alg;
+	MLPPActivation avn;
+	z_test = alg.dot(weights, x) + bias;
+	a_test = (avn.*activationTest_map[activation])(z_test, false);
+}

mlpp/output_layer/output_layer_old.h

@@ -0,0 +1,65 @@
+
+#ifndef MLPP_OUTPUT_LAYER_OLD_H
+#define MLPP_OUTPUT_LAYER_OLD_H
+
+//
+//  OutputLayer.hpp
+//
+//  Created by Marc Melikyan on 11/4/20.
+//
+
+#include "core/math/math_defs.h"
+#include "core/string/ustring.h"
+
+#include "core/object/reference.h"
+
+#include "../activation/activation.h"
+#include "../cost/cost.h"
+#include "../regularization/reg.h"
+#include "../utilities/utilities.h"
+
+#include "../lin_alg/mlpp_matrix.h"
+#include "../lin_alg/mlpp_vector.h"
+
+#include <map>
+#include <string>
+#include <vector>
+
+class MLPPOldOutputLayer {
+public:
+	MLPPOldOutputLayer(int n_hidden, std::string activation, std::string cost, std::vector<std::vector<real_t>> input, std::string weightInit, std::string reg, real_t lambda, real_t alpha);
+
+	int n_hidden;
+	std::string activation;
+	std::string cost;
+
+	std::vector<std::vector<real_t>> input;
+
+	std::vector<real_t> weights;
+	real_t bias;
+
+	std::vector<real_t> z;
+	std::vector<real_t> a;
+
+	std::map<std::string, std::vector<real_t> (MLPPActivation::*)(std::vector<real_t>, bool)> activation_map;
+	std::map<std::string, real_t (MLPPActivation::*)(real_t, bool)> activationTest_map;
+	std::map<std::string, real_t (MLPPCost::*)(std::vector<real_t>, std::vector<real_t>)> cost_map;
+	std::map<std::string, std::vector<real_t> (MLPPCost::*)(std::vector<real_t>, std::vector<real_t>)> costDeriv_map;
+
+	real_t z_test;
+	real_t a_test;
+
+	std::vector<real_t> delta;
+
+	// Regularization Params
+	std::string reg;
+	real_t lambda; /* Regularization Parameter */
+	real_t alpha; /* This is the controlling param for Elastic Net*/
+
+	std::string weightInit;
+
+	void forwardPass();
+	void Test(std::vector<real_t> x);
+};
+
+#endif /* OutputLayer_hpp */

mlpp/wgan/wgan_old.h

@@ -17,8 +17,8 @@
 #include "../lin_alg/mlpp_matrix.h"
 #include "../lin_alg/mlpp_vector.h"
 
-#include "../hidden_layer/hidden_layer.h"
+#include "../hidden_layer/hidden_layer_old.h"
-#include "../output_layer/output_layer.h"
+#include "../output_layer/output_layer_old.h"
 
 #include "../activation/activation.h"
 #include "../cost/cost.h"