Initial HiddenLayer rework.

mlpp/hidden_layer/hidden_layer.cpp

@@ -12,6 +12,107 @@
 #include <iostream>
 #include <random>
 
+/*
+
+void MLPPHiddenLayer::forward_pass() {
+	MLPPLinAlg alg;
+	MLPPActivation avn;
+	z = alg.mat_vec_add(alg.matmult(input, weights), bias);
+	a = (avn.*activation_map[activation])(z, false);
+}
+
+void MLPPHiddenLayer::test(std::vector<real_t> x) {
+	MLPPLinAlg alg;
+	MLPPActivation avn;
+	z_test = alg.addition(alg.mat_vec_mult(alg.transpose(weights), x), bias);
+	a_test = (avn.*activationTest_map[activation])(z_test, 0);
+}
+
+MLPPHiddenLayer::MLPPHiddenLayer(int n_hidden, std::string activation, std::vector<std::vector<real_t>> input, std::string weightInit, std::string reg, real_t lambda, real_t alpha) :
+		n_hidden(n_hidden), activation(activation), input(input), weightInit(weightInit), reg(reg), lambda(lambda), alpha(alpha) {
+	weights = MLPPUtilities::weightInitialization(input[0].size(), n_hidden, weightInit);
+	bias = MLPPUtilities::biasInitialization(n_hidden);
+
+	activation_map["Linear"] = &MLPPActivation::linear;
+	activationTest_map["Linear"] = &MLPPActivation::linear;
+
+	activation_map["Sigmoid"] = &MLPPActivation::sigmoid;
+	activationTest_map["Sigmoid"] = &MLPPActivation::sigmoid;
+
+	activation_map["Swish"] = &MLPPActivation::swish;
+	activationTest_map["Swish"] = &MLPPActivation::swish;
+
+	activation_map["Mish"] = &MLPPActivation::mish;
+	activationTest_map["Mish"] = &MLPPActivation::mish;
+
+	activation_map["SinC"] = &MLPPActivation::sinc;
+	activationTest_map["SinC"] = &MLPPActivation::sinc;
+
+	activation_map["Softplus"] = &MLPPActivation::softplus;
+	activationTest_map["Softplus"] = &MLPPActivation::softplus;
+
+	activation_map["Softsign"] = &MLPPActivation::softsign;
+	activationTest_map["Softsign"] = &MLPPActivation::softsign;
+
+	activation_map["CLogLog"] = &MLPPActivation::cloglog;
+	activationTest_map["CLogLog"] = &MLPPActivation::cloglog;
+
+	activation_map["Logit"] = &MLPPActivation::logit;
+	activationTest_map["Logit"] = &MLPPActivation::logit;
+
+	activation_map["GaussianCDF"] = &MLPPActivation::gaussianCDF;
+	activationTest_map["GaussianCDF"] = &MLPPActivation::gaussianCDF;
+
+	activation_map["RELU"] = &MLPPActivation::RELU;
+	activationTest_map["RELU"] = &MLPPActivation::RELU;
+
+	activation_map["GELU"] = &MLPPActivation::GELU;
+	activationTest_map["GELU"] = &MLPPActivation::GELU;
+
+	activation_map["Sign"] = &MLPPActivation::sign;
+	activationTest_map["Sign"] = &MLPPActivation::sign;
+
+	activation_map["UnitStep"] = &MLPPActivation::unitStep;
+	activationTest_map["UnitStep"] = &MLPPActivation::unitStep;
+
+	activation_map["Sinh"] = &MLPPActivation::sinh;
+	activationTest_map["Sinh"] = &MLPPActivation::sinh;
+
+	activation_map["Cosh"] = &MLPPActivation::cosh;
+	activationTest_map["Cosh"] = &MLPPActivation::cosh;
+
+	activation_map["Tanh"] = &MLPPActivation::tanh;
+	activationTest_map["Tanh"] = &MLPPActivation::tanh;
+
+	activation_map["Csch"] = &MLPPActivation::csch;
+	activationTest_map["Csch"] = &MLPPActivation::csch;
+
+	activation_map["Sech"] = &MLPPActivation::sech;
+	activationTest_map["Sech"] = &MLPPActivation::sech;
+
+	activation_map["Coth"] = &MLPPActivation::coth;
+	activationTest_map["Coth"] = &MLPPActivation::coth;
+
+	activation_map["Arsinh"] = &MLPPActivation::arsinh;
+	activationTest_map["Arsinh"] = &MLPPActivation::arsinh;
+
+	activation_map["Arcosh"] = &MLPPActivation::arcosh;
+	activationTest_map["Arcosh"] = &MLPPActivation::arcosh;
+
+	activation_map["Artanh"] = &MLPPActivation::artanh;
+	activationTest_map["Artanh"] = &MLPPActivation::artanh;
+
+	activation_map["Arcsch"] = &MLPPActivation::arcsch;
+	activationTest_map["Arcsch"] = &MLPPActivation::arcsch;
+
+	activation_map["Arsech"] = &MLPPActivation::arsech;
+	activationTest_map["Arsech"] = &MLPPActivation::arsech;
+
+	activation_map["Arcoth"] = &MLPPActivation::arcoth;
+	activationTest_map["Arcoth"] = &MLPPActivation::arcoth;
+}
+
+*/
 
 MLPPOldHiddenLayer::MLPPOldHiddenLayer(int n_hidden, std::string activation, std::vector<std::vector<real_t>> input, std::string weightInit, std::string reg, real_t lambda, real_t alpha) :
 		n_hidden(n_hidden), activation(activation), input(input), weightInit(weightInit), reg(reg), lambda(lambda), alpha(alpha) {

mlpp/hidden_layer/hidden_layer.h

@@ -8,14 +8,60 @@
 //  Created by Marc Melikyan on 11/4/20.
 //
 
+#include "core/containers/hash_map.h"
 #include "core/math/math_defs.h"
+#include "core/string/ustring.h"
+
+#include "core/object/reference.h"
 
 #include "../activation/activation.h"
 
+#include "../lin_alg/mlpp_matrix.h"
+#include "../lin_alg/mlpp_vector.h"
+
 #include <map>
 #include <string>
 #include <vector>
 
+class MLPPHiddenLayer : public Reference {
+	GDCLASS(MLPPHiddenLayer, Reference);
+
+public:
+	int n_hidden;
+	int activation;
+
+	Ref<MLPPMatrix> input;
+
+	Ref<MLPPMatrix> weights;
+	Ref<MLPPVector> bias;
+
+	Ref<MLPPMatrix> z;
+	Ref<MLPPMatrix> a;
+
+	HashMap<int, Ref<MLPPMatrix> (MLPPActivation::*)(const Ref<MLPPMatrix> &, bool)> activation_map;
+	HashMap<int, Ref<MLPPVector> (MLPPActivation::*)(const Ref<MLPPVector> &, bool)> activation_test_map;
+
+	Ref<MLPPVector> z_test;
+	Ref<MLPPVector> a_test;
+
+	Ref<MLPPMatrix> delta;
+
+	// Regularization Params
+	String reg;
+	real_t lambda; /* Regularization Parameter */
+	real_t alpha; /* This is the controlling param for Elastic Net*/
+
+	String weight_init;
+
+	void forward_pass();
+	void test(const Ref<MLPPVector> &x);
+
+	MLPPHiddenLayer(int p_n_hidden, int p_activation, Ref<MLPPMatrix> p_input, String p_weight_init, String p_reg, real_t p_lambda, real_t p_alpha);
+
+	MLPPHiddenLayer();
+	~MLPPHiddenLayer();
+};
+
 
 class MLPPOldHiddenLayer {
 public:
@@ -51,5 +97,4 @@ public:
 	void Test(std::vector<real_t> x);
 };
 
-
 #endif /* HiddenLayer_hpp */