mirror of
https://github.com/Relintai/pmlpp.git
synced 2024-11-09 13:22:09 +01:00
MLPPSoftmaxNet initial cleanup.
This commit is contained in:
parent
47155163b1
commit
1bb0cab99a
@ -10,7 +10,7 @@
|
||||
#include "core/os/file_access.h"
|
||||
|
||||
#include "../lin_alg/lin_alg.h"
|
||||
#include "../softmax_net/softmax_net.h"
|
||||
#include "../softmax_net/softmax_net_old.h"
|
||||
#include "../stat/stat.h"
|
||||
|
||||
#include <algorithm>
|
||||
@ -1008,11 +1008,11 @@ std::tuple<std::vector<std::vector<real_t>>, std::vector<std::string>> MLPPData:
|
||||
outputSet.push_back(BOW[i]);
|
||||
}
|
||||
MLPPLinAlg alg;
|
||||
MLPPSoftmaxNet *model;
|
||||
MLPPSoftmaxNetOld *model;
|
||||
if (type == "Skipgram") {
|
||||
model = new MLPPSoftmaxNet(outputSet, inputSet, dimension);
|
||||
model = new MLPPSoftmaxNetOld(outputSet, inputSet, dimension);
|
||||
} else { // else = CBOW. We maintain it is a default.
|
||||
model = new MLPPSoftmaxNet(inputSet, outputSet, dimension);
|
||||
model = new MLPPSoftmaxNetOld(inputSet, outputSet, dimension);
|
||||
}
|
||||
model->gradientDescent(learning_rate, max_epoch, 1);
|
||||
|
||||
@ -1074,11 +1074,11 @@ MLPPData::WordsToVecResult MLPPData::word_to_vec(std::vector<std::string> senten
|
||||
outputSet.push_back(BOW[i]);
|
||||
}
|
||||
MLPPLinAlg alg;
|
||||
MLPPSoftmaxNet *model;
|
||||
MLPPSoftmaxNetOld *model;
|
||||
if (type == "Skipgram") {
|
||||
model = new MLPPSoftmaxNet(outputSet, inputSet, dimension);
|
||||
model = new MLPPSoftmaxNetOld(outputSet, inputSet, dimension);
|
||||
} else { // else = CBOW. We maintain it is a default.
|
||||
model = new MLPPSoftmaxNet(inputSet, outputSet, dimension);
|
||||
model = new MLPPSoftmaxNetOld(inputSet, outputSet, dimension);
|
||||
}
|
||||
model->gradientDescent(learning_rate, max_epoch, false);
|
||||
|
||||
|
||||
@ -15,81 +15,114 @@
|
||||
#include <iostream>
|
||||
#include <random>
|
||||
|
||||
MLPPSoftmaxNet::MLPPSoftmaxNet(std::vector<std::vector<real_t>> pinputSet, std::vector<std::vector<real_t>> poutputSet, int pn_hidden, std::string preg, real_t plambda, real_t palpha) {
|
||||
inputSet = pinputSet;
|
||||
outputSet = poutputSet;
|
||||
n = pinputSet.size();
|
||||
k = pinputSet[0].size();
|
||||
n_hidden = pn_hidden;
|
||||
n_class = poutputSet[0].size();
|
||||
reg = preg;
|
||||
lambda = plambda;
|
||||
alpha = palpha;
|
||||
/*
|
||||
Ref<MLPPMatrix> MLPPSoftmaxNet::get_input_set() {
|
||||
return _input_set;
|
||||
}
|
||||
void MLPPSoftmaxNet::set_input_set(const Ref<MLPPMatrix> &val) {
|
||||
_input_set = val;
|
||||
|
||||
y_hat.resize(n);
|
||||
|
||||
weights1 = MLPPUtilities::weightInitialization(k, n_hidden);
|
||||
weights2 = MLPPUtilities::weightInitialization(n_hidden, n_class);
|
||||
bias1 = MLPPUtilities::biasInitialization(n_hidden);
|
||||
bias2 = MLPPUtilities::biasInitialization(n_class);
|
||||
_initialized = false;
|
||||
}
|
||||
|
||||
std::vector<real_t> MLPPSoftmaxNet::modelTest(std::vector<real_t> x) {
|
||||
return Evaluate(x);
|
||||
Ref<MLPPMatrix> MLPPSoftmaxNet::get_output_set() {
|
||||
return _output_set;
|
||||
}
|
||||
void MLPPSoftmaxNet::set_output_set(const Ref<MLPPMatrix> &val) {
|
||||
_output_set = val;
|
||||
|
||||
_initialized = false;
|
||||
}
|
||||
|
||||
std::vector<std::vector<real_t>> MLPPSoftmaxNet::modelSetTest(std::vector<std::vector<real_t>> X) {
|
||||
return Evaluate(X);
|
||||
MLPPReg::RegularizationType MLPPSoftmaxNet::get_reg() {
|
||||
return _reg;
|
||||
}
|
||||
void MLPPSoftmaxNet::set_reg(const MLPPReg::RegularizationType val) {
|
||||
_reg = val;
|
||||
|
||||
_initialized = false;
|
||||
}
|
||||
|
||||
void MLPPSoftmaxNet::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
|
||||
real_t MLPPSoftmaxNet::get_lambda() {
|
||||
return _lambda;
|
||||
}
|
||||
void MLPPSoftmaxNet::set_lambda(const real_t val) {
|
||||
_lambda = val;
|
||||
|
||||
_initialized = false;
|
||||
}
|
||||
|
||||
real_t MLPPSoftmaxNet::get_alpha() {
|
||||
return _alpha;
|
||||
}
|
||||
void MLPPSoftmaxNet::set_alpha(const real_t val) {
|
||||
_alpha = val;
|
||||
|
||||
_initialized = false;
|
||||
}
|
||||
*/
|
||||
|
||||
std::vector<real_t> MLPPSoftmaxNet::model_test(std::vector<real_t> x) {
|
||||
return evaluatev(x);
|
||||
}
|
||||
|
||||
std::vector<std::vector<real_t>> MLPPSoftmaxNet::model_set_test(std::vector<std::vector<real_t>> X) {
|
||||
return evaluatem(X);
|
||||
}
|
||||
|
||||
void MLPPSoftmaxNet::gradient_descent(real_t learning_rate, int max_epoch, bool ui) {
|
||||
MLPPActivation avn;
|
||||
MLPPLinAlg alg;
|
||||
MLPPReg regularization;
|
||||
|
||||
real_t cost_prev = 0;
|
||||
int epoch = 1;
|
||||
forwardPass();
|
||||
|
||||
forward_pass();
|
||||
|
||||
while (true) {
|
||||
cost_prev = Cost(y_hat, outputSet);
|
||||
cost_prev = cost(_y_hat, _output_set);
|
||||
|
||||
// Calculating the errors
|
||||
std::vector<std::vector<real_t>> error = alg.subtraction(y_hat, outputSet);
|
||||
std::vector<std::vector<real_t>> error = alg.subtraction(_y_hat, _output_set);
|
||||
|
||||
// Calculating the weight/bias gradients for layer 2
|
||||
|
||||
std::vector<std::vector<real_t>> D2_1 = alg.matmult(alg.transpose(a2), error);
|
||||
std::vector<std::vector<real_t>> D2_1 = alg.matmult(alg.transpose(_a2), error);
|
||||
|
||||
// weights and bias updation for layer 2
|
||||
weights2 = alg.subtraction(weights2, alg.scalarMultiply(learning_rate, D2_1));
|
||||
weights2 = regularization.regWeights(weights2, lambda, alpha, reg);
|
||||
_weights2 = alg.subtraction(_weights2, alg.scalarMultiply(learning_rate, D2_1));
|
||||
//_reg
|
||||
_weights2 = regularization.regWeights(_weights2, _lambda, _alpha, "None");
|
||||
|
||||
bias2 = alg.subtractMatrixRows(bias2, alg.scalarMultiply(learning_rate, error));
|
||||
_bias2 = alg.subtractMatrixRows(_bias2, alg.scalarMultiply(learning_rate, error));
|
||||
|
||||
//Calculating the weight/bias for layer 1
|
||||
|
||||
std::vector<std::vector<real_t>> D1_1 = alg.matmult(error, alg.transpose(weights2));
|
||||
std::vector<std::vector<real_t>> D1_1 = alg.matmult(error, alg.transpose(_weights2));
|
||||
|
||||
std::vector<std::vector<real_t>> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1));
|
||||
std::vector<std::vector<real_t>> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(_z2, true));
|
||||
|
||||
std::vector<std::vector<real_t>> D1_3 = alg.matmult(alg.transpose(inputSet), D1_2);
|
||||
std::vector<std::vector<real_t>> D1_3 = alg.matmult(alg.transpose(_input_set), D1_2);
|
||||
|
||||
// weight an bias updation for layer 1
|
||||
weights1 = alg.subtraction(weights1, alg.scalarMultiply(learning_rate, D1_3));
|
||||
weights1 = regularization.regWeights(weights1, lambda, alpha, reg);
|
||||
_weights1 = alg.subtraction(_weights1, alg.scalarMultiply(learning_rate, D1_3));
|
||||
//_reg
|
||||
_weights1 = regularization.regWeights(_weights1, _lambda, _alpha, "None");
|
||||
|
||||
bias1 = alg.subtractMatrixRows(bias1, alg.scalarMultiply(learning_rate, D1_2));
|
||||
_bias1 = alg.subtractMatrixRows(_bias1, alg.scalarMultiply(learning_rate, D1_2));
|
||||
|
||||
forwardPass();
|
||||
forward_pass();
|
||||
|
||||
// UI PORTION
|
||||
if (UI) {
|
||||
MLPPUtilities::CostInfo(epoch, cost_prev, Cost(y_hat, outputSet));
|
||||
if (ui) {
|
||||
MLPPUtilities::CostInfo(epoch, cost_prev, cost(_y_hat, _output_set));
|
||||
std::cout << "Layer 1:" << std::endl;
|
||||
MLPPUtilities::UI(weights1, bias1);
|
||||
MLPPUtilities::UI(_weights1, _bias1);
|
||||
std::cout << "Layer 2:" << std::endl;
|
||||
MLPPUtilities::UI(weights2, bias2);
|
||||
MLPPUtilities::UI(_weights2, _bias2);
|
||||
}
|
||||
|
||||
epoch++;
|
||||
|
||||
if (epoch > max_epoch) {
|
||||
@ -98,65 +131,72 @@ void MLPPSoftmaxNet::gradientDescent(real_t learning_rate, int max_epoch, bool U
|
||||
}
|
||||
}
|
||||
|
||||
void MLPPSoftmaxNet::SGD(real_t learning_rate, int max_epoch, bool UI) {
|
||||
void MLPPSoftmaxNet::sgd(real_t learning_rate, int max_epoch, bool ui) {
|
||||
MLPPActivation avn;
|
||||
MLPPLinAlg alg;
|
||||
MLPPReg regularization;
|
||||
|
||||
real_t cost_prev = 0;
|
||||
int epoch = 1;
|
||||
|
||||
std::random_device rd;
|
||||
std::default_random_engine generator(rd());
|
||||
std::uniform_int_distribution<int> distribution(0, int(_n - 1));
|
||||
|
||||
while (true) {
|
||||
std::random_device rd;
|
||||
std::default_random_engine generator(rd());
|
||||
std::uniform_int_distribution<int> distribution(0, int(n - 1));
|
||||
int outputIndex = distribution(generator);
|
||||
|
||||
std::vector<real_t> y_hat = Evaluate(inputSet[outputIndex]);
|
||||
std::vector<real_t> y_hat = evaluatev(_input_set[outputIndex]);
|
||||
|
||||
auto prop_res = propagate(inputSet[outputIndex]);
|
||||
auto prop_res = propagatev(_input_set[outputIndex]);
|
||||
auto z2 = std::get<0>(prop_res);
|
||||
auto a2 = std::get<1>(prop_res);
|
||||
|
||||
cost_prev = Cost({ y_hat }, { outputSet[outputIndex] });
|
||||
std::vector<real_t> error = alg.subtraction(y_hat, outputSet[outputIndex]);
|
||||
cost_prev = cost({ y_hat }, { _output_set[outputIndex] });
|
||||
std::vector<real_t> error = alg.subtraction(y_hat, _output_set[outputIndex]);
|
||||
|
||||
// Weight updation for layer 2
|
||||
std::vector<std::vector<real_t>> D2_1 = alg.outerProduct(error, a2);
|
||||
weights2 = alg.subtraction(weights2, alg.scalarMultiply(learning_rate, alg.transpose(D2_1)));
|
||||
weights2 = regularization.regWeights(weights2, lambda, alpha, reg);
|
||||
_weights2 = alg.subtraction(_weights2, alg.scalarMultiply(learning_rate, alg.transpose(D2_1)));
|
||||
//_reg
|
||||
_weights2 = regularization.regWeights(_weights2, _lambda, _alpha, "None");
|
||||
|
||||
// Bias updation for layer 2
|
||||
bias2 = alg.subtraction(bias2, alg.scalarMultiply(learning_rate, error));
|
||||
_bias2 = alg.subtraction(_bias2, alg.scalarMultiply(learning_rate, error));
|
||||
|
||||
// Weight updation for layer 1
|
||||
std::vector<real_t> D1_1 = alg.mat_vec_mult(weights2, error);
|
||||
std::vector<real_t> D1_1 = alg.mat_vec_mult(_weights2, error);
|
||||
std::vector<real_t> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, true));
|
||||
std::vector<std::vector<real_t>> D1_3 = alg.outerProduct(inputSet[outputIndex], D1_2);
|
||||
std::vector<std::vector<real_t>> D1_3 = alg.outerProduct(_input_set[outputIndex], D1_2);
|
||||
|
||||
weights1 = alg.subtraction(weights1, alg.scalarMultiply(learning_rate, D1_3));
|
||||
weights1 = regularization.regWeights(weights1, lambda, alpha, reg);
|
||||
_weights1 = alg.subtraction(_weights1, alg.scalarMultiply(learning_rate, D1_3));
|
||||
//_reg
|
||||
_weights1 = regularization.regWeights(_weights1, _lambda, _alpha, "None");
|
||||
// Bias updation for layer 1
|
||||
|
||||
bias1 = alg.subtraction(bias1, alg.scalarMultiply(learning_rate, D1_2));
|
||||
_bias1 = alg.subtraction(_bias1, alg.scalarMultiply(learning_rate, D1_2));
|
||||
|
||||
y_hat = Evaluate(inputSet[outputIndex]);
|
||||
if (UI) {
|
||||
MLPPUtilities::CostInfo(epoch, cost_prev, Cost({ y_hat }, { outputSet[outputIndex] }));
|
||||
y_hat = evaluatev(_input_set[outputIndex]);
|
||||
|
||||
if (ui) {
|
||||
MLPPUtilities::CostInfo(epoch, cost_prev, cost({ y_hat }, { _output_set[outputIndex] }));
|
||||
std::cout << "Layer 1:" << std::endl;
|
||||
MLPPUtilities::UI(weights1, bias1);
|
||||
MLPPUtilities::UI(_weights1, _bias1);
|
||||
std::cout << "Layer 2:" << std::endl;
|
||||
MLPPUtilities::UI(weights2, bias2);
|
||||
MLPPUtilities::UI(_weights2, _bias2);
|
||||
}
|
||||
|
||||
epoch++;
|
||||
|
||||
if (epoch > max_epoch) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
forwardPass();
|
||||
|
||||
forward_pass();
|
||||
}
|
||||
|
||||
void MLPPSoftmaxNet::MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI) {
|
||||
void MLPPSoftmaxNet::mbgd(real_t learning_rate, int max_epoch, int mini_batch_size, bool ui) {
|
||||
MLPPActivation avn;
|
||||
MLPPLinAlg alg;
|
||||
MLPPReg regularization;
|
||||
@ -164,40 +204,21 @@ void MLPPSoftmaxNet::MBGD(real_t learning_rate, int max_epoch, int mini_batch_si
|
||||
int epoch = 1;
|
||||
|
||||
// Creating the mini-batches
|
||||
int n_mini_batch = n / mini_batch_size;
|
||||
int n_mini_batch = _n / mini_batch_size;
|
||||
|
||||
auto batches = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch);
|
||||
auto batches = MLPPUtilities::createMiniBatches(_input_set, _output_set, n_mini_batch);
|
||||
auto inputMiniBatches = std::get<0>(batches);
|
||||
auto outputMiniBatches = std::get<1>(batches);
|
||||
|
||||
// Creating the mini-batches
|
||||
for (int i = 0; i < n_mini_batch; i++) {
|
||||
std::vector<std::vector<real_t>> currentInputSet;
|
||||
std::vector<std::vector<real_t>> currentOutputSet;
|
||||
for (int j = 0; j < n / n_mini_batch; j++) {
|
||||
currentInputSet.push_back(inputSet[n / n_mini_batch * i + j]);
|
||||
currentOutputSet.push_back(outputSet[n / n_mini_batch * i + j]);
|
||||
}
|
||||
inputMiniBatches.push_back(currentInputSet);
|
||||
outputMiniBatches.push_back(currentOutputSet);
|
||||
}
|
||||
|
||||
if (real_t(n) / real_t(n_mini_batch) - int(n / n_mini_batch) != 0) {
|
||||
for (int i = 0; i < n - n / n_mini_batch * n_mini_batch; i++) {
|
||||
inputMiniBatches[n_mini_batch - 1].push_back(inputSet[n / n_mini_batch * n_mini_batch + i]);
|
||||
outputMiniBatches[n_mini_batch - 1].push_back(outputSet[n / n_mini_batch * n_mini_batch + i]);
|
||||
}
|
||||
}
|
||||
|
||||
while (true) {
|
||||
for (int i = 0; i < n_mini_batch; i++) {
|
||||
std::vector<std::vector<real_t>> y_hat = Evaluate(inputMiniBatches[i]);
|
||||
std::vector<std::vector<real_t>> y_hat = evaluatem(inputMiniBatches[i]);
|
||||
|
||||
auto propagate_res = propagate(inputMiniBatches[i]);
|
||||
auto propagate_res = propagatem(inputMiniBatches[i]);
|
||||
auto z2 = std::get<0>(propagate_res);
|
||||
auto a2 = std::get<1>(propagate_res);
|
||||
|
||||
cost_prev = Cost(y_hat, outputMiniBatches[i]);
|
||||
cost_prev = cost(y_hat, outputMiniBatches[i]);
|
||||
|
||||
// Calculating the errors
|
||||
std::vector<std::vector<real_t>> error = alg.subtraction(y_hat, outputMiniBatches[i]);
|
||||
@ -207,102 +228,198 @@ void MLPPSoftmaxNet::MBGD(real_t learning_rate, int max_epoch, int mini_batch_si
|
||||
std::vector<std::vector<real_t>> D2_1 = alg.matmult(alg.transpose(a2), error);
|
||||
|
||||
// weights and bias updation for layser 2
|
||||
weights2 = alg.subtraction(weights2, alg.scalarMultiply(learning_rate, D2_1));
|
||||
weights2 = regularization.regWeights(weights2, lambda, alpha, reg);
|
||||
_weights2 = alg.subtraction(_weights2, alg.scalarMultiply(learning_rate, D2_1));
|
||||
//_reg
|
||||
_weights2 = regularization.regWeights(_weights2, _lambda, _alpha, "None");
|
||||
|
||||
// Bias Updation for layer 2
|
||||
bias2 = alg.subtractMatrixRows(bias2, alg.scalarMultiply(learning_rate, error));
|
||||
_bias2 = alg.subtractMatrixRows(_bias2, alg.scalarMultiply(learning_rate, error));
|
||||
|
||||
//Calculating the weight/bias for layer 1
|
||||
|
||||
std::vector<std::vector<real_t>> D1_1 = alg.matmult(error, alg.transpose(weights2));
|
||||
std::vector<std::vector<real_t>> D1_1 = alg.matmult(error, alg.transpose(_weights2));
|
||||
|
||||
std::vector<std::vector<real_t>> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, 1));
|
||||
std::vector<std::vector<real_t>> D1_2 = alg.hadamard_product(D1_1, avn.sigmoid(z2, true));
|
||||
|
||||
std::vector<std::vector<real_t>> D1_3 = alg.matmult(alg.transpose(inputMiniBatches[i]), D1_2);
|
||||
|
||||
// weight an bias updation for layer 1
|
||||
weights1 = alg.subtraction(weights1, alg.scalarMultiply(learning_rate, D1_3));
|
||||
weights1 = regularization.regWeights(weights1, lambda, alpha, reg);
|
||||
_weights1 = alg.subtraction(_weights1, alg.scalarMultiply(learning_rate, D1_3));
|
||||
//_reg
|
||||
_weights1 = regularization.regWeights(_weights1, _lambda, _alpha, "None");
|
||||
|
||||
bias1 = alg.subtractMatrixRows(bias1, alg.scalarMultiply(learning_rate, D1_2));
|
||||
_bias1 = alg.subtractMatrixRows(_bias1, alg.scalarMultiply(learning_rate, D1_2));
|
||||
|
||||
y_hat = Evaluate(inputMiniBatches[i]);
|
||||
y_hat = evaluatem(inputMiniBatches[i]);
|
||||
|
||||
if (UI) {
|
||||
MLPPUtilities::CostInfo(epoch, cost_prev, Cost(y_hat, outputMiniBatches[i]));
|
||||
if (ui) {
|
||||
MLPPUtilities::CostInfo(epoch, cost_prev, cost(y_hat, outputMiniBatches[i]));
|
||||
std::cout << "Layer 1:" << std::endl;
|
||||
MLPPUtilities::UI(weights1, bias1);
|
||||
MLPPUtilities::UI(_weights1, _bias1);
|
||||
std::cout << "Layer 2:" << std::endl;
|
||||
MLPPUtilities::UI(weights2, bias2);
|
||||
MLPPUtilities::UI(_weights2, _bias2);
|
||||
}
|
||||
}
|
||||
|
||||
epoch++;
|
||||
|
||||
if (epoch > max_epoch) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
forwardPass();
|
||||
|
||||
forward_pass();
|
||||
}
|
||||
|
||||
real_t MLPPSoftmaxNet::score() {
|
||||
MLPPUtilities util;
|
||||
return util.performance(y_hat, outputSet);
|
||||
|
||||
return util.performance(_y_hat, _output_set);
|
||||
}
|
||||
|
||||
void MLPPSoftmaxNet::save(std::string fileName) {
|
||||
MLPPUtilities util;
|
||||
util.saveParameters(fileName, weights1, bias1, 0, 1);
|
||||
util.saveParameters(fileName, weights2, bias2, 1, 2);
|
||||
|
||||
util.saveParameters(fileName, _weights1, _bias1, false, 1);
|
||||
util.saveParameters(fileName, _weights2, _bias2, true, 2);
|
||||
}
|
||||
|
||||
std::vector<std::vector<real_t>> MLPPSoftmaxNet::getEmbeddings() {
|
||||
return weights1;
|
||||
std::vector<std::vector<real_t>> MLPPSoftmaxNet::get_embeddings() {
|
||||
return _weights1;
|
||||
}
|
||||
|
||||
real_t MLPPSoftmaxNet::Cost(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
|
||||
bool MLPPSoftmaxNet::is_initialized() {
|
||||
return _initialized;
|
||||
}
|
||||
void MLPPSoftmaxNet::initialize() {
|
||||
if (_initialized) {
|
||||
return;
|
||||
}
|
||||
|
||||
//ERR_FAIL_COND(!_input_set.is_valid() || !_output_set.is_valid());
|
||||
|
||||
_initialized = true;
|
||||
}
|
||||
|
||||
MLPPSoftmaxNet::MLPPSoftmaxNet(std::vector<std::vector<real_t>> p_input_set, std::vector<std::vector<real_t>> p_output_set, int p_n_hidden, MLPPReg::RegularizationType p_reg, real_t p_lambda, real_t p_alpha) {
|
||||
_input_set = p_input_set;
|
||||
_output_set = p_output_set;
|
||||
_n = p_input_set.size();
|
||||
_k = p_input_set[0].size();
|
||||
_n_hidden = p_n_hidden;
|
||||
_n_class = p_output_set[0].size();
|
||||
_reg = p_reg;
|
||||
_lambda = p_lambda;
|
||||
_alpha = p_alpha;
|
||||
|
||||
_y_hat.resize(_n);
|
||||
|
||||
_weights1 = MLPPUtilities::weightInitialization(_k, _n_hidden);
|
||||
_weights2 = MLPPUtilities::weightInitialization(_n_hidden, _n_class);
|
||||
_bias1 = MLPPUtilities::biasInitialization(_n_hidden);
|
||||
_bias2 = MLPPUtilities::biasInitialization(_n_class);
|
||||
|
||||
_initialized = true;
|
||||
}
|
||||
|
||||
MLPPSoftmaxNet::MLPPSoftmaxNet() {
|
||||
_initialized = false;
|
||||
}
|
||||
MLPPSoftmaxNet::~MLPPSoftmaxNet() {
|
||||
}
|
||||
|
||||
real_t MLPPSoftmaxNet::cost(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y) {
|
||||
MLPPReg regularization;
|
||||
MLPPData data;
|
||||
class MLPPCost cost;
|
||||
return cost.CrossEntropy(y_hat, y) + regularization.regTerm(weights1, lambda, alpha, reg) + regularization.regTerm(weights2, lambda, alpha, reg);
|
||||
|
||||
//_reg
|
||||
return cost.CrossEntropy(y_hat, y) + regularization.regTerm(_weights1, _lambda, _alpha, "None") + regularization.regTerm(_weights2, _lambda, _alpha, "None");
|
||||
}
|
||||
|
||||
std::vector<std::vector<real_t>> MLPPSoftmaxNet::Evaluate(std::vector<std::vector<real_t>> X) {
|
||||
std::vector<real_t> MLPPSoftmaxNet::evaluatev(std::vector<real_t> x) {
|
||||
MLPPLinAlg alg;
|
||||
MLPPActivation avn;
|
||||
std::vector<std::vector<real_t>> z2 = alg.mat_vec_add(alg.matmult(X, weights1), bias1);
|
||||
std::vector<std::vector<real_t>> a2 = avn.sigmoid(z2);
|
||||
return avn.adjSoftmax(alg.mat_vec_add(alg.matmult(a2, weights2), bias2));
|
||||
|
||||
std::vector<real_t> z2 = alg.addition(alg.mat_vec_mult(alg.transpose(_weights1), x), _bias1);
|
||||
std::vector<real_t> a2 = avn.sigmoid(z2);
|
||||
|
||||
return avn.adjSoftmax(alg.addition(alg.mat_vec_mult(alg.transpose(_weights2), a2), _bias2));
|
||||
}
|
||||
|
||||
std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> MLPPSoftmaxNet::propagate(std::vector<std::vector<real_t>> X) {
|
||||
std::tuple<std::vector<real_t>, std::vector<real_t>> MLPPSoftmaxNet::propagatev(std::vector<real_t> x) {
|
||||
MLPPLinAlg alg;
|
||||
MLPPActivation avn;
|
||||
std::vector<std::vector<real_t>> z2 = alg.mat_vec_add(alg.matmult(X, weights1), bias1);
|
||||
std::vector<std::vector<real_t>> a2 = avn.sigmoid(z2);
|
||||
|
||||
std::vector<real_t> z2 = alg.addition(alg.mat_vec_mult(alg.transpose(_weights1), x), _bias1);
|
||||
std::vector<real_t> a2 = avn.sigmoid(z2);
|
||||
|
||||
return { z2, a2 };
|
||||
}
|
||||
|
||||
std::vector<real_t> MLPPSoftmaxNet::Evaluate(std::vector<real_t> x) {
|
||||
std::vector<std::vector<real_t>> MLPPSoftmaxNet::evaluatem(std::vector<std::vector<real_t>> X) {
|
||||
MLPPLinAlg alg;
|
||||
MLPPActivation avn;
|
||||
std::vector<real_t> z2 = alg.addition(alg.mat_vec_mult(alg.transpose(weights1), x), bias1);
|
||||
std::vector<real_t> a2 = avn.sigmoid(z2);
|
||||
return avn.adjSoftmax(alg.addition(alg.mat_vec_mult(alg.transpose(weights2), a2), bias2));
|
||||
|
||||
std::vector<std::vector<real_t>> z2 = alg.mat_vec_add(alg.matmult(X, _weights1), _bias1);
|
||||
std::vector<std::vector<real_t>> a2 = avn.sigmoid(z2);
|
||||
|
||||
return avn.adjSoftmax(alg.mat_vec_add(alg.matmult(a2, _weights2), _bias2));
|
||||
}
|
||||
|
||||
std::tuple<std::vector<real_t>, std::vector<real_t>> MLPPSoftmaxNet::propagate(std::vector<real_t> x) {
|
||||
std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> MLPPSoftmaxNet::propagatem(std::vector<std::vector<real_t>> X) {
|
||||
MLPPLinAlg alg;
|
||||
MLPPActivation avn;
|
||||
std::vector<real_t> z2 = alg.addition(alg.mat_vec_mult(alg.transpose(weights1), x), bias1);
|
||||
std::vector<real_t> a2 = avn.sigmoid(z2);
|
||||
|
||||
std::vector<std::vector<real_t>> z2 = alg.mat_vec_add(alg.matmult(X, _weights1), _bias1);
|
||||
std::vector<std::vector<real_t>> a2 = avn.sigmoid(z2);
|
||||
|
||||
return { z2, a2 };
|
||||
}
|
||||
|
||||
void MLPPSoftmaxNet::forwardPass() {
|
||||
void MLPPSoftmaxNet::forward_pass() {
|
||||
MLPPLinAlg alg;
|
||||
MLPPActivation avn;
|
||||
z2 = alg.mat_vec_add(alg.matmult(inputSet, weights1), bias1);
|
||||
a2 = avn.sigmoid(z2);
|
||||
y_hat = avn.adjSoftmax(alg.mat_vec_add(alg.matmult(a2, weights2), bias2));
|
||||
|
||||
_z2 = alg.mat_vec_add(alg.matmult(_input_set, _weights1), _bias1);
|
||||
_a2 = avn.sigmoid(_z2);
|
||||
_y_hat = avn.adjSoftmax(alg.mat_vec_add(alg.matmult(_a2, _weights2), _bias2));
|
||||
}
|
||||
|
||||
void MLPPSoftmaxNet::_bind_methods() {
|
||||
/*
|
||||
ClassDB::bind_method(D_METHOD("get_input_set"), &MLPPSoftmaxNet::get_input_set);
|
||||
ClassDB::bind_method(D_METHOD("set_input_set", "val"), &MLPPSoftmaxNet::set_input_set);
|
||||
ADD_PROPERTY(PropertyInfo(Variant::OBJECT, "input_set", PROPERTY_HINT_RESOURCE_TYPE, "MLPPMatrix"), "set_input_set", "get_input_set");
|
||||
|
||||
ClassDB::bind_method(D_METHOD("get_output_set"), &MLPPSoftmaxNet::get_output_set);
|
||||
ClassDB::bind_method(D_METHOD("set_output_set", "val"), &MLPPSoftmaxNet::set_output_set);
|
||||
ADD_PROPERTY(PropertyInfo(Variant::OBJECT, "output_set", PROPERTY_HINT_RESOURCE_TYPE, "MLPPMatrix"), "set_output_set", "get_output_set");
|
||||
|
||||
ClassDB::bind_method(D_METHOD("get_reg"), &MLPPSoftmaxNet::get_reg);
|
||||
ClassDB::bind_method(D_METHOD("set_reg", "val"), &MLPPSoftmaxNet::set_reg);
|
||||
ADD_PROPERTY(PropertyInfo(Variant::INT, "reg"), "set_reg", "get_reg");
|
||||
|
||||
ClassDB::bind_method(D_METHOD("get_lambda"), &MLPPSoftmaxNet::get_lambda);
|
||||
ClassDB::bind_method(D_METHOD("set_lambda", "val"), &MLPPSoftmaxNet::set_lambda);
|
||||
ADD_PROPERTY(PropertyInfo(Variant::REAL, "lambda"), "set_lambda", "get_lambda");
|
||||
|
||||
ClassDB::bind_method(D_METHOD("get_alpha"), &MLPPSoftmaxNet::get_alpha);
|
||||
ClassDB::bind_method(D_METHOD("set_alpha", "val"), &MLPPSoftmaxNet::set_alpha);
|
||||
ADD_PROPERTY(PropertyInfo(Variant::REAL, "alpha"), "set_alpha", "get_alpha");
|
||||
|
||||
ClassDB::bind_method(D_METHOD("model_test", "x"), &MLPPSoftmaxNet::model_test);
|
||||
ClassDB::bind_method(D_METHOD("model_set_test", "X"), &MLPPSoftmaxNet::model_set_test);
|
||||
|
||||
ClassDB::bind_method(D_METHOD("gradient_descent", "learning_rate", "max_epoch", "ui"), &MLPPSoftmaxNet::gradient_descent, false);
|
||||
ClassDB::bind_method(D_METHOD("sgd", "learning_rate", "max_epoch", "ui"), &MLPPSoftmaxNet::sgd, false);
|
||||
ClassDB::bind_method(D_METHOD("mbgd", "learning_rate", "max_epoch", "mini_batch_size", "ui"), &MLPPSoftmaxNet::mbgd, false);
|
||||
|
||||
ClassDB::bind_method(D_METHOD("score"), &MLPPSoftmaxNet::score);
|
||||
|
||||
ClassDB::bind_method(D_METHOD("save", "file_name"), &MLPPSoftmaxNet::save);
|
||||
|
||||
ClassDB::bind_method(D_METHOD("is_initialized"), &MLPPSoftmaxNet::is_initialized);
|
||||
ClassDB::bind_method(D_METHOD("initialize"), &MLPPSoftmaxNet::initialize);
|
||||
*/
|
||||
}
|
||||
|
||||
@ -9,52 +9,96 @@
|
||||
|
||||
#include "core/math/math_defs.h"
|
||||
|
||||
#include "core/object/reference.h"
|
||||
|
||||
#include "../lin_alg/mlpp_matrix.h"
|
||||
#include "../lin_alg/mlpp_vector.h"
|
||||
|
||||
#include "../regularization/reg.h"
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
class MLPPSoftmaxNet {
|
||||
class MLPPSoftmaxNet : public Reference {
|
||||
GDCLASS(MLPPSoftmaxNet, Reference);
|
||||
|
||||
public:
|
||||
MLPPSoftmaxNet(std::vector<std::vector<real_t>> inputSet, std::vector<std::vector<real_t>> outputSet, int n_hidden, std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
|
||||
std::vector<real_t> modelTest(std::vector<real_t> x);
|
||||
std::vector<std::vector<real_t>> modelSetTest(std::vector<std::vector<real_t>> X);
|
||||
void gradientDescent(real_t learning_rate, int max_epoch, bool UI = false);
|
||||
void SGD(real_t learning_rate, int max_epoch, bool UI = false);
|
||||
void MBGD(real_t learning_rate, int max_epoch, int mini_batch_size, bool UI = false);
|
||||
/*
|
||||
Ref<MLPPMatrix> get_input_set();
|
||||
void set_input_set(const Ref<MLPPMatrix> &val);
|
||||
|
||||
Ref<MLPPMatrix> get_output_set();
|
||||
void set_output_set(const Ref<MLPPMatrix> &val);
|
||||
|
||||
MLPPReg::RegularizationType get_reg();
|
||||
void set_reg(const MLPPReg::RegularizationType val);
|
||||
|
||||
real_t get_lambda();
|
||||
void set_lambda(const real_t val);
|
||||
|
||||
real_t get_alpha();
|
||||
void set_alpha(const real_t val);
|
||||
*/
|
||||
|
||||
std::vector<real_t> model_test(std::vector<real_t> x);
|
||||
std::vector<std::vector<real_t>> model_set_test(std::vector<std::vector<real_t>> X);
|
||||
|
||||
void gradient_descent(real_t learning_rate, int max_epoch, bool ui = false);
|
||||
void sgd(real_t learning_rate, int max_epoch, bool ui = false);
|
||||
void mbgd(real_t learning_rate, int max_epoch, int mini_batch_size, bool ui = false);
|
||||
|
||||
real_t score();
|
||||
|
||||
void save(std::string fileName);
|
||||
|
||||
std::vector<std::vector<real_t>> getEmbeddings(); // This class is used (mostly) for word2Vec. This function returns our embeddings.
|
||||
private:
|
||||
real_t Cost(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
|
||||
std::vector<std::vector<real_t>> get_embeddings(); // This class is used (mostly) for word2Vec. This function returns our embeddings.
|
||||
|
||||
std::vector<std::vector<real_t>> Evaluate(std::vector<std::vector<real_t>> X);
|
||||
std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> propagate(std::vector<std::vector<real_t>> X);
|
||||
std::vector<real_t> Evaluate(std::vector<real_t> x);
|
||||
std::tuple<std::vector<real_t>, std::vector<real_t>> propagate(std::vector<real_t> x);
|
||||
void forwardPass();
|
||||
bool is_initialized();
|
||||
void initialize();
|
||||
|
||||
std::vector<std::vector<real_t>> inputSet;
|
||||
std::vector<std::vector<real_t>> outputSet;
|
||||
std::vector<std::vector<real_t>> y_hat;
|
||||
MLPPSoftmaxNet(std::vector<std::vector<real_t>> p_input_set, std::vector<std::vector<real_t>> p_output_set, int p_n_hidden, MLPPReg::RegularizationType p_reg = MLPPReg::REGULARIZATION_TYPE_NONE, real_t p_lambda = 0.5, real_t p_alpha = 0.5);
|
||||
//MLPPSoftmaxNet(const Ref<MLPPMatrix> &p_input_set, const Ref<MLPPMatrix> &p_output_set, MLPPReg::RegularizationType p_reg = MLPPReg::REGULARIZATION_TYPE_NONE, real_t p_lambda = 0.5, real_t p_alpha = 0.5);
|
||||
|
||||
std::vector<std::vector<real_t>> weights1;
|
||||
std::vector<std::vector<real_t>> weights2;
|
||||
MLPPSoftmaxNet();
|
||||
~MLPPSoftmaxNet();
|
||||
|
||||
std::vector<real_t> bias1;
|
||||
std::vector<real_t> bias2;
|
||||
protected:
|
||||
real_t cost(std::vector<std::vector<real_t>> y_hat, std::vector<std::vector<real_t>> y);
|
||||
|
||||
std::vector<std::vector<real_t>> z2;
|
||||
std::vector<std::vector<real_t>> a2;
|
||||
std::vector<real_t> evaluatev(std::vector<real_t> x);
|
||||
std::tuple<std::vector<real_t>, std::vector<real_t>> propagatev(std::vector<real_t> x);
|
||||
|
||||
int n;
|
||||
int k;
|
||||
int n_class;
|
||||
int n_hidden;
|
||||
std::vector<std::vector<real_t>> evaluatem(std::vector<std::vector<real_t>> X);
|
||||
std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> propagatem(std::vector<std::vector<real_t>> X);
|
||||
|
||||
void forward_pass();
|
||||
|
||||
static void _bind_methods();
|
||||
|
||||
std::vector<std::vector<real_t>> _input_set;
|
||||
std::vector<std::vector<real_t>> _output_set;
|
||||
std::vector<std::vector<real_t>> _y_hat;
|
||||
|
||||
std::vector<std::vector<real_t>> _weights1;
|
||||
std::vector<std::vector<real_t>> _weights2;
|
||||
|
||||
std::vector<real_t> _bias1;
|
||||
std::vector<real_t> _bias2;
|
||||
|
||||
std::vector<std::vector<real_t>> _z2;
|
||||
std::vector<std::vector<real_t>> _a2;
|
||||
|
||||
int _n;
|
||||
int _k;
|
||||
int _n_class;
|
||||
int _n_hidden;
|
||||
|
||||
// Regularization Params
|
||||
std::string reg;
|
||||
real_t lambda;
|
||||
real_t alpha; /* This is the controlling param for Elastic Net*/
|
||||
MLPPReg::RegularizationType _reg;
|
||||
real_t _lambda;
|
||||
real_t _alpha; /* This is the controlling param for Elastic Net*/
|
||||
|
||||
bool _initialized;
|
||||
};
|
||||
|
||||
#endif /* SoftmaxNet_hpp */
|
||||
|
||||
@ -52,12 +52,30 @@
|
||||
#include "../mlpp/outlier_finder/outlier_finder_old.h"
|
||||
#include "../mlpp/pca/pca_old.h"
|
||||
#include "../mlpp/probit_reg/probit_reg_old.h"
|
||||
#include "../mlpp/softmax_net/softmax_net_old.h"
|
||||
#include "../mlpp/softmax_reg/softmax_reg_old.h"
|
||||
#include "../mlpp/svc/svc_old.h"
|
||||
#include "../mlpp/tanh_reg/tanh_reg_old.h"
|
||||
#include "../mlpp/uni_lin_reg/uni_lin_reg_old.h"
|
||||
#include "../mlpp/wgan/wgan_old.h"
|
||||
|
||||
/*
|
||||
#include "../mlpp/ann/ann_old.h"
|
||||
#include "../mlpp/bernoulli_nb/bernoulli_nb_old.h"
|
||||
#include "../mlpp/c_log_log_reg/c_log_log_reg_old.h"
|
||||
#include "../mlpp/dual_svc/dual_svc_old.h"
|
||||
#include "../mlpp/exp_reg/exp_reg_old.h"
|
||||
#include "../mlpp/gan/gan_old.h"
|
||||
#include "../mlpp/gaussian_nb/gaussian_nb_old.h"
|
||||
#include "../mlpp/hidden_layer/hidden_layer_old.h"
|
||||
#include "../mlpp/lin_reg/lin_reg_old.h"
|
||||
#include "../mlpp/log_reg/log_reg_old.h"
|
||||
#include "../mlpp/mann/mann_old.h"
|
||||
#include "../mlpp/multi_output_layer/multi_output_layer_old.h"
|
||||
#include "../mlpp/multinomial_nb/multinomial_nb_old.h"
|
||||
#include "../mlpp/output_layer/output_layer_old.h"
|
||||
*/
|
||||
|
||||
Vector<real_t> dstd_vec_to_vec(const std::vector<real_t> &in) {
|
||||
Vector<real_t> r;
|
||||
|
||||
@ -490,9 +508,14 @@ void MLPPTests::test_soft_max_network(bool ui) {
|
||||
// SOFTMAX NETWORK
|
||||
Ref<MLPPDataComplex> dt = data.load_wine(_wine_data_path);
|
||||
|
||||
MLPPSoftmaxNetOld model_old(dt->get_input()->to_std_vector(), dt->get_output()->to_std_vector(), 1);
|
||||
model_old.gradientDescent(0.01, 100000, ui);
|
||||
alg.printMatrix(model_old.modelSetTest(dt->get_input()->to_std_vector()));
|
||||
std::cout << "ACCURACY: " << 100 * model_old.score() << "%" << std::endl;
|
||||
|
||||
MLPPSoftmaxNet model(dt->get_input()->to_std_vector(), dt->get_output()->to_std_vector(), 1);
|
||||
model.gradientDescent(0.01, 100000, ui);
|
||||
alg.printMatrix(model.modelSetTest(dt->get_input()->to_std_vector()));
|
||||
model.gradient_descent(0.01, 100000, ui);
|
||||
alg.printMatrix(model.model_set_test(dt->get_input()->to_std_vector()));
|
||||
std::cout << "ACCURACY: " << 100 * model.score() << "%" << std::endl;
|
||||
}
|
||||
void MLPPTests::test_autoencoder(bool ui) {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user