Relintai/pmlpp
1
0
Fork 0
mirror of https://github.com/Relintai/pmlpp.git synced 2024-12-22 15:06:47 +01:00
Code Issues Packages Projects Releases Wiki Activity

Reworked MLPPANN to use Vectors of matrices instead of tensors.

Browse Source
This commit is contained in:
Relintai 2023-12-28 00:48:39 +01:00
parent c227786c40
commit 4350722185
4 changed files with 103 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

155
mlpp/ann/ann.cpp
View File

@ -8,6 +8,7 @@
#include "../activation/activation.h"
#include "../cost/cost.h"
#include "../lin_alg/lin_alg.h"
#include "../regularization/reg.h"
#include "../utilities/utilities.h"
#include "core/log/logger.h"
@ -62,6 +63,7 @@ real_t MLPPANN::model_test(const Ref<MLPPVector> &x) {
void MLPPANN::gradient_descent(real_t learning_rate, int max_epoch, bool ui) {
MLPPCost mlpp_cost;
MLPPLinAlg alg;
real_t cost_prev = 0;
int epoch = 1;
@ -75,8 +77,8 @@ void MLPPANN::gradient_descent(real_t learning_rate, int max_epoch, bool ui) {
cost_prev = cost(_y_hat, _output_set);
ComputeGradientsResult grads = compute_gradients(_y_hat, _output_set);
grads.cumulative_hidden_layer_w_grad = alg.scalar_multiplynvt(learning_rate / _n, grads.cumulative_hidden_layer_w_grad);
grads.cumulative_hidden_layer_w_grad->scalar_multiply(learning_rate / _n);
grads.output_w_grad->scalar_multiply(learning_rate / _n);
update_parameters(grads.cumulative_hidden_layer_w_grad, grads.output_w_grad, learning_rate); // subject to change. may want bias to have this matrix too.
@ -97,6 +99,7 @@ void MLPPANN::gradient_descent(real_t learning_rate, int max_epoch, bool ui) {
void MLPPANN::sgd(real_t learning_rate, int max_epoch, bool ui) {
MLPPCost mlpp_cost;
MLPPLinAlg alg;
real_t cost_prev = 0;
int epoch = 1;
@ -134,7 +137,7 @@ void MLPPANN::sgd(real_t learning_rate, int max_epoch, bool ui) {
ComputeGradientsResult grads = compute_gradients(y_hat_row_tmp, output_set_row_tmp);
grads.cumulative_hidden_layer_w_grad->scalar_multiply(learning_rate / _n);
grads.cumulative_hidden_layer_w_grad = alg.scalar_multiplynvt(learning_rate / _n, grads.cumulative_hidden_layer_w_grad);
grads.output_w_grad->scalar_multiply(learning_rate / _n);
update_parameters(grads.cumulative_hidden_layer_w_grad, grads.output_w_grad, learning_rate); // subject to change. may want bias to have this matrix too.
@ -156,6 +159,7 @@ void MLPPANN::sgd(real_t learning_rate, int max_epoch, bool ui) {
void MLPPANN::mbgd(real_t learning_rate, int max_epoch, int mini_batch_size, bool ui) {
MLPPCost mlpp_cost;
MLPPLinAlg alg;
real_t cost_prev = 0;
int epoch = 1;
@ -180,7 +184,7 @@ void MLPPANN::mbgd(real_t learning_rate, int max_epoch, int mini_batch_size, boo
ComputeGradientsResult grads = compute_gradients(y_hat, current_output_batch);
grads.cumulative_hidden_layer_w_grad->scalar_multiply(learning_rate / _n);
grads.cumulative_hidden_layer_w_grad = alg.scalar_multiplynvt(learning_rate / _n, grads.cumulative_hidden_layer_w_grad);
grads.output_w_grad->scalar_multiply(learning_rate / _n);
update_parameters(grads.cumulative_hidden_layer_w_grad, grads.output_w_grad, learning_rate); // subject to change. may want bias to have this matrix too.
@ -203,6 +207,7 @@ void MLPPANN::mbgd(real_t learning_rate, int max_epoch, int mini_batch_size, boo
void MLPPANN::momentum(real_t learning_rate, int max_epoch, int mini_batch_size, real_t gamma, bool nag, bool ui) {
class MLPPCost mlpp_cost;
MLPPLinAlg alg;
real_t cost_prev = 0;
int epoch = 1;
@ -216,8 +221,7 @@ void MLPPANN::momentum(real_t learning_rate, int max_epoch, int mini_batch_size,
MLPPUtilities::CreateMiniBatchMVBatch batches = MLPPUtilities::create_mini_batchesmv(_input_set, _output_set, n_mini_batch);
// Initializing necessary components for Adam.
Ref<MLPPTensor3> v_hidden;
v_hidden.instance();
Vector<Ref<MLPPMatrix>> v_hidden;
Ref<MLPPVector> v_output;
v_output.instance();
@ -234,8 +238,8 @@ void MLPPANN::momentum(real_t learning_rate, int max_epoch, int mini_batch_size,
ComputeGradientsResult grads = compute_gradients(y_hat, current_output_batch);
if (!_network.empty() && v_hidden->size() == Size3i()) { // Initing our tensor
v_hidden->resize(grads.cumulative_hidden_layer_w_grad->size());
if (!_network.empty() && v_hidden.empty()) { // Initing our tensor
alg.resizevt(v_hidden, grads.cumulative_hidden_layer_w_grad);
}
if (v_output->size() == 0) {
@ -246,7 +250,7 @@ void MLPPANN::momentum(real_t learning_rate, int max_epoch, int mini_batch_size,
update_parameters(v_hidden, v_output, 0); // DON'T update bias.
}
v_hidden = v_hidden->scalar_multiplyn(gamma)->addn(grads.cumulative_hidden_layer_w_grad->scalar_multiplyn(learning_rate / _n));
v_hidden = alg.additionnvt(alg.scalar_multiplynvt(gamma, v_hidden), alg.scalar_multiplynvt(learning_rate / _n, grads.cumulative_hidden_layer_w_grad));
v_output = v_output->scalar_multiplyn(gamma)->addn(grads.output_w_grad->scalar_multiplyn(learning_rate / _n));
update_parameters(v_hidden, v_output, learning_rate); // subject to change. may want bias to have this matrix too.
@ -269,6 +273,7 @@ void MLPPANN::momentum(real_t learning_rate, int max_epoch, int mini_batch_size,
void MLPPANN::adagrad(real_t learning_rate, int max_epoch, int mini_batch_size, real_t e, bool ui) {
MLPPCost mlpp_cost;
MLPPLinAlg alg;
real_t cost_prev = 0;
int epoch = 1;
@ -282,8 +287,7 @@ void MLPPANN::adagrad(real_t learning_rate, int max_epoch, int mini_batch_size,
MLPPUtilities::CreateMiniBatchMVBatch batches = MLPPUtilities::create_mini_batchesmv(_input_set, _output_set, n_mini_batch);
// Initializing necessary components for Adam.
Ref<MLPPTensor3> v_hidden;
v_hidden.instance();
Vector<Ref<MLPPMatrix>> v_hidden;
Ref<MLPPVector> v_output;
v_output.instance();
@ -300,18 +304,19 @@ void MLPPANN::adagrad(real_t learning_rate, int max_epoch, int mini_batch_size,
ComputeGradientsResult grads = compute_gradients(y_hat, current_output_batch);
if (!_network.empty() && v_hidden->size() == Size3i()) { // Initing our tensor
v_hidden->resize(grads.cumulative_hidden_layer_w_grad->size());
if (!_network.empty() && v_hidden.empty()) { // Initing our tensor
alg.resizevt(v_hidden, grads.cumulative_hidden_layer_w_grad);
}
if (v_output->size() == 0) {
v_output->resize(grads.output_w_grad->size());
}
v_hidden->add(grads.cumulative_hidden_layer_w_grad->exponentiaten(2));
v_hidden = alg.additionnvt(v_hidden, alg.exponentiatenvt(grads.cumulative_hidden_layer_w_grad, 2));
v_output->add(grads.output_w_grad->exponentiaten(2));
Ref<MLPPTensor3> hidden_layer_updations = grads.cumulative_hidden_layer_w_grad->division_element_wisen(v_hidden->sqrtn()->scalar_addn(e))->scalar_multiplyn(learning_rate / _n);
Vector<Ref<MLPPMatrix>> hidden_layer_updations = alg.scalar_multiplynvt(learning_rate / _n, alg.division_element_wisenvnvt(grads.cumulative_hidden_layer_w_grad, alg.scalar_addnvt(e, alg.sqrtnvt(v_hidden))));
Ref<MLPPVector> output_layer_updation = grads.output_w_grad->division_element_wisen(v_output->sqrtn()->scalar_addn(e))->scalar_multiplyn(learning_rate / _n);
update_parameters(hidden_layer_updations, output_layer_updation, learning_rate); // subject to change. may want bias to have this matrix too.
@ -334,6 +339,7 @@ void MLPPANN::adagrad(real_t learning_rate, int max_epoch, int mini_batch_size,
void MLPPANN::adadelta(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t e, bool ui) {
MLPPCost mlpp_cost;
MLPPLinAlg alg;
real_t cost_prev = 0;
int epoch = 1;
@ -347,8 +353,7 @@ void MLPPANN::adadelta(real_t learning_rate, int max_epoch, int mini_batch_size,
MLPPUtilities::CreateMiniBatchMVBatch batches = MLPPUtilities::create_mini_batchesmv(_input_set, _output_set, n_mini_batch);
// Initializing necessary components for Adam.
Ref<MLPPTensor3> v_hidden;
v_hidden.instance();
Vector<Ref<MLPPMatrix>> v_hidden;
Ref<MLPPVector> v_output;
v_output.instance();
@ -364,18 +369,18 @@ void MLPPANN::adadelta(real_t learning_rate, int max_epoch, int mini_batch_size,
ComputeGradientsResult grads = compute_gradients(y_hat, current_output_batch);
if (!_network.empty() && v_hidden->size() == Size3i()) { // Initing our tensor
v_hidden->resize(grads.cumulative_hidden_layer_w_grad->size());
if (!_network.empty() && v_hidden.empty()) { // Initing our tensor
alg.resizevt(v_hidden, grads.cumulative_hidden_layer_w_grad);
}
if (v_output->size() == 0) {
v_output->resize(grads.output_w_grad->size());
}
v_hidden = v_hidden->scalar_multiplyn(1 - b1)->addn(grads.cumulative_hidden_layer_w_grad->exponentiaten(2)->scalar_multiplyn(b1));
v_hidden = alg.additionnvt(alg.scalar_multiplynvt(1 - b1, v_hidden), alg.scalar_multiplynvt(b1, alg.exponentiatenvt(grads.cumulative_hidden_layer_w_grad, 2)));
v_output->add(grads.output_w_grad->exponentiaten(2));
Ref<MLPPTensor3> hidden_layer_updations = grads.cumulative_hidden_layer_w_grad->division_element_wisen(v_hidden->sqrtn()->scalar_addn(e))->scalar_multiplyn(learning_rate / _n);
Vector<Ref<MLPPMatrix>> hidden_layer_updations = alg.scalar_multiplynvt(learning_rate / _n, alg.division_element_wisenvnvt(grads.cumulative_hidden_layer_w_grad, alg.scalar_addnvt(e, alg.sqrtnvt(v_hidden))));
Ref<MLPPVector> output_layer_updation = grads.output_w_grad->division_element_wisen(v_output->sqrtn()->scalar_addn(e))->scalar_multiplyn(learning_rate / _n);
update_parameters(hidden_layer_updations, output_layer_updation, learning_rate); // subject to change. may want bias to have this matrix too.
@ -398,6 +403,7 @@ void MLPPANN::adadelta(real_t learning_rate, int max_epoch, int mini_batch_size,
void MLPPANN::adam(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t b2, real_t e, bool ui) {
MLPPCost mlpp_cost;
MLPPLinAlg alg;
real_t cost_prev = 0;
int epoch = 1;
@ -411,10 +417,8 @@ void MLPPANN::adam(real_t learning_rate, int max_epoch, int mini_batch_size, rea
MLPPUtilities::CreateMiniBatchMVBatch batches = MLPPUtilities::create_mini_batchesmv(_input_set, _output_set, n_mini_batch);
// Initializing necessary components for Adam.
Ref<MLPPTensor3> m_hidden;
Ref<MLPPTensor3> v_hidden;
m_hidden.instance();
v_hidden.instance();
Vector<Ref<MLPPMatrix>> m_hidden;
Vector<Ref<MLPPMatrix>> v_hidden;
Ref<MLPPVector> m_output;
Ref<MLPPVector> v_output;
@ -432,9 +436,9 @@ void MLPPANN::adam(real_t learning_rate, int max_epoch, int mini_batch_size, rea
ComputeGradientsResult grads = compute_gradients(y_hat, current_output_batch);
if (!_network.empty() && m_hidden->size() == Size3i() && v_hidden->size() == Size3i()) { // Initing our tensor
m_hidden->resize(grads.cumulative_hidden_layer_w_grad->size());
v_hidden->resize(grads.cumulative_hidden_layer_w_grad->size());
if (!_network.empty() && m_hidden.empty() && v_hidden.empty()) { // Initing our tensor
alg.resizevt(m_hidden, grads.cumulative_hidden_layer_w_grad);
alg.resizevt(v_hidden, grads.cumulative_hidden_layer_w_grad);
}
if (m_output->size() == 0 && v_output->size()) {
@ -442,19 +446,19 @@ void MLPPANN::adam(real_t learning_rate, int max_epoch, int mini_batch_size, rea
v_output->resize(grads.output_w_grad->size());
}
m_hidden = m_hidden->scalar_multiplyn(b1)->addn(grads.cumulative_hidden_layer_w_grad->scalar_multiplyn(1 - b1));
v_hidden = v_hidden->scalar_multiplyn(b2)->addn(grads.cumulative_hidden_layer_w_grad->exponentiaten(2)->scalar_multiplyn(1 - b2));
m_hidden = alg.additionnvt(alg.scalar_multiplynvt(b1, m_hidden), alg.scalar_multiplynvt(1 - b1, grads.cumulative_hidden_layer_w_grad));
v_hidden = alg.additionnvt(alg.scalar_multiplynvt(b2, v_hidden), alg.scalar_multiplynvt(1 - b2, alg.exponentiatenvt(grads.cumulative_hidden_layer_w_grad, 2)));
m_output = m_output->scalar_multiplyn(b1)->addn(grads.output_w_grad->scalar_multiplyn(1 - b1));
v_output = v_output->scalar_multiplyn(b2)->addn(grads.output_w_grad->exponentiaten(2)->scalar_multiplyn(1 - b2));
Ref<MLPPTensor3> m_hidden_hat = m_hidden->scalar_multiplyn(1 / (1 - Math::pow(b1, epoch)));
Ref<MLPPTensor3> v_hidden_hat = v_hidden->scalar_multiplyn(1 / (1 - Math::pow(b2, epoch)));
Vector<Ref<MLPPMatrix>> m_hidden_hat = alg.scalar_multiplynvt(1 / (1 - Math::pow(b1, epoch)), m_hidden);
Vector<Ref<MLPPMatrix>> v_hidden_hat = alg.scalar_multiplynvt(1 / (1 - Math::pow(b2, epoch)), v_hidden);
Ref<MLPPVector> m_output_hat = m_output->scalar_multiplyn(1 / (1 - Math::pow(b1, epoch)));
Ref<MLPPVector> v_output_hat = v_output->scalar_multiplyn(1 / (1 - Math::pow(b2, epoch)));
Ref<MLPPTensor3> hidden_layer_updations = m_hidden_hat->division_element_wisen(v_hidden_hat->sqrtn()->scalar_addn(e))->scalar_multiplyn(learning_rate / _n);
Vector<Ref<MLPPMatrix>> hidden_layer_updations = alg.scalar_multiplynvt(learning_rate / _n, alg.division_element_wisenvnvt(m_hidden_hat, alg.scalar_addnvt(e, alg.sqrtnvt(v_hidden_hat))));
Ref<MLPPVector> output_layer_updation = m_output_hat->division_element_wisen(v_output_hat->sqrtn()->scalar_addn(e))->scalar_multiplyn(learning_rate / _n);
update_parameters(hidden_layer_updations, output_layer_updation, learning_rate); // subject to change. may want bias to have this matrix too.
@ -476,6 +480,7 @@ void MLPPANN::adam(real_t learning_rate, int max_epoch, int mini_batch_size, rea
void MLPPANN::adamax(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t b2, real_t e, bool ui) {
MLPPCost mlpp_cost;
MLPPLinAlg alg;
real_t cost_prev = 0;
int epoch = 1;
@ -489,10 +494,8 @@ void MLPPANN::adamax(real_t learning_rate, int max_epoch, int mini_batch_size, r
MLPPUtilities::CreateMiniBatchMVBatch batches = MLPPUtilities::create_mini_batchesmv(_input_set, _output_set, n_mini_batch);
// Initializing necessary components for Adam.
Ref<MLPPTensor3> m_hidden;
Ref<MLPPTensor3> u_hidden;
m_hidden.instance();
u_hidden.instance();
Vector<Ref<MLPPMatrix>> m_hidden;
Vector<Ref<MLPPMatrix>> u_hidden;
Ref<MLPPVector> m_output;
Ref<MLPPVector> u_output;
@ -511,9 +514,9 @@ void MLPPANN::adamax(real_t learning_rate, int max_epoch, int mini_batch_size, r
ComputeGradientsResult grads = compute_gradients(y_hat, current_output_batch);
if (!_network.empty() && m_hidden->size() == Size3i() && u_hidden->size() == Size3i()) { // Initing our tensor
m_hidden->resize(grads.cumulative_hidden_layer_w_grad->size());
u_hidden->resize(grads.cumulative_hidden_layer_w_grad->size());
if (!_network.empty() && m_hidden.empty() && u_hidden.empty()) { // Initing our tensor
alg.resizevt(m_hidden, grads.cumulative_hidden_layer_w_grad);
alg.resizevt(u_hidden, grads.cumulative_hidden_layer_w_grad);
}
if (m_output->size() == 0 && u_output->size() == 0) {
@ -521,16 +524,16 @@ void MLPPANN::adamax(real_t learning_rate, int max_epoch, int mini_batch_size, r
u_output->resize(grads.output_w_grad->size());
}
m_hidden->addb(m_hidden->scalar_multiplyn(b1), grads.cumulative_hidden_layer_w_grad->scalar_multiplyn(1 - b1));
u_hidden->maxb(u_hidden->scalar_multiplyn(b2), grads.cumulative_hidden_layer_w_grad->absn());
m_hidden = alg.additionnvt(alg.scalar_multiplynvt(b1, m_hidden), alg.scalar_multiplynvt(1 - b1, grads.cumulative_hidden_layer_w_grad));
u_hidden = alg.maxnvt(alg.scalar_multiplynvt(b2, u_hidden), alg.absnvt(grads.cumulative_hidden_layer_w_grad));
m_output->addb(m_output->scalar_multiplyn(b1), grads.output_w_grad->scalar_multiplyn(1 - b1));
u_output->maxb(u_output->scalar_multiplyn(b2), grads.output_w_grad->absn());
Ref<MLPPTensor3> m_hidden_hat = m_hidden->scalar_multiplyn(1 / (1 - Math::pow(b1, epoch)));
Vector<Ref<MLPPMatrix>> m_hidden_hat = alg.scalar_multiplynvt(1 / (1 - Math::pow(b1, epoch)), m_hidden);
Ref<MLPPVector> m_output_hat = m_output->scalar_multiplyn(1 / (1 - Math::pow(b1, epoch)));
Ref<MLPPTensor3> hidden_layer_updations = m_hidden_hat->division_element_wisen(u_hidden->scalar_addn(e))->scalar_multiplyn(learning_rate / _n);
Vector<Ref<MLPPMatrix>> hidden_layer_updations = alg.scalar_multiplynvt(learning_rate / _n, alg.division_element_wisenvnvt(m_hidden_hat, alg.scalar_addnvt(e, u_hidden)));
Ref<MLPPVector> output_layer_updation = m_output_hat->division_element_wisen(u_output->scalar_addn(e))->scalar_multiplyn(learning_rate / _n);
update_parameters(hidden_layer_updations, output_layer_updation, learning_rate); // subject to change. may want bias to have this matrix too.
@ -552,6 +555,7 @@ void MLPPANN::adamax(real_t learning_rate, int max_epoch, int mini_batch_size, r
void MLPPANN::nadam(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t b2, real_t e, bool ui) {
MLPPCost mlpp_cost;
MLPPLinAlg alg;
real_t cost_prev = 0;
int epoch = 1;
@ -565,10 +569,8 @@ void MLPPANN::nadam(real_t learning_rate, int max_epoch, int mini_batch_size, re
MLPPUtilities::CreateMiniBatchMVBatch batches = MLPPUtilities::create_mini_batchesmv(_input_set, _output_set, n_mini_batch);
// Initializing necessary components for Adam.
Ref<MLPPTensor3> m_hidden;
Ref<MLPPTensor3> v_hidden;
m_hidden.instance();
v_hidden.instance();
Vector<Ref<MLPPMatrix>> m_hidden;
Vector<Ref<MLPPMatrix>> v_hidden;
Ref<MLPPVector> m_output;
Ref<MLPPVector> v_output;
@ -587,9 +589,9 @@ void MLPPANN::nadam(real_t learning_rate, int max_epoch, int mini_batch_size, re
ComputeGradientsResult grads = compute_gradients(y_hat, current_output_batch);
if (!_network.empty() && m_hidden->size() == Size3i() && v_hidden->size() == Size3i()) { // Initing our tensor
m_hidden->resize(grads.cumulative_hidden_layer_w_grad->size());
v_hidden->resize(grads.cumulative_hidden_layer_w_grad->size());
if (!_network.empty() && m_hidden.empty() && v_hidden.empty()) { // Initing our tensor
alg.resizevt(m_hidden, grads.cumulative_hidden_layer_w_grad);
alg.resizevt(v_hidden, grads.cumulative_hidden_layer_w_grad);
}
if (m_output->size() == 0 && v_output->size() == 0) {
@ -597,21 +599,21 @@ void MLPPANN::nadam(real_t learning_rate, int max_epoch, int mini_batch_size, re
v_output->resize(grads.output_w_grad->size());
}
m_hidden->addb(m_hidden->scalar_multiplyn(b1), grads.cumulative_hidden_layer_w_grad->scalar_multiplyn(1 - b1));
v_hidden->addb(v_hidden->scalar_multiplyn(b2), grads.cumulative_hidden_layer_w_grad->exponentiaten(2)->scalar_multiplyn(1 - b2));
m_hidden = alg.additionnvt(alg.scalar_multiplynvt(b1, m_hidden), alg.scalar_multiplynvt(1 - b1, grads.cumulative_hidden_layer_w_grad));
v_hidden = alg.additionnvt(alg.scalar_multiplynvt(b2, v_hidden), alg.scalar_multiplynvt(1 - b2, alg.exponentiatenvt(grads.cumulative_hidden_layer_w_grad, 2)));
m_output->addb(m_output->scalar_multiplyn(b1), grads.output_w_grad->scalar_multiplyn(1 - b1));
v_output->addb(v_output->scalar_multiplyn(b2), grads.output_w_grad->exponentiaten(2)->scalar_multiplyn(1 - b2));
Ref<MLPPTensor3> m_hidden_hat = m_hidden->scalar_multiplyn(1 / (1.0 - Math::pow(b1, epoch)));
Ref<MLPPTensor3> v_hidden_hat = v_hidden->scalar_multiplyn(1 / (1.0 - Math::pow(b2, epoch)));
Ref<MLPPTensor3> m_hidden_final = m_hidden_hat->scalar_multiplyn(b1)->addn(grads.cumulative_hidden_layer_w_grad->scalar_multiplyn((1 - b1) / (1 - Math::pow(b1, epoch))));
Vector<Ref<MLPPMatrix>> m_hidden_hat = alg.scalar_multiplynvt(1 / (1 - Math::pow(b1, epoch)), m_hidden);
Vector<Ref<MLPPMatrix>> v_hidden_hat = alg.scalar_multiplynvt(1 / (1 - Math::pow(b2, epoch)), v_hidden);
Vector<Ref<MLPPMatrix>> m_hidden_final = alg.additionnvt(alg.scalar_multiplynvt(b1, m_hidden_hat), alg.scalar_multiplynvt((1 - b1) / (1 - Math::pow(b1, epoch)), grads.cumulative_hidden_layer_w_grad));
Ref<MLPPVector> m_output_hat = m_output->scalar_multiplyn(1 / (1.0 - Math::pow(b1, epoch)));
Ref<MLPPVector> v_output_hat = v_output->scalar_multiplyn(1 / (1.0 - Math::pow(b2, epoch)));
Ref<MLPPVector> m_output_final = m_output_hat->scalar_multiplyn(b1)->addn(grads.output_w_grad->scalar_multiplyn((1 - b1) / (1.0 - Math::pow(b1, epoch))));
Ref<MLPPTensor3> hidden_layer_updations = m_hidden_final->division_element_wisen(v_hidden_hat->sqrtn()->scalar_multiplyn(e))->scalar_multiplyn(learning_rate / _n);
Vector<Ref<MLPPMatrix>> hidden_layer_updations = alg.scalar_multiplynvt(learning_rate / _n, alg.division_element_wisenvnvt(m_hidden_final, alg.scalar_addnvt(e, alg.sqrtnvt(v_hidden_hat))));
Ref<MLPPVector> output_layer_updation = m_output_final->division_element_wisen(v_output_hat->sqrtn()->scalar_addn(e))->scalar_multiplyn(learning_rate / _n);
update_parameters(hidden_layer_updations, output_layer_updation, learning_rate); // subject to change. may want bias to have this matrix too.
@ -635,6 +637,7 @@ void MLPPANN::nadam(real_t learning_rate, int max_epoch, int mini_batch_size, re
void MLPPANN::amsgrad(real_t learning_rate, int max_epoch, int mini_batch_size, real_t b1, real_t b2, real_t e, bool ui) {
MLPPCost mlpp_cost;
MLPPLinAlg alg;
real_t cost_prev = 0;
int epoch = 1;
@ -648,13 +651,10 @@ void MLPPANN::amsgrad(real_t learning_rate, int max_epoch, int mini_batch_size,
MLPPUtilities::CreateMiniBatchMVBatch batches = MLPPUtilities::create_mini_batchesmv(_input_set, _output_set, n_mini_batch);
// Initializing necessary components for Adam.
Ref<MLPPTensor3> m_hidden;
Ref<MLPPTensor3> v_hidden;
m_hidden.instance();
v_hidden.instance();
Vector<Ref<MLPPMatrix>> m_hidden;
Vector<Ref<MLPPMatrix>> v_hidden;
Ref<MLPPTensor3> v_hidden_hat;
v_hidden_hat.instance();
Vector<Ref<MLPPMatrix>> v_hidden_hat;
Ref<MLPPVector> m_output;
Ref<MLPPVector> v_output;
@ -676,11 +676,10 @@ void MLPPANN::amsgrad(real_t learning_rate, int max_epoch, int mini_batch_size,
ComputeGradientsResult grads = compute_gradients(y_hat, current_output_batch);
if (!_network.empty() && m_hidden->size() == Size3i() && v_hidden->size() == Size3i()) { // Initing our tensor
m_hidden->resize(grads.cumulative_hidden_layer_w_grad->size());
v_hidden->resize(grads.cumulative_hidden_layer_w_grad->size());
v_hidden_hat->resize(grads.cumulative_hidden_layer_w_grad->size());
if (!_network.empty() && m_hidden.size() == 0 && v_hidden.size() == 0) { // Initing our tensor
alg.resizevt(m_hidden, grads.cumulative_hidden_layer_w_grad);
alg.resizevt(v_hidden, grads.cumulative_hidden_layer_w_grad);
alg.resizevt(v_hidden_hat, grads.cumulative_hidden_layer_w_grad);
}
if (m_output->size() == 0 && v_output->size() == 0) {
@ -689,16 +688,16 @@ void MLPPANN::amsgrad(real_t learning_rate, int max_epoch, int mini_batch_size,
v_output_hat->resize(grads.output_w_grad->size());
}
m_hidden->addb(m_hidden->scalar_multiplyn(b1), grads.cumulative_hidden_layer_w_grad->scalar_multiplyn(1 - b1));
v_hidden->addb(v_hidden->scalar_multiplyn(b2), grads.cumulative_hidden_layer_w_grad->exponentiaten(2)->scalar_multiplyn(1 - b2));
m_hidden = alg.additionnvt(alg.scalar_multiplynvt(b1, m_hidden), alg.scalar_multiplynvt(1 - b1, grads.cumulative_hidden_layer_w_grad));
v_hidden = alg.additionnvt(alg.scalar_multiplynvt(b2, v_hidden), alg.scalar_multiplynvt(1 - b2, alg.exponentiatenvt(grads.cumulative_hidden_layer_w_grad, 2)));
m_output->addb(m_output->scalar_multiplyn(b1), grads.output_w_grad->scalar_multiplyn(1 - b1));
v_output->addb(v_output->scalar_multiplyn(b2), grads.output_w_grad->exponentiaten(2)->scalar_multiplyn(1 - b2));
v_hidden_hat->max(v_hidden);
v_hidden_hat = alg.maxnvt(v_hidden_hat, v_hidden);
v_output_hat->max(v_output);
Ref<MLPPTensor3> hidden_layer_updations = m_hidden->division_element_wisen(v_hidden_hat->sqrtn()->scalar_addn(e))->scalar_multiplyn(learning_rate / _n);
Vector<Ref<MLPPMatrix>> hidden_layer_updations = alg.scalar_multiplynvt(learning_rate / _n, alg.division_element_wisenvnvt(m_hidden, alg.scalar_addnvt(e, alg.sqrtnvt(v_hidden_hat))));
Ref<MLPPVector> output_layer_updation = m_output->division_element_wisen(v_output_hat->sqrtn()->scalar_addn(e))->scalar_multiplyn(learning_rate / _n);
update_parameters(hidden_layer_updations, output_layer_updation, learning_rate); // subject to change. may want bias to have this matrix too.
@ -847,17 +846,16 @@ void MLPPANN::forward_pass() {
_y_hat = _output_layer->get_a();
}
void MLPPANN::update_parameters(const Ref<MLPPTensor3> &hidden_layer_updations, const Ref<MLPPVector> &output_layer_updation, real_t learning_rate) {
void MLPPANN::update_parameters(const Vector<Ref<MLPPMatrix>> &hidden_layer_updations, const Ref<MLPPVector> &output_layer_updation, real_t learning_rate) {
_output_layer->set_weights(_output_layer->get_weights()->subn(output_layer_updation));
_output_layer->set_bias(_output_layer->get_bias() - learning_rate * _output_layer->get_delta()->sum_elements() / _n);
Ref<MLPPMatrix> slice;
slice.instance();
if (!_network.empty()) {
Ref<MLPPHiddenLayer> layer = _network[_network.size() - 1];
hidden_layer_updations->z_slice_get_into_mlpp_matrix(0, slice);
slice = hidden_layer_updations[0];
layer->set_weights(layer->get_weights()->subn(slice));
layer->set_bias(layer->get_bias()->subtract_matrix_rowsn(layer->get_delta()->scalar_multiplyn(learning_rate / _n)));
@ -865,7 +863,7 @@ void MLPPANN::update_parameters(const Ref<MLPPTensor3> &hidden_layer_updations,
for (int i = _network.size() - 2; i >= 0; i--) {
layer = _network[i];
hidden_layer_updations->z_slice_get_into_mlpp_matrix((_network.size() - 2) - i + 1, slice);
slice = hidden_layer_updations[(_network.size() - 2) - i + 1];
layer->set_weights(layer->get_weights()->subn(slice));
layer->set_bias(layer->get_bias()->subtract_matrix_rowsn(layer->get_delta()->scalar_multiplyn(learning_rate / _n)));
@ -893,11 +891,8 @@ MLPPANN::ComputeGradientsResult MLPPANN::compute_gradients(const Ref<MLPPVector>
Ref<MLPPMatrix> hidden_layer_w_grad = layer->get_input()->transposen()->multn(layer->get_delta());
// TODO Are these always uniform in size?
res.cumulative_hidden_layer_w_grad->resize(Size3i(hidden_layer_w_grad->size().x, hidden_layer_w_grad->size().y, 0));
// Adding to our cumulative hidden layer grads. Maintain reg terms as well.
res.cumulative_hidden_layer_w_grad->z_slice_add_mlpp_matrix(hidden_layer_w_grad->addn(regularization.reg_deriv_termm(layer->get_weights(), layer->get_lambda(), layer->get_alpha(), layer->get_reg())));
res.cumulative_hidden_layer_w_grad.push_back(hidden_layer_w_grad->addn(regularization.reg_deriv_termm(layer->get_weights(), layer->get_lambda(), layer->get_alpha(), layer->get_reg())));
for (int i = _network.size() - 2; i >= 0; i--) {
layer = _network[i];
@ -905,7 +900,7 @@ MLPPANN::ComputeGradientsResult MLPPANN::compute_gradients(const Ref<MLPPVector>
layer->set_delta(next_layer->get_delta()->multn(next_layer->get_weights()->transposen())->hadamard_productn(avn.run_activation_deriv_matrix(layer->get_activation(), layer->get_z())));
hidden_layer_w_grad = layer->get_input()->transposen()->multn(layer->get_delta());
res.cumulative_hidden_layer_w_grad->z_slice_add_mlpp_matrix(hidden_layer_w_grad->addn(regularization.reg_deriv_termm(layer->get_weights(), layer->get_lambda(), layer->get_alpha(), layer->get_reg()))); // Adding to our cumulative hidden layer grads. Maintain reg terms as well.
res.cumulative_hidden_layer_w_grad.push_back(hidden_layer_w_grad->addn(regularization.reg_deriv_termm(layer->get_weights(), layer->get_lambda(), layer->get_alpha(), layer->get_reg()))); // Adding to our cumulative hidden layer grads. Maintain reg terms as well.
}
}

5
mlpp/ann/ann.h
View File

@ -70,14 +70,13 @@ protected:
real_t cost(const Ref<MLPPVector> &y_hat, const Ref<MLPPVector> &y);
void forward_pass();
void update_parameters(const Ref<MLPPTensor3> &hidden_layer_updations, const Ref<MLPPVector> &output_layer_updation, real_t learning_rate);
void update_parameters(const Vector<Ref<MLPPMatrix>> &hidden_layer_updations, const Ref<MLPPVector> &output_layer_updation, real_t learning_rate);
struct ComputeGradientsResult {
Ref<MLPPTensor3> cumulative_hidden_layer_w_grad;
Vector<Ref<MLPPMatrix>> cumulative_hidden_layer_w_grad;
Ref<MLPPVector> output_w_grad;
ComputeGradientsResult() {
cumulative_hidden_layer_w_grad.instance();
output_w_grad.instance();
}
};

26
mlpp/lin_alg/lin_alg.cpp
View File

@ -1768,6 +1768,14 @@ Vector<Ref<MLPPMatrix>> MLPPLinAlg::additionnvt(const Vector<Ref<MLPPMatrix>> &A
return res;
}
void MLPPLinAlg::division_element_wisevt(const Vector<Ref<MLPPMatrix>> &A, const Vector<Ref<MLPPMatrix>> &B) {
for (int i = 0; i < A.size(); i++) {
Ref<MLPPMatrix> m = A[i];
m->division_element_wise(B[i]);
}
}
Vector<Ref<MLPPMatrix>> MLPPLinAlg::division_element_wisenvnvt(const Vector<Ref<MLPPMatrix>> &A, const Vector<Ref<MLPPMatrix>> &B) {
Vector<Ref<MLPPMatrix>> res;
res.resize(A.size());
@ -1841,14 +1849,26 @@ Vector<Ref<MLPPMatrix>> MLPPLinAlg::scalar_addnvt(real_t scalar, Vector<Ref<MLPP
return A;
}
Vector<Ref<MLPPMatrix>> MLPPLinAlg::resizenvt(const Vector<Ref<MLPPMatrix>> &A, const Vector<Ref<MLPPMatrix>> &B) {
void MLPPLinAlg::resizevt(Vector<Ref<MLPPMatrix>> &r_target, const Vector<Ref<MLPPMatrix>> &A) {
r_target.resize(A.size());
for (int i = 0; i < r_target.size(); i++) {
Ref<MLPPMatrix> m;
m.instance();
m->resize(A[i]->size());
r_target.write[i] = m;
}
}
Vector<Ref<MLPPMatrix>> MLPPLinAlg::resizencvt(const Vector<Ref<MLPPMatrix>> &A) {
Vector<Ref<MLPPMatrix>> res;
res.resize(B.size());
res.resize(A.size());
for (int i = 0; i < res.size(); i++) {
Ref<MLPPMatrix> m;
m.instance();
m->resize(B[i]->size());
m->resize(A[i]->size());
res.write[i] = m;
}

4
mlpp/lin_alg/lin_alg.h
View File

@ -219,6 +219,7 @@ public:
// TENSOR FUNCTIONS
Vector<Ref<MLPPMatrix>> additionnvt(const Vector<Ref<MLPPMatrix>> &A, const Vector<Ref<MLPPMatrix>> &B);
void division_element_wisevt(const Vector<Ref<MLPPMatrix>> &A, const Vector<Ref<MLPPMatrix>> &B);
Vector<Ref<MLPPMatrix>> division_element_wisenvnvt(const Vector<Ref<MLPPMatrix>> &A, const Vector<Ref<MLPPMatrix>> &B);
Vector<Ref<MLPPMatrix>> sqrtnvt(const Vector<Ref<MLPPMatrix>> &A);
@ -232,7 +233,8 @@ public:
Vector<Ref<MLPPMatrix>> scalar_multiplynvt(real_t scalar, Vector<Ref<MLPPMatrix>> A);
Vector<Ref<MLPPMatrix>> scalar_addnvt(real_t scalar, Vector<Ref<MLPPMatrix>> A);
Vector<Ref<MLPPMatrix>> resizenvt(const Vector<Ref<MLPPMatrix>> &A, const Vector<Ref<MLPPMatrix>> &B);
void resizevt(Vector<Ref<MLPPMatrix>> &r_target, const Vector<Ref<MLPPMatrix>> &A);
Vector<Ref<MLPPMatrix>> resizencvt(const Vector<Ref<MLPPMatrix>> &A);
//std::vector<std::vector<std::vector<real_t>>> hadamard_product(std::vector<std::vector<std::vector<real_t>>> A, std::vector<std::vector<std::vector<real_t>>> B);