Relintai/pmlpp
1
0
Fork 0
mirror of https://github.com/Relintai/pmlpp.git synced 2024-12-31 16:17:10 +01:00
Code Issues Packages Projects Releases Wiki Activity

Api cleanups to LinAlg.

Browse Source
This commit is contained in:
Relintai 2023-04-22 13:17:54 +02:00
parent bfc1c40a0c
commit 5a4ff2f19e
5 changed files with 57 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
mlpp/activation/activation.cpp
View File

@ -851,7 +851,7 @@ real_t MLPPActivation::sigmoid_normr(real_t z) {
} }
Ref<MLPPVector> MLPPActivation::sigmoid_normv(const Ref<MLPPVector> &z) { Ref<MLPPVector> MLPPActivation::sigmoid_normv(const Ref<MLPPVector> &z) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.element_wise_division(alg.onevecv(z->size()), alg.additionnv(alg.onevecv(z->size()), alg.expv(alg.scalar_multiplynv(-1, z)))); return alg.element_wise_division(alg.onevecv(z->size()), alg.additionnv(alg.onevecv(z->size()), alg.expnv(alg.scalar_multiplynv(-1, z))));
} }
Ref<MLPPMatrix> MLPPActivation::sigmoid_normm(const Ref<MLPPMatrix> &z) { Ref<MLPPMatrix> MLPPActivation::sigmoid_normm(const Ref<MLPPMatrix> &z) {
MLPPLinAlg alg; MLPPLinAlg alg;
@ -893,7 +893,7 @@ Ref<MLPPVector> MLPPActivation::softmax_normv(const Ref<MLPPVector> &z) {
a.instance(); a.instance();
a->resize(z_size); a->resize(z_size);
Ref<MLPPVector> exp_z = alg.expv(z); Ref<MLPPVector> exp_z = alg.expnv(z);
real_t sum = 0; real_t sum = 0;
const real_t *exp_z_ptr = exp_z->ptr(); const real_t *exp_z_ptr = exp_z->ptr();
@ -943,7 +943,7 @@ Ref<MLPPVector> MLPPActivation::softmax_derivv(const Ref<MLPPVector> &z) {
a.instance(); a.instance();
a->resize(z_size); a->resize(z_size);
Ref<MLPPVector> exp_z = alg.expv(z); Ref<MLPPVector> exp_z = alg.expnv(z);
real_t sum = 0; real_t sum = 0;
const real_t *exp_z_ptr = exp_z->ptr(); const real_t *exp_z_ptr = exp_z->ptr();
@ -1219,12 +1219,12 @@ real_t MLPPActivation::softplus_normr(real_t z) {
Ref<MLPPVector> MLPPActivation::softplus_normv(const Ref<MLPPVector> &z) { Ref<MLPPVector> MLPPActivation::softplus_normv(const Ref<MLPPVector> &z) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.logv(alg.additionnv(alg.onevecv(z->size()), alg.expv(z))); return alg.lognv(alg.additionnv(alg.onevecv(z->size()), alg.expnv(z)));
} }
Ref<MLPPMatrix> MLPPActivation::softplus_normm(const Ref<MLPPMatrix> &z) { Ref<MLPPMatrix> MLPPActivation::softplus_normm(const Ref<MLPPMatrix> &z) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.logv(alg.additionnv(alg.onematm(z->size().x, z->size().y), alg.expv(z))); return alg.lognv(alg.additionnv(alg.onematm(z->size().x, z->size().y), alg.expnv(z)));
} }
real_t MLPPActivation::softplus_derivr(real_t z) { real_t MLPPActivation::softplus_derivr(real_t z) {
@ -1259,12 +1259,12 @@ real_t MLPPActivation::softsign_derivr(real_t z) {
Ref<MLPPVector> MLPPActivation::softsign_derivv(const Ref<MLPPVector> &z) { Ref<MLPPVector> MLPPActivation::softsign_derivv(const Ref<MLPPVector> &z) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.element_wise_division(alg.onevecv(z->size()), alg.exponentiatev(alg.additionnv(alg.onevecv(z->size()), alg.absv(z)), 2)); return alg.element_wise_division(alg.onevecv(z->size()), alg.exponentiatenv(alg.additionnv(alg.onevecv(z->size()), alg.absv(z)), 2));
} }
Ref<MLPPMatrix> MLPPActivation::softsign_derivm(const Ref<MLPPMatrix> &z) { Ref<MLPPMatrix> MLPPActivation::softsign_derivm(const Ref<MLPPMatrix> &z) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.element_wise_divisionm(alg.onematm(z->size().x, z->size().y), alg.exponentiatev(alg.additionm(alg.onematm(z->size().x, z->size().y), alg.absm(z)), 2)); return alg.element_wise_divisionm(alg.onematm(z->size().x, z->size().y), alg.exponentiatenv(alg.additionm(alg.onematm(z->size().x, z->size().y), alg.absm(z)), 2));
} }
//GAUSSIANCDF //GAUSSIANCDF
@ -1275,7 +1275,7 @@ real_t MLPPActivation::gaussian_cdf_normr(real_t z) {
Ref<MLPPVector> MLPPActivation::gaussian_cdf_normv(const Ref<MLPPVector> &z) { Ref<MLPPVector> MLPPActivation::gaussian_cdf_normv(const Ref<MLPPVector> &z) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.scalar_multiplynv(0.5, alg.additionnv(alg.onevecv(z->size()), alg.erfv(alg.scalar_multiplynv(1 / sqrt(2), z)))); return alg.scalar_multiplynv(0.5, alg.additionnv(alg.onevecv(z->size()), alg.erfnv(alg.scalar_multiplynv(1 / sqrt(2), z))));
} }
Ref<MLPPMatrix> MLPPActivation::gaussian_cdf_normm(const Ref<MLPPMatrix> &z) { Ref<MLPPMatrix> MLPPActivation::gaussian_cdf_normm(const Ref<MLPPMatrix> &z) {
@ -1290,7 +1290,7 @@ real_t MLPPActivation::gaussian_cdf_derivr(real_t z) {
Ref<MLPPVector> MLPPActivation::gaussian_cdf_derivv(const Ref<MLPPVector> &z) { Ref<MLPPVector> MLPPActivation::gaussian_cdf_derivv(const Ref<MLPPVector> &z) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.scalar_multiplynv(1 / Math::sqrt(2 * M_PI), alg.expv(alg.scalar_multiplynv(-1 / 2.0, alg.hadamard_productnv(z, z)))); return alg.scalar_multiplynv(1 / Math::sqrt(2 * M_PI), alg.expnv(alg.scalar_multiplynv(-1 / 2.0, alg.hadamard_productnv(z, z))));
} }
Ref<MLPPMatrix> MLPPActivation::gaussian_cdf_derivm(const Ref<MLPPMatrix> &z) { Ref<MLPPMatrix> MLPPActivation::gaussian_cdf_derivm(const Ref<MLPPMatrix> &z) {
@ -1307,7 +1307,7 @@ real_t MLPPActivation::cloglog_normr(real_t z) {
Ref<MLPPVector> MLPPActivation::cloglog_normv(const Ref<MLPPVector> &z) { Ref<MLPPVector> MLPPActivation::cloglog_normv(const Ref<MLPPVector> &z) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.scalar_multiplynv(-1, alg.scalar_addnv(-1, alg.expv(alg.scalar_multiplynv(-1, alg.expv(z))))); return alg.scalar_multiplynv(-1, alg.scalar_addnv(-1, alg.expnv(alg.scalar_multiplynv(-1, alg.expnv(z)))));
} }
Ref<MLPPMatrix> MLPPActivation::cloglog_normm(const Ref<MLPPMatrix> &z) { Ref<MLPPMatrix> MLPPActivation::cloglog_normm(const Ref<MLPPMatrix> &z) {
@ -1322,7 +1322,7 @@ real_t MLPPActivation::cloglog_derivr(real_t z) {
Ref<MLPPVector> MLPPActivation::cloglog_derivv(const Ref<MLPPVector> &z) { Ref<MLPPVector> MLPPActivation::cloglog_derivv(const Ref<MLPPVector> &z) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.expv(alg.scalar_multiplynv(-1, alg.expv(z))); return alg.expnv(alg.scalar_multiplynv(-1, alg.expnv(z)));
} }
Ref<MLPPMatrix> MLPPActivation::cloglog_derivm(const Ref<MLPPMatrix> &z) { Ref<MLPPMatrix> MLPPActivation::cloglog_derivm(const Ref<MLPPMatrix> &z) {
@ -1339,7 +1339,7 @@ real_t MLPPActivation::logit_normr(real_t z) {
Ref<MLPPVector> MLPPActivation::logit_normv(const Ref<MLPPVector> &z) { Ref<MLPPVector> MLPPActivation::logit_normv(const Ref<MLPPVector> &z) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.logv(alg.element_wise_division(z, alg.subtractionnv(alg.onevecv(z->size()), z))); return alg.lognv(alg.element_wise_division(z, alg.subtractionnv(alg.onevecv(z->size()), z)));
} }
Ref<MLPPMatrix> MLPPActivation::logit_normm(const Ref<MLPPMatrix> &z) { Ref<MLPPMatrix> MLPPActivation::logit_normm(const Ref<MLPPMatrix> &z) {
MLPPLinAlg alg; MLPPLinAlg alg;
@ -2002,7 +2002,7 @@ real_t MLPPActivation::sinh_normr(real_t z) {
} }
Ref<MLPPVector> MLPPActivation::sinh_normv(const Ref<MLPPVector> &z) { Ref<MLPPVector> MLPPActivation::sinh_normv(const Ref<MLPPVector> &z) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.scalar_multiplynv(0.5, alg.subtractionnv(alg.expv(z), alg.expv(alg.scalar_multiplynv(-1, z)))); return alg.scalar_multiplynv(0.5, alg.subtractionnv(alg.expnv(z), alg.expnv(alg.scalar_multiplynv(-1, z))));
} }
Ref<MLPPMatrix> MLPPActivation::sinh_normm(const Ref<MLPPMatrix> &z) { Ref<MLPPMatrix> MLPPActivation::sinh_normm(const Ref<MLPPMatrix> &z) {
MLPPLinAlg alg; MLPPLinAlg alg;
@ -2026,7 +2026,7 @@ real_t MLPPActivation::cosh_normr(real_t z) {
} }
Ref<MLPPVector> MLPPActivation::cosh_normv(const Ref<MLPPVector> &z) { Ref<MLPPVector> MLPPActivation::cosh_normv(const Ref<MLPPVector> &z) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.scalar_multiplynv(0.5, alg.additionnv(alg.expv(z), alg.expv(alg.scalar_multiplynv(-1, z)))); return alg.scalar_multiplynv(0.5, alg.additionnv(alg.expnv(z), alg.expnv(alg.scalar_multiplynv(-1, z))));
} }
Ref<MLPPMatrix> MLPPActivation::cosh_normm(const Ref<MLPPMatrix> &z) { Ref<MLPPMatrix> MLPPActivation::cosh_normm(const Ref<MLPPMatrix> &z) {
MLPPLinAlg alg; MLPPLinAlg alg;
@ -2051,7 +2051,7 @@ real_t MLPPActivation::tanh_normr(real_t z) {
Ref<MLPPVector> MLPPActivation::tanh_normv(const Ref<MLPPVector> &z) { Ref<MLPPVector> MLPPActivation::tanh_normv(const Ref<MLPPVector> &z) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.element_wise_division(alg.subtractionnv(alg.expv(z), alg.expv(alg.scalar_multiplynv(-1, z))), alg.additionnv(alg.expv(z), alg.expv(alg.scalar_multiplynv(-1, z)))); return alg.element_wise_division(alg.subtractionnv(alg.expnv(z), alg.expnv(alg.scalar_multiplynv(-1, z))), alg.additionnv(alg.expnv(z), alg.expnv(alg.scalar_multiplynv(-1, z))));
} }
Ref<MLPPMatrix> MLPPActivation::tanh_normm(const Ref<MLPPMatrix> &z) { Ref<MLPPMatrix> MLPPActivation::tanh_normm(const Ref<MLPPMatrix> &z) {
MLPPLinAlg alg; MLPPLinAlg alg;
@ -2180,7 +2180,7 @@ real_t MLPPActivation::arsinh_normr(real_t z) {
Ref<MLPPVector> MLPPActivation::arsinh_normv(const Ref<MLPPVector> &z) { Ref<MLPPVector> MLPPActivation::arsinh_normv(const Ref<MLPPVector> &z) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.logv(alg.additionnv(z, alg.sqrtv(alg.additionnv(alg.hadamard_productnv(z, z), alg.onevecv(z->size()))))); return alg.lognv(alg.additionnv(z, alg.sqrtnv(alg.additionnv(alg.hadamard_productnv(z, z), alg.onevecv(z->size())))));
} }
Ref<MLPPMatrix> MLPPActivation::arsinh_normm(const Ref<MLPPMatrix> &z) { Ref<MLPPMatrix> MLPPActivation::arsinh_normm(const Ref<MLPPMatrix> &z) {
@ -2196,7 +2196,7 @@ real_t MLPPActivation::arsinh_derivr(real_t z) {
Ref<MLPPVector> MLPPActivation::arsinh_derivv(const Ref<MLPPVector> &z) { Ref<MLPPVector> MLPPActivation::arsinh_derivv(const Ref<MLPPVector> &z) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.element_wise_division(alg.onevecv(z->size()), alg.sqrtv(alg.additionnv(alg.hadamard_productnv(z, z), alg.onevecv(z->size())))); return alg.element_wise_division(alg.onevecv(z->size()), alg.sqrtnv(alg.additionnv(alg.hadamard_productnv(z, z), alg.onevecv(z->size()))));
} }
Ref<MLPPMatrix> MLPPActivation::arsinh_derivm(const Ref<MLPPMatrix> &z) { Ref<MLPPMatrix> MLPPActivation::arsinh_derivm(const Ref<MLPPMatrix> &z) {
@ -2213,7 +2213,7 @@ real_t MLPPActivation::arcosh_normr(real_t z) {
Ref<MLPPVector> MLPPActivation::arcosh_normv(const Ref<MLPPVector> &z) { Ref<MLPPVector> MLPPActivation::arcosh_normv(const Ref<MLPPVector> &z) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.logv(alg.additionnv(z, alg.sqrtv(alg.subtractionnv(alg.hadamard_productnv(z, z), alg.onevecv(z->size()))))); return alg.lognv(alg.additionnv(z, alg.sqrtnv(alg.subtractionnv(alg.hadamard_productnv(z, z), alg.onevecv(z->size())))));
} }
Ref<MLPPMatrix> MLPPActivation::arcosh_normm(const Ref<MLPPMatrix> &z) { Ref<MLPPMatrix> MLPPActivation::arcosh_normm(const Ref<MLPPMatrix> &z) {
@ -2228,7 +2228,7 @@ real_t MLPPActivation::arcosh_derivr(real_t z) {
Ref<MLPPVector> MLPPActivation::arcosh_derivv(const Ref<MLPPVector> &z) { Ref<MLPPVector> MLPPActivation::arcosh_derivv(const Ref<MLPPVector> &z) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.element_wise_division(alg.onevecv(z->size()), alg.sqrtv(alg.subtractionnv(alg.hadamard_productnv(z, z), alg.onevecv(z->size())))); return alg.element_wise_division(alg.onevecv(z->size()), alg.sqrtnv(alg.subtractionnv(alg.hadamard_productnv(z, z), alg.onevecv(z->size()))));
} }
Ref<MLPPMatrix> MLPPActivation::arcosh_derivm(const Ref<MLPPMatrix> &z) { Ref<MLPPMatrix> MLPPActivation::arcosh_derivm(const Ref<MLPPMatrix> &z) {
@ -2245,7 +2245,7 @@ real_t MLPPActivation::artanh_normr(real_t z) {
Ref<MLPPVector> MLPPActivation::artanh_normv(const Ref<MLPPVector> &z) { Ref<MLPPVector> MLPPActivation::artanh_normv(const Ref<MLPPVector> &z) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.scalar_multiplynv(0.5, alg.logv(alg.element_wise_division(alg.additionnv(alg.onevecv(z->size()), z), alg.subtractionnv(alg.onevecv(z->size()), z)))); return alg.scalar_multiplynv(0.5, alg.lognv(alg.element_wise_division(alg.additionnv(alg.onevecv(z->size()), z), alg.subtractionnv(alg.onevecv(z->size()), z))));
} }
Ref<MLPPMatrix> MLPPActivation::artanh_normm(const Ref<MLPPMatrix> &z) { Ref<MLPPMatrix> MLPPActivation::artanh_normm(const Ref<MLPPMatrix> &z) {
@ -2277,9 +2277,9 @@ real_t MLPPActivation::arcsch_normr(real_t z) {
Ref<MLPPVector> MLPPActivation::arcsch_normv(const Ref<MLPPVector> &z) { Ref<MLPPVector> MLPPActivation::arcsch_normv(const Ref<MLPPVector> &z) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.logv( return alg.lognv(
alg.additionnv( alg.additionnv(
alg.sqrtv( alg.sqrtnv(
alg.additionnv( alg.additionnv(
alg.onevecv(z->size()), alg.onevecv(z->size()),
alg.element_wise_division(alg.onevecv(z->size()), alg.hadamard_productnv(z, z)))), alg.element_wise_division(alg.onevecv(z->size()), alg.hadamard_productnv(z, z)))),
@ -2306,7 +2306,7 @@ Ref<MLPPVector> MLPPActivation::arcsch_derivv(const Ref<MLPPVector> &z) {
alg.fullv(z->size(), -1), alg.fullv(z->size(), -1),
alg.hadamard_productm( alg.hadamard_productm(
alg.hadamard_productnv(z, z), alg.hadamard_productnv(z, z),
alg.sqrtv(alg.additionnv(alg.onevecv(z->size()), alg.element_wise_division(alg.onevecv(z->size()), alg.hadamard_productnv(z, z)))))); alg.sqrtnv(alg.additionnv(alg.onevecv(z->size()), alg.element_wise_division(alg.onevecv(z->size()), alg.hadamard_productnv(z, z))))));
} }
Ref<MLPPMatrix> MLPPActivation::arcsch_derivm(const Ref<MLPPMatrix> &z) { Ref<MLPPMatrix> MLPPActivation::arcsch_derivm(const Ref<MLPPMatrix> &z) {
MLPPLinAlg alg; MLPPLinAlg alg;
@ -2327,7 +2327,7 @@ real_t MLPPActivation::arsech_normr(real_t z) {
Ref<MLPPVector> MLPPActivation::arsech_normv(const Ref<MLPPVector> &z) { Ref<MLPPVector> MLPPActivation::arsech_normv(const Ref<MLPPVector> &z) {
MLPPLinAlg alg; MLPPLinAlg alg;
return alg.logv( return alg.lognv(
alg.additionnv( alg.additionnv(
alg.element_wise_division( alg.element_wise_division(
alg.onevecv(z->size()), z), alg.onevecv(z->size()), z),
@ -2365,7 +2365,7 @@ Ref<MLPPVector> MLPPActivation::arsech_derivv(const Ref<MLPPVector> &z) {
alg.fullv(z->size(), -1), alg.fullv(z->size(), -1),
alg.hadamard_productnv( alg.hadamard_productnv(
z, z,
alg.sqrtv( alg.sqrtnv(
alg.subtractionnv(alg.onevecv(z->size()), alg.hadamard_productnv(z, z))))); alg.subtractionnv(alg.onevecv(z->size()), alg.hadamard_productnv(z, z)))));
} }
@ -2389,7 +2389,7 @@ Ref<MLPPVector> MLPPActivation::arcoth_normv(const Ref<MLPPVector> &z) {
return alg.scalar_multiplynv( return alg.scalar_multiplynv(
0.5, 0.5,
alg.logv(alg.element_wise_division(alg.additionnv(alg.onevecv(z->size()), z), alg.subtractionnv(z, alg.onevecv(z->size()))))); alg.lognv(alg.element_wise_division(alg.additionnv(alg.onevecv(z->size()), z), alg.subtractionnv(z, alg.onevecv(z->size())))));
} }
Ref<MLPPMatrix> MLPPActivation::arcoth_normm(const Ref<MLPPMatrix> &z) { Ref<MLPPMatrix> MLPPActivation::arcoth_normm(const Ref<MLPPMatrix> &z) {

20
mlpp/ann/ann.cpp
View File

@ -312,10 +312,10 @@ void MLPPANN::adagrad(real_t learning_rate, int max_epoch, int mini_batch_size,
} }
v_hidden = alg.addition_vt(v_hidden, alg.exponentiate_vt(grads.cumulative_hidden_layer_w_grad, 2)); v_hidden = alg.addition_vt(v_hidden, alg.exponentiate_vt(grads.cumulative_hidden_layer_w_grad, 2));
v_output = alg.additionnv(v_output, alg.exponentiatev(grads.output_w_grad, 2)); v_output = alg.additionnv(v_output, alg.exponentiatenv(grads.output_w_grad, 2));
Vector<Ref<MLPPMatrix>> hidden_layer_updations = alg.scalar_multiply_vm(learning_rate / _n, alg.element_wise_division_vt(grads.cumulative_hidden_layer_w_grad, alg.scalar_add_vm(e, alg.sqrt_vt(v_hidden)))); Vector<Ref<MLPPMatrix>> hidden_layer_updations = alg.scalar_multiply_vm(learning_rate / _n, alg.element_wise_division_vt(grads.cumulative_hidden_layer_w_grad, alg.scalar_add_vm(e, alg.sqrt_vt(v_hidden))));
Ref<MLPPVector> output_layer_updation = alg.scalar_multiplynv(learning_rate / _n, alg.element_wise_division(grads.output_w_grad, alg.scalar_addnv(e, alg.sqrtv(v_output)))); Ref<MLPPVector> output_layer_updation = alg.scalar_multiplynv(learning_rate / _n, alg.element_wise_division(grads.output_w_grad, alg.scalar_addnv(e, alg.sqrtnv(v_output))));
update_parameters(hidden_layer_updations, output_layer_updation, learning_rate); // subject to change. may want bias to have this matrix too. update_parameters(hidden_layer_updations, output_layer_updation, learning_rate); // subject to change. may want bias to have this matrix too.
y_hat = model_set_test(current_input_batch); y_hat = model_set_test(current_input_batch);
@ -376,10 +376,10 @@ void MLPPANN::adadelta(real_t learning_rate, int max_epoch, int mini_batch_size,
} }
v_hidden = alg.addition_vt(alg.scalar_multiply_vm(1 - b1, v_hidden), alg.scalar_multiply_vm(b1, alg.exponentiate_vt(grads.cumulative_hidden_layer_w_grad, 2))); v_hidden = alg.addition_vt(alg.scalar_multiply_vm(1 - b1, v_hidden), alg.scalar_multiply_vm(b1, alg.exponentiate_vt(grads.cumulative_hidden_layer_w_grad, 2)));
v_output = alg.additionnv(v_output, alg.exponentiatev(grads.output_w_grad, 2)); v_output = alg.additionnv(v_output, alg.exponentiatenv(grads.output_w_grad, 2));
Vector<Ref<MLPPMatrix>> hidden_layer_updations = alg.scalar_multiply_vm(learning_rate / _n, alg.element_wise_division_vt(grads.cumulative_hidden_layer_w_grad, alg.scalar_add_vm(e, alg.sqrt_vt(v_hidden)))); Vector<Ref<MLPPMatrix>> hidden_layer_updations = alg.scalar_multiply_vm(learning_rate / _n, alg.element_wise_division_vt(grads.cumulative_hidden_layer_w_grad, alg.scalar_add_vm(e, alg.sqrt_vt(v_hidden))));
Ref<MLPPVector> output_layer_updation = alg.scalar_multiplynv(learning_rate / _n, alg.element_wise_division(grads.output_w_grad, alg.scalar_addnv(e, alg.sqrtv(v_output)))); Ref<MLPPVector> output_layer_updation = alg.scalar_multiplynv(learning_rate / _n, alg.element_wise_division(grads.output_w_grad, alg.scalar_addnv(e, alg.sqrtnv(v_output))));
update_parameters(hidden_layer_updations, output_layer_updation, learning_rate); // subject to change. may want bias to have this matrix too. update_parameters(hidden_layer_updations, output_layer_updation, learning_rate); // subject to change. may want bias to have this matrix too.
y_hat = model_set_test(current_input_batch); y_hat = model_set_test(current_input_batch);
@ -448,7 +448,7 @@ void MLPPANN::adam(real_t learning_rate, int max_epoch, int mini_batch_size, rea
v_hidden = alg.addition_vt(alg.scalar_multiply_vm(b2, v_hidden), alg.scalar_multiply_vm(1 - b2, alg.exponentiate_vt(grads.cumulative_hidden_layer_w_grad, 2))); v_hidden = alg.addition_vt(alg.scalar_multiply_vm(b2, v_hidden), alg.scalar_multiply_vm(1 - b2, alg.exponentiate_vt(grads.cumulative_hidden_layer_w_grad, 2)));
m_output = alg.additionnv(alg.scalar_multiplynv(b1, m_output), alg.scalar_multiplynv(1 - b1, grads.output_w_grad)); m_output = alg.additionnv(alg.scalar_multiplynv(b1, m_output), alg.scalar_multiplynv(1 - b1, grads.output_w_grad));
v_output = alg.additionnv(alg.scalar_multiplynv(b2, v_output), alg.scalar_multiplynv(1 - b2, alg.exponentiatev(grads.output_w_grad, 2))); v_output = alg.additionnv(alg.scalar_multiplynv(b2, v_output), alg.scalar_multiplynv(1 - b2, alg.exponentiatenv(grads.output_w_grad, 2)));
Vector<Ref<MLPPMatrix>> m_hidden_hat = alg.scalar_multiply_vm(1 / (1 - Math::pow(b1, epoch)), m_hidden); Vector<Ref<MLPPMatrix>> m_hidden_hat = alg.scalar_multiply_vm(1 / (1 - Math::pow(b1, epoch)), m_hidden);
Vector<Ref<MLPPMatrix>> v_hidden_hat = alg.scalar_multiply_vm(1 / (1 - Math::pow(b2, epoch)), v_hidden); Vector<Ref<MLPPMatrix>> v_hidden_hat = alg.scalar_multiply_vm(1 / (1 - Math::pow(b2, epoch)), v_hidden);
@ -457,7 +457,7 @@ void MLPPANN::adam(real_t learning_rate, int max_epoch, int mini_batch_size, rea
Ref<MLPPVector> v_output_hat = alg.scalar_multiplynv(1 / (1 - Math::pow(b2, epoch)), v_output); Ref<MLPPVector> v_output_hat = alg.scalar_multiplynv(1 / (1 - Math::pow(b2, epoch)), v_output);
Vector<Ref<MLPPMatrix>> hidden_layer_updations = alg.scalar_multiply_vm(learning_rate / _n, alg.element_wise_division_vt(m_hidden_hat, alg.scalar_add_vm(e, alg.sqrt_vt(v_hidden_hat)))); Vector<Ref<MLPPMatrix>> hidden_layer_updations = alg.scalar_multiply_vm(learning_rate / _n, alg.element_wise_division_vt(m_hidden_hat, alg.scalar_add_vm(e, alg.sqrt_vt(v_hidden_hat))));
Ref<MLPPVector> output_layer_updation = alg.scalar_multiplynv(learning_rate / _n, alg.element_wise_division(m_output_hat, alg.scalar_addnv(e, alg.sqrtv(v_output_hat)))); Ref<MLPPVector> output_layer_updation = alg.scalar_multiplynv(learning_rate / _n, alg.element_wise_division(m_output_hat, alg.scalar_addnv(e, alg.sqrtnv(v_output_hat))));
update_parameters(hidden_layer_updations, output_layer_updation, learning_rate); // subject to change. may want bias to have this matrix too. update_parameters(hidden_layer_updations, output_layer_updation, learning_rate); // subject to change. may want bias to have this matrix too.
y_hat = model_set_test(current_input_batch); y_hat = model_set_test(current_input_batch);
@ -596,7 +596,7 @@ void MLPPANN::nadam(real_t learning_rate, int max_epoch, int mini_batch_size, re
v_hidden = alg.addition_vt(alg.scalar_multiply_vm(b2, v_hidden), alg.scalar_multiply_vm(1 - b2, alg.exponentiate_vt(grads.cumulative_hidden_layer_w_grad, 2))); v_hidden = alg.addition_vt(alg.scalar_multiply_vm(b2, v_hidden), alg.scalar_multiply_vm(1 - b2, alg.exponentiate_vt(grads.cumulative_hidden_layer_w_grad, 2)));
m_output = alg.additionnv(alg.scalar_multiplynv(b1, m_output), alg.scalar_multiplynv(1 - b1, grads.output_w_grad)); m_output = alg.additionnv(alg.scalar_multiplynv(b1, m_output), alg.scalar_multiplynv(1 - b1, grads.output_w_grad));
v_output = alg.additionnv(alg.scalar_multiplynv(b2, v_output), alg.scalar_multiplynv(1 - b2, alg.exponentiatev(grads.output_w_grad, 2))); v_output = alg.additionnv(alg.scalar_multiplynv(b2, v_output), alg.scalar_multiplynv(1 - b2, alg.exponentiatenv(grads.output_w_grad, 2)));
Vector<Ref<MLPPMatrix>> m_hidden_hat = alg.scalar_multiply_vm(1 / (1.0 - Math::pow(b1, epoch)), m_hidden); Vector<Ref<MLPPMatrix>> m_hidden_hat = alg.scalar_multiply_vm(1 / (1.0 - Math::pow(b1, epoch)), m_hidden);
Vector<Ref<MLPPMatrix>> v_hidden_hat = alg.scalar_multiply_vm(1 / (1.0 - Math::pow(b2, epoch)), v_hidden); Vector<Ref<MLPPMatrix>> v_hidden_hat = alg.scalar_multiply_vm(1 / (1.0 - Math::pow(b2, epoch)), v_hidden);
@ -607,7 +607,7 @@ void MLPPANN::nadam(real_t learning_rate, int max_epoch, int mini_batch_size, re
Ref<MLPPVector> m_output_final = alg.additionnv(alg.scalar_multiplynv(b1, m_output_hat), alg.scalar_multiplynv((1 - b1) / (1.0 - Math::pow(b1, epoch)), grads.output_w_grad)); Ref<MLPPVector> m_output_final = alg.additionnv(alg.scalar_multiplynv(b1, m_output_hat), alg.scalar_multiplynv((1 - b1) / (1.0 - Math::pow(b1, epoch)), grads.output_w_grad));
Vector<Ref<MLPPMatrix>> hidden_layer_updations = alg.scalar_multiply_vm(learning_rate / _n, alg.element_wise_division_vt(m_hidden_final, alg.scalar_add_vm(e, alg.sqrt_vt(v_hidden_hat)))); Vector<Ref<MLPPMatrix>> hidden_layer_updations = alg.scalar_multiply_vm(learning_rate / _n, alg.element_wise_division_vt(m_hidden_final, alg.scalar_add_vm(e, alg.sqrt_vt(v_hidden_hat))));
Ref<MLPPVector> output_layer_updation = alg.scalar_multiplynv(learning_rate / _n, alg.element_wise_divisionm(m_output_final, alg.scalar_addnv(e, alg.sqrtv(v_output_hat)))); Ref<MLPPVector> output_layer_updation = alg.scalar_multiplynv(learning_rate / _n, alg.element_wise_divisionm(m_output_final, alg.scalar_addnv(e, alg.sqrtnv(v_output_hat))));
update_parameters(hidden_layer_updations, output_layer_updation, learning_rate); // subject to change. may want bias to have this matrix too. update_parameters(hidden_layer_updations, output_layer_updation, learning_rate); // subject to change. may want bias to have this matrix too.
@ -681,13 +681,13 @@ void MLPPANN::amsgrad(real_t learning_rate, int max_epoch, int mini_batch_size,
v_hidden = alg.addition_vt(alg.scalar_multiply_vm(b2, v_hidden), alg.scalar_multiply_vm(1 - b2, alg.exponentiate_vt(grads.cumulative_hidden_layer_w_grad, 2))); v_hidden = alg.addition_vt(alg.scalar_multiply_vm(b2, v_hidden), alg.scalar_multiply_vm(1 - b2, alg.exponentiate_vt(grads.cumulative_hidden_layer_w_grad, 2)));
m_output = alg.additionnv(alg.scalar_multiplynv(b1, m_output), alg.scalar_multiplynv(1 - b1, grads.output_w_grad)); m_output = alg.additionnv(alg.scalar_multiplynv(b1, m_output), alg.scalar_multiplynv(1 - b1, grads.output_w_grad));
v_output = alg.additionnv(alg.scalar_multiplynv(b2, v_output), alg.scalar_multiplynv(1 - b2, alg.exponentiatev(grads.output_w_grad, 2))); v_output = alg.additionnv(alg.scalar_multiplynv(b2, v_output), alg.scalar_multiplynv(1 - b2, alg.exponentiatenv(grads.output_w_grad, 2)));
v_hidden_hat = alg.max_vt(v_hidden_hat, v_hidden); v_hidden_hat = alg.max_vt(v_hidden_hat, v_hidden);
v_output_hat = alg.maxnvv(v_output_hat, v_output); v_output_hat = alg.maxnvv(v_output_hat, v_output);
Vector<Ref<MLPPMatrix>> hidden_layer_updations = alg.scalar_multiply_vm(learning_rate / _n, alg.element_wise_division_vt(m_hidden, alg.scalar_add_vm(e, alg.sqrt_vt(v_hidden_hat)))); Vector<Ref<MLPPMatrix>> hidden_layer_updations = alg.scalar_multiply_vm(learning_rate / _n, alg.element_wise_division_vt(m_hidden, alg.scalar_add_vm(e, alg.sqrt_vt(v_hidden_hat))));
Ref<MLPPVector> output_layer_updation = alg.scalar_multiplynv(learning_rate / _n, alg.element_wise_division(m_output, alg.scalar_addnv(e, alg.sqrtv(v_output_hat)))); Ref<MLPPVector> output_layer_updation = alg.scalar_multiplynv(learning_rate / _n, alg.element_wise_division(m_output, alg.scalar_addnv(e, alg.sqrtnv(v_output_hat))));
update_parameters(hidden_layer_updations, output_layer_updation, learning_rate); // subject to change. may want bias to have this matrix too. update_parameters(hidden_layer_updations, output_layer_updation, learning_rate); // subject to change. may want bias to have this matrix too.
y_hat = model_set_test(current_input_batch); y_hat = model_set_test(current_input_batch);

16
mlpp/lin_alg/lin_alg.cpp
View File

@ -2171,7 +2171,7 @@ std::vector<real_t> MLPPLinAlg::cbrt(std::vector<real_t> a) {
return exponentiate(a, real_t(1) / real_t(3)); return exponentiate(a, real_t(1) / real_t(3));
} }
Ref<MLPPVector> MLPPLinAlg::logv(const Ref<MLPPVector> &a) { Ref<MLPPVector> MLPPLinAlg::lognv(const Ref<MLPPVector> &a) {
ERR_FAIL_COND_V(!a.is_valid(), Ref<MLPPVector>()); ERR_FAIL_COND_V(!a.is_valid(), Ref<MLPPVector>());
Ref<MLPPVector> out; Ref<MLPPVector> out;
@ -2189,7 +2189,7 @@ Ref<MLPPVector> MLPPLinAlg::logv(const Ref<MLPPVector> &a) {
return out; return out;
} }
Ref<MLPPVector> MLPPLinAlg::log10v(const Ref<MLPPVector> &a) { Ref<MLPPVector> MLPPLinAlg::log10nv(const Ref<MLPPVector> &a) {
ERR_FAIL_COND_V(!a.is_valid(), Ref<MLPPVector>()); ERR_FAIL_COND_V(!a.is_valid(), Ref<MLPPVector>());
Ref<MLPPVector> out; Ref<MLPPVector> out;
@ -2207,7 +2207,7 @@ Ref<MLPPVector> MLPPLinAlg::log10v(const Ref<MLPPVector> &a) {
return out; return out;
} }
Ref<MLPPVector> MLPPLinAlg::expv(const Ref<MLPPVector> &a) { Ref<MLPPVector> MLPPLinAlg::expnv(const Ref<MLPPVector> &a) {
ERR_FAIL_COND_V(!a.is_valid(), Ref<MLPPVector>()); ERR_FAIL_COND_V(!a.is_valid(), Ref<MLPPVector>());
Ref<MLPPVector> out; Ref<MLPPVector> out;
@ -2225,7 +2225,7 @@ Ref<MLPPVector> MLPPLinAlg::expv(const Ref<MLPPVector> &a) {
return out; return out;
} }
Ref<MLPPVector> MLPPLinAlg::erfv(const Ref<MLPPVector> &a) { Ref<MLPPVector> MLPPLinAlg::erfnv(const Ref<MLPPVector> &a) {
ERR_FAIL_COND_V(!a.is_valid(), Ref<MLPPVector>()); ERR_FAIL_COND_V(!a.is_valid(), Ref<MLPPVector>());
Ref<MLPPVector> out; Ref<MLPPVector> out;
@ -2243,7 +2243,7 @@ Ref<MLPPVector> MLPPLinAlg::erfv(const Ref<MLPPVector> &a) {
return out; return out;
} }
Ref<MLPPVector> MLPPLinAlg::exponentiatev(const Ref<MLPPVector> &a, real_t p) { Ref<MLPPVector> MLPPLinAlg::exponentiatenv(const Ref<MLPPVector> &a, real_t p) {
ERR_FAIL_COND_V(!a.is_valid(), Ref<MLPPVector>()); ERR_FAIL_COND_V(!a.is_valid(), Ref<MLPPVector>());
Ref<MLPPVector> out; Ref<MLPPVector> out;
@ -2261,7 +2261,7 @@ Ref<MLPPVector> MLPPLinAlg::exponentiatev(const Ref<MLPPVector> &a, real_t p) {
return out; return out;
} }
Ref<MLPPVector> MLPPLinAlg::sqrtv(const Ref<MLPPVector> &a) { Ref<MLPPVector> MLPPLinAlg::sqrtnv(const Ref<MLPPVector> &a) {
ERR_FAIL_COND_V(!a.is_valid(), Ref<MLPPVector>()); ERR_FAIL_COND_V(!a.is_valid(), Ref<MLPPVector>());
Ref<MLPPVector> out; Ref<MLPPVector> out;
@ -2279,8 +2279,8 @@ Ref<MLPPVector> MLPPLinAlg::sqrtv(const Ref<MLPPVector> &a) {
return out; return out;
} }
Ref<MLPPVector> MLPPLinAlg::cbrtv(const Ref<MLPPVector> &a) { Ref<MLPPVector> MLPPLinAlg::cbrtnv(const Ref<MLPPVector> &a) {
return exponentiatev(a, static_cast<real_t>(1) / static_cast<real_t>(3)); return exponentiatenv(a, static_cast<real_t>(1) / static_cast<real_t>(3));
} }
real_t MLPPLinAlg::dot(std::vector<real_t> a, std::vector<real_t> b) { real_t MLPPLinAlg::dot(std::vector<real_t> a, std::vector<real_t> b) {

14
mlpp/lin_alg/lin_alg.h
View File

@ -235,13 +235,13 @@ public:
std::vector<real_t> sqrt(std::vector<real_t> a); std::vector<real_t> sqrt(std::vector<real_t> a);
std::vector<real_t> cbrt(std::vector<real_t> a); std::vector<real_t> cbrt(std::vector<real_t> a);
Ref<MLPPVector> logv(const Ref<MLPPVector> &a); Ref<MLPPVector> lognv(const Ref<MLPPVector> &a);
Ref<MLPPVector> log10v(const Ref<MLPPVector> &a); Ref<MLPPVector> log10nv(const Ref<MLPPVector> &a);
Ref<MLPPVector> expv(const Ref<MLPPVector> &a); Ref<MLPPVector> expnv(const Ref<MLPPVector> &a);
Ref<MLPPVector> erfv(const Ref<MLPPVector> &a); Ref<MLPPVector> erfnv(const Ref<MLPPVector> &a);
Ref<MLPPVector> exponentiatev(const Ref<MLPPVector> &a, real_t p); Ref<MLPPVector> exponentiatenv(const Ref<MLPPVector> &a, real_t p);
Ref<MLPPVector> sqrtv(const Ref<MLPPVector> &a); Ref<MLPPVector> sqrtnv(const Ref<MLPPVector> &a);
Ref<MLPPVector> cbrtv(const Ref<MLPPVector> &a); Ref<MLPPVector> cbrtnv(const Ref<MLPPVector> &a);
real_t dot(std::vector<real_t> a, std::vector<real_t> b); real_t dot(std::vector<real_t> a, std::vector<real_t> b);
real_t dotv(const Ref<MLPPVector> &a, const Ref<MLPPVector> &b); real_t dotv(const Ref<MLPPVector> &a, const Ref<MLPPVector> &b);

12
mlpp/lin_reg/lin_reg.cpp
View File

@ -405,7 +405,7 @@ void MLPPLinReg::adagrad(real_t learning_rate, int max_epoch, int mini_batch_siz
v = alg.hadamard_productnv(weight_grad, weight_grad); v = alg.hadamard_productnv(weight_grad, weight_grad);
_weights = alg.subtractionnv(_weights, alg.scalar_multiplynv(learning_rate, alg.element_wise_division(weight_grad, alg.sqrtv(alg.scalar_addnv(e, v))))); _weights = alg.subtractionnv(_weights, alg.scalar_multiplynv(learning_rate, alg.element_wise_division(weight_grad, alg.sqrtnv(alg.scalar_addnv(e, v)))));
// Calculating the bias gradients // Calculating the bias gradients
_bias -= learning_rate * alg.sum_elementsv(error) / current_output_mini_batch->size(); // As normal _bias -= learning_rate * alg.sum_elementsv(error) / current_output_mini_batch->size(); // As normal
@ -460,7 +460,7 @@ void MLPPLinReg::adadelta(real_t learning_rate, int max_epoch, int mini_batch_si
v = alg.additionnv(alg.scalar_multiplynv(b1, v), alg.scalar_multiplynv(1 - b1, alg.hadamard_productnv(weight_grad, weight_grad))); v = alg.additionnv(alg.scalar_multiplynv(b1, v), alg.scalar_multiplynv(1 - b1, alg.hadamard_productnv(weight_grad, weight_grad)));
_weights = alg.subtractionnv(_weights, alg.scalar_multiplynv(learning_rate, alg.element_wise_division(weight_grad, alg.sqrtv(alg.scalar_addnv(e, v))))); _weights = alg.subtractionnv(_weights, alg.scalar_multiplynv(learning_rate, alg.element_wise_division(weight_grad, alg.sqrtnv(alg.scalar_addnv(e, v)))));
// Calculating the bias gradients // Calculating the bias gradients
_bias -= learning_rate * alg.sum_elementsv(error) / current_output_mini_batch->size(); // As normal _bias -= learning_rate * alg.sum_elementsv(error) / current_output_mini_batch->size(); // As normal
@ -514,12 +514,12 @@ void MLPPLinReg::adam(real_t learning_rate, int max_epoch, int mini_batch_size,
Ref<MLPPVector> weight_grad = alg.additionnv(gradient, reg_deriv_term); // Weight_grad_final Ref<MLPPVector> weight_grad = alg.additionnv(gradient, reg_deriv_term); // Weight_grad_final
m = alg.additionnv(alg.scalar_multiplynv(b1, m), alg.scalar_multiplynv(1 - b1, weight_grad)); m = alg.additionnv(alg.scalar_multiplynv(b1, m), alg.scalar_multiplynv(1 - b1, weight_grad));
v = alg.additionnv(alg.scalar_multiplynv(b2, v), alg.scalar_multiplynv(1 - b2, alg.exponentiatev(weight_grad, 2))); v = alg.additionnv(alg.scalar_multiplynv(b2, v), alg.scalar_multiplynv(1 - b2, alg.exponentiatenv(weight_grad, 2)));
Ref<MLPPVector> m_hat = alg.scalar_multiplynv(1 / (1 - Math::pow(b1, epoch)), m); Ref<MLPPVector> m_hat = alg.scalar_multiplynv(1 / (1 - Math::pow(b1, epoch)), m);
Ref<MLPPVector> v_hat = alg.scalar_multiplynv(1 / (1 - Math::pow(b2, epoch)), v); Ref<MLPPVector> v_hat = alg.scalar_multiplynv(1 / (1 - Math::pow(b2, epoch)), v);
_weights = alg.subtractionnv(_weights, alg.scalar_multiplynv(learning_rate, alg.element_wise_divisionm(m_hat, alg.scalar_addnv(e, alg.sqrtv(v_hat))))); _weights = alg.subtractionnv(_weights, alg.scalar_multiplynv(learning_rate, alg.element_wise_divisionm(m_hat, alg.scalar_addnv(e, alg.sqrtnv(v_hat)))));
// Calculating the bias gradients // Calculating the bias gradients
_bias -= learning_rate * alg.sum_elementsv(error) / current_output_mini_batch->size(); // As normal _bias -= learning_rate * alg.sum_elementsv(error) / current_output_mini_batch->size(); // As normal
@ -631,13 +631,13 @@ void MLPPLinReg::nadam(real_t learning_rate, int max_epoch, int mini_batch_size,
Ref<MLPPVector> weight_grad = alg.additionnv(gradient, reg_deriv_term); // Weight_grad_final Ref<MLPPVector> weight_grad = alg.additionnv(gradient, reg_deriv_term); // Weight_grad_final
m = alg.additionnv(alg.scalar_multiplynv(b1, m), alg.scalar_multiplynv(1 - b1, weight_grad)); m = alg.additionnv(alg.scalar_multiplynv(b1, m), alg.scalar_multiplynv(1 - b1, weight_grad));
v = alg.additionnv(alg.scalar_multiplynv(b2, v), alg.scalar_multiplynv(1 - b2, alg.exponentiatev(weight_grad, 2))); v = alg.additionnv(alg.scalar_multiplynv(b2, v), alg.scalar_multiplynv(1 - b2, alg.exponentiatenv(weight_grad, 2)));
m_final = alg.additionnv(alg.scalar_multiplynv(b1, m), alg.scalar_multiplynv((1 - b1) / (1 - Math::pow(b1, epoch)), weight_grad)); m_final = alg.additionnv(alg.scalar_multiplynv(b1, m), alg.scalar_multiplynv((1 - b1) / (1 - Math::pow(b1, epoch)), weight_grad));
Ref<MLPPVector> m_hat = alg.scalar_multiplynv(1 / (1 - Math::pow(b1, epoch)), m); Ref<MLPPVector> m_hat = alg.scalar_multiplynv(1 / (1 - Math::pow(b1, epoch)), m);
Ref<MLPPVector> v_hat = alg.scalar_multiplynv(1 / (1 - Math::pow(b2, epoch)), v); Ref<MLPPVector> v_hat = alg.scalar_multiplynv(1 / (1 - Math::pow(b2, epoch)), v);
_weights = alg.subtractionnv(_weights, alg.scalar_multiplynv(learning_rate, alg.element_wise_division(m_final, alg.scalar_addnv(e, alg.sqrtv(v_hat))))); _weights = alg.subtractionnv(_weights, alg.scalar_multiplynv(learning_rate, alg.element_wise_division(m_final, alg.scalar_addnv(e, alg.sqrtnv(v_hat)))));
// Calculating the bias gradients // Calculating the bias gradients
_bias -= learning_rate * alg.sum_elementsv(error) / current_output_mini_batch->size(); // As normal _bias -= learning_rate * alg.sum_elementsv(error) / current_output_mini_batch->size(); // As normal