diff --git a/mlpp/activation/activation.cpp b/mlpp/activation/activation.cpp index fe0d6e2..35d03e2 100644 --- a/mlpp/activation/activation.cpp +++ b/mlpp/activation/activation.cpp @@ -56,64 +56,64 @@ Ref MLPPActivation::activation(const Ref &z, real_t (*fu //ACTIVATION FUNCTIONS //LINEAR -real_t MLPPActivation::linear_norm(real_t z) { +real_t MLPPActivation::linear_normr(real_t z) { return z; } -Ref MLPPActivation::linear_norm(const Ref &z) { +Ref MLPPActivation::linear_normv(const Ref &z) { return z->duplicate(); } -Ref MLPPActivation::linear_norm(const Ref &z) { +Ref MLPPActivation::linear_normm(const Ref &z) { return z->duplicate(); } -real_t MLPPActivation::linear_deriv(real_t z) { +real_t MLPPActivation::linear_derivr(real_t z) { return 1; } -Ref MLPPActivation::linear_deriv(const Ref &z) { +Ref MLPPActivation::linear_derivv(const Ref &z) { MLPPLinAlg alg; return alg.onevecv(z->size()); } -Ref MLPPActivation::linear_deriv(const Ref &z) { +Ref MLPPActivation::linear_derivm(const Ref &z) { MLPPLinAlg alg; return alg.onematm(z->size().x, z->size().y); } //SIGMOID -real_t MLPPActivation::sigmoid_norm(real_t z) { +real_t MLPPActivation::sigmoid_normr(real_t z) { return 1 / (1 + exp(-z)); } -Ref MLPPActivation::sigmoid_norm(const Ref &z) { +Ref MLPPActivation::sigmoid_normv(const Ref &z) { MLPPLinAlg alg; return alg.element_wise_division(alg.onevecv(z->size()), alg.additionm(alg.onevecv(z->size()), alg.expv(alg.scalar_multiplynv(-1, z)))); } -Ref MLPPActivation::sigmoid_norm(const Ref &z) { +Ref MLPPActivation::sigmoid_normm(const Ref &z) { MLPPLinAlg alg; return alg.element_wise_division(alg.onematm(z->size().x, z->size().y), alg.additionm(alg.onematm(z->size().x, z->size().y), alg.expv(alg.scalar_multiplynv(-1, z)))); } -real_t MLPPActivation::sigmoid_deriv(real_t z) { - real_t sig_norm = sigmoid_norm(z); +real_t MLPPActivation::sigmoid_derivr(real_t z) { + real_t sig_norm = sigmoid_normr(z); return sig_norm * (1 - sig_norm); } -Ref MLPPActivation::sigmoid_deriv(const Ref &z) { +Ref MLPPActivation::sigmoid_derivv(const Ref &z) { MLPPLinAlg alg; - Ref sig_norm = sigmoid_norm(z); + Ref sig_norm = sigmoid_normv(z); return alg.subtractionnv(sig_norm, alg.hadamard_productnv(sig_norm, sig_norm)); } -Ref MLPPActivation::sigmoid_deriv(const Ref &z) { +Ref MLPPActivation::sigmoid_derivm(const Ref &z) { MLPPLinAlg alg; - Ref sig_norm = sigmoid_norm(z); + Ref sig_norm = sigmoid_normm(z); return alg.subtractionnv(sig_norm, alg.hadamard_productnv(sig_norm, sig_norm)); } //SOFTMAX -Ref MLPPActivation::softmax_norm(const Ref &z) { +Ref MLPPActivation::softmax_normv(const Ref &z) { MLPPLinAlg alg; int z_size = z->size(); @@ -138,7 +138,7 @@ Ref MLPPActivation::softmax_norm(const Ref &z) { return a; } -Ref MLPPActivation::softmax_norm(const Ref &z) { +Ref MLPPActivation::softmax_normm(const Ref &z) { MLPPLinAlg alg; Size2i z_size = z->size(); @@ -154,7 +154,7 @@ Ref MLPPActivation::softmax_norm(const Ref &z) { for (int i = 0; i < z_size.y; ++i) { z->get_row_into_mlpp_vector(i, row_tmp); - Ref sfn = softmax_norm(z); + Ref sfn = softmax_normv(z); a->set_row_mlpp_vector(i, sfn); } @@ -162,7 +162,7 @@ Ref MLPPActivation::softmax_norm(const Ref &z) { return a; } -Ref MLPPActivation::softmax_deriv(const Ref &z) { +Ref MLPPActivation::softmax_derivv(const Ref &z) { MLPPLinAlg alg; int z_size = z->size(); @@ -187,7 +187,7 @@ Ref MLPPActivation::softmax_deriv(const Ref &z) { return a; } -Ref MLPPActivation::softmax_deriv(const Ref &z) { +Ref MLPPActivation::softmax_derivm(const Ref &z) { MLPPLinAlg alg; Size2i z_size = z->size(); @@ -203,7 +203,7 @@ Ref MLPPActivation::softmax_deriv(const Ref &z) { for (int i = 0; i < z_size.y; ++i) { z->get_row_into_mlpp_vector(i, row_tmp); - Ref sfn = softmax_deriv(z); + Ref sfn = softmax_derivm(z); a->set_row_mlpp_vector(i, sfn); } @@ -213,7 +213,7 @@ Ref MLPPActivation::softmax_deriv(const Ref &z) { //ADJ_SOFTMAX -Ref MLPPActivation::adj_softmax_norm(const Ref &z) { +Ref MLPPActivation::adj_softmax_normv(const Ref &z) { MLPPLinAlg alg; int size = z->size(); @@ -232,9 +232,9 @@ Ref MLPPActivation::adj_softmax_norm(const Ref &z) { Ref n = alg.scalar_addnv(c, z); - return softmax_norm(n); + return softmax_normv(n); } -Ref MLPPActivation::adj_softmax_norm(const Ref &z) { +Ref MLPPActivation::adj_softmax_normm(const Ref &z) { MLPPLinAlg alg; Ref n = z->duplicate(); @@ -248,7 +248,7 @@ Ref MLPPActivation::adj_softmax_norm(const Ref &z) { for (int i = 0; i < size.y; ++i) { z->get_row_into_mlpp_vector(i, row_rmp); - Ref nv = adj_softmax_norm(row_rmp); + Ref nv = adj_softmax_normv(row_rmp); n->set_row_mlpp_vector(i, nv); } @@ -256,7 +256,7 @@ Ref MLPPActivation::adj_softmax_norm(const Ref &z) { return n; } -Ref MLPPActivation::adj_softmax_deriv(const Ref &z) { +Ref MLPPActivation::adj_softmax_derivv(const Ref &z) { MLPPLinAlg alg; int size = z->size(); @@ -275,9 +275,9 @@ Ref MLPPActivation::adj_softmax_deriv(const Ref &z) { Ref n = alg.scalar_addnv(c, z); - return adj_softmax_deriv(n); + return adj_softmax_derivv(n); } -Ref MLPPActivation::adj_softmax_deriv(const Ref &z) { +Ref MLPPActivation::adj_softmax_derivm(const Ref &z) { MLPPLinAlg alg; Ref n = z->duplicate(); @@ -291,7 +291,7 @@ Ref MLPPActivation::adj_softmax_deriv(const Ref &z) { for (int i = 0; i < size.y; ++i) { z->get_row_into_mlpp_vector(i, row_rmp); - Ref nv = adj_softmax_deriv(row_rmp); + Ref nv = adj_softmax_derivv(row_rmp); n->set_row_mlpp_vector(i, nv); } @@ -301,10 +301,10 @@ Ref MLPPActivation::adj_softmax_deriv(const Ref &z) { //SOFTMAX DERIV -Ref MLPPActivation::softmax_deriv_norm(const Ref &z) { +Ref MLPPActivation::softmax_deriv_normv(const Ref &z) { MLPPLinAlg alg; - Ref a = softmax_norm(z); + Ref a = softmax_normv(z); int z_size = z->size(); int a_size = a->size(); @@ -327,12 +327,12 @@ Ref MLPPActivation::softmax_deriv_norm(const Ref &z) { return deriv; } -Vector> MLPPActivation::softmax_deriv_norm(const Ref &z) { +Vector> MLPPActivation::softmax_deriv_normm(const Ref &z) { MLPPLinAlg alg; int z_size_y = z->size().y; - Ref a = softmax_norm(z); + Ref a = softmax_normm(z); int a_size_y = a->size().y; int a_size_x = a->size().x; @@ -371,10 +371,10 @@ Vector> MLPPActivation::softmax_deriv_norm(const Ref return deriv; } -Ref MLPPActivation::softmax_deriv_deriv(const Ref &z) { +Ref MLPPActivation::softmax_deriv_derivv(const Ref &z) { MLPPLinAlg alg; - Ref a = softmax_norm(z); + Ref a = softmax_normv(z); int z_size = z->size(); int a_size = a->size(); @@ -397,12 +397,12 @@ Ref MLPPActivation::softmax_deriv_deriv(const Ref &z) { return deriv; } -Vector> MLPPActivation::softmax_deriv_deriv(const Ref &z) { +Vector> MLPPActivation::softmax_deriv_derivm(const Ref &z) { MLPPLinAlg alg; int z_size_y = z->size().y; - Ref a = softmax_norm(z); + Ref a = softmax_normm(z); int a_size_y = a->size().y; int a_size_x = a->size().x; @@ -443,55 +443,55 @@ Vector> MLPPActivation::softmax_deriv_deriv(const Ref MLPPActivation::softplus_norm(const Ref &z) { +Ref MLPPActivation::softplus_normv(const Ref &z) { MLPPLinAlg alg; return alg.logv(alg.additionnv(alg.onevecv(z->size()), alg.expv(z))); } -Ref MLPPActivation::softplus_norm(const Ref &z) { +Ref MLPPActivation::softplus_normm(const Ref &z) { MLPPLinAlg alg; return alg.logv(alg.additionnv(alg.onematm(z->size().x, z->size().y), alg.expv(z))); } -real_t MLPPActivation::softplus_deriv(real_t z) { - return sigmoid_norm(z); +real_t MLPPActivation::softplus_derivr(real_t z) { + return sigmoid_normr(z); } -Ref MLPPActivation::softplus_deriv(const Ref &z) { - return sigmoid_norm(z); +Ref MLPPActivation::softplus_derivv(const Ref &z) { + return sigmoid_normv(z); } -Ref MLPPActivation::softplus_deriv(const Ref &z) { - return sigmoid_norm(z); +Ref MLPPActivation::softplus_derivm(const Ref &z) { + return sigmoid_normm(z); } //SOFTSIGN -real_t MLPPActivation::softsign_norm(real_t z) { +real_t MLPPActivation::softsign_normr(real_t z) { return z / (1 + abs(z)); } -Ref MLPPActivation::softsign_norm(const Ref &z) { +Ref MLPPActivation::softsign_normv(const Ref &z) { MLPPLinAlg alg; return alg.element_wise_division(z, alg.additionnv(alg.onevecv(z->size()), alg.absv(z))); } -Ref MLPPActivation::softsign_norm(const Ref &z) { +Ref MLPPActivation::softsign_normm(const Ref &z) { MLPPLinAlg alg; return alg.element_wise_divisionm(z, alg.additionnv(alg.onematm(z->size().x, z->size().y), alg.absm(z))); } -real_t MLPPActivation::softsign_deriv(real_t z) { +real_t MLPPActivation::softsign_derivr(real_t z) { return 1 / ((1 + abs(z)) * (1 + abs(z))); } -Ref MLPPActivation::softsign_deriv(const Ref &z) { +Ref MLPPActivation::softsign_derivv(const Ref &z) { MLPPLinAlg alg; return alg.element_wise_division(alg.onevecv(z->size()), alg.exponentiatev(alg.additionnv(alg.onevecv(z->size()), alg.absv(z)), 2)); } -Ref MLPPActivation::softsign_deriv(const Ref &z) { +Ref MLPPActivation::softsign_derivm(const Ref &z) { MLPPLinAlg alg; return alg.element_wise_divisionm(alg.onematm(z->size().x, z->size().y), alg.exponentiatev(alg.additionm(alg.onematm(z->size().x, z->size().y), alg.absm(z)), 2)); @@ -499,31 +499,31 @@ Ref MLPPActivation::softsign_deriv(const Ref &z) { //GAUSSIANCDF -real_t MLPPActivation::gaussian_cdf_norm(real_t z) { +real_t MLPPActivation::gaussian_cdf_normr(real_t z) { return 0.5 * (1 + erf(z / sqrt(2))); } -Ref MLPPActivation::gaussian_cdf_norm(const Ref &z) { +Ref MLPPActivation::gaussian_cdf_normv(const Ref &z) { MLPPLinAlg alg; return alg.scalar_multiplynv(0.5, alg.additionnv(alg.onevecv(z->size()), alg.erfv(alg.scalar_multiplynv(1 / sqrt(2), z)))); } -Ref MLPPActivation::gaussian_cdf_norm(const Ref &z) { +Ref MLPPActivation::gaussian_cdf_normm(const Ref &z) { MLPPLinAlg alg; return alg.scalar_multiplym(0.5, alg.additionm(alg.onematm(z->size().x, z->size().y), alg.erfm(alg.scalar_multiplym(1 / sqrt(2), z)))); } -real_t MLPPActivation::gaussian_cdf_deriv(real_t z) { +real_t MLPPActivation::gaussian_cdf_derivr(real_t z) { return (1 / sqrt(2 * M_PI)) * exp(-z * z / 2); } -Ref MLPPActivation::gaussian_cdf_deriv(const Ref &z) { +Ref MLPPActivation::gaussian_cdf_derivv(const Ref &z) { MLPPLinAlg alg; return alg.scalar_multiplynv(1 / Math::sqrt(2 * M_PI), alg.expv(alg.scalar_multiplynv(-1 / 2.0, alg.hadamard_productnv(z, z)))); } -Ref MLPPActivation::gaussian_cdf_deriv(const Ref &z) { +Ref MLPPActivation::gaussian_cdf_derivm(const Ref &z) { MLPPLinAlg alg; return alg.scalar_multiplym(1 / Math::sqrt(2 * M_PI), alg.expm(alg.scalar_multiplym(-1 / 2.0, alg.hadamard_productm(z, z)))); @@ -531,31 +531,31 @@ Ref MLPPActivation::gaussian_cdf_deriv(const Ref &z) { //CLOGLOG -real_t MLPPActivation::cloglog_norm(real_t z) { +real_t MLPPActivation::cloglog_normr(real_t z) { return 1 - exp(-exp(z)); } -Ref MLPPActivation::cloglog_norm(const Ref &z) { +Ref MLPPActivation::cloglog_normv(const Ref &z) { MLPPLinAlg alg; return alg.scalar_multiplynv(-1, alg.scalar_addnv(-1, alg.expv(alg.scalar_multiplynv(-1, alg.expv(z))))); } -Ref MLPPActivation::cloglog_norm(const Ref &z) { +Ref MLPPActivation::cloglog_normm(const Ref &z) { MLPPLinAlg alg; return alg.scalar_multiplym(-1, alg.scalar_addm(-1, alg.expm(alg.scalar_multiplym(-1, alg.expm(z))))); } -real_t MLPPActivation::cloglog_deriv(real_t z) { +real_t MLPPActivation::cloglog_derivr(real_t z) { return exp(z - exp(z)); } -Ref MLPPActivation::cloglog_deriv(const Ref &z) { +Ref MLPPActivation::cloglog_derivv(const Ref &z) { MLPPLinAlg alg; return alg.expv(alg.scalar_multiplynv(-1, alg.expv(z))); } -Ref MLPPActivation::cloglog_deriv(const Ref &z) { +Ref MLPPActivation::cloglog_derivm(const Ref &z) { MLPPLinAlg alg; return alg.expm(alg.scalar_multiplym(-1, alg.expm(z))); @@ -563,31 +563,31 @@ Ref MLPPActivation::cloglog_deriv(const Ref &z) { //LOGIT -real_t MLPPActivation::logit_norm(real_t z) { +real_t MLPPActivation::logit_normr(real_t z) { return std::log(z / (1 - z)); } -Ref MLPPActivation::logit_norm(const Ref &z) { +Ref MLPPActivation::logit_normv(const Ref &z) { MLPPLinAlg alg; return alg.logv(alg.element_wise_division(z, alg.subtractionnv(alg.onevecv(z->size()), z))); } -Ref MLPPActivation::logit_norm(const Ref &z) { +Ref MLPPActivation::logit_normm(const Ref &z) { MLPPLinAlg alg; return alg.logm(alg.element_wise_divisionm(z, alg.subtractionm(alg.onematm(z->size().x, z->size().y), z))); } -real_t MLPPActivation::logit_deriv(real_t z) { +real_t MLPPActivation::logit_derivr(real_t z) { return 1 / z - 1 / (z - 1); } -Ref MLPPActivation::logit_deriv(const Ref &z) { +Ref MLPPActivation::logit_derivv(const Ref &z) { MLPPLinAlg alg; return alg.subtractionnv( alg.element_wise_division(alg.onevecv(z->size()), z), alg.element_wise_division(alg.onevecv(z->size()), alg.subtractionnv(z, alg.onevecv(z->size())))); } -Ref MLPPActivation::logit_deriv(const Ref &z) { +Ref MLPPActivation::logit_derivm(const Ref &z) { MLPPLinAlg alg; return alg.subtractionm( @@ -599,10 +599,10 @@ Ref MLPPActivation::logit_deriv(const Ref &z) { //UNITSTEP -real_t MLPPActivation::unit_step_norm(real_t z) { +real_t MLPPActivation::unit_step_normr(real_t z) { return z < 0 ? 0 : 1; } -Ref MLPPActivation::unit_step_norm(const Ref &z) { +Ref MLPPActivation::unit_step_normv(const Ref &z) { MLPPLinAlg alg; Ref a; @@ -615,12 +615,12 @@ Ref MLPPActivation::unit_step_norm(const Ref &z) { real_t *a_ptr = a->ptrw(); for (int i = 0; i < z_size; ++i) { - a_ptr[i] = unit_step_norm(z_ptr[i]); + a_ptr[i] = unit_step_normr(z_ptr[i]); } return a; } -Ref MLPPActivation::unit_step_norm(const Ref &z) { +Ref MLPPActivation::unit_step_normm(const Ref &z) { MLPPLinAlg alg; Ref a; @@ -633,16 +633,16 @@ Ref MLPPActivation::unit_step_norm(const Ref &z) { real_t *a_ptr = a->ptrw(); for (int i = 0; i < z_data_size; ++i) { - a_ptr[i] = unit_step_norm(z_ptr[i]); + a_ptr[i] = unit_step_normr(z_ptr[i]); } return a; } -real_t MLPPActivation::unit_step_deriv(real_t z) { +real_t MLPPActivation::unit_step_derivr(real_t z) { return 0; } -Ref MLPPActivation::unit_step_deriv(const Ref &z) { +Ref MLPPActivation::unit_step_derivv(const Ref &z) { Ref a; a.instance(); a->resize(z->size()); @@ -650,7 +650,7 @@ Ref MLPPActivation::unit_step_deriv(const Ref &z) { return a; } -Ref MLPPActivation::unit_step_deriv(const Ref &z) { +Ref MLPPActivation::unit_step_derivm(const Ref &z) { Ref a; a.instance(); a->resize(z->size()); @@ -661,103 +661,103 @@ Ref MLPPActivation::unit_step_deriv(const Ref &z) { //SWISH -real_t MLPPActivation::swish_norm(real_t z) { - return z * sigmoid_norm(z); +real_t MLPPActivation::swish_normr(real_t z) { + return z * sigmoid_normr(z); } -Ref MLPPActivation::swish_norm(const Ref &z) { +Ref MLPPActivation::swish_normv(const Ref &z) { MLPPLinAlg alg; - return alg.hadamard_productnv(z, sigmoid_norm(z)); + return alg.hadamard_productnv(z, sigmoid_normv(z)); } -Ref MLPPActivation::swish_norm(const Ref &z) { +Ref MLPPActivation::swish_normm(const Ref &z) { MLPPLinAlg alg; - return alg.hadamard_productnv(z, sigmoid_norm(z)); + return alg.hadamard_productnv(z, sigmoid_normm(z)); } -real_t MLPPActivation::swish_deriv(real_t z) { - return swish_norm(z) + sigmoid_norm(z) * (1 - swish_norm(z)); +real_t MLPPActivation::swish_derivr(real_t z) { + return swish_normr(z) + sigmoid_normr(z) * (1 - swish_normr(z)); } -Ref MLPPActivation::swish_deriv(const Ref &z) { +Ref MLPPActivation::swish_derivv(const Ref &z) { MLPPLinAlg alg; - alg.additionnv(swish_norm(z), alg.subtractionnv(sigmoid_norm(z), alg.hadamard_productnv(sigmoid_norm(z), swish_norm(z)))); + alg.additionnv(swish_normv(z), alg.subtractionnv(sigmoid_normv(z), alg.hadamard_productnv(sigmoid_normv(z), swish_normv(z)))); } -Ref MLPPActivation::swish_deriv(const Ref &z) { +Ref MLPPActivation::swish_derivm(const Ref &z) { MLPPLinAlg alg; - alg.additionnv(swish_norm(z), alg.subtractionnv(sigmoid_norm(z), alg.hadamard_productm(sigmoid_norm(z), swish_norm(z)))); + alg.additionnv(swish_normm(z), alg.subtractionnv(sigmoid_normm(z), alg.hadamard_productm(sigmoid_normm(z), swish_normm(z)))); } //MISH -real_t MLPPActivation::mish_norm(real_t z) { +real_t MLPPActivation::mish_normr(real_t z) { return z * tanh(softplus(z)); } -Ref MLPPActivation::mish_norm(const Ref &z) { +Ref MLPPActivation::mish_normv(const Ref &z) { MLPPLinAlg alg; - return alg.hadamard_productnv(z, tanh_norm(softplus_norm(z))); + return alg.hadamard_productnv(z, tanh_normv(softplus_normv(z))); } -Ref MLPPActivation::mish_norm(const Ref &z) { +Ref MLPPActivation::mish_normm(const Ref &z) { MLPPLinAlg alg; - return alg.hadamard_productm(z, tanh_norm(softplus_norm(z))); + return alg.hadamard_productm(z, tanh_normm(softplus_normm(z))); } -real_t MLPPActivation::mish_deriv(real_t z) { - return sech(softplus_norm(z)) * sech(softplus_norm(z)) * z * sigmoid_norm(z) + mish_norm(z) / z; +real_t MLPPActivation::mish_derivr(real_t z) { + return sech(softplus_normr(z)) * sech(softplus_normr(z)) * z * sigmoid_normr(z) + mish_normr(z) / z; } -Ref MLPPActivation::mish_deriv(const Ref &z) { +Ref MLPPActivation::mish_derivv(const Ref &z) { MLPPLinAlg alg; return alg.additionnv( alg.hadamard_productnv( alg.hadamard_productnv( alg.hadamard_productnv( - sech_norm(softplus_norm(z)), sech_norm(softplus_norm(z))), + sech_normv(softplus_normv(z)), sech_normv(softplus_normv(z))), z), - sigmoid_norm(z)), - alg.element_wise_division(mish_norm(z), z)); + sigmoid_normv(z)), + alg.element_wise_division(mish_normv(z), z)); } -Ref MLPPActivation::mish_deriv(const Ref &z) { +Ref MLPPActivation::mish_derivm(const Ref &z) { MLPPLinAlg alg; return alg.additionnv( alg.hadamard_productm( alg.hadamard_productm( alg.hadamard_productm( - sech_norm(softplus_norm(z)), sech_norm(softplus_norm(z))), + sech_normm(softplus_normm(z)), sech_normm(softplus_normm(z))), z), - sigmoid_norm(z)), - alg.element_wise_divisionm(mish_norm(z), z)); + sigmoid_normm(z)), + alg.element_wise_divisionm(mish_normm(z), z)); } //SINC -real_t MLPPActivation::sinc_norm(real_t z) { +real_t MLPPActivation::sinc_normr(real_t z) { return std::sin(z) / z; } -Ref MLPPActivation::sinc_norm(const Ref &z) { +Ref MLPPActivation::sinc_normv(const Ref &z) { MLPPLinAlg alg; return alg.element_wise_division(alg.sinv(z), z); } -Ref MLPPActivation::sinc_norm(const Ref &z) { +Ref MLPPActivation::sinc_normm(const Ref &z) { MLPPLinAlg alg; return alg.element_wise_divisionm(alg.sinm(z), z); } -real_t MLPPActivation::sinc_deriv(real_t z) { +real_t MLPPActivation::sinc_derivr(real_t z) { return (z * std::cos(z) - std::sin(z)) / (z * z); } -Ref MLPPActivation::sinc_deriv(const Ref &z) { +Ref MLPPActivation::sinc_derivv(const Ref &z) { MLPPLinAlg alg; return alg.element_wise_division(alg.subtractionnv(alg.hadamard_productnv(z, alg.cosv(z)), alg.sinv(z)), alg.hadamard_productnv(z, z)); } -Ref MLPPActivation::sinc_deriv(const Ref &z) { +Ref MLPPActivation::sinc_derivm(const Ref &z) { MLPPLinAlg alg; return alg.element_wise_divisionm(alg.subtractionm(alg.hadamard_productm(z, alg.cosm(z)), alg.sinm(z)), alg.hadamard_productm(z, z)); @@ -765,10 +765,10 @@ Ref MLPPActivation::sinc_deriv(const Ref &z) { //RELU -real_t MLPPActivation::relu_norm(real_t z) { +real_t MLPPActivation::relu_normr(real_t z) { return fmax(0, z); } -Ref MLPPActivation::relu_norm(const Ref &z) { +Ref MLPPActivation::relu_normv(const Ref &z) { MLPPLinAlg alg; Ref a; @@ -781,12 +781,12 @@ Ref MLPPActivation::relu_norm(const Ref &z) { real_t *a_ptr = a->ptrw(); for (int i = 0; i < z_size; ++i) { - a_ptr[i] = relu_norm(z_ptr[i]); + a_ptr[i] = relu_normr(z_ptr[i]); } return a; } -Ref MLPPActivation::relu_norm(const Ref &z) { +Ref MLPPActivation::relu_normm(const Ref &z) { MLPPLinAlg alg; Ref a; @@ -799,20 +799,20 @@ Ref MLPPActivation::relu_norm(const Ref &z) { real_t *a_ptr = a->ptrw(); for (int i = 0; i < z_data_size; ++i) { - a_ptr[i] = relu_norm(z_ptr[i]); + a_ptr[i] = relu_normr(z_ptr[i]); } return a; } -real_t MLPPActivation::relu_deriv(real_t z) { +real_t MLPPActivation::relu_derivr(real_t z) { if (z <= 0) { return 0; } else { return 1; } } -Ref MLPPActivation::relu_deriv(const Ref &z) { +Ref MLPPActivation::relu_derivv(const Ref &z) { MLPPLinAlg alg; Ref a; @@ -825,12 +825,12 @@ Ref MLPPActivation::relu_deriv(const Ref &z) { real_t *a_ptr = a->ptrw(); for (int i = 0; i < z_size; ++i) { - a_ptr[i] = relu_deriv(z_ptr[i]); + a_ptr[i] = relu_derivr(z_ptr[i]); } return a; } -Ref MLPPActivation::relu_deriv(const Ref &z) { +Ref MLPPActivation::relu_derivm(const Ref &z) { MLPPLinAlg alg; Ref a; @@ -843,7 +843,7 @@ Ref MLPPActivation::relu_deriv(const Ref &z) { real_t *a_ptr = a->ptrw(); for (int i = 0; i < z_data_size; ++i) { - a_ptr[i] = relu_deriv(z_ptr[i]); + a_ptr[i] = relu_derivr(z_ptr[i]); } return a; @@ -851,10 +851,10 @@ Ref MLPPActivation::relu_deriv(const Ref &z) { //LEAKYRELU -real_t MLPPActivation::leaky_relu_norm(real_t z, real_t c) { +real_t MLPPActivation::leaky_relu_normr(real_t z, real_t c) { return fmax(c * z, z); } -Ref MLPPActivation::leaky_relu_norm(const Ref &z, real_t c) { +Ref MLPPActivation::leaky_relu_normv(const Ref &z, real_t c) { MLPPLinAlg alg; Ref a; @@ -867,12 +867,12 @@ Ref MLPPActivation::leaky_relu_norm(const Ref &z, real_t real_t *a_ptr = a->ptrw(); for (int i = 0; i < z_size; ++i) { - a_ptr[i] = leaky_relu_norm(z_ptr[i], c); + a_ptr[i] = leaky_relu_normr(z_ptr[i], c); } return a; } -Ref MLPPActivation::leaky_relu_norm(const Ref &z, real_t c) { +Ref MLPPActivation::leaky_relu_normm(const Ref &z, real_t c) { MLPPLinAlg alg; Ref a; @@ -885,20 +885,20 @@ Ref MLPPActivation::leaky_relu_norm(const Ref &z, real_t real_t *a_ptr = a->ptrw(); for (int i = 0; i < z_data_size; ++i) { - a_ptr[i] = leaky_relu_norm(z_ptr[i], c); + a_ptr[i] = leaky_relu_normr(z_ptr[i], c); } return a; } -real_t MLPPActivation::leaky_relu_deriv(real_t z, real_t c) { +real_t MLPPActivation::leaky_relu_derivr(real_t z, real_t c) { if (z <= 0) { return c; } else { return 1; } } -Ref MLPPActivation::leaky_relu_deriv(const Ref &z, real_t c) { +Ref MLPPActivation::leaky_relu_derivv(const Ref &z, real_t c) { MLPPLinAlg alg; Ref a; @@ -911,12 +911,12 @@ Ref MLPPActivation::leaky_relu_deriv(const Ref &z, real_ real_t *a_ptr = a->ptrw(); for (int i = 0; i < z_size; ++i) { - a_ptr[i] = leaky_relu_deriv(z_ptr[i], c); + a_ptr[i] = leaky_relu_derivr(z_ptr[i], c); } return a; } -Ref MLPPActivation::leaky_relu_deriv(const Ref &z, real_t c) { +Ref MLPPActivation::leaky_relu_derivm(const Ref &z, real_t c) { MLPPLinAlg alg; Ref a; @@ -929,7 +929,7 @@ Ref MLPPActivation::leaky_relu_deriv(const Ref &z, real_ real_t *a_ptr = a->ptrw(); for (int i = 0; i < z_data_size; ++i) { - a_ptr[i] = leaky_relu_deriv(z_ptr[i], c); + a_ptr[i] = leaky_relu_derivr(z_ptr[i], c); } return a; @@ -937,14 +937,14 @@ Ref MLPPActivation::leaky_relu_deriv(const Ref &z, real_ //ELU -real_t MLPPActivation::elu_norm(real_t z, real_t c) { +real_t MLPPActivation::elu_normr(real_t z, real_t c) { if (z >= 0) { return z; } else { return c * (exp(z) - 1); } } -Ref MLPPActivation::elu_norm(const Ref &z, real_t c) { +Ref MLPPActivation::elu_normv(const Ref &z, real_t c) { MLPPLinAlg alg; Ref a; @@ -957,12 +957,12 @@ Ref MLPPActivation::elu_norm(const Ref &z, real_t c) { real_t *a_ptr = a->ptrw(); for (int i = 0; i < z_size; ++i) { - a_ptr[i] = elu_norm(z_ptr[i], c); + a_ptr[i] = elu_normr(z_ptr[i], c); } return a; } -Ref MLPPActivation::elu_norm(const Ref &z, real_t c) { +Ref MLPPActivation::elu_normm(const Ref &z, real_t c) { MLPPLinAlg alg; Ref a; @@ -975,20 +975,20 @@ Ref MLPPActivation::elu_norm(const Ref &z, real_t c) { real_t *a_ptr = a->ptrw(); for (int i = 0; i < z_data_size; ++i) { - a_ptr[i] = elu_norm(z_ptr[i], c); + a_ptr[i] = elu_normr(z_ptr[i], c); } return a; } -real_t MLPPActivation::elu_deriv(real_t z, real_t c) { +real_t MLPPActivation::elu_derivr(real_t z, real_t c) { if (z <= 0) { return c * exp(z); } else { return 1; } } -Ref MLPPActivation::elu_deriv(const Ref &z, real_t c) { +Ref MLPPActivation::elu_derivv(const Ref &z, real_t c) { MLPPLinAlg alg; Ref a; @@ -1001,12 +1001,12 @@ Ref MLPPActivation::elu_deriv(const Ref &z, real_t c) { real_t *a_ptr = a->ptrw(); for (int i = 0; i < z_size; ++i) { - a_ptr[i] = elu_deriv(z_ptr[i], c); + a_ptr[i] = elu_derivr(z_ptr[i], c); } return a; } -Ref MLPPActivation::elu_deriv(const Ref &z, real_t c) { +Ref MLPPActivation::elu_derivm(const Ref &z, real_t c) { MLPPLinAlg alg; Ref a; @@ -1019,7 +1019,7 @@ Ref MLPPActivation::elu_deriv(const Ref &z, real_t c) { real_t *a_ptr = a->ptrw(); for (int i = 0; i < z_data_size; ++i) { - a_ptr[i] = elu_deriv(z_ptr[i], c); + a_ptr[i] = elu_derivr(z_ptr[i], c); } return a; @@ -1027,10 +1027,10 @@ Ref MLPPActivation::elu_deriv(const Ref &z, real_t c) { //SELU -real_t MLPPActivation::selu_norm(real_t z, real_t lambda, real_t c) { +real_t MLPPActivation::selu_normr(real_t z, real_t lambda, real_t c) { return lambda * ELU(z, c); } -Ref MLPPActivation::selu_norm(const Ref &z, real_t lambda, real_t c) { +Ref MLPPActivation::selu_normv(const Ref &z, real_t lambda, real_t c) { MLPPLinAlg alg; Ref a; @@ -1043,12 +1043,12 @@ Ref MLPPActivation::selu_norm(const Ref &z, real_t lambd real_t *a_ptr = a->ptrw(); for (int i = 0; i < z_size; ++i) { - a_ptr[i] = selu_norm(z_ptr[i], lambda, c); + a_ptr[i] = selu_normr(z_ptr[i], lambda, c); } return a; } -Ref MLPPActivation::selu_norm(const Ref &z, real_t lambda, real_t c) { +Ref MLPPActivation::selu_normm(const Ref &z, real_t lambda, real_t c) { MLPPLinAlg alg; Ref a; @@ -1061,16 +1061,16 @@ Ref MLPPActivation::selu_norm(const Ref &z, real_t lambd real_t *a_ptr = a->ptrw(); for (int i = 0; i < z_data_size; ++i) { - a_ptr[i] = selu_norm(z_ptr[i], lambda, c); + a_ptr[i] = selu_normr(z_ptr[i], lambda, c); } return a; } -real_t MLPPActivation::selu_deriv(real_t z, real_t lambda, real_t c) { - return elu_deriv(z, c); +real_t MLPPActivation::selu_derivr(real_t z, real_t lambda, real_t c) { + return elu_derivr(z, c); } -Ref MLPPActivation::selu_deriv(const Ref &z, real_t lambda, real_t c) { +Ref MLPPActivation::selu_derivv(const Ref &z, real_t lambda, real_t c) { MLPPLinAlg alg; Ref a; @@ -1083,12 +1083,12 @@ Ref MLPPActivation::selu_deriv(const Ref &z, real_t lamb real_t *a_ptr = a->ptrw(); for (int i = 0; i < z_size; ++i) { - a_ptr[i] = selu_deriv(z_ptr[i], lambda, c); + a_ptr[i] = selu_derivr(z_ptr[i], lambda, c); } return a; } -Ref MLPPActivation::selu_deriv(const Ref &z, real_t lambda, real_t c) { +Ref MLPPActivation::selu_derivm(const Ref &z, real_t lambda, real_t c) { MLPPLinAlg alg; Ref a; @@ -1101,7 +1101,7 @@ Ref MLPPActivation::selu_deriv(const Ref &z, real_t lamb real_t *a_ptr = a->ptrw(); for (int i = 0; i < z_data_size; ++i) { - a_ptr[i] = selu_deriv(z_ptr[i], lambda, c); + a_ptr[i] = selu_derivr(z_ptr[i], lambda, c); } return a; @@ -1109,10 +1109,10 @@ Ref MLPPActivation::selu_deriv(const Ref &z, real_t lamb //GELU -real_t MLPPActivation::gelu_norm(real_t z) { +real_t MLPPActivation::gelu_normr(real_t z) { return 0.5 * z * (1 + tanh(sqrt(2 / M_PI) * (z + 0.044715 * Math::pow(z, 3)))); } -Ref MLPPActivation::gelu_norm(const Ref &z) { +Ref MLPPActivation::gelu_normv(const Ref &z) { MLPPLinAlg alg; Ref a; @@ -1125,12 +1125,12 @@ Ref MLPPActivation::gelu_norm(const Ref &z) { real_t *a_ptr = a->ptrw(); for (int i = 0; i < z_size; ++i) { - a_ptr[i] = gelu_norm(z_ptr[i]); + a_ptr[i] = gelu_normr(z_ptr[i]); } return a; } -Ref MLPPActivation::gelu_norm(const Ref &z) { +Ref MLPPActivation::gelu_normm(const Ref &z) { MLPPLinAlg alg; Ref a; @@ -1143,16 +1143,16 @@ Ref MLPPActivation::gelu_norm(const Ref &z) { real_t *a_ptr = a->ptrw(); for (int i = 0; i < z_data_size; ++i) { - a_ptr[i] = gelu_norm(z_ptr[i]); + a_ptr[i] = gelu_normr(z_ptr[i]); } return a; } -real_t MLPPActivation::gelu_deriv(real_t z) { +real_t MLPPActivation::gelu_derivr(real_t z) { return 0.5 * tanh(0.0356774 * std::pow(z, 3) + 0.797885 * z) + (0.0535161 * std::pow(z, 3) + 0.398942 * z) * std::pow(sech(0.0356774 * std::pow(z, 3) + 0.797885 * z), 2) + 0.5; } -Ref MLPPActivation::gelu_deriv(const Ref &z) { +Ref MLPPActivation::gelu_derivv(const Ref &z) { MLPPLinAlg alg; Ref a; @@ -1165,12 +1165,12 @@ Ref MLPPActivation::gelu_deriv(const Ref &z) { real_t *a_ptr = a->ptrw(); for (int i = 0; i < z_size; ++i) { - a_ptr[i] = gelu_deriv(z_ptr[i]); + a_ptr[i] = gelu_derivr(z_ptr[i]); } return a; } -Ref MLPPActivation::gelu_deriv(const Ref &z) { +Ref MLPPActivation::gelu_derivm(const Ref &z) { MLPPLinAlg alg; Ref a; @@ -1183,7 +1183,7 @@ Ref MLPPActivation::gelu_deriv(const Ref &z) { real_t *a_ptr = a->ptrw(); for (int i = 0; i < z_data_size; ++i) { - a_ptr[i] = gelu_deriv(z_ptr[i]); + a_ptr[i] = gelu_derivr(z_ptr[i]); } return a; @@ -1191,7 +1191,7 @@ Ref MLPPActivation::gelu_deriv(const Ref &z) { //SIGN -real_t MLPPActivation::sign_norm(real_t z) { +real_t MLPPActivation::sign_normr(real_t z) { if (z < 0) { return -1; } else if (z == 0) { @@ -1200,7 +1200,7 @@ real_t MLPPActivation::sign_norm(real_t z) { return 1; } } -Ref MLPPActivation::sign_norm(const Ref &z) { +Ref MLPPActivation::sign_normv(const Ref &z) { MLPPLinAlg alg; Ref a; @@ -1213,12 +1213,12 @@ Ref MLPPActivation::sign_norm(const Ref &z) { real_t *a_ptr = a->ptrw(); for (int i = 0; i < z_size; ++i) { - a_ptr[i] = sign_norm(z_ptr[i]); + a_ptr[i] = sign_normr(z_ptr[i]); } return a; } -Ref MLPPActivation::sign_norm(const Ref &z) { +Ref MLPPActivation::sign_normm(const Ref &z) { MLPPLinAlg alg; Ref a; @@ -1231,16 +1231,16 @@ Ref MLPPActivation::sign_norm(const Ref &z) { real_t *a_ptr = a->ptrw(); for (int i = 0; i < z_data_size; ++i) { - a_ptr[i] = sign_norm(z_ptr[i]); + a_ptr[i] = sign_normr(z_ptr[i]); } return a; } -real_t MLPPActivation::sign_deriv(real_t z) { +real_t MLPPActivation::sign_derivr(real_t z) { return 0; } -Ref MLPPActivation::sign_deriv(const Ref &z) { +Ref MLPPActivation::sign_derivv(const Ref &z) { MLPPLinAlg alg; Ref a; @@ -1253,12 +1253,12 @@ Ref MLPPActivation::sign_deriv(const Ref &z) { real_t *a_ptr = a->ptrw(); for (int i = 0; i < z_size; ++i) { - a_ptr[i] = sign_deriv(z_ptr[i]); + a_ptr[i] = sign_derivr(z_ptr[i]); } return a; } -Ref MLPPActivation::sign_deriv(const Ref &z) { +Ref MLPPActivation::sign_derivm(const Ref &z) { MLPPLinAlg alg; Ref a; @@ -1271,7 +1271,7 @@ Ref MLPPActivation::sign_deriv(const Ref &z) { real_t *a_ptr = a->ptrw(); for (int i = 0; i < z_data_size; ++i) { - a_ptr[i] = sign_deriv(z_ptr[i]); + a_ptr[i] = sign_derivr(z_ptr[i]); } return a; @@ -1279,209 +1279,209 @@ Ref MLPPActivation::sign_deriv(const Ref &z) { //SINH -real_t MLPPActivation::sinh_norm(real_t z) { +real_t MLPPActivation::sinh_normr(real_t z) { return 0.5 * (Math::exp(z) - Math::exp(-z)); } -Ref MLPPActivation::sinh_norm(const Ref &z) { +Ref MLPPActivation::sinh_normv(const Ref &z) { MLPPLinAlg alg; return alg.scalar_multiplynv(0.5, alg.subtractionnv(alg.expv(z), alg.expv(alg.scalar_multiplynv(-1, z)))); } -Ref MLPPActivation::sinh_norm(const Ref &z) { +Ref MLPPActivation::sinh_normm(const Ref &z) { MLPPLinAlg alg; return alg.scalar_multiplym(0.5, alg.subtractionm(alg.expm(z), alg.expm(alg.scalar_multiplym(-1, z)))); } -real_t MLPPActivation::sinh_deriv(real_t z) { - return cosh_norm(z); +real_t MLPPActivation::sinh_derivr(real_t z) { + return cosh_normr(z); } -Ref MLPPActivation::sinh_deriv(const Ref &z) { - return cosh_norm(z); +Ref MLPPActivation::sinh_derivv(const Ref &z) { + return cosh_normv(z); } -Ref MLPPActivation::sinh_deriv(const Ref &z) { - return cosh_norm(z); +Ref MLPPActivation::sinh_derivm(const Ref &z) { + return cosh_normm(z); } //COSH -real_t MLPPActivation::cosh_norm(real_t z) { +real_t MLPPActivation::cosh_normr(real_t z) { return 0.5 * (Math::exp(z) + Math::exp(-z)); } -Ref MLPPActivation::cosh_norm(const Ref &z) { +Ref MLPPActivation::cosh_normv(const Ref &z) { MLPPLinAlg alg; return alg.scalar_multiplynv(0.5, alg.additionnv(alg.expv(z), alg.expv(alg.scalar_multiplynv(-1, z)))); } -Ref MLPPActivation::cosh_norm(const Ref &z) { +Ref MLPPActivation::cosh_normm(const Ref &z) { MLPPLinAlg alg; return alg.scalar_multiplym(0.5, alg.additionnv(alg.expm(z), alg.expm(alg.scalar_multiplym(-1, z)))); } -real_t MLPPActivation::cosh_deriv(real_t z) { - return sinh_norm(z); +real_t MLPPActivation::cosh_derivr(real_t z) { + return sinh_normr(z); } -Ref MLPPActivation::cosh_deriv(const Ref &z) { - return sinh_norm(z); +Ref MLPPActivation::cosh_derivv(const Ref &z) { + return sinh_normv(z); } -Ref MLPPActivation::cosh_deriv(const Ref &z) { - return sinh_norm(z); +Ref MLPPActivation::cosh_derivm(const Ref &z) { + return sinh_normm(z); } //TANH -real_t MLPPActivation::tanh_norm(real_t z) { +real_t MLPPActivation::tanh_normr(real_t z) { return (Math::exp(z) - Math::exp(-z)) / (Math::exp(z) + Math::exp(-z)); } -Ref MLPPActivation::tanh_norm(const Ref &z) { +Ref MLPPActivation::tanh_normv(const Ref &z) { MLPPLinAlg alg; return alg.element_wise_division(alg.subtractionnv(alg.expv(z), alg.expv(alg.scalar_multiplynv(-1, z))), alg.additionnv(alg.expv(z), alg.expv(alg.scalar_multiplynv(-1, z)))); } -Ref MLPPActivation::tanh_norm(const Ref &z) { +Ref MLPPActivation::tanh_normm(const Ref &z) { MLPPLinAlg alg; return alg.element_wise_divisionm(alg.subtractionm(alg.expm(z), alg.expm(alg.scalar_multiplym(-1, z))), alg.additionm(alg.expm(z), alg.expm(alg.scalar_multiplym(-1, z)))); } -real_t MLPPActivation::tanh_deriv(real_t z) { +real_t MLPPActivation::tanh_derivr(real_t z) { return 1 - tanh(z) * tanh(z); } -Ref MLPPActivation::tanh_deriv(const Ref &z) { +Ref MLPPActivation::tanh_derivv(const Ref &z) { MLPPLinAlg alg; - return alg.scalar_multiplynv(-1, alg.scalar_addnv(-1, alg.hadamard_productnv(tanh_norm(z), tanh_norm(z)))); + return alg.scalar_multiplynv(-1, alg.scalar_addnv(-1, alg.hadamard_productnv(tanh_normv(z), tanh_normv(z)))); } -Ref MLPPActivation::tanh_deriv(const Ref &z) { +Ref MLPPActivation::tanh_derivm(const Ref &z) { MLPPLinAlg alg; - return alg.scalar_multiplym(-1, alg.scalar_addm(-1, alg.hadamard_productm(tanh_norm(z), tanh_norm(z)))); + return alg.scalar_multiplym(-1, alg.scalar_addm(-1, alg.hadamard_productm(tanh_normm(z), tanh_normm(z)))); } //CSCH -real_t MLPPActivation::csch_norm(real_t z) { +real_t MLPPActivation::csch_normr(real_t z) { return 1 / sinh(z); } -Ref MLPPActivation::csch_norm(const Ref &z) { +Ref MLPPActivation::csch_normv(const Ref &z) { MLPPLinAlg alg; - return alg.element_wise_division(alg.onevecv(z->size()), sinh_norm(z)); + return alg.element_wise_division(alg.onevecv(z->size()), sinh_normv(z)); } -Ref MLPPActivation::csch_norm(const Ref &z) { +Ref MLPPActivation::csch_normm(const Ref &z) { MLPPLinAlg alg; - return alg.element_wise_divisionm(alg.onematm(z->size().x, z->size().y), sinh_norm(z)); + return alg.element_wise_divisionm(alg.onematm(z->size().x, z->size().y), sinh_normm(z)); } -real_t MLPPActivation::csch_deriv(real_t z) { +real_t MLPPActivation::csch_derivr(real_t z) { return -csch(z) * coth(z); } -Ref MLPPActivation::csch_deriv(const Ref &z) { +Ref MLPPActivation::csch_derivv(const Ref &z) { MLPPLinAlg alg; - return alg.hadamard_productnv(alg.scalar_multiplynv(-1, csch_norm(z)), coth_norm(z)); + return alg.hadamard_productnv(alg.scalar_multiplynv(-1, csch_normv(z)), coth_normv(z)); } -Ref MLPPActivation::csch_deriv(const Ref &z) { +Ref MLPPActivation::csch_derivm(const Ref &z) { MLPPLinAlg alg; - return alg.hadamard_productm(alg.scalar_multiplym(-1, csch_norm(z)), coth_norm(z)); + return alg.hadamard_productm(alg.scalar_multiplym(-1, csch_normm(z)), coth_normm(z)); } //SECH -real_t MLPPActivation::sech_norm(real_t z) { +real_t MLPPActivation::sech_normr(real_t z) { return 1 / cosh(z); } -Ref MLPPActivation::sech_norm(const Ref &z) { +Ref MLPPActivation::sech_normv(const Ref &z) { MLPPLinAlg alg; - return alg.element_wise_division(alg.onevecv(z->size()), cosh_norm(z)); + return alg.element_wise_division(alg.onevecv(z->size()), cosh_normv(z)); // return activation(z, deriv, static_cast(&sech)); } -Ref MLPPActivation::sech_norm(const Ref &z) { +Ref MLPPActivation::sech_normm(const Ref &z) { MLPPLinAlg alg; - return alg.element_wise_divisionm(alg.onematm(z->size().x, z->size().y), cosh_norm(z)); + return alg.element_wise_divisionm(alg.onematm(z->size().x, z->size().y), cosh_normm(z)); // return activation(z, deriv, static_cast(&sech)); } -real_t MLPPActivation::sech_deriv(real_t z) { +real_t MLPPActivation::sech_derivr(real_t z) { return -sech(z) * tanh(z); } -Ref MLPPActivation::sech_deriv(const Ref &z) { +Ref MLPPActivation::sech_derivv(const Ref &z) { MLPPLinAlg alg; - return alg.hadamard_productnv(alg.scalar_multiplynv(-1, sech_norm(z)), tanh_norm(z)); + return alg.hadamard_productnv(alg.scalar_multiplynv(-1, sech_normv(z)), tanh_normv(z)); } -Ref MLPPActivation::sech_deriv(const Ref &z) { +Ref MLPPActivation::sech_derivm(const Ref &z) { MLPPLinAlg alg; - return alg.hadamard_productm(alg.scalar_multiplym(-1, sech_norm(z)), tanh_norm(z)); + return alg.hadamard_productm(alg.scalar_multiplym(-1, sech_normm(z)), tanh_normm(z)); } //COTH -real_t MLPPActivation::coth_norm(real_t z) { +real_t MLPPActivation::coth_normr(real_t z) { return 1 / tanh(z); } -Ref MLPPActivation::coth_norm(const Ref &z) { +Ref MLPPActivation::coth_normv(const Ref &z) { MLPPLinAlg alg; - return alg.element_wise_division(alg.onevecv(z->size()), tanh_norm(z)); + return alg.element_wise_division(alg.onevecv(z->size()), tanh_normv(z)); } -Ref MLPPActivation::coth_norm(const Ref &z) { +Ref MLPPActivation::coth_normm(const Ref &z) { MLPPLinAlg alg; - return alg.element_wise_divisionm(alg.onematm(z->size().x, z->size().y), tanh_norm(z)); + return alg.element_wise_divisionm(alg.onematm(z->size().x, z->size().y), tanh_normm(z)); } -real_t MLPPActivation::coth_deriv(real_t z) { - return -csch_norm(z) * csch_norm(z); +real_t MLPPActivation::coth_derivr(real_t z) { + return -csch_normr(z) * csch_normr(z); } -Ref MLPPActivation::coth_deriv(const Ref &z) { +Ref MLPPActivation::coth_derivv(const Ref &z) { MLPPLinAlg alg; - return alg.hadamard_productnv(alg.scalar_multiplynv(-1, csch_norm(z)), csch_norm(z)); + return alg.hadamard_productnv(alg.scalar_multiplynv(-1, csch_normv(z)), csch_normv(z)); } -Ref MLPPActivation::coth_deriv(const Ref &z) { +Ref MLPPActivation::coth_derivm(const Ref &z) { MLPPLinAlg alg; - return alg.hadamard_productm(alg.scalar_multiplym(-1, csch_norm(z)), csch_norm(z)); + return alg.hadamard_productm(alg.scalar_multiplym(-1, csch_normm(z)), csch_normm(z)); } //ARSINH -real_t MLPPActivation::arsinh_norm(real_t z) { +real_t MLPPActivation::arsinh_normr(real_t z) { return std::log(z + sqrt(z * z + 1)); } -Ref MLPPActivation::arsinh_norm(const Ref &z) { +Ref MLPPActivation::arsinh_normv(const Ref &z) { MLPPLinAlg alg; return alg.logv(alg.additionnv(z, alg.sqrtv(alg.additionnv(alg.hadamard_productnv(z, z), alg.onevecv(z->size()))))); } -Ref MLPPActivation::arsinh_norm(const Ref &z) { +Ref MLPPActivation::arsinh_normm(const Ref &z) { MLPPLinAlg alg; return alg.logm(alg.additionm(z, alg.sqrtm(alg.additionm(alg.hadamard_productm(z, z), alg.onematm(z->size().x, z->size().y))))); } -real_t MLPPActivation::arsinh_deriv(real_t z) { +real_t MLPPActivation::arsinh_derivr(real_t z) { return 1 / sqrt(z * z + 1); } -Ref MLPPActivation::arsinh_deriv(const Ref &z) { +Ref MLPPActivation::arsinh_derivv(const Ref &z) { MLPPLinAlg alg; return alg.element_wise_division(alg.onevecv(z->size()), alg.sqrtv(alg.additionnv(alg.hadamard_productnv(z, z), alg.onevecv(z->size())))); } -Ref MLPPActivation::arsinh_deriv(const Ref &z) { +Ref MLPPActivation::arsinh_derivm(const Ref &z) { MLPPLinAlg alg; return alg.element_wise_divisionm(alg.onematm(z->size().x, z->size().y), alg.sqrtm(alg.additionm(alg.hadamard_productm(z, z), alg.onematm(z->size().x, z->size().y)))); @@ -1489,31 +1489,31 @@ Ref MLPPActivation::arsinh_deriv(const Ref &z) { //ARCOSH -real_t MLPPActivation::arcosh_norm(real_t z) { +real_t MLPPActivation::arcosh_normr(real_t z) { return std::log(z + sqrt(z * z - 1)); } -Ref MLPPActivation::arcosh_norm(const Ref &z) { +Ref MLPPActivation::arcosh_normv(const Ref &z) { MLPPLinAlg alg; return alg.logv(alg.additionnv(z, alg.sqrtv(alg.subtractionnv(alg.hadamard_productnv(z, z), alg.onevecv(z->size()))))); } -Ref MLPPActivation::arcosh_norm(const Ref &z) { +Ref MLPPActivation::arcosh_normm(const Ref &z) { MLPPLinAlg alg; return alg.logm(alg.additionm(z, alg.sqrtm(alg.subtractionm(alg.hadamard_productm(z, z), alg.onematm(z->size().x, z->size().y))))); } -real_t MLPPActivation::arcosh_deriv(real_t z) { +real_t MLPPActivation::arcosh_derivr(real_t z) { return 1 / sqrt(z * z - 1); } -Ref MLPPActivation::arcosh_deriv(const Ref &z) { +Ref MLPPActivation::arcosh_derivv(const Ref &z) { MLPPLinAlg alg; return alg.element_wise_division(alg.onevecv(z->size()), alg.sqrtv(alg.subtractionnv(alg.hadamard_productnv(z, z), alg.onevecv(z->size())))); } -Ref MLPPActivation::arcosh_deriv(const Ref &z) { +Ref MLPPActivation::arcosh_derivm(const Ref &z) { MLPPLinAlg alg; return alg.element_wise_divisionm(alg.onematm(z->size().x, z->size().y), alg.sqrtm(alg.subtractionm(alg.hadamard_productm(z, z), alg.onematm(z->size().x, z->size().y)))); @@ -1521,31 +1521,31 @@ Ref MLPPActivation::arcosh_deriv(const Ref &z) { //ARTANH -real_t MLPPActivation::artanh_norm(real_t z) { +real_t MLPPActivation::artanh_normr(real_t z) { return 0.5 * std::log((1 + z) / (1 - z)); } -Ref MLPPActivation::artanh_norm(const Ref &z) { +Ref MLPPActivation::artanh_normv(const Ref &z) { MLPPLinAlg alg; return alg.scalar_multiplynv(0.5, alg.logv(alg.element_wise_division(alg.additionnv(alg.onevecv(z->size()), z), alg.subtractionnv(alg.onevecv(z->size()), z)))); } -Ref MLPPActivation::artanh_norm(const Ref &z) { +Ref MLPPActivation::artanh_normm(const Ref &z) { MLPPLinAlg alg; return alg.scalar_multiplym(0.5, alg.logm(alg.element_wise_divisionm(alg.additionm(alg.onematm(z->size().x, z->size().y), z), alg.subtractionm(alg.onematm(z->size().x, z->size().y), z)))); } -real_t MLPPActivation::artanh_deriv(real_t z) { +real_t MLPPActivation::artanh_derivr(real_t z) { return 1 / (1 - z * z); } -Ref MLPPActivation::artanh_deriv(const Ref &z) { +Ref MLPPActivation::artanh_derivv(const Ref &z) { MLPPLinAlg alg; return alg.element_wise_division(alg.onevecv(z->size()), alg.subtractionnv(alg.onevecv(z->size()), alg.hadamard_productnv(z, z))); } -Ref MLPPActivation::artanh_deriv(const Ref &z) { +Ref MLPPActivation::artanh_derivm(const Ref &z) { MLPPLinAlg alg; return alg.element_wise_divisionm(alg.onematm(z->size().x, z->size().y), alg.subtractionnv(alg.onematm(z->size().x, z->size().y), alg.hadamard_productm(z, z))); @@ -1553,10 +1553,10 @@ Ref MLPPActivation::artanh_deriv(const Ref &z) { //ARCSCH -real_t MLPPActivation::arcsch_norm(real_t z) { +real_t MLPPActivation::arcsch_normr(real_t z) { return std::log(sqrt(1 + (1 / (z * z))) + (1 / z)); } -Ref MLPPActivation::arcsch_norm(const Ref &z) { +Ref MLPPActivation::arcsch_normv(const Ref &z) { MLPPLinAlg alg; return alg.logv( @@ -1567,7 +1567,7 @@ Ref MLPPActivation::arcsch_norm(const Ref &z) { alg.element_wise_division(alg.onevecv(z->size()), alg.hadamard_productnv(z, z)))), alg.element_wise_division(alg.onevecv(z->size()), z))); } -Ref MLPPActivation::arcsch_norm(const Ref &z) { +Ref MLPPActivation::arcsch_normm(const Ref &z) { MLPPLinAlg alg; return alg.logm( @@ -1578,10 +1578,10 @@ Ref MLPPActivation::arcsch_norm(const Ref &z) { alg.element_wise_divisionm(alg.onematm(z->size().x, z->size().y), z))); } -real_t MLPPActivation::arcsch_deriv(real_t z) { +real_t MLPPActivation::arcsch_derivr(real_t z) { return -1 / ((z * z) * sqrt(1 + (1 / (z * z)))); } -Ref MLPPActivation::arcsch_deriv(const Ref &z) { +Ref MLPPActivation::arcsch_derivv(const Ref &z) { MLPPLinAlg alg; return alg.element_wise_division( @@ -1590,7 +1590,7 @@ Ref MLPPActivation::arcsch_deriv(const Ref &z) { alg.hadamard_productnv(z, z), alg.sqrtv(alg.additionnv(alg.onevecv(z->size()), alg.element_wise_division(alg.onevecv(z->size()), alg.hadamard_productnv(z, z)))))); } -Ref MLPPActivation::arcsch_deriv(const Ref &z) { +Ref MLPPActivation::arcsch_derivm(const Ref &z) { MLPPLinAlg alg; return alg.element_wise_divisionm( @@ -1602,11 +1602,11 @@ Ref MLPPActivation::arcsch_deriv(const Ref &z) { //ARSECH -real_t MLPPActivation::arsech_norm(real_t z) { +real_t MLPPActivation::arsech_normr(real_t z) { return std::log((1 / z) + ((1 / z) + 1) * ((1 / z) - 1)); } -Ref MLPPActivation::arsech_norm(const Ref &z) { +Ref MLPPActivation::arsech_normv(const Ref &z) { MLPPLinAlg alg; return alg.logv( @@ -1618,7 +1618,7 @@ Ref MLPPActivation::arsech_norm(const Ref &z) { alg.subtractionnv(alg.element_wise_division(alg.onevecv(z->size()), z), alg.onevecv(z->size()))))); } -Ref MLPPActivation::arsech_norm(const Ref &z) { +Ref MLPPActivation::arsech_normm(const Ref &z) { MLPPLinAlg alg; return alg.logm( @@ -1636,11 +1636,11 @@ Ref MLPPActivation::arsech_norm(const Ref &z) { alg.onematm(z->size().x, z->size().y))))); } -real_t MLPPActivation::arsech_deriv(real_t z) { +real_t MLPPActivation::arsech_derivr(real_t z) { return -1 / (z * sqrt(1 - z * z)); } -Ref MLPPActivation::arsech_deriv(const Ref &z) { +Ref MLPPActivation::arsech_derivv(const Ref &z) { MLPPLinAlg alg; return alg.element_wise_division( @@ -1651,7 +1651,7 @@ Ref MLPPActivation::arsech_deriv(const Ref &z) { alg.subtractionnv(alg.onevecv(z->size()), alg.hadamard_productnv(z, z))))); } -Ref MLPPActivation::arsech_deriv(const Ref &z) { +Ref MLPPActivation::arsech_derivm(const Ref &z) { MLPPLinAlg alg; return alg.element_wise_divisionm( @@ -1663,10 +1663,10 @@ Ref MLPPActivation::arsech_deriv(const Ref &z) { //ARCOTH -real_t MLPPActivation::arcoth_norm(real_t z) { +real_t MLPPActivation::arcoth_normr(real_t z) { return 0.5 * std::log((1 + z) / (z - 1)); } -Ref MLPPActivation::arcoth_norm(const Ref &z) { +Ref MLPPActivation::arcoth_normv(const Ref &z) { MLPPLinAlg alg; return alg.scalar_multiplynv( @@ -1674,7 +1674,7 @@ Ref MLPPActivation::arcoth_norm(const Ref &z) { alg.logv(alg.element_wise_division(alg.additionnv(alg.onevecv(z->size()), z), alg.subtractionnv(z, alg.onevecv(z->size()))))); } -Ref MLPPActivation::arcoth_norm(const Ref &z) { +Ref MLPPActivation::arcoth_normm(const Ref &z) { MLPPLinAlg alg; return alg.scalar_multiplym( @@ -1682,16 +1682,16 @@ Ref MLPPActivation::arcoth_norm(const Ref &z) { alg.logm(alg.element_wise_divisionm(alg.additionm(alg.onematm(z->size().x, z->size().y), z), alg.subtractionm(z, alg.onematm(z->size().x, z->size().y))))); } -real_t MLPPActivation::arcoth_deriv(real_t z) { +real_t MLPPActivation::arcoth_derivr(real_t z) { return 1 / (1 - z * z); } -Ref MLPPActivation::arcoth_deriv(const Ref &z) { +Ref MLPPActivation::arcoth_derivv(const Ref &z) { MLPPLinAlg alg; return alg.element_wise_division(alg.onevecv(z->size()), alg.subtractionnv(alg.onevecv(z->size()), alg.hadamard_productnv(z, z))); } -Ref MLPPActivation::arcoth_deriv(const Ref &z) { +Ref MLPPActivation::arcoth_derivm(const Ref &z) { MLPPLinAlg alg; return alg.element_wise_divisionm(alg.onematm(z->size().x, z->size().y), alg.subtractionm(alg.onematm(z->size().x, z->size().y), alg.hadamard_productm(z, z))); diff --git a/mlpp/activation/activation.h b/mlpp/activation/activation.h index afc198e..68470d3 100644 --- a/mlpp/activation/activation.h +++ b/mlpp/activation/activation.h @@ -10,8 +10,8 @@ #include "core/math/math_defs.h" -#include "core/object/reference.h" #include "core/object/func_ref.h" +#include "core/object/reference.h" #include "../lin_alg/mlpp_matrix.h" #include "../lin_alg/mlpp_vector.h" @@ -76,317 +76,317 @@ public: //LINEAR - real_t linear_norm(real_t z); - Ref linear_norm(const Ref &z); - Ref linear_norm(const Ref &z); + real_t linear_normr(real_t z); + Ref linear_normv(const Ref &z); + Ref linear_normm(const Ref &z); - real_t linear_deriv(real_t z); - Ref linear_deriv(const Ref &z); - Ref linear_deriv(const Ref &z); + real_t linear_derivr(real_t z); + Ref linear_derivv(const Ref &z); + Ref linear_derivm(const Ref &z); //SIGMOID - real_t sigmoid_norm(real_t z); - Ref sigmoid_norm(const Ref &z); - Ref sigmoid_norm(const Ref &z); + real_t sigmoid_normr(real_t z); + Ref sigmoid_normv(const Ref &z); + Ref sigmoid_normm(const Ref &z); - real_t sigmoid_deriv(real_t z); - Ref sigmoid_deriv(const Ref &z); - Ref sigmoid_deriv(const Ref &z); + real_t sigmoid_derivr(real_t z); + Ref sigmoid_derivv(const Ref &z); + Ref sigmoid_derivm(const Ref &z); //SOFTMAX - Ref softmax_norm(const Ref &z); - Ref softmax_norm(const Ref &z); + Ref softmax_normv(const Ref &z); + Ref softmax_normm(const Ref &z); - Ref softmax_deriv(const Ref &z); - Ref softmax_deriv(const Ref &z); + Ref softmax_derivv(const Ref &z); + Ref softmax_derivm(const Ref &z); //ADJ_SOFTMAX - Ref adj_softmax_norm(const Ref &z); - Ref adj_softmax_norm(const Ref &z); + Ref adj_softmax_normv(const Ref &z); + Ref adj_softmax_normm(const Ref &z); - Ref adj_softmax_deriv(const Ref &z); - Ref adj_softmax_deriv(const Ref &z); + Ref adj_softmax_derivv(const Ref &z); + Ref adj_softmax_derivm(const Ref &z); //SOFTMAX DERIV - Ref softmax_deriv_norm(const Ref &z); - Vector> softmax_deriv_norm(const Ref &z); + Ref softmax_deriv_normv(const Ref &z); + Vector> softmax_deriv_normm(const Ref &z); - Ref softmax_deriv_deriv(const Ref &z); - Vector> softmax_deriv_deriv(const Ref &z); + Ref softmax_deriv_derivv(const Ref &z); + Vector> softmax_deriv_derivm(const Ref &z); //SOFTPLUS - real_t softplus_norm(real_t z); - Ref softplus_norm(const Ref &z); - Ref softplus_norm(const Ref &z); + real_t softplus_normr(real_t z); + Ref softplus_normv(const Ref &z); + Ref softplus_normm(const Ref &z); - real_t softplus_deriv(real_t z); - Ref softplus_deriv(const Ref &z); - Ref softplus_deriv(const Ref &z); + real_t softplus_derivr(real_t z); + Ref softplus_derivv(const Ref &z); + Ref softplus_derivm(const Ref &z); //SOFTSIGN - real_t softsign_norm(real_t z); - Ref softsign_norm(const Ref &z); - Ref softsign_norm(const Ref &z); + real_t softsign_normr(real_t z); + Ref softsign_normv(const Ref &z); + Ref softsign_normm(const Ref &z); - real_t softsign_deriv(real_t z); - Ref softsign_deriv(const Ref &z); - Ref softsign_deriv(const Ref &z); + real_t softsign_derivr(real_t z); + Ref softsign_derivv(const Ref &z); + Ref softsign_derivm(const Ref &z); //GAUSSIANCDF - real_t gaussian_cdf_norm(real_t z); - Ref gaussian_cdf_norm(const Ref &z); - Ref gaussian_cdf_norm(const Ref &z); + real_t gaussian_cdf_normr(real_t z); + Ref gaussian_cdf_normv(const Ref &z); + Ref gaussian_cdf_normm(const Ref &z); - real_t gaussian_cdf_deriv(real_t z); - Ref gaussian_cdf_deriv(const Ref &z); - Ref gaussian_cdf_deriv(const Ref &z); + real_t gaussian_cdf_derivr(real_t z); + Ref gaussian_cdf_derivv(const Ref &z); + Ref gaussian_cdf_derivm(const Ref &z); //CLOGLOG - real_t cloglog_norm(real_t z); - Ref cloglog_norm(const Ref &z); - Ref cloglog_norm(const Ref &z); + real_t cloglog_normr(real_t z); + Ref cloglog_normv(const Ref &z); + Ref cloglog_normm(const Ref &z); - real_t cloglog_deriv(real_t z); - Ref cloglog_deriv(const Ref &z); - Ref cloglog_deriv(const Ref &z); + real_t cloglog_derivr(real_t z); + Ref cloglog_derivv(const Ref &z); + Ref cloglog_derivm(const Ref &z); //LOGIT - real_t logit_norm(real_t z); - Ref logit_norm(const Ref &z); - Ref logit_norm(const Ref &z); + real_t logit_normr(real_t z); + Ref logit_normv(const Ref &z); + Ref logit_normm(const Ref &z); - real_t logit_deriv(real_t z); - Ref logit_deriv(const Ref &z); - Ref logit_deriv(const Ref &z); + real_t logit_derivr(real_t z); + Ref logit_derivv(const Ref &z); + Ref logit_derivm(const Ref &z); //UNITSTEP - real_t unit_step_norm(real_t z); - Ref unit_step_norm(const Ref &z); - Ref unit_step_norm(const Ref &z); + real_t unit_step_normr(real_t z); + Ref unit_step_normv(const Ref &z); + Ref unit_step_normm(const Ref &z); - real_t unit_step_deriv(real_t z); - Ref unit_step_deriv(const Ref &z); - Ref unit_step_deriv(const Ref &z); + real_t unit_step_derivr(real_t z); + Ref unit_step_derivv(const Ref &z); + Ref unit_step_derivm(const Ref &z); //SWISH - real_t swish_norm(real_t z); - Ref swish_norm(const Ref &z); - Ref swish_norm(const Ref &z); + real_t swish_normr(real_t z); + Ref swish_normv(const Ref &z); + Ref swish_normm(const Ref &z); - real_t swish_deriv(real_t z); - Ref swish_deriv(const Ref &z); - Ref swish_deriv(const Ref &z); + real_t swish_derivr(real_t z); + Ref swish_derivv(const Ref &z); + Ref swish_derivm(const Ref &z); //MISH - real_t mish_norm(real_t z); - Ref mish_norm(const Ref &z); - Ref mish_norm(const Ref &z); + real_t mish_normr(real_t z); + Ref mish_normv(const Ref &z); + Ref mish_normm(const Ref &z); - real_t mish_deriv(real_t z); - Ref mish_deriv(const Ref &z); - Ref mish_deriv(const Ref &z); + real_t mish_derivr(real_t z); + Ref mish_derivv(const Ref &z); + Ref mish_derivm(const Ref &z); //SINC - real_t sinc_norm(real_t z); - Ref sinc_norm(const Ref &z); - Ref sinc_norm(const Ref &z); + real_t sinc_normr(real_t z); + Ref sinc_normv(const Ref &z); + Ref sinc_normm(const Ref &z); - real_t sinc_deriv(real_t z); - Ref sinc_deriv(const Ref &z); - Ref sinc_deriv(const Ref &z); + real_t sinc_derivr(real_t z); + Ref sinc_derivv(const Ref &z); + Ref sinc_derivm(const Ref &z); //RELU - real_t relu_norm(real_t z); - Ref relu_norm(const Ref &z); - Ref relu_norm(const Ref &z); + real_t relu_normr(real_t z); + Ref relu_normv(const Ref &z); + Ref relu_normm(const Ref &z); - real_t relu_deriv(real_t z); - Ref relu_deriv(const Ref &z); - Ref relu_deriv(const Ref &z); + real_t relu_derivr(real_t z); + Ref relu_derivv(const Ref &z); + Ref relu_derivm(const Ref &z); //LEAKYRELU - real_t leaky_relu_norm(real_t z, real_t c); - Ref leaky_relu_norm(const Ref &z, real_t c); - Ref leaky_relu_norm(const Ref &z, real_t c); + real_t leaky_relu_normr(real_t z, real_t c); + Ref leaky_relu_normv(const Ref &z, real_t c); + Ref leaky_relu_normm(const Ref &z, real_t c); - real_t leaky_relu_deriv(real_t z, real_t c); - Ref leaky_relu_deriv(const Ref &z, real_t c); - Ref leaky_relu_deriv(const Ref &z, real_t c); + real_t leaky_relu_derivr(real_t z, real_t c); + Ref leaky_relu_derivv(const Ref &z, real_t c); + Ref leaky_relu_derivm(const Ref &z, real_t c); //ELU - real_t elu_norm(real_t z, real_t c); - Ref elu_norm(const Ref &z, real_t c); - Ref elu_norm(const Ref &z, real_t c); + real_t elu_normr(real_t z, real_t c); + Ref elu_normv(const Ref &z, real_t c); + Ref elu_normm(const Ref &z, real_t c); - real_t elu_deriv(real_t z, real_t c); - Ref elu_deriv(const Ref &z, real_t c); - Ref elu_deriv(const Ref &z, real_t c); + real_t elu_derivr(real_t z, real_t c); + Ref elu_derivv(const Ref &z, real_t c); + Ref elu_derivm(const Ref &z, real_t c); //SELU - real_t selu_norm(real_t z, real_t lambda, real_t c); - Ref selu_norm(const Ref &z, real_t lambda, real_t c); - Ref selu_norm(const Ref &z, real_t lambda, real_t c); + real_t selu_normr(real_t z, real_t lambda, real_t c); + Ref selu_normv(const Ref &z, real_t lambda, real_t c); + Ref selu_normm(const Ref &z, real_t lambda, real_t c); - real_t selu_deriv(real_t z, real_t lambda, real_t c); - Ref selu_deriv(const Ref &z, real_t lambda, real_t c); - Ref selu_deriv(const Ref &z, real_t lambda, real_t c); + real_t selu_derivr(real_t z, real_t lambda, real_t c); + Ref selu_derivv(const Ref &z, real_t lambda, real_t c); + Ref selu_derivm(const Ref &z, real_t lambda, real_t c); //GELU - real_t gelu_norm(real_t z); - Ref gelu_norm(const Ref &z); - Ref gelu_norm(const Ref &z); + real_t gelu_normr(real_t z); + Ref gelu_normv(const Ref &z); + Ref gelu_normm(const Ref &z); - real_t gelu_deriv(real_t z); - Ref gelu_deriv(const Ref &z); - Ref gelu_deriv(const Ref &z); + real_t gelu_derivr(real_t z); + Ref gelu_derivv(const Ref &z); + Ref gelu_derivm(const Ref &z); //SIGN - real_t sign_norm(real_t z); - Ref sign_norm(const Ref &z); - Ref sign_norm(const Ref &z); + real_t sign_normr(real_t z); + Ref sign_normv(const Ref &z); + Ref sign_normm(const Ref &z); - real_t sign_deriv(real_t z); - Ref sign_deriv(const Ref &z); - Ref sign_deriv(const Ref &z); + real_t sign_derivr(real_t z); + Ref sign_derivv(const Ref &z); + Ref sign_derivm(const Ref &z); //SINH - real_t sinh_norm(real_t z); - Ref sinh_norm(const Ref &z); - Ref sinh_norm(const Ref &z); + real_t sinh_normr(real_t z); + Ref sinh_normv(const Ref &z); + Ref sinh_normm(const Ref &z); - real_t sinh_deriv(real_t z); - Ref sinh_deriv(const Ref &z); - Ref sinh_deriv(const Ref &z); + real_t sinh_derivr(real_t z); + Ref sinh_derivv(const Ref &z); + Ref sinh_derivm(const Ref &z); //COSH - real_t cosh_norm(real_t z); - Ref cosh_norm(const Ref &z); - Ref cosh_norm(const Ref &z); + real_t cosh_normr(real_t z); + Ref cosh_normv(const Ref &z); + Ref cosh_normm(const Ref &z); - real_t cosh_deriv(real_t z); - Ref cosh_deriv(const Ref &z); - Ref cosh_deriv(const Ref &z); + real_t cosh_derivr(real_t z); + Ref cosh_derivv(const Ref &z); + Ref cosh_derivm(const Ref &z); //TANH - real_t tanh_norm(real_t z); - Ref tanh_norm(const Ref &z); - Ref tanh_norm(const Ref &z); + real_t tanh_normr(real_t z); + Ref tanh_normv(const Ref &z); + Ref tanh_normm(const Ref &z); - real_t tanh_deriv(real_t z); - Ref tanh_deriv(const Ref &z); - Ref tanh_deriv(const Ref &z); + real_t tanh_derivr(real_t z); + Ref tanh_derivv(const Ref &z); + Ref tanh_derivm(const Ref &z); //CSCH - real_t csch_norm(real_t z); - Ref csch_norm(const Ref &z); - Ref csch_norm(const Ref &z); + real_t csch_normr(real_t z); + Ref csch_normv(const Ref &z); + Ref csch_normm(const Ref &z); - real_t csch_deriv(real_t z); - Ref csch_deriv(const Ref &z); - Ref csch_deriv(const Ref &z); + real_t csch_derivr(real_t z); + Ref csch_derivv(const Ref &z); + Ref csch_derivm(const Ref &z); //SECH - real_t sech_norm(real_t z); - Ref sech_norm(const Ref &z); - Ref sech_norm(const Ref &z); + real_t sech_normr(real_t z); + Ref sech_normv(const Ref &z); + Ref sech_normm(const Ref &z); - real_t sech_deriv(real_t z); - Ref sech_deriv(const Ref &z); - Ref sech_deriv(const Ref &z); + real_t sech_derivr(real_t z); + Ref sech_derivv(const Ref &z); + Ref sech_derivm(const Ref &z); //COTH - real_t coth_norm(real_t z); - Ref coth_norm(const Ref &z); - Ref coth_norm(const Ref &z); + real_t coth_normr(real_t z); + Ref coth_normv(const Ref &z); + Ref coth_normm(const Ref &z); - real_t coth_deriv(real_t z); - Ref coth_deriv(const Ref &z); - Ref coth_deriv(const Ref &z); + real_t coth_derivr(real_t z); + Ref coth_derivv(const Ref &z); + Ref coth_derivm(const Ref &z); //ARSINH - real_t arsinh_norm(real_t z); - Ref arsinh_norm(const Ref &z); - Ref arsinh_norm(const Ref &z); + real_t arsinh_normr(real_t z); + Ref arsinh_normv(const Ref &z); + Ref arsinh_normm(const Ref &z); - real_t arsinh_deriv(real_t z); - Ref arsinh_deriv(const Ref &z); - Ref arsinh_deriv(const Ref &z); + real_t arsinh_derivr(real_t z); + Ref arsinh_derivv(const Ref &z); + Ref arsinh_derivm(const Ref &z); //ARCOSH - real_t arcosh_norm(real_t z); - Ref arcosh_norm(const Ref &z); - Ref arcosh_norm(const Ref &z); + real_t arcosh_normr(real_t z); + Ref arcosh_normv(const Ref &z); + Ref arcosh_normm(const Ref &z); - real_t arcosh_deriv(real_t z); - Ref arcosh_deriv(const Ref &z); - Ref arcosh_deriv(const Ref &z); + real_t arcosh_derivr(real_t z); + Ref arcosh_derivv(const Ref &z); + Ref arcosh_derivm(const Ref &z); //ARTANH - real_t artanh_norm(real_t z); - Ref artanh_norm(const Ref &z); - Ref artanh_norm(const Ref &z); + real_t artanh_normr(real_t z); + Ref artanh_normv(const Ref &z); + Ref artanh_normm(const Ref &z); - real_t artanh_deriv(real_t z); - Ref artanh_deriv(const Ref &z); - Ref artanh_deriv(const Ref &z); + real_t artanh_derivr(real_t z); + Ref artanh_derivv(const Ref &z); + Ref artanh_derivm(const Ref &z); //ARCSCH - real_t arcsch_norm(real_t z); - Ref arcsch_norm(const Ref &z); - Ref arcsch_norm(const Ref &z); + real_t arcsch_normr(real_t z); + Ref arcsch_normv(const Ref &z); + Ref arcsch_normm(const Ref &z); - real_t arcsch_deriv(real_t z); - Ref arcsch_deriv(const Ref &z); - Ref arcsch_deriv(const Ref &z); + real_t arcsch_derivr(real_t z); + Ref arcsch_derivv(const Ref &z); + Ref arcsch_derivm(const Ref &z); //ARSECH - real_t arsech_norm(real_t z); - Ref arsech_norm(const Ref &z); - Ref arsech_norm(const Ref &z); + real_t arsech_normr(real_t z); + Ref arsech_normv(const Ref &z); + Ref arsech_normm(const Ref &z); - real_t arsech_deriv(real_t z); - Ref arsech_deriv(const Ref &z); - Ref arsech_deriv(const Ref &z); + real_t arsech_derivr(real_t z); + Ref arsech_derivv(const Ref &z); + Ref arsech_derivm(const Ref &z); //ARCOTH - real_t arcoth_norm(real_t z); - Ref arcoth_norm(const Ref &z); - Ref arcoth_norm(const Ref &z); + real_t arcoth_normr(real_t z); + Ref arcoth_normv(const Ref &z); + Ref arcoth_normm(const Ref &z); - real_t arcoth_deriv(real_t z); - Ref arcoth_deriv(const Ref &z); - Ref arcoth_deriv(const Ref &z); + real_t arcoth_derivr(real_t z); + Ref arcoth_derivv(const Ref &z); + Ref arcoth_derivm(const Ref &z); // ========= OLD ===========