From 048dc20f6add4a78486ffba95f7a467e87de61d9 Mon Sep 17 00:00:00 2001 From: Relintai Date: Tue, 31 Jan 2023 01:22:50 +0100 Subject: [PATCH] INitial Activation rework. --- mlpp/activation/activation.cpp | 1256 +++++++++++++++++++++++++++++++- mlpp/activation/activation.h | 553 +++++++++++--- 2 files changed, 1717 insertions(+), 92 deletions(-) diff --git a/mlpp/activation/activation.cpp b/mlpp/activation/activation.cpp index 2414c72..8d4e490 100644 --- a/mlpp/activation/activation.cpp +++ b/mlpp/activation/activation.cpp @@ -10,6 +10,1258 @@ #include #include +MLPPActivation::ActivationFunctionPointer MLPPActivation::get_activation_function_ptr(const ActivationFunction func, const bool deriv) { + return NULL; +} + +Ref MLPPActivation::run_activation_vector(const ActivationFunction func, const Ref &z, const bool deriv) { + return Ref(); +} +Ref MLPPActivation::run_activation_matrix(const ActivationFunction func, const Ref &z, const bool deriv) { + return Ref(); +} + +Ref MLPPActivation::run_activation_norm_vector(const ActivationFunction func, const Ref &z) { + return Ref(); +} +Ref MLPPActivation::run_activation_norm_matrix(const ActivationFunction func, const Ref &z) { + return Ref(); +} + +Ref MLPPActivation::run_activation_deriv_vector(const ActivationFunction func, const Ref &z) { + return Ref(); +} +Ref MLPPActivation::run_activation_deriv_matrix(const ActivationFunction func, const Ref &z) { + return Ref(); +} + +Ref MLPPActivation::activation(const Ref &z, real_t (*function)(real_t), const bool deriv) { + return Ref(); +} +Ref MLPPActivation::activation_norm(const Ref &z, real_t (*function)(real_t)) { + return Ref(); +} +Ref MLPPActivation::activation_deriv(const Ref &z, real_t (*function)(real_t)) { + return Ref(); +} + +/* + +// TO DO: Implement this template activation +std::vector MLPPActivation::activation(std::vector z, bool deriv, real_t (*function)(real_t, bool)) { + if (deriv) { + std::vector deriv; + deriv.resize(z.size()); + for (int i = 0; i < z.size(); i++) { + deriv[i] = function(z[i], true); + } + return deriv; + } + std::vector a; + a.resize(z.size()); + for (int i = 0; i < z.size(); i++) { + a[i] = function(z[i], deriv); + } + return a; +} + +//ACTIVATION FUNCTIONS + +//LINEAR +real_t MLPPActivation::linear_norm(real_t z) { + return z; +} +Ref MLPPActivation::linear_norm(const Ref &z) { + return z->duplicate(); +} +Ref MLPPActivation::linear_norm(const Ref &z) { + return z->duplicate(); +} + +real_t MLPPActivation::linear_deriv(real_t z) { + return 1; +} +Ref MLPPActivation::linear_deriv(const Ref &z) { + MLPPLinAlg alg; + return alg.onevec(z.size()); +} +Ref MLPPActivation::linear_deriv(const Ref &z) { + MLPPLinAlg alg; + return alg.onemat(z.size(), z[0].size()); +} + +//SIGMOID +real_t MLPPActivation::sigmoid_norm(real_t z) { + return 1 / (1 + exp(-z)); +} +Ref MLPPActivation::sigmoid_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.elementWiseDivision(alg.onevec(z.size()), alg.addition(alg.onevec(z.size()), alg.exp(alg.scalarMultiply(-1, z)))); +} +Ref MLPPActivation::sigmoid_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.addition(alg.onemat(z.size(), z[0].size()), alg.exp(alg.scalarMultiply(-1, z)))); +} + +real_t MLPPActivation::sigmoid_deriv(real_t z) { + return sigmoid_norm(z) * (1 - sigmoid_norm(z)); +} +Ref MLPPActivation::sigmoid_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.subtraction(sigmoid(z), alg.hadamard_product(sigmoid(z), sigmoid(z))); +} +Ref MLPPActivation::sigmoid_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.subtraction(sigmoid(z), alg.hadamard_product(sigmoid(z), sigmoid(z))); +} + +//SOFTMAX +Ref MLPPActivation::softmax_norm(const Ref &z) { + MLPPLinAlg alg; + std::vector a; + a.resize(z.size()); + std::vector expZ = alg.exp(z); + real_t sum = 0; + + for (int i = 0; i < z.size(); i++) { + sum += expZ[i]; + } + + for (int i = 0; i < z.size(); i++) { + a[i] = expZ[i] / sum; + } + + return a; +} +Ref MLPPActivation::softmax_norm(const Ref &z) { + MLPPLinAlg alg; + std::vector> a; + a.resize(z.size()); + + for (int i = 0; i < z.size(); i++) { + a[i] = softmax(z[i]); + } + + return a; +} + +Ref MLPPActivation::softmax_deriv(const Ref &z) { + MLPPLinAlg alg; + std::vector a; + a.resize(z.size()); + std::vector expZ = alg.exp(z); + real_t sum = 0; + + for (int i = 0; i < z.size(); i++) { + sum += expZ[i]; + } + + for (int i = 0; i < z.size(); i++) { + a[i] = expZ[i] / sum; + } + + return a; +} +Ref MLPPActivation::softmax_deriv(const Ref &z) { + MLPPLinAlg alg; + std::vector> a; + a.resize(z.size()); + + for (int i = 0; i < z.size(); i++) { + a[i] = softmax(z[i]); + } + + return a; +} + +//ADJ_SOFTMAX + +Ref MLPPActivation::adj_softmax_norm(const Ref &z) { + MLPPLinAlg alg; + + std::vector a; + real_t C = -*std::max_element(z.begin(), z.end()); + z = alg.scalarAdd(C, z); + + return softmax(z); +} +Ref MLPPActivation::adj_softmax_norm(const Ref &z) { + MLPPLinAlg alg; + + std::vector> a; + a.resize(z.size()); + + for (int i = 0; i < z.size(); i++) { + a[i] = adjSoftmax(z[i]); + } + + return a; +} + +Ref MLPPActivation::adj_softmax(const Ref &z) { + MLPPLinAlg alg; + + std::vector a; + real_t C = -*std::max_element(z.begin(), z.end()); + z = alg.scalarAdd(C, z); + + return softmax(z); +} +Ref MLPPActivation::adj_softmax(const Ref &z) { + MLPPLinAlg alg; + + std::vector> a; + a.resize(z.size()); + + for (int i = 0; i < z.size(); i++) { + a[i] = adjSoftmax(z[i]); + } + + return a; +} + +//SOFTMAX DERIV + +Ref MLPPActivation::softmax_deriv_norm(const Ref &z) { + MLPPLinAlg alg; + + std::vector> deriv; + std::vector a = softmax(z); + deriv.resize(a.size()); + + for (int i = 0; i < deriv.size(); i++) { + deriv[i].resize(a.size()); + } + + for (int i = 0; i < a.size(); i++) { + for (int j = 0; j < z.size(); j++) { + if (i == j) { + deriv[i][j] = a[i] * (1 - a[i]); + } else { + deriv[i][j] = -a[i] * a[j]; + } + } + } + + return deriv; +} +std::vector> MLPPActivation::softmax_deriv_norm(const Ref &z) { + MLPPLinAlg alg; + + std::vector>> deriv; + std::vector> a = softmax(z); + + deriv.resize(a.size()); + for (int i = 0; i < deriv.size(); i++) { + deriv[i].resize(a.size()); + } + + for (int i = 0; i < a.size(); i++) { + for (int j = 0; j < z.size(); j++) { + if (i == j) { + deriv[i][j] = alg.subtraction(a[i], alg.hadamard_product(a[i], a[i])); + } else { + deriv[i][j] = alg.scalarMultiply(-1, alg.hadamard_product(a[i], a[j])); + } + } + } + + return deriv; +} + +Ref MLPPActivation::softmax_deriv_deriv(const Ref &z) { + MLPPLinAlg alg; + + std::vector> deriv; + std::vector a = softmax(z); + deriv.resize(a.size()); + + for (int i = 0; i < deriv.size(); i++) { + deriv[i].resize(a.size()); + } + + for (int i = 0; i < a.size(); i++) { + for (int j = 0; j < z.size(); j++) { + if (i == j) { + deriv[i][j] = a[i] * (1 - a[i]); + } else { + deriv[i][j] = -a[i] * a[j]; + } + } + } + + return deriv; +} +std::vector> MLPPActivation::softmax_deriv_deriv(const Ref &z) { + MLPPLinAlg alg; + + std::vector>> deriv; + std::vector> a = softmax(z); + + deriv.resize(a.size()); + for (int i = 0; i < deriv.size(); i++) { + deriv[i].resize(a.size()); + } + + for (int i = 0; i < a.size(); i++) { + for (int j = 0; j < z.size(); j++) { + if (i == j) { + deriv[i][j] = alg.subtraction(a[i], alg.hadamard_product(a[i], a[i])); + } else { + deriv[i][j] = alg.scalarMultiply(-1, alg.hadamard_product(a[i], a[j])); + } + } + } + + return deriv; +} + +//SOFTPLUS + +real_t MLPPActivation::softplus_norm(real_t z) { + return std::log(1 + exp(z)); +} +Ref MLPPActivation::softplus_norm(const Ref &z) { + MLPPLinAlg alg; + return alg.log(alg.addition(alg.onevec(z.size()), alg.exp(z))); +} +Ref MLPPActivation::softplus_norm(const Ref &z) { + MLPPLinAlg alg; + return alg.log(alg.addition(alg.onemat(z.size(), z[0].size()), alg.exp(z))); +} + +real_t MLPPActivation::softplus_deriv(real_t z) { + return sigmoid(z); +} +Ref MLPPActivation::softplus_deriv(const Ref &z) { + return sigmoid(z); +} +Ref MLPPActivation::softplus_deriv(const Ref &z) { + return sigmoid(z); +} + +//SOFTSIGN + +real_t MLPPActivation::softsign_norm(real_t z) { + return z / (1 + abs(z)); +} +Ref MLPPActivation::softsign_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.elementWiseDivision(z, alg.addition(alg.onevec(z.size()), alg.abs(z))); +} +Ref MLPPActivation::softsign_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.elementWiseDivision(z, alg.addition(alg.onemat(z.size(), z[0].size()), alg.abs(z))); +} + +real_t MLPPActivation::softsign_deriv(real_t z) { + return 1 / ((1 + abs(z)) * (1 + abs(z))); +} +Ref MLPPActivation::softsign_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.elementWiseDivision(alg.onevec(z.size()), alg.exponentiate(alg.addition(alg.onevec(z.size()), alg.abs(z)), 2)); +} +Ref MLPPActivation::softsign_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.exponentiate(alg.addition(alg.onemat(z.size(), z[0].size()), alg.abs(z)), 2)); +} + +//GAUSSIANCDF + +real_t MLPPActivation::gaussian_cdf_norm(real_t z) { + return 0.5 * (1 + erf(z / sqrt(2))); +} +Ref MLPPActivation::gaussian_cdf_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.scalarMultiply(0.5, alg.addition(alg.onevec(z.size()), alg.erf(alg.scalarMultiply(1 / sqrt(2), z)))); +} + +Ref MLPPActivation::gaussian_cdf_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.scalarMultiply(0.5, alg.addition(alg.onemat(z.size(), z[0].size()), alg.erf(alg.scalarMultiply(1 / sqrt(2), z)))); +} + +real_t MLPPActivation::gaussian_cdf_deriv(real_t z) { + return (1 / sqrt(2 * M_PI)) * exp(-z * z / 2); +} +Ref MLPPActivation::gaussian_cdf_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.scalarMultiply(1 / sqrt(2 * M_PI), alg.exp(alg.scalarMultiply(-1 / 2, alg.hadamard_product(z, z)))); +} + +Ref MLPPActivation::gaussian_cdf_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.scalarMultiply(1 / sqrt(2 * M_PI), alg.exp(alg.scalarMultiply(-1 / 2, alg.hadamard_product(z, z)))); +} + +//CLOGLOG + +real_t MLPPActivation::cloglog_norm(real_t z) { + return 1 - exp(-exp(z)); +} +Ref MLPPActivation::cloglog_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.scalarMultiply(-1, alg.scalarAdd(-1, alg.exp(alg.scalarMultiply(-1, alg.exp(z))))); +} + +Ref MLPPActivation::cloglog_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.scalarMultiply(-1, alg.scalarAdd(-1, alg.exp(alg.scalarMultiply(-1, alg.exp(z))))); +} + +real_t MLPPActivation::cloglog_deriv(real_t z) { + return exp(z - exp(z)); +} +Ref MLPPActivation::cloglog_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.exp(alg.scalarMultiply(-1, alg.exp(z))); +} + +Ref MLPPActivation::cloglog_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.exp(alg.scalarMultiply(-1, alg.exp(z))); +} + +//LOGIT + +real_t MLPPActivation::logit_norm(real_t z) { + return std::log(z / (1 - z)); +} +Ref MLPPActivation::logit_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.log(alg.elementWiseDivision(z, alg.subtraction(alg.onevec(z.size()), z))); +} +Ref MLPPActivation::logit_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.log(alg.elementWiseDivision(z, alg.subtraction(alg.onemat(z.size(), z[0].size()), z))); +} + +real_t MLPPActivation::logit_deriv(real_t z) { + return 1 / z - 1 / (z - 1); +} +Ref MLPPActivation::logit_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.subtraction(alg.elementWiseDivision(alg.onevec(z.size()), z), alg.elementWiseDivision(alg.onevec(z.size()), alg.subtraction(z, alg.onevec(z.size())))); +} +Ref MLPPActivation::logit_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.subtraction(alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), z), alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.subtraction(z, alg.onemat(z.size(), z[0].size())))); +} + +//UNITSTEP + +real_t MLPPActivation::unit_step_norm(real_t z) { + return z < 0 ? 0 : 1; +} +Ref MLPPActivation::unit_step_norm(const Ref &z) { + std::vector a; + a.resize(z.size()); + + for (int i = 0; i < a.size(); i++) { + a[i] = unitStep(z[i]); + } + return a; +} +Ref MLPPActivation::unit_step_norm(const Ref &z) { + std::vector> a; + a.resize(z.size()); + + for (int i = 0; i < a.size(); i++) { + a[i] = unitStep(z[i]); + } + return a; +} + +real_t MLPPActivation::unit_step_deriv(real_t z) { + return 0; +} +Ref MLPPActivation::unit_step_deriv(const Ref &z) { + std::vector deriv; + deriv.resize(z.size()); + for (int i = 0; i < z.size(); i++) { + deriv[i] = unitStep(z[i], 1); + } + return deriv; +} +Ref MLPPActivation::unit_step_deriv(const Ref &z) { + std::vector> deriv; + deriv.resize(z.size()); + for (int i = 0; i < z.size(); i++) { + deriv[i] = unitStep(z[i], 1); + } + return deriv; +} + +//SWISH + +real_t MLPPActivation::swish_norm(real_t z) { + return z * sigmoid(z); +} +Ref MLPPActivation::swish_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.hadamard_product(z, sigmoid(z)); +} +Ref MLPPActivation::swish_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.hadamard_product(z, sigmoid(z)); +} + +real_t MLPPActivation::swish_deriv(real_t z) { + return swish(z) + sigmoid(z) * (1 - swish(z)); +} +Ref MLPPActivation::swish_deriv(const Ref &z) { + MLPPLinAlg alg; + + alg.addition(swish(z), alg.subtraction(sigmoid(z), alg.hadamard_product(sigmoid(z), swish(z)))); +} +Ref MLPPActivation::swish_deriv(const Ref &z) { + MLPPLinAlg alg; + + alg.addition(swish(z), alg.subtraction(sigmoid(z), alg.hadamard_product(sigmoid(z), swish(z)))); +} + +//MISH + +real_t MLPPActivation::mish_norm(real_t z) { + return z * tanh(softplus(z)); +} +Ref MLPPActivation::mish_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.hadamard_product(z, tanh(softplus(z))); +} +Ref MLPPActivation::mish_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.hadamard_product(z, tanh(softplus(z))); +} + +real_t MLPPActivation::mish_deriv(real_t z) { + return sech(softplus(z)) * sech(softplus(z)) * z * sigmoid(z) + mish(z) / z; +} +Ref MLPPActivation::mish_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.addition(alg.hadamard_product(alg.hadamard_product(alg.hadamard_product(sech(softplus(z)), sech(softplus(z))), z), sigmoid(z)), alg.elementWiseDivision(mish(z), z)); +} +Ref MLPPActivation::mish_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.addition(alg.hadamard_product(alg.hadamard_product(alg.hadamard_product(sech(softplus(z)), sech(softplus(z))), z), sigmoid(z)), alg.elementWiseDivision(mish(z), z)); +} + +//SINC + +real_t MLPPActivation::sinc_norm(real_t z) { + return std::sin(z) / z; +} +Ref MLPPActivation::sinc_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.elementWiseDivision(alg.sin(z), z); +} +Ref MLPPActivation::sinc_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.elementWiseDivision(alg.sin(z), z); +} + +real_t MLPPActivation::sinc_deriv(real_t z) { + return (z * std::cos(z) - std::sin(z)) / (z * z); +} +Ref MLPPActivation::sinc_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.elementWiseDivision(alg.subtraction(alg.hadamard_product(z, alg.cos(z)), alg.sin(z)), alg.hadamard_product(z, z)); +} +Ref MLPPActivation::sinc_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.elementWiseDivision(alg.subtraction(alg.hadamard_product(z, alg.cos(z)), alg.sin(z)), alg.hadamard_product(z, z)); +} + +//RELU + +real_t MLPPActivation::relu_norm(real_t z) { + return fmax(0, z); +} +Ref MLPPActivation::relu_norm(const Ref &z) { + std::vector a; + a.resize(z.size()); + + for (int i = 0; i < a.size(); i++) { + a[i] = RELU(z[i]); + } + return a; +} +Ref MLPPActivation::relu_norm(const Ref &z) { + std::vector> a; + a.resize(z.size()); + + for (int i = 0; i < a.size(); i++) { + a[i] = RELU(z[i]); + } + return a; +} + +real_t MLPPActivation::relu_deriv(real_t z) { + if (z <= 0) { + return 0; + } else { + return 1; + } +} +Ref MLPPActivation::relu_deriv(const Ref &z) { + std::vector deriv; + deriv.resize(z.size()); + for (int i = 0; i < z.size(); i++) { + deriv[i] = RELU(z[i], 1); + } + return deriv; +} +Ref MLPPActivation::relu_deriv(const Ref &z) { + std::vector> deriv; + deriv.resize(z.size()); + for (int i = 0; i < z.size(); i++) { + deriv[i] = RELU(z[i], 1); + } + return deriv; +} + +//LEAKYRELU + +real_t MLPPActivation::leaky_relu_norm(real_t z, real_t c) { + return fmax(c * z, z); +} +Ref MLPPActivation::leaky_relu_norm(const Ref &z, real_t c) { + std::vector a; + a.resize(z.size()); + + for (int i = 0; i < a.size(); i++) { + a[i] = leakyReLU(z[i], c); + } + return a; +} +Ref MLPPActivation::leaky_relu_norm(const Ref &z, real_t c) { + std::vector> a; + a.resize(z.size()); + + for (int i = 0; i < a.size(); i++) { + a[i] = leakyReLU(z[i], c); + } + return a; +} + +real_t MLPPActivation::leaky_relu_deriv(real_t z, real_t c) { + if (z <= 0) { + return c; + } else { + return 1; + } +} +Ref MLPPActivation::leaky_relu_deriv(const Ref &z, real_t c) { + std::vector deriv; + deriv.resize(z.size()); + for (int i = 0; i < z.size(); i++) { + deriv[i] = leakyReLU(z[i], c, 1); + } + return deriv; +} +Ref MLPPActivation::leaky_relu_deriv(const Ref &z, real_t c) { + std::vector> deriv; + deriv.resize(z.size()); + for (int i = 0; i < z.size(); i++) { + deriv[i] = leakyReLU(z[i], c, 1); + } + return deriv; +} + +//ELU + +real_t MLPPActivation::elu_norm(real_t z, real_t c) { + if (z >= 0) { + return z; + } else { + return c * (exp(z) - 1); + } +} +Ref MLPPActivation::elu_norm(const Ref &z, real_t c) { + std::vector a; + a.resize(z.size()); + + for (int i = 0; i < a.size(); i++) { + a[i] = ELU(z[i], c); + } + return a; +} +Ref MLPPActivation::elu_norm(const Ref &z, real_t c) { + std::vector> a; + a.resize(z.size()); + + for (int i = 0; i < a.size(); i++) { + a[i] = ELU(z[i], c); + } + return a; +} + +real_t MLPPActivation::elu_deriv(real_t z, real_t c) { + if (z <= 0) { + return c * exp(z); + } else { + return 1; + } +} +Ref MLPPActivation::elu_deriv(const Ref &z, real_t c) { + std::vector deriv; + deriv.resize(z.size()); + for (int i = 0; i < z.size(); i++) { + deriv[i] = ELU(z[i], c, 1); + } + return deriv; +} +Ref MLPPActivation::elu_deriv(const Ref &z, real_t c) { + std::vector> deriv; + deriv.resize(z.size()); + for (int i = 0; i < z.size(); i++) { + deriv[i] = ELU(z[i], c, 1); + } + return deriv; +} + +//SELU + +real_t MLPPActivation::selu_norm(real_t z, real_t lambda, real_t c) { + return lambda * ELU(z, c); +} +Ref MLPPActivation::selu_norm(const Ref &z, real_t lambda, real_t c) { + std::vector a; + a.resize(z.size()); + + for (int i = 0; i < a.size(); i++) { + a[i] = SELU(z[i], lambda, c); + } + return a; +} +Ref MLPPActivation::selu_norm(Ref, real_t lambda, real_t c) { + std::vector> a; + a.resize(z.size()); + + for (int i = 0; i < a.size(); i++) { + a[i] = SELU(z[i], lambda, c); + } + return a; +} + +real_t MLPPActivation::selu_deriv(real_t z, real_t lambda, real_t c) { + return ELU(z, c, 1); +} +Ref MLPPActivation::selu_deriv(const Ref &z, real_t lambda, real_t c) { + std::vector deriv; + deriv.resize(z.size()); + for (int i = 0; i < z.size(); i++) { + deriv[i] = SELU(z[i], lambda, c, 1); + } + return deriv; +} +Ref MLPPActivation::selu_deriv(Ref, real_t lambda, real_t c) { + std::vector> deriv; + deriv.resize(z.size()); + for (int i = 0; i < z.size(); i++) { + deriv[i] = SELU(z[i], lambda, c, 1); + } + return deriv; +} + +//GELU + +real_t MLPPActivation::gelu_norm(real_t z) { + return 0.5 * z * (1 + tanh(sqrt(2 / M_PI) * (z + 0.044715 * std::pow(z, 3)))); +} +Ref MLPPActivation::gelu_norm(const Ref &z) { + std::vector a; + a.resize(z.size()); + + for (int i = 0; i < a.size(); i++) { + a[i] = GELU(z[i]); + } + return a; +} +Ref MLPPActivation::gelu_norm(const Ref &z) { + std::vector> a; + a.resize(z.size()); + + for (int i = 0; i < a.size(); i++) { + a[i] = GELU(z[i]); + } + return a; +} + +real_t MLPPActivation::gelu_deriv(real_t z) { + return 0.5 * tanh(0.0356774 * std::pow(z, 3) + 0.797885 * z) + (0.0535161 * std::pow(z, 3) + 0.398942 * z) * std::pow(sech(0.0356774 * std::pow(z, 3) + 0.797885 * z), 2) + 0.5; +} +Ref MLPPActivation::gelu_deriv(const Ref &z) { + std::vector deriv; + deriv.resize(z.size()); + for (int i = 0; i < z.size(); i++) { + deriv[i] = GELU(z[i], 1); + } + return deriv; +} +Ref MLPPActivation::gelu_deriv(const Ref &z) { + std::vector> deriv; + deriv.resize(z.size()); + for (int i = 0; i < z.size(); i++) { + deriv[i] = GELU(z[i], 1); + } + return deriv; +} + +//SIGN + +real_t MLPPActivation::sign_norm(real_t z) { + if (z < 0) { + return -1; + } else if (z == 0) { + return 0; + } else { + return 1; + } +} +Ref MLPPActivation::sign_norm(const Ref &z) { + std::vector a; + a.resize(z.size()); + + for (int i = 0; i < a.size(); i++) { + a[i] = sign(z[i]); + } + return a; +} +Ref MLPPActivation::sign_norm(const Ref &z) { + std::vector> a; + a.resize(z.size()); + + for (int i = 0; i < a.size(); i++) { + a[i] = sign(z[i]); + } + return a; +} + +real_t MLPPActivation::sign_deriv(real_t z) { + return 0; +} +Ref MLPPActivation::sign_deriv(const Ref &z) { + std::vector deriv; + deriv.resize(z.size()); + for (int i = 0; i < z.size(); i++) { + deriv[i] = sign(z[i], 1); + } + return deriv; +} +Ref MLPPActivation::sign_deriv(const Ref &z) { + std::vector> deriv; + deriv.resize(z.size()); + for (int i = 0; i < z.size(); i++) { + deriv[i] = sign(z[i], 1); + } + return deriv; +} + +//SINH + +real_t MLPPActivation::sinh_norm(real_t z) { + return 0.5 * (exp(z) - exp(-z)); +} +Ref MLPPActivation::sinh_norm(const Ref &z) { + MLPPLinAlg alg; + return alg.scalarMultiply(0.5, alg.subtraction(alg.exp(z), alg.exp(alg.scalarMultiply(-1, z)))); +} +Ref MLPPActivation::sinh_norm(const Ref &z) { + MLPPLinAlg alg; + return alg.scalarMultiply(0.5, alg.subtraction(alg.exp(z), alg.exp(alg.scalarMultiply(-1, z)))); +} + +real_t MLPPActivation::sinh_deriv(real_t z) { + return cosh(z); +} +Ref MLPPActivation::sinh_deriv(const Ref &z) { + return cosh(z); +} +Ref MLPPActivation::sinh_deriv(const Ref &z) { + return cosh(z); +} + +//COSH + +real_t MLPPActivation::cosh_norm(real_t z) { + return 0.5 * (exp(z) + exp(-z)); +} +Ref MLPPActivation::cosh_norm(const Ref &z) { + MLPPLinAlg alg; + return alg.scalarMultiply(0.5, alg.addition(alg.exp(z), alg.exp(alg.scalarMultiply(-1, z)))); +} +Ref MLPPActivation::cosh_norm(const Ref &z) { + MLPPLinAlg alg; + return alg.scalarMultiply(0.5, alg.addition(alg.exp(z), alg.exp(alg.scalarMultiply(-1, z)))); +} + +real_t MLPPActivation::cosh_deriv(real_t z) { + return sinh(z); +} +Ref MLPPActivation::cosh_deriv(const Ref &z) { + return sinh(z); +} +Ref MLPPActivation::cosh_deriv(const Ref &z) { + return sinh(z); +} + +//TANH + +real_t MLPPActivation::tanh_norm(real_t z) { + return (exp(z) - exp(-z)) / (exp(z) + exp(-z)); +} +Ref MLPPActivation::tanh_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.elementWiseDivision(alg.subtraction(alg.exp(z), alg.exp(alg.scalarMultiply(-1, z))), alg.addition(alg.exp(z), alg.exp(alg.scalarMultiply(-1, z)))); +} +Ref MLPPActivation::tanh_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.elementWiseDivision(alg.subtraction(alg.exp(z), alg.exp(alg.scalarMultiply(-1, z))), alg.addition(alg.exp(z), alg.exp(alg.scalarMultiply(-1, z)))); +} + +real_t MLPPActivation::tanh_deriv(real_t z) { + return 1 - tanh(z) * tanh(z); +} +Ref MLPPActivation::tanh_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.scalarMultiply(-1, alg.scalarAdd(-1, alg.hadamard_product(tanh(z), tanh(z)))); +} +Ref MLPPActivation::tanh_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.scalarMultiply(-1, alg.scalarAdd(-1, alg.hadamard_product(tanh(z), tanh(z)))); +} + +//CSCH + +real_t MLPPActivation::csch_norm(real_t z) { + return 1 / sinh(z); +} +Ref MLPPActivation::csch_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.elementWiseDivision(alg.onevec(z.size()), sinh(z)); +} + +Ref MLPPActivation::csch_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), sinh(z)); +} + +real_t MLPPActivation::csch_deriv(real_t z) { + return -csch(z) * coth(z); +} +Ref MLPPActivation::csch_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.hadamard_product(alg.scalarMultiply(-1, csch(z)), coth(z)); +} + +Ref MLPPActivation::csch_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.hadamard_product(alg.scalarMultiply(-1, csch(z)), coth(z)); +} + +//SECH + +real_t MLPPActivation::sech_norm(real_t z) { + return 1 / cosh(z); +} + +Ref MLPPActivation::sech_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.elementWiseDivision(alg.onevec(z.size()), cosh(z)); + + // return activation(z, deriv, static_cast(&sech)); +} +Ref MLPPActivation::sech_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), cosh(z)); + + // return activation(z, deriv, static_cast(&sech)); +} + +real_t MLPPActivation::sech_deriv(real_t z) { + return -sech(z) * tanh(z); +} + +Ref MLPPActivation::sech_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.hadamard_product(alg.scalarMultiply(-1, sech(z)), tanh(z)); +} +Ref MLPPActivation::sech_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.hadamard_product(alg.scalarMultiply(-1, sech(z)), tanh(z)); +} + +//COTH + +real_t MLPPActivation::coth_norm(real_t z) { + return 1 / tanh(z); +} +Ref MLPPActivation::coth_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.elementWiseDivision(alg.onevec(z.size()), tanh(z)); +} +Ref MLPPActivation::coth_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), tanh(z)); +} + +real_t MLPPActivation::coth_deriv(real_t z) { + return -csch(z) * csch(z); +} +Ref MLPPActivation::coth_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.hadamard_product(alg.scalarMultiply(-1, csch(z)), csch(z)); +} +Ref MLPPActivation::coth_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.hadamard_product(alg.scalarMultiply(-1, csch(z)), csch(z)); +} + +//ARSINH + +real_t MLPPActivation::arsinh_norm(real_t z) { + return std::log(z + sqrt(z * z + 1)); +} + +Ref MLPPActivation::arsinh_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.log(alg.addition(z, alg.sqrt(alg.addition(alg.hadamard_product(z, z), alg.onevec(z.size()))))); +} + +Ref MLPPActivation::arsinh_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.log(alg.addition(z, alg.sqrt(alg.addition(alg.hadamard_product(z, z), alg.onemat(z.size(), z[0].size()))))); +} + +real_t MLPPActivation::arsinh_deriv(real_t z) { + return 1 / sqrt(z * z + 1); +} + +Ref MLPPActivation::arsinh_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.elementWiseDivision(alg.onevec(z.size()), alg.sqrt(alg.addition(alg.hadamard_product(z, z), alg.onevec(z.size())))); +} + +Ref MLPPActivation::arsinh_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.sqrt(alg.addition(alg.hadamard_product(z, z), alg.onemat(z.size(), z[0].size())))); +} + +//ARCOSH + +real_t MLPPActivation::arcosh_norm(real_t z) { + return std::log(z + sqrt(z * z - 1)); +} +Ref MLPPActivation::arcosh_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.log(alg.addition(z, alg.sqrt(alg.subtraction(alg.hadamard_product(z, z), alg.onevec(z.size()))))); +} + +Ref MLPPActivation::arcosh_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.log(alg.addition(z, alg.sqrt(alg.subtraction(alg.hadamard_product(z, z), alg.onemat(z.size(), z[0].size()))))); +} + +real_t MLPPActivation::arcosh_deriv(real_t z) { + return 1 / sqrt(z * z - 1); +} +Ref MLPPActivation::arcosh_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.elementWiseDivision(alg.onevec(z.size()), alg.sqrt(alg.subtraction(alg.hadamard_product(z, z), alg.onevec(z.size())))); +} + +Ref MLPPActivation::arcosh_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.sqrt(alg.subtraction(alg.hadamard_product(z, z), alg.onemat(z.size(), z[0].size())))); +} + +//ARTANH + +real_t MLPPActivation::artanh_norm(real_t z) { + return 0.5 * std::log((1 + z) / (1 - z)); +} +Ref MLPPActivation::artanh_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.scalarMultiply(0.5, alg.log(alg.elementWiseDivision(alg.addition(alg.onevec(z.size()), z), alg.subtraction(alg.onevec(z.size()), z)))); +} + +Ref MLPPActivation::artanh_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.scalarMultiply(0.5, alg.log(alg.elementWiseDivision(alg.addition(alg.onemat(z.size(), z[0].size()), z), alg.subtraction(alg.onemat(z.size(), z[0].size()), z)))); +} + +real_t MLPPActivation::artanh_deriv(real_t z) { + return 1 / (1 - z * z); +} +Ref MLPPActivation::artanh_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.elementWiseDivision(alg.onevec(z.size()), alg.subtraction(alg.onevec(z.size()), alg.hadamard_product(z, z))); +} + +Ref MLPPActivation::artanh_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.subtraction(alg.onemat(z.size(), z[0].size()), alg.hadamard_product(z, z))); +} + +//ARCSCH + +real_t MLPPActivation::arcsch_norm(real_t z) { + return std::log(sqrt(1 + (1 / (z * z))) + (1 / z)); +} +Ref MLPPActivation::arcsch_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.log(alg.addition(alg.sqrt(alg.addition(alg.onevec(z.size()), alg.elementWiseDivision(alg.onevec(z.size()), alg.hadamard_product(z, z)))), alg.elementWiseDivision(alg.onevec(z.size()), z))); +} +Ref MLPPActivation::arcsch_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.log(alg.addition(alg.sqrt(alg.addition(alg.onemat(z.size(), z[0].size()), alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.hadamard_product(z, z)))), alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), z))); +} + +real_t MLPPActivation::arcsch_deriv(real_t z) { + return -1 / ((z * z) * sqrt(1 + (1 / (z * z)))); +} +Ref MLPPActivation::arcsch_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.elementWiseDivision(alg.full(z.size(), -1), alg.hadamard_product(alg.hadamard_product(z, z), alg.sqrt(alg.addition(alg.onevec(z.size()), alg.elementWiseDivision(alg.onevec(z.size()), alg.hadamard_product(z, z)))))); +} +Ref MLPPActivation::arcsch_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.elementWiseDivision(alg.full(z.size(), z[0].size(), -1), alg.hadamard_product(alg.hadamard_product(z, z), alg.sqrt(alg.addition(alg.onemat(z.size(), z[0].size()), alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.hadamard_product(z, z)))))); +} + +//ARSECH + +real_t MLPPActivation::arsech_norm(real_t z) { + return std::log((1 / z) + ((1 / z) + 1) * ((1 / z) - 1)); +} + +Ref MLPPActivation::arsech_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.log(alg.addition(alg.elementWiseDivision(alg.onevec(z.size()), z), alg.hadamard_product(alg.addition(alg.elementWiseDivision(alg.onevec(z.size()), z), alg.onevec(z.size())), alg.subtraction(alg.elementWiseDivision(alg.onevec(z.size()), z), alg.onevec(z.size()))))); +} + +Ref MLPPActivation::arsech_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.log(alg.addition(alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), z), alg.hadamard_product(alg.addition(alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), z), alg.onemat(z.size(), z[0].size())), alg.subtraction(alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), z), alg.onemat(z.size(), z[0].size()))))); +} + +real_t MLPPActivation::arsech_deriv(real_t z) { + return -1 / (z * sqrt(1 - z * z)); +} + +Ref MLPPActivation::arsech_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.elementWiseDivision(alg.full(z.size(), -1), alg.hadamard_product(z, alg.sqrt(alg.subtraction(alg.onevec(z.size()), alg.hadamard_product(z, z))))); +} + +Ref MLPPActivation::arsech_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.elementWiseDivision(alg.full(z.size(), z[0].size(), -1), alg.hadamard_product(z, alg.sqrt(alg.subtraction(alg.onemat(z.size(), z[0].size()), alg.hadamard_product(z, z))))); +} + +//ARCOTH + +real_t MLPPActivation::arcoth_norm(real_t z) { + return 0.5 * std::log((1 + z) / (z - 1)); +} +Ref MLPPActivation::arcoth_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.scalarMultiply(0.5, alg.log(alg.elementWiseDivision(alg.addition(alg.onevec(z.size()), z), alg.subtraction(z, alg.onevec(z.size()))))); +} + +Ref MLPPActivation::arcoth_norm(const Ref &z) { + MLPPLinAlg alg; + + return alg.scalarMultiply(0.5, alg.log(alg.elementWiseDivision(alg.addition(alg.onemat(z.size(), z[0].size()), z), alg.subtraction(z, alg.onemat(z.size(), z[0].size()))))); +} + +real_t MLPPActivation::arcoth_deriv(real_t z) { + return 1 / (1 - z * z); +} +Ref MLPPActivation::arcoth_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.elementWiseDivision(alg.onevec(z.size()), alg.subtraction(alg.onevec(z.size()), alg.hadamard_product(z, z))); +} + +Ref MLPPActivation::arcoth_deriv(const Ref &z) { + MLPPLinAlg alg; + + return alg.elementWiseDivision(alg.onemat(z.size(), z[0].size()), alg.subtraction(alg.onemat(z.size(), z[0].size()), alg.hadamard_product(z, z))); +} + +*/ + +//======================== OLD ============================= + real_t MLPPActivation::linear(real_t z, bool deriv) { if (deriv) { return 1; @@ -518,7 +1770,7 @@ std::vector> MLPPActivation::ELU(std::vector MLPPActivation::activation(std::vector z, bool deriv std::vector deriv; deriv.resize(z.size()); for (int i = 0; i < z.size(); i++) { - deriv[i] = function(z[i], 1); + deriv[i] = function(z[i], true); } return deriv; } diff --git a/mlpp/activation/activation.h b/mlpp/activation/activation.h index 5def9fe..d648194 100644 --- a/mlpp/activation/activation.h +++ b/mlpp/activation/activation.h @@ -10,20 +10,391 @@ #include "core/math/math_defs.h" +#include "core/object/reference.h" + +#include "../lin_alg/mlpp_matrix.h" +#include "../lin_alg/mlpp_vector.h" + #include -class MLPPActivation { +//TODO this should probably be a singleton +//TODO Activation functions should either have a variant which does not allocate, or they should just be reworked altogether + +class MLPPActivation : public Reference { + GDCLASS(MLPPActivation, Reference); + public: - real_t linear(real_t z, bool deriv = 0); - std::vector linear(std::vector z, bool deriv = 0); - std::vector> linear(std::vector> z, bool deriv = 0); + enum ActivationFunction { + ACTIVATION_FUNCTION_LINEAR = 0, + ACTIVATION_FUNCTION_SIGMOID, + ACTIVATION_FUNCTION_SWISH, + ACTIVATION_FUNCTION_MISH, + ACTIVATION_FUNCTION_SIN_C, + ACTIVATION_FUNCTION_SOFTPLUS, + ACTIVATION_FUNCTION_SOFTSIGN, + ACTIVATION_FUNCTION_C_LOG_LOG, + ACTIVATION_FUNCTION_LOGIT, + ACTIVATION_FUNCTION_GAUSSIAN_CDF, + ACTIVATION_FUNCTION_RELU, + ACTIVATION_FUNCTION_GELU, + ACTIVATION_FUNCTION_SIGN, + ACTIVATION_FUNCTION_UNIT_STEP, + ACTIVATION_FUNCTION_SINH, + ACTIVATION_FUNCTION_COSH, + ACTIVATION_FUNCTION_TANH, + ACTIVATION_FUNCTION_CSCH, + ACTIVATION_FUNCTION_SECH, + ACTIVATION_FUNCTION_COTH, + ACTIVATION_FUNCTION_ARSINH, + ACTIVATION_FUNCTION_ARCOSH, + ACTIVATION_FUNCTION_ARTANH, + ACTIVATION_FUNCTION_ARCSCH, + ACTIVATION_FUNCTION_ARSECH, + ACTIVATION_FUNCTION_ARCOTH, + }; - real_t sigmoid(real_t z, bool deriv = 0); - std::vector sigmoid(std::vector z, bool deriv = 0); - std::vector> sigmoid(std::vector> z, bool deriv = 0); +public: + typedef Ref (MLPPActivation::*ActivationFunctionPointer)(const Ref &); + ActivationFunctionPointer get_activation_function_ptr(const ActivationFunction func, const bool deriv = false); - std::vector softmax(std::vector z, bool deriv = 0); - std::vector> softmax(std::vector> z, bool deriv = 0); + Ref run_activation_vector(const ActivationFunction func, const Ref &z, const bool deriv = false); + Ref run_activation_matrix(const ActivationFunction func, const Ref &z, const bool deriv = false); + + Ref run_activation_norm_vector(const ActivationFunction func, const Ref &z); + Ref run_activation_norm_matrix(const ActivationFunction func, const Ref &z); + + Ref run_activation_deriv_vector(const ActivationFunction func, const Ref &z); + Ref run_activation_deriv_matrix(const ActivationFunction func, const Ref &z); + + Ref activation(const Ref &z, real_t (*function)(real_t), const bool deriv = false); + Ref activation_norm(const Ref &z, real_t (*function)(real_t)); + Ref activation_deriv(const Ref &z, real_t (*function)(real_t)); + + //ACTIVATION FUNCTIONS + + //LINEAR + real_t linear_norm(real_t z); + Ref linear_norm(const Ref &z); + Ref linear_norm(const Ref &z); + + real_t linear_deriv(real_t z); + Ref linear_deriv(const Ref &z); + Ref linear_deriv(const Ref &z); + + //SIGMOID + real_t sigmoid_norm(real_t z); + Ref sigmoid_norm(const Ref &z); + Ref sigmoid_norm(const Ref &z); + + real_t sigmoid_deriv(real_t z); + Ref sigmoid_deriv(const Ref &z); + Ref sigmoid_deriv(const Ref &z); + + //SOFTMAX + Ref softmax_norm(const Ref &z); + Ref softmax_norm(const Ref &z); + + Ref softmax_deriv(const Ref &z); + Ref softmax_deriv(const Ref &z); + + //ADJ_SOFTMAX + + Ref adj_softmax_norm(const Ref &z); + Ref adj_softmax_norm(const Ref &z); + + Ref adj_softmax(const Ref &z); + Ref adj_softmax(const Ref &z); + + //SOFTMAX DERIV + + Ref softmax_deriv_norm(const Ref &z); + std::vector> softmax_deriv_norm(const Ref &z); + + Ref softmax_deriv_deriv(const Ref &z); + std::vector> softmax_deriv_deriv(const Ref &z); + + //SOFTPLUS + + real_t softplus_norm(real_t z); + Ref softplus_norm(const Ref &z); + Ref softplus_norm(const Ref &z); + + real_t softplus_deriv(real_t z); + Ref softplus_deriv(const Ref &z); + Ref softplus_deriv(const Ref &z); + + //SOFTSIGN + + real_t softsign_norm(real_t z); + Ref softsign_norm(const Ref &z); + Ref softsign_norm(const Ref &z); + + real_t softsign_deriv(real_t z); + Ref softsign_deriv(const Ref &z); + Ref softsign_deriv(const Ref &z); + + //GAUSSIANCDF + + real_t gaussian_cdf_norm(real_t z); + Ref gaussian_cdf_norm(const Ref &z); + Ref gaussian_cdf_norm(const Ref &z); + + real_t gaussian_cdf_deriv(real_t z); + Ref gaussian_cdf_deriv(const Ref &z); + Ref gaussian_cdf_deriv(const Ref &z); + + //CLOGLOG + + real_t cloglog_norm(real_t z); + Ref cloglog_norm(const Ref &z); + Ref cloglog_norm(const Ref &z); + + real_t cloglog_deriv(real_t z); + Ref cloglog_deriv(const Ref &z); + Ref cloglog_deriv(const Ref &z); + + //LOGIT + + real_t logit_norm(real_t z); + Ref logit_norm(const Ref &z); + Ref logit_norm(const Ref &z); + + real_t logit_deriv(real_t z); + Ref logit_deriv(const Ref &z); + Ref logit_deriv(const Ref &z); + + //UNITSTEP + + real_t unit_step_norm(real_t z); + Ref unit_step_norm(const Ref &z); + Ref unit_step_norm(const Ref &z); + + real_t unit_step_deriv(real_t z); + Ref unit_step_deriv(const Ref &z); + Ref unit_step_deriv(const Ref &z); + + //SWISH + + real_t swish_norm(real_t z); + Ref swish_norm(const Ref &z); + Ref swish_norm(const Ref &z); + + real_t swish_deriv(real_t z); + Ref swish_deriv(const Ref &z); + Ref swish_deriv(const Ref &z); + + //MISH + + real_t mish_norm(real_t z); + Ref mish_norm(const Ref &z); + Ref mish_norm(const Ref &z); + + real_t mish_deriv(real_t z); + Ref mish_deriv(const Ref &z); + Ref mish_deriv(const Ref &z); + + //SINC + + real_t sinc_norm(real_t z); + Ref sinc_norm(const Ref &z); + Ref sinc_norm(const Ref &z); + + real_t sinc_deriv(real_t z); + Ref sinc_deriv(const Ref &z); + Ref sinc_deriv(const Ref &z); + + //RELU + + real_t relu_norm(real_t z); + Ref relu_norm(const Ref &z); + Ref relu_norm(const Ref &z); + + real_t relu_deriv(real_t z); + Ref relu_deriv(const Ref &z); + Ref relu_deriv(const Ref &z); + + //LEAKYRELU + + real_t leaky_relu_norm(real_t z, real_t c); + Ref leaky_relu_norm(const Ref &z, real_t c); + Ref leaky_relu_norm(const Ref &z, real_t c); + + real_t leaky_relu_deriv(real_t z, real_t c); + Ref leaky_relu_deriv(const Ref &z, real_t c); + Ref leaky_relu_deriv(const Ref &z, real_t c); + + //ELU + + real_t elu_norm(real_t z, real_t c); + Ref elu_norm(const Ref &z, real_t c); + Ref elu_norm(const Ref &z, real_t c); + + real_t elu_deriv(real_t z, real_t c); + Ref elu_deriv(const Ref &z, real_t c); + Ref elu_deriv(const Ref &z, real_t c); + + //SELU + + real_t selu_norm(real_t z, real_t lambda, real_t c); + Ref selu_norm(const Ref &z, real_t lambda, real_t c); + Ref selu_norm(Ref, real_t lambda, real_t c); + + real_t selu_deriv(real_t z, real_t lambda, real_t c); + Ref selu_deriv(const Ref &z, real_t lambda, real_t c); + Ref selu_deriv(Ref, real_t lambda, real_t c); + + //GELU + + real_t gelu_norm(real_t z); + Ref gelu_norm(const Ref &z); + Ref gelu_norm(const Ref &z); + + real_t gelu_deriv(real_t z); + Ref gelu_deriv(const Ref &z); + Ref gelu_deriv(const Ref &z); + + //SIGN + + real_t sign_norm(real_t z); + Ref sign_norm(const Ref &z); + Ref sign_norm(const Ref &z); + + real_t sign_deriv(real_t z); + Ref sign_deriv(const Ref &z); + Ref sign_deriv(const Ref &z); + + //SINH + + real_t sinh_norm(real_t z); + Ref sinh_norm(const Ref &z); + Ref sinh_norm(const Ref &z); + + real_t sinh_deriv(real_t z); + Ref sinh_deriv(const Ref &z); + Ref sinh_deriv(const Ref &z); + + //COSH + + real_t cosh_norm(real_t z); + Ref cosh_norm(const Ref &z); + Ref cosh_norm(const Ref &z); + + real_t cosh_deriv(real_t z); + Ref cosh_deriv(const Ref &z); + Ref cosh_deriv(const Ref &z); + + //TANH + + real_t tanh_norm(real_t z); + Ref tanh_norm(const Ref &z); + Ref tanh_norm(const Ref &z); + + real_t tanh_deriv(real_t z); + Ref tanh_deriv(const Ref &z); + Ref tanh_deriv(const Ref &z); + + //CSCH + + real_t csch_norm(real_t z); + Ref csch_norm(const Ref &z); + Ref csch_norm(const Ref &z); + + real_t csch_deriv(real_t z); + Ref csch_deriv(const Ref &z); + Ref csch_deriv(const Ref &z); + + //SECH + + real_t sech_norm(real_t z); + Ref sech_norm(const Ref &z); + Ref sech_norm(const Ref &z); + + real_t sech_deriv(real_t z); + Ref sech_deriv(const Ref &z); + Ref sech_deriv(const Ref &z); + + //COTH + + real_t coth_norm(real_t z); + Ref coth_norm(const Ref &z); + Ref coth_norm(const Ref &z); + + real_t coth_deriv(real_t z); + Ref coth_deriv(const Ref &z); + Ref coth_deriv(const Ref &z); + + //ARSINH + + real_t arsinh_norm(real_t z); + Ref arsinh_norm(const Ref &z); + Ref arsinh_norm(const Ref &z); + + real_t arsinh_deriv(real_t z); + Ref arsinh_deriv(const Ref &z); + Ref arsinh_deriv(const Ref &z); + + //ARCOSH + + real_t arcosh_norm(real_t z); + Ref arcosh_norm(const Ref &z); + Ref arcosh_norm(const Ref &z); + + real_t arcosh_deriv(real_t z); + Ref arcosh_deriv(const Ref &z); + Ref arcosh_deriv(const Ref &z); + + //ARTANH + + real_t artanh_norm(real_t z); + Ref artanh_norm(const Ref &z); + Ref artanh_norm(const Ref &z); + + real_t artanh_deriv(real_t z); + Ref artanh_deriv(const Ref &z); + Ref artanh_deriv(const Ref &z); + + //ARCSCH + + real_t arcsch_norm(real_t z); + Ref arcsch_norm(const Ref &z); + Ref arcsch_norm(const Ref &z); + + real_t arcsch_deriv(real_t z); + Ref arcsch_deriv(const Ref &z); + Ref arcsch_deriv(const Ref &z); + + //ARSECH + + real_t arsech_norm(real_t z); + Ref arsech_norm(const Ref &z); + Ref arsech_norm(const Ref &z); + + real_t arsech_deriv(real_t z); + Ref arsech_deriv(const Ref &z); + Ref arsech_deriv(const Ref &z); + + //ARCOTH + + real_t arcoth_norm(real_t z); + Ref arcoth_norm(const Ref &z); + Ref arcoth_norm(const Ref &z); + + real_t arcoth_deriv(real_t z); + Ref arcoth_deriv(const Ref &z); + Ref arcoth_deriv(const Ref &z); + + // ========= OLD =========== + + real_t linear(real_t z, bool deriv = false); + std::vector linear(std::vector z, bool deriv = false); + std::vector> linear(std::vector> z, bool deriv = false); + + real_t sigmoid(real_t z, bool deriv = false); + std::vector sigmoid(std::vector z, bool deriv = false); + std::vector> sigmoid(std::vector> z, bool deriv = false); + + std::vector softmax(std::vector z, bool deriv = false); + std::vector> softmax(std::vector> z, bool deriv = false); std::vector adjSoftmax(std::vector z); std::vector> adjSoftmax(std::vector> z); @@ -31,117 +402,119 @@ public: std::vector> softmaxDeriv(std::vector z); std::vector>> softmaxDeriv(std::vector> z); - real_t softplus(real_t z, bool deriv = 0); - std::vector softplus(std::vector z, bool deriv = 0); - std::vector> softplus(std::vector> z, bool deriv = 0); + real_t softplus(real_t z, bool deriv = false); + std::vector softplus(std::vector z, bool deriv = false); + std::vector> softplus(std::vector> z, bool deriv = false); - real_t softsign(real_t z, bool deriv = 0); - std::vector softsign(std::vector z, bool deriv = 0); - std::vector> softsign(std::vector> z, bool deriv = 0); + real_t softsign(real_t z, bool deriv = false); + std::vector softsign(std::vector z, bool deriv = false); + std::vector> softsign(std::vector> z, bool deriv = false); - real_t gaussianCDF(real_t z, bool deriv = 0); - std::vector gaussianCDF(std::vector z, bool deriv = 0); - std::vector> gaussianCDF(std::vector> z, bool deriv = 0); + real_t gaussianCDF(real_t z, bool deriv = false); + std::vector gaussianCDF(std::vector z, bool deriv = false); + std::vector> gaussianCDF(std::vector> z, bool deriv = false); - real_t cloglog(real_t z, bool deriv = 0); - std::vector cloglog(std::vector z, bool deriv = 0); - std::vector> cloglog(std::vector> z, bool deriv = 0); + real_t cloglog(real_t z, bool deriv = false); + std::vector cloglog(std::vector z, bool deriv = false); + std::vector> cloglog(std::vector> z, bool deriv = false); - real_t logit(real_t z, bool deriv = 0); - std::vector logit(std::vector z, bool deriv = 0); - std::vector> logit(std::vector> z, bool deriv = 0); + real_t logit(real_t z, bool deriv = false); + std::vector logit(std::vector z, bool deriv = false); + std::vector> logit(std::vector> z, bool deriv = false); - real_t unitStep(real_t z, bool deriv = 0); - std::vector unitStep(std::vector z, bool deriv = 0); - std::vector> unitStep(std::vector> z, bool deriv = 0); + real_t unitStep(real_t z, bool deriv = false); + std::vector unitStep(std::vector z, bool deriv = false); + std::vector> unitStep(std::vector> z, bool deriv = false); - real_t swish(real_t z, bool deriv = 0); - std::vector swish(std::vector z, bool deriv = 0); - std::vector> swish(std::vector> z, bool deriv = 0); + real_t swish(real_t z, bool deriv = false); + std::vector swish(std::vector z, bool deriv = false); + std::vector> swish(std::vector> z, bool deriv = false); - real_t mish(real_t z, bool deriv = 0); - std::vector mish(std::vector z, bool deriv = 0); - std::vector> mish(std::vector> z, bool deriv = 0); + real_t mish(real_t z, bool deriv = false); + std::vector mish(std::vector z, bool deriv = false); + std::vector> mish(std::vector> z, bool deriv = false); - real_t sinc(real_t z, bool deriv = 0); - std::vector sinc(std::vector z, bool deriv = 0); - std::vector> sinc(std::vector> z, bool deriv = 0); + real_t sinc(real_t z, bool deriv = false); + std::vector sinc(std::vector z, bool deriv = false); + std::vector> sinc(std::vector> z, bool deriv = false); - real_t RELU(real_t z, bool deriv = 0); - std::vector RELU(std::vector z, bool deriv = 0); - std::vector> RELU(std::vector> z, bool deriv = 0); + real_t RELU(real_t z, bool deriv = false); + std::vector RELU(std::vector z, bool deriv = false); + std::vector> RELU(std::vector> z, bool deriv = false); - real_t leakyReLU(real_t z, real_t c, bool deriv = 0); - std::vector leakyReLU(std::vector z, real_t c, bool deriv = 0); - std::vector> leakyReLU(std::vector> z, real_t c, bool deriv = 0); + real_t leakyReLU(real_t z, real_t c, bool deriv = false); + std::vector leakyReLU(std::vector z, real_t c, bool deriv = false); + std::vector> leakyReLU(std::vector> z, real_t c, bool deriv = false); - real_t ELU(real_t z, real_t c, bool deriv = 0); - std::vector ELU(std::vector z, real_t c, bool deriv = 0); - std::vector> ELU(std::vector> z, real_t c, bool deriv = 0); + real_t ELU(real_t z, real_t c, bool deriv = false); + std::vector ELU(std::vector z, real_t c, bool deriv = false); + std::vector> ELU(std::vector> z, real_t c, bool deriv = false); - real_t SELU(real_t z, real_t lambda, real_t c, bool deriv = 0); - std::vector SELU(std::vector z, real_t lambda, real_t c, bool deriv = 0); - std::vector> SELU(std::vector>, real_t lambda, real_t c, bool deriv = 0); + real_t SELU(real_t z, real_t lambda, real_t c, bool deriv = false); + std::vector SELU(std::vector z, real_t lambda, real_t c, bool deriv = false); + std::vector> SELU(std::vector>, real_t lambda, real_t c, bool deriv = false); - real_t GELU(real_t z, bool deriv = 0); - std::vector GELU(std::vector z, bool deriv = 0); - std::vector> GELU(std::vector> z, bool deriv = 0); + real_t GELU(real_t z, bool deriv = false); + std::vector GELU(std::vector z, bool deriv = false); + std::vector> GELU(std::vector> z, bool deriv = false); - real_t sign(real_t z, bool deriv = 0); - std::vector sign(std::vector z, bool deriv = 0); - std::vector> sign(std::vector> z, bool deriv = 0); + real_t sign(real_t z, bool deriv = false); + std::vector sign(std::vector z, bool deriv = false); + std::vector> sign(std::vector> z, bool deriv = false); - real_t sinh(real_t z, bool deriv = 0); - std::vector sinh(std::vector z, bool deriv = 0); - std::vector> sinh(std::vector> z, bool deriv = 0); + real_t sinh(real_t z, bool deriv = false); + std::vector sinh(std::vector z, bool deriv = false); + std::vector> sinh(std::vector> z, bool deriv = false); - real_t cosh(real_t z, bool deriv = 0); - std::vector cosh(std::vector z, bool deriv = 0); - std::vector> cosh(std::vector> z, bool deriv = 0); + real_t cosh(real_t z, bool deriv = false); + std::vector cosh(std::vector z, bool deriv = false); + std::vector> cosh(std::vector> z, bool deriv = false); - real_t tanh(real_t z, bool deriv = 0); - std::vector tanh(std::vector z, bool deriv = 0); - std::vector> tanh(std::vector> z, bool deriv = 0); + real_t tanh(real_t z, bool deriv = false); + std::vector tanh(std::vector z, bool deriv = false); + std::vector> tanh(std::vector> z, bool deriv = false); - real_t csch(real_t z, bool deriv = 0); - std::vector csch(std::vector z, bool deriv = 0); - std::vector> csch(std::vector> z, bool deriv = 0); + real_t csch(real_t z, bool deriv = false); + std::vector csch(std::vector z, bool deriv = false); + std::vector> csch(std::vector> z, bool deriv = false); - real_t sech(real_t z, bool deriv = 0); - std::vector sech(std::vector z, bool deriv = 0); - std::vector> sech(std::vector> z, bool deriv = 0); + real_t sech(real_t z, bool deriv = false); + std::vector sech(std::vector z, bool deriv = false); + std::vector> sech(std::vector> z, bool deriv = false); - real_t coth(real_t z, bool deriv = 0); - std::vector coth(std::vector z, bool deriv = 0); - std::vector> coth(std::vector> z, bool deriv = 0); + real_t coth(real_t z, bool deriv = false); + std::vector coth(std::vector z, bool deriv = false); + std::vector> coth(std::vector> z, bool deriv = false); - real_t arsinh(real_t z, bool deriv = 0); - std::vector arsinh(std::vector z, bool deriv = 0); - std::vector> arsinh(std::vector> z, bool deriv = 0); + real_t arsinh(real_t z, bool deriv = false); + std::vector arsinh(std::vector z, bool deriv = false); + std::vector> arsinh(std::vector> z, bool deriv = false); - real_t arcosh(real_t z, bool deriv = 0); - std::vector arcosh(std::vector z, bool deriv = 0); - std::vector> arcosh(std::vector> z, bool deriv = 0); + real_t arcosh(real_t z, bool deriv = false); + std::vector arcosh(std::vector z, bool deriv = false); + std::vector> arcosh(std::vector> z, bool deriv = false); - real_t artanh(real_t z, bool deriv = 0); - std::vector artanh(std::vector z, bool deriv = 0); - std::vector> artanh(std::vector> z, bool deriv = 0); + real_t artanh(real_t z, bool deriv = false); + std::vector artanh(std::vector z, bool deriv = false); + std::vector> artanh(std::vector> z, bool deriv = false); - real_t arcsch(real_t z, bool deriv = 0); - std::vector arcsch(std::vector z, bool deriv = 0); - std::vector> arcsch(std::vector> z, bool deriv = 0); + real_t arcsch(real_t z, bool deriv = false); + std::vector arcsch(std::vector z, bool deriv = false); + std::vector> arcsch(std::vector> z, bool deriv = false); - real_t arsech(real_t z, bool deriv = 0); - std::vector arsech(std::vector z, bool deriv = 0); - std::vector> arsech(std::vector> z, bool deriv = 0); + real_t arsech(real_t z, bool deriv = false); + std::vector arsech(std::vector z, bool deriv = false); + std::vector> arsech(std::vector> z, bool deriv = false); - real_t arcoth(real_t z, bool deriv = 0); - std::vector arcoth(std::vector z, bool deriv = 0); - std::vector> arcoth(std::vector> z, bool deriv = 0); + real_t arcoth(real_t z, bool deriv = false); + std::vector arcoth(std::vector z, bool deriv = false); + std::vector> arcoth(std::vector> z, bool deriv = false); std::vector activation(std::vector z, bool deriv, real_t (*function)(real_t, bool)); private: }; +VARIANT_ENUM_CAST(MLPPActivation::ActivationFunction); + #endif /* Activation_hpp */