From e368a4fadb574705dc51ab81dcd171aef3cf8f74 Mon Sep 17 00:00:00 2001 From: Relintai Date: Mon, 13 Feb 2023 16:46:27 +0100 Subject: [PATCH] Added MLPPRegOld. --- SCsub | 1 + mlpp/regularization/reg_old.cpp | 364 ++++++++++++++++++++++++++++++++ mlpp/regularization/reg_old.h | 72 +++++++ 3 files changed, 437 insertions(+) create mode 100644 mlpp/regularization/reg_old.cpp create mode 100644 mlpp/regularization/reg_old.h diff --git a/SCsub b/SCsub index 0034c1c..b6950d9 100644 --- a/SCsub +++ b/SCsub @@ -76,6 +76,7 @@ sources = [ "mlpp/bernoulli_nb/bernoulli_nb_old.cpp", "mlpp/ann/ann_old.cpp", "mlpp/numerical_analysis/numerical_analysis_old.cpp", + "mlpp/regularization/reg_old.cpp", "test/mlpp_tests.cpp", ] diff --git a/mlpp/regularization/reg_old.cpp b/mlpp/regularization/reg_old.cpp new file mode 100644 index 0000000..514a432 --- /dev/null +++ b/mlpp/regularization/reg_old.cpp @@ -0,0 +1,364 @@ +// +// Reg.cpp +// +// Created by Marc Melikyan on 1/16/21. +// + +#include "reg_old.h" + +#include "core/math/math_defs.h" + +#include "../activation/activation.h" +#include "../lin_alg/lin_alg.h" + +#include +#include + +real_t MLPPRegOld::reg_termv(const Ref &weights, real_t lambda, real_t alpha, MLPPRegOld::RegularizationType p_reg) { + int size = weights->size(); + const real_t *weights_ptr = weights->ptr(); + + if (p_reg == REGULARIZATION_TYPE_RIDGE) { + real_t reg = 0; + for (int i = 0; i < size; ++i) { + real_t wi = weights_ptr[i]; + reg += wi * wi; + } + return reg * lambda / 2; + } else if (p_reg == REGULARIZATION_TYPE_LASSO) { + real_t reg = 0; + for (int i = 0; i < size; ++i) { + reg += ABS(weights_ptr[i]); + } + return reg * lambda; + } else if (p_reg == REGULARIZATION_TYPE_ELASTIC_NET) { + real_t reg = 0; + for (int i = 0; i < size; ++i) { + real_t wi = weights_ptr[i]; + reg += alpha * ABS(wi); // Lasso Reg + reg += ((1 - alpha) / 2) * wi * wi; // Ridge Reg + } + return reg * lambda; + } + + return 0; +} +real_t MLPPRegOld::reg_termm(const Ref &weights, real_t lambda, real_t alpha, MLPPRegOld::RegularizationType p_reg) { + int size = weights->data_size(); + const real_t *weights_ptr = weights->ptr(); + + if (p_reg == REGULARIZATION_TYPE_RIDGE) { + real_t reg = 0; + for (int i = 0; i < size; ++i) { + real_t wi = weights_ptr[i]; + reg += wi * wi; + } + return reg * lambda / 2; + } else if (p_reg == REGULARIZATION_TYPE_LASSO) { + real_t reg = 0; + for (int i = 0; i < size; ++i) { + reg += ABS(weights_ptr[i]); + } + return reg * lambda; + } else if (p_reg == REGULARIZATION_TYPE_ELASTIC_NET) { + real_t reg = 0; + for (int i = 0; i < size; ++i) { + real_t wi = weights_ptr[i]; + reg += alpha * ABS(wi); // Lasso Reg + reg += ((1 - alpha) / 2) * wi * wi; // Ridge Reg + } + return reg * lambda; + } + + return 0; +} + +Ref MLPPRegOld::reg_weightsv(const Ref &weights, real_t lambda, real_t alpha, MLPPRegOld::RegularizationType p_reg) { + MLPPLinAlg alg; + + if (p_reg == REGULARIZATION_TYPE_WEIGHT_CLIPPING) { + return reg_deriv_termv(weights, lambda, alpha, p_reg); + } + + return alg.subtractionnv(weights, reg_deriv_termv(weights, lambda, alpha, p_reg)); + + // for(int i = 0; i < weights.size(); i++){ + // weights[i] -= regDerivTerm(weights, lambda, alpha, reg, i); + // } + // return weights; +} +Ref MLPPRegOld::reg_weightsm(const Ref &weights, real_t lambda, real_t alpha, MLPPRegOld::RegularizationType reg) { + MLPPLinAlg alg; + + if (reg == REGULARIZATION_TYPE_WEIGHT_CLIPPING) { + return reg_deriv_termm(weights, lambda, alpha, reg); + } + + return alg.subtractionm(weights, reg_deriv_termm(weights, lambda, alpha, reg)); + + // for(int i = 0; i < weights.size(); i++){ + // for(int j = 0; j < weights[i].size(); j++){ + // weights[i][j] -= regDerivTerm(weights, lambda, alpha, reg, i, j); + // } + // } + // return weights; +} + +Ref MLPPRegOld::reg_deriv_termv(const Ref &weights, real_t lambda, real_t alpha, MLPPRegOld::RegularizationType reg) { + Ref reg_driv; + reg_driv.instance(); + + int size = weights->size(); + + reg_driv->resize(size); + + real_t *reg_driv_ptr = reg_driv->ptrw(); + + for (int i = 0; i < size; ++i) { + reg_driv_ptr[i] = reg_deriv_termvr(weights, lambda, alpha, reg, i); + } + + return reg_driv; +} +Ref MLPPRegOld::reg_deriv_termm(const Ref &weights, real_t lambda, real_t alpha, MLPPRegOld::RegularizationType reg) { + Ref reg_driv; + reg_driv.instance(); + + Size2i size = weights->size(); + + reg_driv->resize(size); + + real_t *reg_driv_ptr = reg_driv->ptrw(); + + for (int i = 0; i < size.y; ++i) { + for (int j = 0; j < size.x; ++j) { + reg_driv_ptr[reg_driv->calculate_index(i, j)] = reg_deriv_termmr(weights, lambda, alpha, reg, i, j); + } + } + + return reg_driv; +} + +MLPPRegOld::MLPPRegOld() { +} +MLPPRegOld::~MLPPRegOld() { +} + +void MLPPRegOld::_bind_methods() { + ClassDB::bind_method(D_METHOD("reg_termv", "weights", "lambda", "alpha", "reg"), &MLPPRegOld::reg_termv); + ClassDB::bind_method(D_METHOD("reg_termm", "weights", "lambda", "alpha", "reg"), &MLPPRegOld::reg_termm); + + ClassDB::bind_method(D_METHOD("reg_weightsv", "weights", "lambda", "alpha", "reg"), &MLPPRegOld::reg_weightsv); + ClassDB::bind_method(D_METHOD("reg_weightsm", "weights", "lambda", "alpha", "reg"), &MLPPRegOld::reg_weightsm); + + ClassDB::bind_method(D_METHOD("reg_deriv_termv", "weights", "lambda", "alpha", "reg"), &MLPPRegOld::reg_deriv_termv); + ClassDB::bind_method(D_METHOD("reg_deriv_termm", "weights", "lambda", "alpha", "reg"), &MLPPRegOld::reg_deriv_termm); + + BIND_ENUM_CONSTANT(REGULARIZATION_TYPE_NONE); + BIND_ENUM_CONSTANT(REGULARIZATION_TYPE_RIDGE); + BIND_ENUM_CONSTANT(REGULARIZATION_TYPE_LASSO); + BIND_ENUM_CONSTANT(REGULARIZATION_TYPE_ELASTIC_NET); + BIND_ENUM_CONSTANT(REGULARIZATION_TYPE_WEIGHT_CLIPPING); +} + +real_t MLPPRegOld::reg_deriv_termvr(const Ref &weights, real_t lambda, real_t alpha, MLPPRegOld::RegularizationType reg, int j) { + MLPPActivation act; + + real_t wj = weights->get_element(j); + + if (reg == REGULARIZATION_TYPE_RIDGE) { + return lambda * wj; + } else if (reg == REGULARIZATION_TYPE_LASSO) { + return lambda * act.sign(wj); + } else if (reg == REGULARIZATION_TYPE_ELASTIC_NET) { + return alpha * lambda * act.sign(wj) + (1 - alpha) * lambda * wj; + } else if (reg == REGULARIZATION_TYPE_WEIGHT_CLIPPING) { // Preparation for Wasserstein GANs. + // We assume lambda is the lower clipping threshold, while alpha is the higher clipping threshold. + // alpha > lambda. + if (wj > alpha) { + return alpha; + } else if (wj < lambda) { + return lambda; + } else { + return wj; + } + } else { + return 0; + } +} +real_t MLPPRegOld::reg_deriv_termmr(const Ref &weights, real_t lambda, real_t alpha, MLPPRegOld::RegularizationType reg, int i, int j) { + MLPPActivation act; + + real_t wj = weights->get_element(i, j); + + if (reg == REGULARIZATION_TYPE_RIDGE) { + return lambda * wj; + } else if (reg == REGULARIZATION_TYPE_LASSO) { + return lambda * act.sign(wj); + } else if (reg == REGULARIZATION_TYPE_ELASTIC_NET) { + return alpha * lambda * act.sign(wj) + (1 - alpha) * lambda * wj; + } else if (reg == REGULARIZATION_TYPE_WEIGHT_CLIPPING) { // Preparation for Wasserstein GANs. + // We assume lambda is the lower clipping threshold, while alpha is the higher clipping threshold. + // alpha > lambda. + if (wj > alpha) { + return alpha; + } else if (wj < lambda) { + return lambda; + } else { + return wj; + } + } else { + return 0; + } +} + +real_t MLPPRegOld::regTerm(std::vector weights, real_t lambda, real_t alpha, std::string p_reg) { + if (p_reg == "Ridge") { + real_t reg = 0; + for (uint32_t i = 0; i < weights.size(); i++) { + reg += weights[i] * weights[i]; + } + return reg * lambda / 2; + } else if (p_reg == "Lasso") { + real_t reg = 0; + for (uint32_t i = 0; i < weights.size(); i++) { + reg += abs(weights[i]); + } + return reg * lambda; + } else if (p_reg == "ElasticNet") { + real_t reg = 0; + for (uint32_t i = 0; i < weights.size(); i++) { + reg += alpha * abs(weights[i]); // Lasso Reg + reg += ((1 - alpha) / 2) * weights[i] * weights[i]; // Ridge Reg + } + return reg * lambda; + } + return 0; +} + +real_t MLPPRegOld::regTerm(std::vector> weights, real_t lambda, real_t alpha, std::string p_reg) { + if (p_reg == "Ridge") { + real_t reg = 0; + for (uint32_t i = 0; i < weights.size(); i++) { + for (uint32_t j = 0; j < weights[i].size(); j++) { + reg += weights[i][j] * weights[i][j]; + } + } + return reg * lambda / 2; + } else if (p_reg == "Lasso") { + real_t reg = 0; + for (uint32_t i = 0; i < weights.size(); i++) { + for (uint32_t j = 0; j < weights[i].size(); j++) { + reg += abs(weights[i][j]); + } + } + return reg * lambda; + } else if (p_reg == "ElasticNet") { + real_t reg = 0; + for (uint32_t i = 0; i < weights.size(); i++) { + for (uint32_t j = 0; j < weights[i].size(); j++) { + reg += alpha * abs(weights[i][j]); // Lasso Reg + reg += ((1 - alpha) / 2) * weights[i][j] * weights[i][j]; // Ridge Reg + } + } + return reg * lambda; + } + return 0; +} + +std::vector MLPPRegOld::regWeights(std::vector weights, real_t lambda, real_t alpha, std::string reg) { + MLPPLinAlg alg; + if (reg == "WeightClipping") { + return regDerivTerm(weights, lambda, alpha, reg); + } + return alg.subtraction(weights, regDerivTerm(weights, lambda, alpha, reg)); + // for(int i = 0; i < weights.size(); i++){ + // weights[i] -= regDerivTerm(weights, lambda, alpha, reg, i); + // } + // return weights; +} + +std::vector> MLPPRegOld::regWeights(std::vector> weights, real_t lambda, real_t alpha, std::string reg) { + MLPPLinAlg alg; + if (reg == "WeightClipping") { + return regDerivTerm(weights, lambda, alpha, reg); + } + return alg.subtraction(weights, regDerivTerm(weights, lambda, alpha, reg)); + // for(int i = 0; i < weights.size(); i++){ + // for(int j = 0; j < weights[i].size(); j++){ + // weights[i][j] -= regDerivTerm(weights, lambda, alpha, reg, i, j); + // } + // } + // return weights; +} + +std::vector MLPPRegOld::regDerivTerm(std::vector weights, real_t lambda, real_t alpha, std::string reg) { + std::vector regDeriv; + regDeriv.resize(weights.size()); + + for (uint32_t i = 0; i < regDeriv.size(); i++) { + regDeriv[i] = regDerivTerm(weights, lambda, alpha, reg, i); + } + return regDeriv; +} + +std::vector> MLPPRegOld::regDerivTerm(std::vector> weights, real_t lambda, real_t alpha, std::string reg) { + std::vector> regDeriv; + regDeriv.resize(weights.size()); + for (uint32_t i = 0; i < regDeriv.size(); i++) { + regDeriv[i].resize(weights[0].size()); + } + + for (uint32_t i = 0; i < regDeriv.size(); i++) { + for (uint32_t j = 0; j < regDeriv[i].size(); j++) { + regDeriv[i][j] = regDerivTerm(weights, lambda, alpha, reg, i, j); + } + } + return regDeriv; +} + +real_t MLPPRegOld::regDerivTerm(std::vector weights, real_t lambda, real_t alpha, std::string reg, int j) { + MLPPActivation act; + if (reg == "Ridge") { + return lambda * weights[j]; + } else if (reg == "Lasso") { + return lambda * act.sign(weights[j]); + } else if (reg == "ElasticNet") { + return alpha * lambda * act.sign(weights[j]) + (1 - alpha) * lambda * weights[j]; + } else if (reg == "WeightClipping") { // Preparation for Wasserstein GANs. + // We assume lambda is the lower clipping threshold, while alpha is the higher clipping threshold. + // alpha > lambda. + if (weights[j] > alpha) { + return alpha; + } else if (weights[j] < lambda) { + return lambda; + } else { + return weights[j]; + } + } else { + return 0; + } +} + +real_t MLPPRegOld::regDerivTerm(std::vector> weights, real_t lambda, real_t alpha, std::string reg, int i, int j) { + MLPPActivation act; + if (reg == "Ridge") { + return lambda * weights[i][j]; + } else if (reg == "Lasso") { + return lambda * act.sign(weights[i][j]); + } else if (reg == "ElasticNet") { + return alpha * lambda * act.sign(weights[i][j]) + (1 - alpha) * lambda * weights[i][j]; + } else if (reg == "WeightClipping") { // Preparation for Wasserstein GANs. + // We assume lambda is the lower clipping threshold, while alpha is the higher clipping threshold. + // alpha > lambda. + if (weights[i][j] > alpha) { + return alpha; + } else if (weights[i][j] < lambda) { + return lambda; + } else { + return weights[i][j]; + } + } else { + return 0; + } +} diff --git a/mlpp/regularization/reg_old.h b/mlpp/regularization/reg_old.h new file mode 100644 index 0000000..8aef595 --- /dev/null +++ b/mlpp/regularization/reg_old.h @@ -0,0 +1,72 @@ + + +#ifndef MLPP_REG_OLD_H +#define MLPP_REG_OLD_H + +// +// Reg.hpp +// +// Created by Marc Melikyan on 1/16/21. +// + +#include "core/math/math_defs.h" + +#include "core/object/reference.h" + +#include "../lin_alg/mlpp_matrix.h" +#include "../lin_alg/mlpp_vector.h" + +#include +#include + +class MLPPRegOld : public Reference { + GDCLASS(MLPPRegOld, Reference); + +public: + enum RegularizationType { + REGULARIZATION_TYPE_NONE = 0, + REGULARIZATION_TYPE_RIDGE, + REGULARIZATION_TYPE_LASSO, + REGULARIZATION_TYPE_ELASTIC_NET, + REGULARIZATION_TYPE_WEIGHT_CLIPPING, + }; + + real_t reg_termv(const Ref &weights, real_t lambda, real_t alpha, RegularizationType reg); + real_t reg_termm(const Ref &weights, real_t lambda, real_t alpha, RegularizationType reg); + + Ref reg_weightsv(const Ref &weights, real_t lambda, real_t alpha, RegularizationType reg); + Ref reg_weightsm(const Ref &weights, real_t lambda, real_t alpha, RegularizationType reg); + + Ref reg_deriv_termv(const Ref &weights, real_t lambda, real_t alpha, RegularizationType reg); + Ref reg_deriv_termm(const Ref &weights, real_t lambda, real_t alpha, RegularizationType reg); + + MLPPRegOld(); + ~MLPPRegOld(); + +protected: + static void _bind_methods(); + +private: + real_t reg_deriv_termvr(const Ref &weights, real_t lambda, real_t alpha, RegularizationType reg, int j); + real_t reg_deriv_termmr(const Ref &weights, real_t lambda, real_t alpha, RegularizationType reg, int i, int j); + +public: + // ======== OLD ========= + + real_t regTerm(std::vector weights, real_t lambda, real_t alpha, std::string reg); + real_t regTerm(std::vector> weights, real_t lambda, real_t alpha, std::string reg); + + std::vector regWeights(std::vector weights, real_t lambda, real_t alpha, std::string reg); + std::vector> regWeights(std::vector> weights, real_t lambda, real_t alpha, std::string reg); + + std::vector regDerivTerm(std::vector weights, real_t lambda, real_t alpha, std::string reg); + std::vector> regDerivTerm(std::vector>, real_t lambda, real_t alpha, std::string reg); + +private: + real_t regDerivTerm(std::vector weights, real_t lambda, real_t alpha, std::string reg, int j); + real_t regDerivTerm(std::vector> weights, real_t lambda, real_t alpha, std::string reg, int i, int j); +}; + +VARIANT_ENUM_CAST(MLPPRegOld::RegularizationType); + +#endif /* Reg_hpp */