Relintai/pmlpp
1
0
Fork 0
mirror of https://github.com/Relintai/pmlpp.git synced 2024-12-22 15:06:47 +01:00
Code Issues Packages Projects Releases Wiki Activity

Initial cleanup.

Browse Source
This commit is contained in:
Relintai 2023-02-05 18:46:12 +01:00
parent 8c3671fc8f
commit 35f4a01bac
2 changed files with 80 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
mlpp/wgan/wgan.cpp
View File

@ -14,31 +14,23 @@
#include <cmath> #include <cmath>
#include <iostream> #include <iostream>
std::vector<std::vector<real_t>> MLPPWGAN::generate_example(int n) {
MLPPWGAN::MLPPWGAN(real_t k, std::vector<std::vector<real_t>> outputSet) :
outputSet(outputSet), n(outputSet.size()), k(k) {
}
MLPPWGAN::~MLPPWGAN() {
delete outputLayer;
}
std::vector<std::vector<real_t>> MLPPWGAN::generateExample(int n) {
MLPPLinAlg alg; MLPPLinAlg alg;
return modelSetTestGenerator(alg.gaussianNoise(n, k)); return model_set_test_generator(alg.gaussianNoise(n, k));
} }
void MLPPWGAN::gradientDescent(real_t learning_rate, int max_epoch, bool UI) { void MLPPWGAN::gradient_descent(real_t learning_rate, int max_epoch, bool UI) {
class MLPPCost cost; //MLPPCost mlpp_cost;
MLPPLinAlg alg; MLPPLinAlg alg;
real_t cost_prev = 0; real_t cost_prev = 0;
int epoch = 1; int epoch = 1;
forwardPass();
forward_pass();
const int CRITIC_INTERATIONS = 5; // Wasserstein GAN specific parameter. const int CRITIC_INTERATIONS = 5; // Wasserstein GAN specific parameter.
while (true) { while (true) {
cost_prev = Cost(y_hat, alg.onevec(n)); cost_prev = cost(y_hat, alg.onevec(n));
std::vector<std::vector<real_t>> generatorInputSet; std::vector<std::vector<real_t>> generatorInputSet;
std::vector<std::vector<real_t>> discriminatorInputSet; std::vector<std::vector<real_t>> discriminatorInputSet;
@ -49,36 +41,37 @@ void MLPPWGAN::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
// Training of the discriminator. // Training of the discriminator.
for (int i = 0; i < CRITIC_INTERATIONS; i++) { for (int i = 0; i < CRITIC_INTERATIONS; i++) {
generatorInputSet = alg.gaussianNoise(n, k); generatorInputSet = alg.gaussianNoise(n, k);
discriminatorInputSet = modelSetTestGenerator(generatorInputSet); discriminatorInputSet = model_set_test_generator(generatorInputSet);
discriminatorInputSet.insert(discriminatorInputSet.end(), MLPPWGAN::outputSet.begin(), MLPPWGAN::outputSet.end()); // Fake + real inputs. discriminatorInputSet.insert(discriminatorInputSet.end(), MLPPWGAN::outputSet.begin(), MLPPWGAN::outputSet.end()); // Fake + real inputs.
y_hat = modelSetTestDiscriminator(discriminatorInputSet); y_hat = model_set_test_discriminator(discriminatorInputSet);
outputSet = alg.scalarMultiply(-1, alg.onevec(n)); // WGAN changes y_i = 1 and y_i = 0 to y_i = 1 and y_i = -1 outputSet = alg.scalarMultiply(-1, alg.onevec(n)); // WGAN changes y_i = 1 and y_i = 0 to y_i = 1 and y_i = -1
std::vector<real_t> outputSetReal = alg.onevec(n); std::vector<real_t> outputSetReal = alg.onevec(n);
outputSet.insert(outputSet.end(), outputSetReal.begin(), outputSetReal.end()); // Fake + real output scores. outputSet.insert(outputSet.end(), outputSetReal.begin(), outputSetReal.end()); // Fake + real output scores.
auto discriminator_gradient_results = computeDiscriminatorGradients(y_hat, outputSet); auto discriminator_gradient_results = compute_discriminator_gradients(y_hat, outputSet);
auto cumulativeDiscriminatorHiddenLayerWGrad = std::get<0>(discriminator_gradient_results); auto cumulativeDiscriminatorHiddenLayerWGrad = std::get<0>(discriminator_gradient_results);
auto outputDiscriminatorWGrad = std::get<1>(discriminator_gradient_results); auto outputDiscriminatorWGrad = std::get<1>(discriminator_gradient_results);
cumulativeDiscriminatorHiddenLayerWGrad = alg.scalarMultiply(learning_rate / n, cumulativeDiscriminatorHiddenLayerWGrad); cumulativeDiscriminatorHiddenLayerWGrad = alg.scalarMultiply(learning_rate / n, cumulativeDiscriminatorHiddenLayerWGrad);
outputDiscriminatorWGrad = alg.scalarMultiply(learning_rate / n, outputDiscriminatorWGrad); outputDiscriminatorWGrad = alg.scalarMultiply(learning_rate / n, outputDiscriminatorWGrad);
updateDiscriminatorParameters(cumulativeDiscriminatorHiddenLayerWGrad, outputDiscriminatorWGrad, learning_rate); update_discriminator_parameters(cumulativeDiscriminatorHiddenLayerWGrad, outputDiscriminatorWGrad, learning_rate);
} }
// Training of the generator. // Training of the generator.
generatorInputSet = alg.gaussianNoise(n, k); generatorInputSet = alg.gaussianNoise(n, k);
discriminatorInputSet = modelSetTestGenerator(generatorInputSet); discriminatorInputSet = model_set_test_generator(generatorInputSet);
y_hat = modelSetTestDiscriminator(discriminatorInputSet); y_hat = model_set_test_discriminator(discriminatorInputSet);
outputSet = alg.onevec(n); outputSet = alg.onevec(n);
std::vector<std::vector<std::vector<real_t>>> cumulativeGeneratorHiddenLayerWGrad = computeGeneratorGradients(y_hat, outputSet); std::vector<std::vector<std::vector<real_t>>> cumulativeGeneratorHiddenLayerWGrad = compute_generator_gradients(y_hat, outputSet);
cumulativeGeneratorHiddenLayerWGrad = alg.scalarMultiply(learning_rate / n, cumulativeGeneratorHiddenLayerWGrad); cumulativeGeneratorHiddenLayerWGrad = alg.scalarMultiply(learning_rate / n, cumulativeGeneratorHiddenLayerWGrad);
updateGeneratorParameters(cumulativeGeneratorHiddenLayerWGrad, learning_rate); update_generator_parameters(cumulativeGeneratorHiddenLayerWGrad, learning_rate);
forward_pass();
forwardPass();
if (UI) { if (UI) {
MLPPWGAN::UI(epoch, cost_prev, MLPPWGAN::y_hat, alg.onevec(n)); handle_ui(epoch, cost_prev, MLPPWGAN::y_hat, alg.onevec(n));
} }
epoch++; epoch++;
@ -91,7 +84,7 @@ void MLPPWGAN::gradientDescent(real_t learning_rate, int max_epoch, bool UI) {
real_t MLPPWGAN::score() { real_t MLPPWGAN::score() {
MLPPLinAlg alg; MLPPLinAlg alg;
MLPPUtilities util; MLPPUtilities util;
forwardPass(); forward_pass();
return util.performance(y_hat, alg.onevec(n)); return util.performance(y_hat, alg.onevec(n));
} }
@ -108,7 +101,7 @@ void MLPPWGAN::save(std::string fileName) {
} }
} }
void MLPPWGAN::addLayer(int n_hidden, std::string activation, std::string weightInit, std::string reg, real_t lambda, real_t alpha) { void MLPPWGAN::add_layer(int n_hidden, std::string activation, std::string weightInit, std::string reg, real_t lambda, real_t alpha) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (network.empty()) { if (network.empty()) {
network.push_back(MLPPOldHiddenLayer(n_hidden, activation, alg.gaussianNoise(n, k), weightInit, reg, lambda, alpha)); network.push_back(MLPPOldHiddenLayer(n_hidden, activation, alg.gaussianNoise(n, k), weightInit, reg, lambda, alpha));
@ -119,7 +112,7 @@ void MLPPWGAN::addLayer(int n_hidden, std::string activation, std::string weight
} }
} }
void MLPPWGAN::addOutputLayer(std::string weightInit, std::string reg, real_t lambda, real_t alpha) { void MLPPWGAN::add_output_layer(std::string weightInit, std::string reg, real_t lambda, real_t alpha) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (!network.empty()) { if (!network.empty()) {
outputLayer = new MLPPOldOutputLayer(network[network.size() - 1].n_hidden, "Linear", "WassersteinLoss", network[network.size() - 1].a, weightInit, "WeightClipping", -0.01, 0.01); outputLayer = new MLPPOldOutputLayer(network[network.size() - 1].n_hidden, "Linear", "WassersteinLoss", network[network.size() - 1].a, weightInit, "WeightClipping", -0.01, 0.01);
@ -128,7 +121,18 @@ void MLPPWGAN::addOutputLayer(std::string weightInit, std::string reg, real_t la
} }
} }
std::vector<std::vector<real_t>> MLPPWGAN::modelSetTestGenerator(std::vector<std::vector<real_t>> X) { MLPPWGAN::MLPPWGAN(real_t k, std::vector<std::vector<real_t>> outputSet) :
outputSet(outputSet), n(outputSet.size()), k(k) {
}
MLPPWGAN::MLPPWGAN() {
}
MLPPWGAN::~MLPPWGAN() {
delete outputLayer;
}
std::vector<std::vector<real_t>> MLPPWGAN::model_set_test_generator(std::vector<std::vector<real_t>> X) {
if (!network.empty()) { if (!network.empty()) {
network[0].input = X; network[0].input = X;
network[0].forwardPass(); network[0].forwardPass();
@ -141,7 +145,7 @@ std::vector<std::vector<real_t>> MLPPWGAN::modelSetTestGenerator(std::vector<std
return network[network.size() / 2].a; return network[network.size() / 2].a;
} }
std::vector<real_t> MLPPWGAN::modelSetTestDiscriminator(std::vector<std::vector<real_t>> X) { std::vector<real_t> MLPPWGAN::model_set_test_discriminator(std::vector<std::vector<real_t>> X) {
if (!network.empty()) { if (!network.empty()) {
for (uint32_t i = network.size() / 2 + 1; i < network.size(); i++) { for (uint32_t i = network.size() / 2 + 1; i < network.size(); i++) {
if (i == network.size() / 2 + 1) { if (i == network.size() / 2 + 1) {
@ -157,7 +161,7 @@ std::vector<real_t> MLPPWGAN::modelSetTestDiscriminator(std::vector<std::vector<
return outputLayer->a; return outputLayer->a;
} }
real_t MLPPWGAN::Cost(std::vector<real_t> y_hat, std::vector<real_t> y) { real_t MLPPWGAN::cost(std::vector<real_t> y_hat, std::vector<real_t> y) {
MLPPReg regularization; MLPPReg regularization;
class MLPPCost cost; class MLPPCost cost;
real_t totalRegTerm = 0; real_t totalRegTerm = 0;
@ -171,7 +175,7 @@ real_t MLPPWGAN::Cost(std::vector<real_t> y_hat, std::vector<real_t> y) {
return (cost.*cost_function)(y_hat, y) + totalRegTerm + regularization.regTerm(outputLayer->weights, outputLayer->lambda, outputLayer->alpha, outputLayer->reg); return (cost.*cost_function)(y_hat, y) + totalRegTerm + regularization.regTerm(outputLayer->weights, outputLayer->lambda, outputLayer->alpha, outputLayer->reg);
} }
void MLPPWGAN::forwardPass() { void MLPPWGAN::forward_pass() {
MLPPLinAlg alg; MLPPLinAlg alg;
if (!network.empty()) { if (!network.empty()) {
network[0].input = alg.gaussianNoise(n, k); network[0].input = alg.gaussianNoise(n, k);
@ -189,7 +193,7 @@ void MLPPWGAN::forwardPass() {
y_hat = outputLayer->a; y_hat = outputLayer->a;
} }
void MLPPWGAN::updateDiscriminatorParameters(std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations, std::vector<real_t> outputLayerUpdation, real_t learning_rate) { void MLPPWGAN::update_discriminator_parameters(std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations, std::vector<real_t> outputLayerUpdation, real_t learning_rate) {
MLPPLinAlg alg; MLPPLinAlg alg;
outputLayer->weights = alg.subtraction(outputLayer->weights, outputLayerUpdation); outputLayer->weights = alg.subtraction(outputLayer->weights, outputLayerUpdation);
@ -206,7 +210,7 @@ void MLPPWGAN::updateDiscriminatorParameters(std::vector<std::vector<std::vector
} }
} }
void MLPPWGAN::updateGeneratorParameters(std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations, real_t learning_rate) { void MLPPWGAN::update_generator_parameters(std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations, real_t learning_rate) {
MLPPLinAlg alg; MLPPLinAlg alg;
if (!network.empty()) { if (!network.empty()) {
@ -219,7 +223,7 @@ void MLPPWGAN::updateGeneratorParameters(std::vector<std::vector<std::vector<rea
} }
} }
std::tuple<std::vector<std::vector<std::vector<real_t>>>, std::vector<real_t>> MLPPWGAN::computeDiscriminatorGradients(std::vector<real_t> y_hat, std::vector<real_t> outputSet) { std::tuple<std::vector<std::vector<std::vector<real_t>>>, std::vector<real_t>> MLPPWGAN::compute_discriminator_gradients(std::vector<real_t> y_hat, std::vector<real_t> outputSet) {
class MLPPCost cost; class MLPPCost cost;
MLPPActivation avn; MLPPActivation avn;
MLPPLinAlg alg; MLPPLinAlg alg;
@ -255,7 +259,7 @@ std::tuple<std::vector<std::vector<std::vector<real_t>>>, std::vector<real_t>> M
return { cumulativeHiddenLayerWGrad, outputWGrad }; return { cumulativeHiddenLayerWGrad, outputWGrad };
} }
std::vector<std::vector<std::vector<real_t>>> MLPPWGAN::computeGeneratorGradients(std::vector<real_t> y_hat, std::vector<real_t> outputSet) { std::vector<std::vector<std::vector<real_t>>> MLPPWGAN::compute_generator_gradients(std::vector<real_t> y_hat, std::vector<real_t> outputSet) {
class MLPPCost cost; class MLPPCost cost;
MLPPActivation avn; MLPPActivation avn;
MLPPLinAlg alg; MLPPLinAlg alg;
@ -284,8 +288,8 @@ std::vector<std::vector<std::vector<real_t>>> MLPPWGAN::computeGeneratorGradient
return cumulativeHiddenLayerWGrad; return cumulativeHiddenLayerWGrad;
} }
void MLPPWGAN::UI(int epoch, real_t cost_prev, std::vector<real_t> y_hat, std::vector<real_t> outputSet) { void MLPPWGAN::handle_ui(int epoch, real_t cost_prev, std::vector<real_t> y_hat, std::vector<real_t> outputSet) {
MLPPUtilities::CostInfo(epoch, cost_prev, Cost(y_hat, outputSet)); MLPPUtilities::CostInfo(epoch, cost_prev, cost(y_hat, outputSet));
std::cout << "Layer " << network.size() + 1 << ": " << std::endl; std::cout << "Layer " << network.size() + 1 << ": " << std::endl;
MLPPUtilities::UI(outputLayer->weights, outputLayer->bias); MLPPUtilities::UI(outputLayer->weights, outputLayer->bias);
if (!network.empty()) { if (!network.empty()) {
@ -296,6 +300,11 @@ void MLPPWGAN::UI(int epoch, real_t cost_prev, std::vector<real_t> y_hat, std::v
} }
} }
void MLPPWGAN::_bind_methods() {
//ClassDB::bind_method(D_METHOD("get_input_set"), &MLPPWGAN::get_input_set);
//ClassDB::bind_method(D_METHOD("set_input_set", "val"), &MLPPWGAN::set_input_set);
//ADD_PROPERTY(PropertyInfo(Variant::OBJECT, "input_set", PROPERTY_HINT_RESOURCE_TYPE, "MLPPMatrix"), "set_input_set", "get_input_set");
}
// ======== OLD ========== // ======== OLD ==========

50
mlpp/wgan/wgan.h
View File

@ -8,6 +8,15 @@
// Created by Marc Melikyan on 11/4/20. // Created by Marc Melikyan on 11/4/20.
// //
#include "core/containers/vector.h"
#include "core/math/math_defs.h"
#include "core/string/ustring.h"
#include "core/object/reference.h"
#include "../lin_alg/mlpp_matrix.h"
#include "../lin_alg/mlpp_vector.h"
#include "../hidden_layer/hidden_layer.h" #include "../hidden_layer/hidden_layer.h"
#include "../output_layer/output_layer.h" #include "../output_layer/output_layer.h"
@ -15,31 +24,38 @@
#include <tuple> #include <tuple>
#include <vector> #include <vector>
class MLPPWGAN { class MLPPWGAN : public Reference {
GDCLASS(MLPPWGAN, Reference);
public: public:
MLPPWGAN(real_t k, std::vector<std::vector<real_t>> outputSet); std::vector<std::vector<real_t>> generate_example(int n);
~MLPPWGAN(); void gradient_descent(real_t learning_rate, int max_epoch, bool UI = false);
std::vector<std::vector<real_t>> generateExample(int n);
void gradientDescent(real_t learning_rate, int max_epoch, bool UI = false);
real_t score(); real_t score();
void save(std::string fileName); void save(std::string fileName);
void addLayer(int n_hidden, std::string activation, std::string weightInit = "Default", std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5); void add_layer(int n_hidden, std::string activation, std::string weightInit = "Default", std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
void addOutputLayer(std::string weightInit = "Default", std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5); void add_output_layer(std::string weightInit = "Default", std::string reg = "None", real_t lambda = 0.5, real_t alpha = 0.5);
private: MLPPWGAN(real_t k, std::vector<std::vector<real_t>> outputSet);
std::vector<std::vector<real_t>> modelSetTestGenerator(std::vector<std::vector<real_t>> X); // Evaluator for the generator of the WGAN.
std::vector<real_t> modelSetTestDiscriminator(std::vector<std::vector<real_t>> X); // Evaluator for the discriminator of the WGAN.
real_t Cost(std::vector<real_t> y_hat, std::vector<real_t> y); MLPPWGAN();
~MLPPWGAN();
void forwardPass(); protected:
void updateDiscriminatorParameters(std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations, std::vector<real_t> outputLayerUpdation, real_t learning_rate); std::vector<std::vector<real_t>> model_set_test_generator(std::vector<std::vector<real_t>> X); // Evaluator for the generator of the WGAN.
void updateGeneratorParameters(std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations, real_t learning_rate); std::vector<real_t> model_set_test_discriminator(std::vector<std::vector<real_t>> X); // Evaluator for the discriminator of the WGAN.
std::tuple<std::vector<std::vector<std::vector<real_t>>>, std::vector<real_t>> computeDiscriminatorGradients(std::vector<real_t> y_hat, std::vector<real_t> outputSet);
std::vector<std::vector<std::vector<real_t>>> computeGeneratorGradients(std::vector<real_t> y_hat, std::vector<real_t> outputSet);
void UI(int epoch, real_t cost_prev, std::vector<real_t> y_hat, std::vector<real_t> outputSet); real_t cost(std::vector<real_t> y_hat, std::vector<real_t> y);
void forward_pass();
void update_discriminator_parameters(std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations, std::vector<real_t> outputLayerUpdation, real_t learning_rate);
void update_generator_parameters(std::vector<std::vector<std::vector<real_t>>> hiddenLayerUpdations, real_t learning_rate);
std::tuple<std::vector<std::vector<std::vector<real_t>>>, std::vector<real_t>> compute_discriminator_gradients(std::vector<real_t> y_hat, std::vector<real_t> outputSet);
std::vector<std::vector<std::vector<real_t>>> compute_generator_gradients(std::vector<real_t> y_hat, std::vector<real_t> outputSet);
void handle_ui(int epoch, real_t cost_prev, std::vector<real_t> y_hat, std::vector<real_t> outputSet);
static void _bind_methods();
std::vector<std::vector<real_t>> outputSet; std::vector<std::vector<real_t>> outputSet;
std::vector<real_t> y_hat; std::vector<real_t> y_hat;