// // SoftmaxReg.cpp // // Created by Marc Melikyan on 10/2/20. // #include "SoftmaxReg.hpp" #include "LinAlg/LinAlg.hpp" #include "Regularization/Reg.hpp" #include "Activation/Activation.hpp" #include "Utilities/Utilities.hpp" #include "Cost/Cost.hpp" #include #include namespace MLPP{ SoftmaxReg::SoftmaxReg(std::vector> inputSet, std::vector> outputSet, std::string reg, double lambda, double alpha) : inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), n_class(outputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) { y_hat.resize(n); weights = Utilities::weightInitialization(k, n_class); bias = Utilities::biasInitialization(n_class); } std::vector SoftmaxReg::modelTest(std::vector x){ return Evaluate(x); } std::vector> SoftmaxReg::modelSetTest(std::vector> X){ return Evaluate(X); } void SoftmaxReg::gradientDescent(double learning_rate, int max_epoch, bool UI){ LinAlg alg; Reg regularization; double cost_prev = 0; int epoch = 1; forwardPass(); while(true){ cost_prev = Cost(y_hat, outputSet); std::vector> error = alg.subtraction(y_hat, outputSet); //Calculating the weight gradients std::vector> w_gradient = alg.matmult(alg.transpose(inputSet), error); //Weight updation weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate, w_gradient)); weights = regularization.regWeights(weights, lambda, alpha, reg); // Calculating the bias gradients //double b_gradient = alg.sum_elements(error); // Bias Updation bias = alg.subtractMatrixRows(bias, alg.scalarMultiply(learning_rate, error)); forwardPass(); // UI PORTION if(UI) { Utilities::CostInfo(epoch, cost_prev, Cost(y_hat, outputSet)); Utilities::UI(weights, bias); } epoch++; if(epoch > max_epoch) { break; } } } void SoftmaxReg::SGD(double learning_rate, int max_epoch, bool UI){ LinAlg alg; Reg regularization; double cost_prev = 0; int epoch = 1; while(true){ std::random_device rd; std::default_random_engine generator(rd()); std::uniform_int_distribution distribution(0, int(n - 1)); double outputIndex = distribution(generator); std::vector y_hat = Evaluate(inputSet[outputIndex]); cost_prev = Cost({y_hat}, {outputSet[outputIndex]}); // Calculating the weight gradients std::vector> w_gradient = alg.outerProduct(inputSet[outputIndex], alg.subtraction(y_hat, outputSet[outputIndex])); // Weight Updation weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate, w_gradient)); weights = regularization.regWeights(weights, lambda, alpha, reg); // Calculating the bias gradients std::vector b_gradient = alg.subtraction(y_hat, outputSet[outputIndex]); // Bias updation bias = alg.subtraction(bias, alg.scalarMultiply(learning_rate, b_gradient)); y_hat = Evaluate({inputSet[outputIndex]}); if(UI) { Utilities::CostInfo(epoch, cost_prev, Cost({y_hat}, {outputSet[outputIndex]})); Utilities::UI(weights, bias); } epoch++; if(epoch > max_epoch) { break; } } forwardPass(); } void SoftmaxReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI){ LinAlg alg; Reg regularization; double cost_prev = 0; int epoch = 1; // Creating the mini-batches int n_mini_batch = n/mini_batch_size; auto [inputMiniBatches, outputMiniBatches] = Utilities::createMiniBatches(inputSet, outputSet, n_mini_batch); while(true){ for(int i = 0; i < n_mini_batch; i++){ std::vector> y_hat = Evaluate(inputMiniBatches[i]); cost_prev = Cost(y_hat, outputMiniBatches[i]); std::vector> error = alg.subtraction(y_hat, outputMiniBatches[i]); // Calculating the weight gradients std::vector> w_gradient = alg.matmult(alg.transpose(inputMiniBatches[i]), error); //Weight updation weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate, w_gradient)); weights = regularization.regWeights(weights, lambda, alpha, reg); // Calculating the bias gradients bias = alg.subtractMatrixRows(bias, alg.scalarMultiply(learning_rate, error)); y_hat = Evaluate(inputMiniBatches[i]); if(UI) { Utilities::CostInfo(epoch, cost_prev, Cost(y_hat, outputMiniBatches[i])); Utilities::UI(weights, bias); } } epoch++; if(epoch > max_epoch) { break; } } forwardPass(); } double SoftmaxReg::score(){ Utilities util; return util.performance(y_hat, outputSet); } void SoftmaxReg::save(std::string fileName){ Utilities util; util.saveParameters(fileName, weights, bias); } double SoftmaxReg::Cost(std::vector> y_hat, std::vector> y){ Reg regularization; class Cost cost; return cost.CrossEntropy(y_hat, y) + regularization.regTerm(weights, lambda, alpha, reg); } std::vector SoftmaxReg::Evaluate(std::vector x){ LinAlg alg; Activation avn; return avn.softmax(alg.addition(bias, alg.mat_vec_mult(alg.transpose(weights), x))); } std::vector> SoftmaxReg::Evaluate(std::vector> X){ LinAlg alg; Activation avn; return avn.softmax(alg.mat_vec_add(alg.matmult(X, weights), bias)); } // softmax ( wTx + b ) void SoftmaxReg::forwardPass(){ LinAlg alg; Activation avn; y_hat = avn.softmax(alg.mat_vec_add(alg.matmult(inputSet, weights), bias)); } }