diff --git a/mlpp/lin_reg/lin_reg.cpp b/mlpp/lin_reg/lin_reg.cpp index c5d26db..9baec0b 100644 --- a/mlpp/lin_reg/lin_reg.cpp +++ b/mlpp/lin_reg/lin_reg.cpp @@ -15,8 +15,6 @@ #include #include - - MLPPLinReg::MLPPLinReg(std::vector> inputSet, std::vector outputSet, std::string reg, double lambda, double alpha) : inputSet(inputSet), outputSet(outputSet), n(inputSet.size()), k(inputSet[0].size()), reg(reg), lambda(lambda), alpha(alpha) { y_hat.resize(n); @@ -173,9 +171,344 @@ void MLPPLinReg::MBGD(double learning_rate, int max_epoch, int mini_batch_size, forwardPass(); } +void MLPPLinReg::Momentum(double learning_rate, int max_epoch, int mini_batch_size, double gamma, bool UI) { + MLPPLinAlg alg; + MLPPReg regularization; + double cost_prev = 0; + int epoch = 1; + + // Creating the mini-batches + int n_mini_batch = n / mini_batch_size; + auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch); + + // Initializing necessary components for Momentum. + std::vector v = alg.zerovec(weights.size()); + while (true) { + for (int i = 0; i < n_mini_batch; i++) { + std::vector y_hat = Evaluate(inputMiniBatches[i]); + cost_prev = Cost(y_hat, outputMiniBatches[i]); + + std::vector error = alg.subtraction(y_hat, outputMiniBatches[i]); + + // Calculating the weight gradients + std::vector gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error)); + std::vector RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg); + std::vector weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final + + v = alg.addition(alg.scalarMultiply(gamma, v), alg.scalarMultiply(learning_rate, weight_grad)); + + weights = alg.subtraction(weights, v); + + // Calculating the bias gradients + bias -= learning_rate * alg.sum_elements(error) / outputMiniBatches[i].size(); // As normal + y_hat = Evaluate(inputMiniBatches[i]); + + if (UI) { + MLPPUtilities::CostInfo(epoch, cost_prev, Cost(y_hat, outputMiniBatches[i])); + MLPPUtilities::UI(weights, bias); + } + } + epoch++; + if (epoch > max_epoch) { + break; + } + } + forwardPass(); +} + +void MLPPLinReg::NAG(double learning_rate, int max_epoch, int mini_batch_size, double gamma, bool UI) { + MLPPLinAlg alg; + MLPPReg regularization; + double cost_prev = 0; + int epoch = 1; + + // Creating the mini-batches + int n_mini_batch = n / mini_batch_size; + auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch); + + // Initializing necessary components for Momentum. + std::vector v = alg.zerovec(weights.size()); + while (true) { + for (int i = 0; i < n_mini_batch; i++) { + weights = alg.subtraction(weights, alg.scalarMultiply(gamma, v)); // "Aposterori" calculation + + std::vector y_hat = Evaluate(inputMiniBatches[i]); + cost_prev = Cost(y_hat, outputMiniBatches[i]); + + std::vector error = alg.subtraction(y_hat, outputMiniBatches[i]); + + // Calculating the weight gradients + std::vector gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error)); + std::vector RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg); + std::vector weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final + + v = alg.addition(alg.scalarMultiply(gamma, v), alg.scalarMultiply(learning_rate, weight_grad)); + + weights = alg.subtraction(weights, v); + + // Calculating the bias gradients + bias -= learning_rate * alg.sum_elements(error) / outputMiniBatches[i].size(); // As normal + y_hat = Evaluate(inputMiniBatches[i]); + + if (UI) { + MLPPUtilities::CostInfo(epoch, cost_prev, Cost(y_hat, outputMiniBatches[i])); + MLPPUtilities::UI(weights, bias); + } + } + epoch++; + if (epoch > max_epoch) { + break; + } + } + forwardPass(); +} + +void MLPPLinReg::Adagrad(double learning_rate, int max_epoch, int mini_batch_size, double e, bool UI) { + MLPPLinAlg alg; + MLPPReg regularization; + double cost_prev = 0; + int epoch = 1; + + // Creating the mini-batches + int n_mini_batch = n / mini_batch_size; + auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch); + + // Initializing necessary components for Adagrad. + std::vector v = alg.zerovec(weights.size()); + while (true) { + for (int i = 0; i < n_mini_batch; i++) { + std::vector y_hat = Evaluate(inputMiniBatches[i]); + cost_prev = Cost(y_hat, outputMiniBatches[i]); + + std::vector error = alg.subtraction(y_hat, outputMiniBatches[i]); + + // Calculating the weight gradients + std::vector gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error)); + std::vector RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg); + std::vector weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final + + v = alg.hadamard_product(weight_grad, weight_grad); + + weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate, alg.elementWiseDivision(weight_grad, alg.sqrt(alg.scalarAdd(e, v))))); + + // Calculating the bias gradients + bias -= learning_rate * alg.sum_elements(error) / outputMiniBatches[i].size(); // As normal + y_hat = Evaluate(inputMiniBatches[i]); + + if (UI) { + MLPPUtilities::CostInfo(epoch, cost_prev, Cost(y_hat, outputMiniBatches[i])); + MLPPUtilities::UI(weights, bias); + } + } + epoch++; + if (epoch > max_epoch) { + break; + } + } + forwardPass(); +} + +void MLPPLinReg::Adadelta(double learning_rate, int max_epoch, int mini_batch_size, double b1, double e, bool UI) { + // Adagrad upgrade. Momentum is applied. + MLPPLinAlg alg; + MLPPReg regularization; + double cost_prev = 0; + int epoch = 1; + + // Creating the mini-batches + int n_mini_batch = n / mini_batch_size; + auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch); + + // Initializing necessary components for Adagrad. + std::vector v = alg.zerovec(weights.size()); + while (true) { + for (int i = 0; i < n_mini_batch; i++) { + std::vector y_hat = Evaluate(inputMiniBatches[i]); + cost_prev = Cost(y_hat, outputMiniBatches[i]); + + std::vector error = alg.subtraction(y_hat, outputMiniBatches[i]); + + // Calculating the weight gradients + std::vector gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error)); + std::vector RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg); + std::vector weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final + + v = alg.addition(alg.scalarMultiply(b1, v), alg.scalarMultiply(1 - b1, alg.hadamard_product(weight_grad, weight_grad))); + + weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate, alg.elementWiseDivision(weight_grad, alg.sqrt(alg.scalarAdd(e, v))))); + + // Calculating the bias gradients + bias -= learning_rate * alg.sum_elements(error) / outputMiniBatches[i].size(); // As normal + y_hat = Evaluate(inputMiniBatches[i]); + + if (UI) { + MLPPUtilities::CostInfo(epoch, cost_prev, Cost(y_hat, outputMiniBatches[i])); + MLPPUtilities::UI(weights, bias); + } + } + epoch++; + if (epoch > max_epoch) { + break; + } + } + forwardPass(); +} + +void MLPPLinReg::Adam(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI) { + MLPPLinAlg alg; + MLPPReg regularization; + double cost_prev = 0; + int epoch = 1; + + // Creating the mini-batches + int n_mini_batch = n / mini_batch_size; + auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch); + + // Initializing necessary components for Adam. + std::vector m = alg.zerovec(weights.size()); + + std::vector v = alg.zerovec(weights.size()); + while (true) { + for (int i = 0; i < n_mini_batch; i++) { + std::vector y_hat = Evaluate(inputMiniBatches[i]); + cost_prev = Cost(y_hat, outputMiniBatches[i]); + + std::vector error = alg.subtraction(y_hat, outputMiniBatches[i]); + + // Calculating the weight gradients + std::vector gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error)); + std::vector RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg); + std::vector weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final + + m = alg.addition(alg.scalarMultiply(b1, m), alg.scalarMultiply(1 - b1, weight_grad)); + v = alg.addition(alg.scalarMultiply(b2, v), alg.scalarMultiply(1 - b2, alg.exponentiate(weight_grad, 2))); + + std::vector m_hat = alg.scalarMultiply(1 / (1 - pow(b1, epoch)), m); + std::vector v_hat = alg.scalarMultiply(1 / (1 - pow(b2, epoch)), v); + + weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate, alg.elementWiseDivision(m_hat, alg.scalarAdd(e, alg.sqrt(v_hat))))); + + // Calculating the bias gradients + bias -= learning_rate * alg.sum_elements(error) / outputMiniBatches[i].size(); // As normal + y_hat = Evaluate(inputMiniBatches[i]); + + if (UI) { + MLPPUtilities::CostInfo(epoch, cost_prev, Cost(y_hat, outputMiniBatches[i])); + MLPPUtilities::UI(weights, bias); + } + } + epoch++; + if (epoch > max_epoch) { + break; + } + } + forwardPass(); +} + +void MLPPLinReg::Adamax(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI) { + MLPPLinAlg alg; + MLPPReg regularization; + double cost_prev = 0; + int epoch = 1; + + // Creating the mini-batches + int n_mini_batch = n / mini_batch_size; + auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch); + + std::vector m = alg.zerovec(weights.size()); + + std::vector u = alg.zerovec(weights.size()); + while (true) { + for (int i = 0; i < n_mini_batch; i++) { + std::vector y_hat = Evaluate(inputMiniBatches[i]); + cost_prev = Cost(y_hat, outputMiniBatches[i]); + + std::vector error = alg.subtraction(y_hat, outputMiniBatches[i]); + + // Calculating the weight gradients + std::vector gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error)); + std::vector RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg); + std::vector weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final + + m = alg.addition(alg.scalarMultiply(b1, m), alg.scalarMultiply(1 - b1, weight_grad)); + u = alg.max(alg.scalarMultiply(b2, u), alg.abs(weight_grad)); + + std::vector m_hat = alg.scalarMultiply(1 / (1 - pow(b1, epoch)), m); + + weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate, alg.elementWiseDivision(m_hat, u))); + + // Calculating the bias gradients + bias -= learning_rate * alg.sum_elements(error) / outputMiniBatches[i].size(); // As normal + y_hat = Evaluate(inputMiniBatches[i]); + + if (UI) { + MLPPUtilities::CostInfo(epoch, cost_prev, Cost(y_hat, outputMiniBatches[i])); + MLPPUtilities::UI(weights, bias); + } + } + epoch++; + if (epoch > max_epoch) { + break; + } + } + forwardPass(); +} + +void MLPPLinReg::Nadam(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI) { + MLPPLinAlg alg; + MLPPReg regularization; + double cost_prev = 0; + int epoch = 1; + + // Creating the mini-batches + int n_mini_batch = n / mini_batch_size; + auto [inputMiniBatches, outputMiniBatches] = MLPPUtilities::createMiniBatches(inputSet, outputSet, n_mini_batch); + + // Initializing necessary components for Adam. + std::vector m = alg.zerovec(weights.size()); + std::vector v = alg.zerovec(weights.size()); + std::vector m_final = alg.zerovec(weights.size()); + while (true) { + for (int i = 0; i < n_mini_batch; i++) { + std::vector y_hat = Evaluate(inputMiniBatches[i]); + cost_prev = Cost(y_hat, outputMiniBatches[i]); + + std::vector error = alg.subtraction(y_hat, outputMiniBatches[i]); + + // Calculating the weight gradients + std::vector gradient = alg.scalarMultiply(1 / outputMiniBatches[i].size(), alg.mat_vec_mult(alg.transpose(inputMiniBatches[i]), error)); + std::vector RegDerivTerm = regularization.regDerivTerm(weights, lambda, alpha, reg); + std::vector weight_grad = alg.addition(gradient, RegDerivTerm); // Weight_grad_final + + m = alg.addition(alg.scalarMultiply(b1, m), alg.scalarMultiply(1 - b1, weight_grad)); + v = alg.addition(alg.scalarMultiply(b2, v), alg.scalarMultiply(1 - b2, alg.exponentiate(weight_grad, 2))); + m_final = alg.addition(alg.scalarMultiply(b1, m), alg.scalarMultiply((1 - b1) / (1 - pow(b1, epoch)), weight_grad)); + + std::vector m_hat = alg.scalarMultiply(1 / (1 - pow(b1, epoch)), m); + std::vector v_hat = alg.scalarMultiply(1 / (1 - pow(b2, epoch)), v); + + weights = alg.subtraction(weights, alg.scalarMultiply(learning_rate, alg.elementWiseDivision(m_final, alg.scalarAdd(e, alg.sqrt(v_hat))))); + + // Calculating the bias gradients + bias -= learning_rate * alg.sum_elements(error) / outputMiniBatches[i].size(); // As normal + y_hat = Evaluate(inputMiniBatches[i]); + + if (UI) { + MLPPUtilities::CostInfo(epoch, cost_prev, Cost(y_hat, outputMiniBatches[i])); + MLPPUtilities::UI(weights, bias); + } + } + epoch++; + if (epoch > max_epoch) { + break; + } + } + forwardPass(); +} + void MLPPLinReg::normalEquation() { MLPPLinAlg alg; - MLPPStat stat; + MLPPStat stat; std::vector x_means; std::vector> inputSetT = alg.transpose(inputSet); @@ -185,35 +518,37 @@ void MLPPLinReg::normalEquation() { } //try { - std::vector temp; - temp.resize(k); - temp = alg.mat_vec_mult(alg.inverse(alg.matmult(alg.transpose(inputSet), inputSet)), alg.mat_vec_mult(alg.transpose(inputSet), outputSet)); - if (std::isnan(temp[0])) { - //throw 99; - //TODO ERR_FAIL_COND + std::vector temp; + temp.resize(k); + temp = alg.mat_vec_mult(alg.inverse(alg.matmult(alg.transpose(inputSet), inputSet)), alg.mat_vec_mult(alg.transpose(inputSet), outputSet)); + if (std::isnan(temp[0])) { + //throw 99; + //TODO ERR_FAIL_COND + std::cout << "ERR: Resulting matrix was noninvertible/degenerate, and so the normal equation could not be performed. Try utilizing gradient descent." << std::endl; + return; + } else { + if (reg == "Ridge") { + weights = alg.mat_vec_mult(alg.inverse(alg.addition(alg.matmult(alg.transpose(inputSet), inputSet), alg.scalarMultiply(lambda, alg.identity(k)))), alg.mat_vec_mult(alg.transpose(inputSet), outputSet)); } else { - if (reg == "Ridge") { - weights = alg.mat_vec_mult(alg.inverse(alg.addition(alg.matmult(alg.transpose(inputSet), inputSet), alg.scalarMultiply(lambda, alg.identity(k)))), alg.mat_vec_mult(alg.transpose(inputSet), outputSet)); - } else { - weights = alg.mat_vec_mult(alg.inverse(alg.matmult(alg.transpose(inputSet), inputSet)), alg.mat_vec_mult(alg.transpose(inputSet), outputSet)); - } - - bias = stat.mean(outputSet) - alg.dot(weights, x_means); - - forwardPass(); + weights = alg.mat_vec_mult(alg.inverse(alg.matmult(alg.transpose(inputSet), inputSet)), alg.mat_vec_mult(alg.transpose(inputSet), outputSet)); } + + bias = stat.mean(outputSet) - alg.dot(weights, x_means); + + forwardPass(); + } //} catch (int err_num) { // std::cout << "ERR " << err_num << ": Resulting matrix was noninvertible/degenerate, and so the normal equation could not be performed. Try utilizing gradient descent." << std::endl; //} } double MLPPLinReg::score() { - MLPPUtilities util; + MLPPUtilities util; return util.performance(y_hat, outputSet); } void MLPPLinReg::save(std::string fileName) { - MLPPUtilities util; + MLPPUtilities util; util.saveParameters(fileName, weights, bias); } diff --git a/mlpp/lin_reg/lin_reg.h b/mlpp/lin_reg/lin_reg.h index af45fe4..2882fe7 100644 --- a/mlpp/lin_reg/lin_reg.h +++ b/mlpp/lin_reg/lin_reg.h @@ -11,7 +11,6 @@ #include #include - class MLPPLinReg { public: MLPPLinReg(std::vector> inputSet, std::vector outputSet, std::string reg = "None", double lambda = 0.5, double alpha = 0.5); @@ -20,6 +19,15 @@ public: void NewtonRaphson(double learning_rate, int max_epoch, bool UI); void gradientDescent(double learning_rate, int max_epoch, bool UI = 1); void SGD(double learning_rate, int max_epoch, bool UI = 1); + + void Momentum(double learning_rate, int max_epoch, int mini_batch_size, double gamma, bool UI = 1); + void NAG(double learning_rate, int max_epoch, int mini_batch_size, double gamma, bool UI = 1); + void Adagrad(double learning_rate, int max_epoch, int mini_batch_size, double e, bool UI = 1); + void Adadelta(double learning_rate, int max_epoch, int mini_batch_size, double b1, double e, bool UI = 1); + void Adam(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI = 1); + void Adamax(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI = 1); + void Nadam(double learning_rate, int max_epoch, int mini_batch_size, double b1, double b2, double e, bool UI = 1); + void MBGD(double learning_rate, int max_epoch, int mini_batch_size, bool UI = 1); void normalEquation(); double score(); @@ -47,5 +55,4 @@ private: int alpha; /* This is the controlling param for Elastic Net*/ }; - #endif /* LinReg_hpp */