pmlpp/mlpp/data/data.h

247 lines
8.2 KiB
C
Raw Normal View History

2023-01-24 18:57:18 +01:00
#ifndef MLPP_DATA_H
#define MLPP_DATA_H
//
// Data.hpp
// MLP
//
// Created by Marc Melikyan on 11/4/20.
//
2023-01-27 13:01:16 +01:00
#include "core/math/math_defs.h"
#include "core/string/ustring.h"
2023-01-26 14:52:49 +01:00
#include "core/variant/array.h"
#include "core/object/reference.h"
2023-02-08 01:26:37 +01:00
#include "../lin_alg/mlpp_matrix.h"
#include "../lin_alg/mlpp_vector.h"
#include <string>
2023-01-24 19:00:54 +01:00
#include <tuple>
#include <vector>
class MLPPDataESimple : public Reference {
GDCLASS(MLPPDataESimple, Reference);
public:
Ref<MLPPVector> get_input();
void set_input(const Ref<MLPPVector> &val);
Ref<MLPPVector> get_output();
void set_output(const Ref<MLPPVector> &val);
void instance_data();
protected:
static void _bind_methods();
Ref<MLPPVector> _input;
Ref<MLPPVector> _output;
};
class MLPPDataSimple : public Reference {
GDCLASS(MLPPDataSimple, Reference);
public:
Ref<MLPPMatrix> get_input();
void set_input(const Ref<MLPPMatrix> &val);
Ref<MLPPVector> get_output();
void set_output(const Ref<MLPPVector> &val);
void instance_data();
protected:
static void _bind_methods();
Ref<MLPPMatrix> _input;
Ref<MLPPVector> _output;
};
class MLPPDataComplex : public Reference {
GDCLASS(MLPPDataComplex, Reference);
public:
Ref<MLPPMatrix> get_input();
void set_input(const Ref<MLPPMatrix> &val);
Ref<MLPPMatrix> get_output();
void set_output(const Ref<MLPPMatrix> &val);
void instance_data();
protected:
static void _bind_methods();
Ref<MLPPMatrix> _input;
Ref<MLPPMatrix> _output;
};
class MLPPData : public Reference {
GDCLASS(MLPPData, Reference);
2023-01-24 19:20:18 +01:00
2023-01-24 19:00:54 +01:00
public:
// Load Datasets
Ref<MLPPDataSimple> load_breast_cancer(const String &path);
Ref<MLPPDataSimple> load_breast_cancer_svc(const String &path);
Ref<MLPPDataComplex> load_iris(const String &path);
Ref<MLPPDataComplex> load_wine(const String &path);
Ref<MLPPDataComplex> load_mnist_train(const String &path);
Ref<MLPPDataComplex> load_mnist_test(const String &path);
Ref<MLPPDataSimple> load_california_housing(const String &path);
Ref<MLPPDataESimple> load_fires_and_crime(const String &path);
void set_data_supervised(int k, const String &file_name, Ref<MLPPMatrix> input_set, Ref<MLPPVector> output_set);
void set_data_unsupervised(int k, const String &file_name, Ref<MLPPMatrix> input_set);
void set_data_simple(const String &file_name, Ref<MLPPVector> input_set, Ref<MLPPVector> output_set);
2023-01-26 14:52:49 +01:00
struct SplitComplexData {
Ref<MLPPDataComplex> train;
Ref<MLPPDataComplex> test;
};
SplitComplexData train_test_split(Ref<MLPPDataComplex> data, real_t test_size);
2023-01-27 13:01:16 +01:00
Array train_test_split_bind(const Ref<MLPPDataComplex> &data, real_t test_size);
2023-01-26 14:52:49 +01:00
2023-01-24 19:00:54 +01:00
// Load Datasets
2023-01-27 13:01:16 +01:00
std::tuple<std::vector<std::vector<real_t>>, std::vector<real_t>> loadBreastCancer();
std::tuple<std::vector<std::vector<real_t>>, std::vector<real_t>> loadBreastCancerSVC();
std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> loadIris();
std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> loadWine();
std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> loadMnistTrain();
std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> loadMnistTest();
std::tuple<std::vector<std::vector<real_t>>, std::vector<real_t>> loadCaliforniaHousing();
std::tuple<std::vector<real_t>, std::vector<real_t>> loadFiresAndCrime();
2023-01-24 19:00:54 +01:00
2023-01-27 13:01:16 +01:00
std::tuple<std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>, std::vector<std::vector<real_t>>> trainTestSplit(std::vector<std::vector<real_t>> inputSet, std::vector<std::vector<real_t>> outputSet, real_t testSize);
2023-01-24 19:00:54 +01:00
// Supervised
2023-01-27 13:01:16 +01:00
void setData(int k, std::string fileName, std::vector<std::vector<real_t>> &inputSet, std::vector<real_t> &outputSet);
void printData(std::vector<std::string> inputName, std::string outputName, std::vector<std::vector<real_t>> inputSet, std::vector<real_t> outputSet);
2023-01-24 19:00:54 +01:00
// Unsupervised
2023-01-27 13:01:16 +01:00
void setData(int k, std::string fileName, std::vector<std::vector<real_t>> &inputSet);
void printData(std::vector<std::string> inputName, std::vector<std::vector<real_t>> inputSet);
2023-01-24 19:00:54 +01:00
// Simple
2023-01-27 13:01:16 +01:00
void setData(std::string fileName, std::vector<real_t> &inputSet, std::vector<real_t> &outputSet);
void printData(std::string &inputName, std::string &outputName, std::vector<real_t> &inputSet, std::vector<real_t> &outputSet);
2023-01-24 19:00:54 +01:00
// Images
2023-01-27 13:01:16 +01:00
std::vector<std::vector<real_t>> rgb2gray(std::vector<std::vector<std::vector<real_t>>> input);
std::vector<std::vector<std::vector<real_t>>> rgb2ycbcr(std::vector<std::vector<std::vector<real_t>>> input);
std::vector<std::vector<std::vector<real_t>>> rgb2hsv(std::vector<std::vector<std::vector<real_t>>> input);
std::vector<std::vector<std::vector<real_t>>> rgb2xyz(std::vector<std::vector<std::vector<real_t>>> input);
std::vector<std::vector<std::vector<real_t>>> xyz2rgb(std::vector<std::vector<std::vector<real_t>>> input);
2023-01-24 19:00:54 +01:00
// Text-Based & NLP
std::string toLower(std::string text);
std::vector<char> split(std::string text);
std::vector<std::string> splitSentences(std::string data);
std::vector<std::string> removeSpaces(std::vector<std::string> data);
std::vector<std::string> removeNullByte(std::vector<std::string> data);
std::vector<std::string> segment(std::string text);
2023-01-27 13:01:16 +01:00
std::vector<real_t> tokenize(std::string text);
2023-01-24 19:00:54 +01:00
std::vector<std::string> removeStopWords(std::string text);
std::vector<std::string> removeStopWords(std::vector<std::string> segmented_data);
std::string stemming(std::string text);
2023-01-27 13:01:16 +01:00
std::vector<std::vector<real_t>> BOW(std::vector<std::string> sentences, std::string = "Default");
std::vector<std::vector<real_t>> TFIDF(std::vector<std::string> sentences);
2023-01-26 14:52:49 +01:00
2023-01-27 13:01:16 +01:00
std::tuple<std::vector<std::vector<real_t>>, std::vector<std::string>> word2Vec(std::vector<std::string> sentences, std::string type, int windowSize, int dimension, real_t learning_rate, int max_epoch);
2023-01-26 14:52:49 +01:00
struct WordsToVecResult {
2023-01-27 13:01:16 +01:00
std::vector<std::vector<real_t>> word_embeddings;
2023-01-26 14:52:49 +01:00
std::vector<std::string> word_list;
};
2023-01-27 13:01:16 +01:00
WordsToVecResult word_to_vec(std::vector<std::string> sentences, std::string type, int windowSize, int dimension, real_t learning_rate, int max_epoch);
2023-01-26 14:52:49 +01:00
2023-01-27 13:01:16 +01:00
std::vector<std::vector<real_t>> LSA(std::vector<std::string> sentences, int dim);
2023-01-24 19:00:54 +01:00
std::vector<std::string> createWordList(std::vector<std::string> sentences);
// Extra
void setInputNames(std::string fileName, std::vector<std::string> &inputNames);
2023-01-27 13:01:16 +01:00
std::vector<std::vector<real_t>> featureScaling(std::vector<std::vector<real_t>> X);
std::vector<std::vector<real_t>> meanNormalization(std::vector<std::vector<real_t>> X);
std::vector<std::vector<real_t>> meanCentering(std::vector<std::vector<real_t>> X);
std::vector<std::vector<real_t>> oneHotRep(std::vector<real_t> tempOutputSet, int n_class);
std::vector<real_t> reverseOneHot(std::vector<std::vector<real_t>> tempOutputSet);
2023-02-08 01:26:37 +01:00
Ref<MLPPMatrix> mean_centering(const Ref<MLPPMatrix> &X);
Ref<MLPPMatrix> one_hot_rep(const Ref<MLPPVector> &temp_output_set, int n_class);
2023-02-08 01:26:37 +01:00
2023-01-24 19:00:54 +01:00
template <class T>
std::vector<T> vecToSet(std::vector<T> inputSet) {
std::vector<T> setInputSet;
2023-02-12 18:03:17 +01:00
for (uint32_t i = 0; i < inputSet.size(); i++) {
2023-01-24 19:00:54 +01:00
bool new_element = true;
2023-02-12 18:03:17 +01:00
for (uint32_t j = 0; j < setInputSet.size(); j++) {
2023-01-24 19:00:54 +01:00
if (setInputSet[j] == inputSet[i]) {
new_element = false;
}
}
if (new_element) {
setInputSet.push_back(inputSet[i]);
}
}
return setInputSet;
}
template <class T>
Vector<T> vec_to_set(Vector<T> input_set) {
Vector<T> set_input_set;
for (int i = 0; i < input_set.size(); i++) {
bool new_element = true;
for (int j = 0; j < set_input_set.size(); j++) {
if (set_input_set[j] == input_set[i]) {
new_element = false;
}
}
if (new_element) {
set_input_set.push_back(input_set[i]);
}
}
return set_input_set;
}
Ref<MLPPVector> vec_to_setnv(const Ref<MLPPVector> &input_set) {
Vector<real_t> set_input_set;
for (int i = 0; i < input_set->size(); i++) {
bool new_element = true;
for (int j = 0; j < set_input_set.size(); j++) {
if (set_input_set[j] == input_set->element_get(i)) {
new_element = false;
}
}
if (new_element) {
set_input_set.push_back(input_set->element_get(i));
}
}
Ref<MLPPVector> ret;
ret.instance();
ret->set_from_vector(set_input_set);
return ret;
}
protected:
static void _bind_methods();
2023-01-24 19:00:54 +01:00
};
2023-01-24 19:20:18 +01:00
#endif /* Data_hpp */