data.h

 
#ifndef STOCHTREE_DATA_H_
#define STOCHTREE_DATA_H_
 
#include <Eigen/Dense>
#include <stochtree/io.h>
#include <stochtree/log.h>
#include <stochtree/meta.h>
 
namespace StochTree {
 
static inline void ExtractMultipleFeaturesFromMemory(std::vector<std::string>* text_data, const Parser* parser,
                                                     std::vector<int>& column_indices, Eigen::MatrixXd& data,
                                                     data_size_t num_rows) {
  std::vector<std::pair<int, double>> oneline_features;
  auto& ref_text_data = *text_data;
  int feature_counter;
  bool column_matched;
  for (data_size_t i = 0; i < num_rows; ++i) {
    // unpack the vector of textlines read from file into a vector of (int, double) tuples
    oneline_features.clear();
    parser->ParseOneLine(ref_text_data[i].c_str(), &oneline_features);
 
    // free processed line:
    ref_text_data[i].clear();
 
    // unload the data from oneline_features vector into the dataset variables containers
    int feature_counter = 0;
    for (auto& inner_data : oneline_features) {
      int feature_idx = inner_data.first;
      column_matched = (std::find(column_indices.begin(), column_indices.end(), feature_idx) != column_indices.end());
      if (column_matched) {
        data(i, feature_counter) = inner_data.second;
        feature_counter += 1;
      }
    }
  }
  // free text data after use
  text_data->clear();
}
 
static inline void ExtractSingleFeatureFromMemory(std::vector<std::string>* text_data, const Parser* parser,
                                                  int column_index, Eigen::VectorXd& data, data_size_t num_rows) {
  std::vector<std::pair<int, double>> oneline_features;
  auto& ref_text_data = *text_data;
  bool column_matched;
  for (data_size_t i = 0; i < num_rows; ++i) {
    // unpack the vector of textlines read from file into a vector of (int, double) tuples
    oneline_features.clear();
    parser->ParseOneLine(ref_text_data[i].c_str(), &oneline_features);
 
    // free processed line:
    ref_text_data[i].clear();
 
    // unload the data from oneline_features vector into the dataset variables containers
    for (auto& inner_data : oneline_features) {
      int feature_idx = inner_data.first;
      if (column_index == feature_idx) {
        data(i) = inner_data.second;
      }
    }
  }
  // free text data after use
  text_data->clear();
}
 
static inline std::vector<std::string> LoadTextDataToMemory(const char* filename, int* num_global_data, bool header) {
  size_t file_load_progress_interval_bytes = size_t(10) * 1024 * 1024 * 1024;
  TextReader<data_size_t> text_reader(filename, header, file_load_progress_interval_bytes);
  // read all lines
  *num_global_data = text_reader.ReadAllLines();
  return std::move(text_reader.Lines());
}
 
static inline void FeatureUnpack(std::vector<int>& categorical_variables, const char* var_id) {
  std::string var_clean = Common::RemoveQuotationSymbol(Common::Trim(var_id));
  int out;
  bool success = Common::AtoiAndCheck(var_clean.c_str(), &out);
  if (success) {
    categorical_variables.push_back(out);
  } else {
    Log::Warning("Parsed variable index %s cannot be cast to an integer", var_clean.c_str());
  }
}
 
static inline std::vector<int> Str2FeatureVec(const char* parameters) {
  std::vector<int> feature_vec;
  auto args = Common::Split(parameters, ",");
  for (const auto& arg : args) {
    FeatureUnpack(feature_vec, Common::Trim(arg).c_str());
  }
  return feature_vec;
}
 
class ColumnMatrix {
 public:
  ColumnMatrix() {}
  ColumnMatrix(double* data_ptr, data_size_t num_row, int num_col, bool is_row_major);
  ColumnMatrix(std::string filename, std::string column_index_string, bool header = true, bool precise_float_parser = false);
  ~ColumnMatrix() {}
  double GetElement(data_size_t row_num, int col_num) { return data_(row_num, col_num); }
  void SetElement(data_size_t row_num, int col_num, double value) { data_(row_num, col_num) = value; }
  void LoadData(double* data_ptr, data_size_t num_row, int num_col, bool is_row_major);
  inline data_size_t NumRows() { return data_.rows(); }
  inline int NumCols() { return data_.cols(); }
  inline Eigen::MatrixXd& GetData() { return data_; }
 
 private:
  Eigen::MatrixXd data_;
};
 
class ColumnVector {
 public:
  ColumnVector() {}
  ColumnVector(double* data_ptr, data_size_t num_row);
  ColumnVector(std::string filename, int column_index, bool header = true, bool precise_float_parser = false);
  ~ColumnVector() {}
  double GetElement(data_size_t row) { return data_(row); }
  void SetElement(data_size_t row, double value) { data_(row) = value; }
  void LoadData(double* data_ptr, data_size_t num_row);
  void AddToData(double* data_ptr, data_size_t num_row);
  void SubtractFromData(double* data_ptr, data_size_t num_row);
  void OverwriteData(double* data_ptr, data_size_t num_row);
  inline data_size_t NumRows() { return data_.size(); }
  inline Eigen::VectorXd& GetData() { return data_; }
 
 private:
  Eigen::VectorXd data_;
  void UpdateData(double* data_ptr, data_size_t num_row, std::function<double(double, double)> op);
};
 
class ForestDataset {
 public:
  ForestDataset() {}
  ~ForestDataset() {}
  void AddCovariates(double* data_ptr, data_size_t num_row, int num_col, bool is_row_major) {
    covariates_ = ColumnMatrix(data_ptr, num_row, num_col, is_row_major);
    num_observations_ = num_row;
    num_covariates_ = num_col;
    has_covariates_ = true;
  }
  void AddBasis(double* data_ptr, data_size_t num_row, int num_col, bool is_row_major) {
    basis_ = ColumnMatrix(data_ptr, num_row, num_col, is_row_major);
    num_basis_ = num_col;
    has_basis_ = true;
  }
  void AddVarianceWeights(double* data_ptr, data_size_t num_row) {
    var_weights_ = ColumnVector(data_ptr, num_row);
    has_var_weights_ = true;
  }
  void AddCovariatesFromCSV(std::string filename, std::string column_index_string, bool header = true, bool precise_float_parser = false) {
    covariates_ = ColumnMatrix(filename, column_index_string, header, precise_float_parser);
    num_observations_ = covariates_.NumRows();
    num_covariates_ = covariates_.NumCols();
    has_covariates_ = true;
  }
  void AddBasisFromCSV(std::string filename, std::string column_index_string, bool header = true, bool precise_float_parser = false) {
    basis_ = ColumnMatrix(filename, column_index_string, header, precise_float_parser);
    num_basis_ = basis_.NumCols();
    has_basis_ = true;
  }
  void AddVarianceWeightsFromCSV(std::string filename, int column_index, bool header = true, bool precise_float_parser = false) {
    var_weights_ = ColumnVector(filename, column_index, header, precise_float_parser);
    has_var_weights_ = true;
  }
  inline bool HasCovariates() { return has_covariates_; }
  inline bool HasBasis() { return has_basis_; }
  inline bool HasVarWeights() { return has_var_weights_; }
  inline data_size_t NumObservations() { return num_observations_; }
  inline int NumCovariates() { return num_covariates_; }
  inline int NumBasis() { return num_basis_; }
  inline double CovariateValue(data_size_t row, int col) { return covariates_.GetElement(row, col); }
  inline double BasisValue(data_size_t row, int col) { return basis_.GetElement(row, col); }
  inline double VarWeightValue(data_size_t row) { return var_weights_.GetElement(row); }
  inline Eigen::MatrixXd& GetCovariates() { return covariates_.GetData(); }
  inline Eigen::MatrixXd& GetBasis() { return basis_.GetData(); }
  inline Eigen::VectorXd& GetVarWeights() { return var_weights_.GetData(); }
  void UpdateBasis(double* data_ptr, data_size_t num_row, int num_col, bool is_row_major) {
    CHECK(has_basis_);
    CHECK_EQ(num_col, num_basis_);
    // Copy data from R / Python process memory to Eigen matrix
    double temp_value;
    for (data_size_t i = 0; i < num_row; ++i) {
      for (int j = 0; j < num_col; ++j) {
        if (is_row_major) {
          // Numpy 2-d arrays are stored in "row major" order
          temp_value = static_cast<double>(*(data_ptr + static_cast<data_size_t>(num_col) * i + j));
        } else {
          // R matrices are stored in "column major" order
          temp_value = static_cast<double>(*(data_ptr + static_cast<data_size_t>(num_row) * j + i));
        }
        basis_.SetElement(i, j, temp_value);
      }
    }
  }
  void UpdateVarWeights(double* data_ptr, data_size_t num_row, bool exponentiate = true) {
    CHECK(has_var_weights_);
    // Copy data from R / Python process memory to Eigen vector
    double temp_value;
    for (data_size_t i = 0; i < num_row; ++i) {
      if (exponentiate)
        temp_value = std::exp(static_cast<double>(*(data_ptr + i)));
      else
        temp_value = static_cast<double>(*(data_ptr + i));
      var_weights_.SetElement(i, temp_value);
    }
  }
  void SetCovariateValue(data_size_t row_id, int col, double new_value) {
    covariates_.SetElement(row_id, col, new_value);
  }
  void SetBasisValue(data_size_t row_id, int col, double new_value) {
    CHECK(has_basis_);
    basis_.SetElement(row_id, col, new_value);
  }
  void SetVarWeightValue(data_size_t row_id, double new_value, bool exponentiate = true) {
    CHECK(has_var_weights_);
    if (exponentiate)
      var_weights_.SetElement(row_id, std::exp(new_value));
    else
      var_weights_.SetElement(row_id, new_value);
  }
  void AddAuxiliaryDimension(int dim_size) {
    if (!has_auxiliary_data_) has_auxiliary_data_ = true;
    auxiliary_data_.resize(num_auxiliary_dims_ + 1);
    auxiliary_data_[num_auxiliary_dims_].resize(dim_size);
    num_auxiliary_dims_++;
  }
  double GetAuxiliaryDataValue(int dim_idx, data_size_t element_idx) {
    return auxiliary_data_[dim_idx][element_idx];
  }
  void SetAuxiliaryDataValue(int dim_idx, data_size_t element_idx, double value) {
    auxiliary_data_[dim_idx][element_idx] = value;
  }
  std::vector<double>& GetAuxiliaryDataVector(int dim_idx) {
    return auxiliary_data_[dim_idx];
  }
  const std::vector<double>& GetAuxiliaryDataVectorConst(int dim_idx) {
    return auxiliary_data_[dim_idx];
  }
  bool HasAuxiliaryDimension(int dim_idx) {
    return (num_auxiliary_dims_ > dim_idx) & (dim_idx >= 0);
  }
 
 private:
  ColumnMatrix covariates_;
  ColumnMatrix basis_;
  ColumnVector var_weights_;
  data_size_t num_observations_{0};
  int num_covariates_{0};
  int num_basis_{0};
  bool has_covariates_{false};
  bool has_basis_{false};
  bool has_var_weights_{false};
 
  std::vector<std::vector<double>> auxiliary_data_;
  int num_auxiliary_dims_{0};
  bool has_auxiliary_data_{false};
};
 
class RandomEffectsDataset {
 public:
  RandomEffectsDataset() {}
  ~RandomEffectsDataset() {}
  void AddBasis(double* data_ptr, data_size_t num_row, int num_col, bool is_row_major) {
    basis_ = ColumnMatrix(data_ptr, num_row, num_col, is_row_major);
    num_basis_ = num_col;
    has_basis_ = true;
  }
  void AddVarianceWeights(double* data_ptr, data_size_t num_row) {
    var_weights_ = ColumnVector(data_ptr, num_row);
    has_var_weights_ = true;
  }
  void UpdateBasis(double* data_ptr, data_size_t num_row, int num_col, bool is_row_major) {
    CHECK(has_basis_);
    CHECK_EQ(num_col, num_basis_);
    // Copy data from R / Python process memory to Eigen matrix
    double temp_value;
    for (data_size_t i = 0; i < num_row; ++i) {
      for (int j = 0; j < num_col; ++j) {
        if (is_row_major) {
          // Numpy 2-d arrays are stored in "row major" order
          temp_value = static_cast<double>(*(data_ptr + static_cast<data_size_t>(num_col) * i + j));
        } else {
          // R matrices are stored in "column major" order
          temp_value = static_cast<double>(*(data_ptr + static_cast<data_size_t>(num_row) * j + i));
        }
        basis_.SetElement(i, j, temp_value);
      }
    }
  }
  void UpdateVarWeights(double* data_ptr, data_size_t num_row, bool exponentiate = true) {
    CHECK(has_var_weights_);
    // Copy data from R / Python process memory to Eigen vector
    double temp_value;
    for (data_size_t i = 0; i < num_row; ++i) {
      if (exponentiate)
        temp_value = std::exp(static_cast<double>(*(data_ptr + i)));
      else
        temp_value = static_cast<double>(*(data_ptr + i));
      var_weights_.SetElement(i, temp_value);
    }
  }
  void UpdateGroupLabels(std::vector<int>& group_labels, data_size_t num_row) {
    CHECK(has_group_labels_);
    CHECK_EQ(this->NumObservations(), num_row)
    // Copy data from R / Python process memory to internal vector
    for (data_size_t i = 0; i < num_row; ++i) {
      group_labels_[i] = group_labels[i];
    }
  }
  void AddGroupLabels(std::vector<int>& group_labels) {
    group_labels_ = group_labels;
    has_group_labels_ = true;
  }
  void AddGroupLabels(int* group_labels, int num_rows) {
    group_labels_.resize(num_rows);
    for (int i = 0; i < num_rows; ++i) {
      group_labels_[i] = group_labels[i];
    }
    has_group_labels_ = true;
  }
  inline data_size_t NumObservations() { return basis_.NumRows(); }
  inline int NumBases() { return basis_.NumCols(); }
  inline bool HasBasis() { return has_basis_; }
  inline bool HasVarWeights() { return has_var_weights_; }
  inline bool HasGroupLabels() { return has_group_labels_; }
  inline double BasisValue(data_size_t row, int col) { return basis_.GetElement(row, col); }
  inline double VarWeightValue(data_size_t row) { return var_weights_.GetElement(row); }
  inline int GroupId(data_size_t row) { return group_labels_[row]; }
  inline Eigen::MatrixXd& GetBasis() { return basis_.GetData(); }
  inline Eigen::VectorXd& GetVarWeights() { return var_weights_.GetData(); }
  inline std::vector<int>& GetGroupLabels() { return group_labels_; }
 
 private:
  ColumnMatrix basis_;
  ColumnVector var_weights_;
  std::vector<int> group_labels_;
  int num_basis_{0};
  bool has_basis_{false};
  bool has_var_weights_{false};
  bool has_group_labels_{false};
};
 
 
}  // namespace StochTree
 
#endif  // STOCHTREE_DATA_H_