csv.h

Go to the documentation of this file.

 
 
#ifndef THEORETICA_IO_CSV_H
#define THEORETICA_IO_CSV_H
 
#include <fstream>
#include <iomanip>
#include <algorithm>
 
#include "./error.h"
#include "../algebra/vec.h"
#include "../algebra/mat.h"
#include "../statistics/histogram.h"
#include "./data_table.h"
#include "./strings.h"
 
 
namespace theoretica {
namespace io {
 
 
    inline std::vector<std::string> parse_csv(const std::string& line, char delimiter = ',') {
 
        std::vector<std::string> fields;
        std::string field;
        bool quoted = false;
 
        for (size_t i = 0; i < line.length(); ++i) {
 
            char c = line[i];
 
            if (c == '"') {
                quoted = !quoted;
            } else if (c == delimiter && !quoted) {
                fields.emplace_back(field);
                field.clear();
            } else if(!std::isspace(c) || quoted) {
                field += c;
            }
        }
 
        fields.emplace_back(field);
        return fields;
    }
 
 
    inline std::string quote_csv(const std::string& str) {
 
        bool has_whitespace = false;
        for (char c : str) {
 
            if (std::isspace(c)) {
                has_whitespace = true;
                break;
            }
        }
 
        if (str.find(',') != std::string::npos || has_whitespace)
            return "\"" + str + "\"";
        else
            return str;
    }
 
    
    template<typename Type, unsigned int N>
    inline void write_csv(
        const std::string& filename, const vec<Type, N>& v, unsigned int precision = 8) {
 
        std::ofstream file (filename);
        if (!file.is_open()) {
            TH_IO_ERROR("io::write_csv", filename, IoError::FileNotFound);
            return;
        }
 
        for (size_t i = 0; i < v.size(); ++i)
            file << std::setprecision(precision) << v[i] << std::endl;
    }
 
 
    template<typename Type, unsigned int N>
    inline void write_csv(
        const std::string& filename, const std::string& header,
        const vec<Type, N>& v, unsigned int precision = 8) {
 
        std::ofstream file (filename);
        if (!file.is_open()) {
            // TODO: throw another exception ?
            TH_IO_ERROR("io::write_csv", filename, IoError::FileNotFound);
            return;
        }
 
        file << "\"" << header << "\"" << std::endl;
        for (size_t i = 0; i < v.size(); ++i)
            file << std::setprecision(precision) << v[i] << std::endl;
    }
 
 
    template<typename Type, unsigned int N, enable_real<Type> = true>
    inline void read_csv(const std::string& filename, vec<Type, N>& v) {
 
        std::ifstream file (filename);
        std::string line;
 
        if (!file.is_open()) {
            TH_IO_ERROR("io::read_csv", filename, IoError::ReadError);
            return;
        }
 
        // Check for header
        if (!std::getline(file, line))
            return;
 
        line = io::unquote(io::trim(line));
 
        // Resulting column vector
        std::vector<real> col;
 
        if (io::is_number(line)) {
 
            real first;
 
            try {
                std::replace(line.begin(), line.end(), ',', '.');
                first = std::stod(line);
                col.emplace_back(first);
            } catch (const std::invalid_argument& e) {
                // Do nothing, the entry is not a number
            } catch(const std::out_of_range& e) {
                TH_MATH_ERROR("io::read_csv", first, MathError::OutOfRange);
            }
        }
 
        // All remaining lines are data
        while (std::getline(file, line)) {
 
            line = io::unquote(io::trim(line));
            std::replace(line.begin(), line.end(), ',', '.');
 
            try {
                real val = std::stod(line);
                col.emplace_back(val);
            } catch (const std::exception& e) {
                col.emplace_back(nan());
            }
        }
 
        // Handle mismatched sizes with empty values (NaN)
        if (v.size() > col.size()) {
 
            for (size_t i = 0; i < col.size(); i++)
                v[i] = col[i];
 
            for (size_t i = col.size(); i < v.size(); i++)
                v[i] = nan();
 
        } else {
            algebra::vec_copy(v, col);
        }
    }
 
 
    template<typename Type, unsigned int N, enable_real<Type> = true>
    inline void read_csv(
        const std::string& filename, const std::string& col_name,
        vec<Type, N>& v, bool trim_nan = true) {
 
        std::ifstream file (filename);
        std::string line;
 
        if (!file.is_open()) {
            TH_IO_ERROR("io::read_csv", filename, IoError::ReadError);
            algebra::vec_error(v);
            return;
        }
 
        // Read header
        if (!std::getline(file, line))
            return;
 
        // Find the index of the specified column
        std::vector<std::string> headers = parse_csv(line);
        int col_index = -1;
        for (size_t i = 0; i < headers.size(); ++i) {
 
            if (headers[i] == col_name) {
                col_index = i;
                break;
            }
        }
 
        // No column was found
        if (col_index == -1) {
 
            TH_IO_ERROR("io::read_csv", filename, IoError::FormatError);
 
            if (!v.size())
                v.resize(1);
 
            for (size_t i = 0; i < v.size(); ++i)
                v[i] = nan();
 
            return;
        }
 
        // Read data from the specified column
        std::vector<real> data;
        std::vector<std::string> cells;
 
        while (std::getline(file, line)) {
 
            cells = parse_csv(line);
 
            if (size_t(col_index) < cells.size()) {
 
                std::string cell = cells[col_index];
                std::replace(cell.begin(), cell.end(), ',', '.');
 
                try {
                    const real val = std::stod(cell);
                    data.emplace_back(val);
                } catch (const std::exception& e) {
                    data.emplace_back(nan());
                }
            } else {
                data.emplace_back(nan());
            }
        }
 
        // Trim trailing NaN values if enabled
        size_t actual_size = data.size();
 
        if (trim_nan)
            while (actual_size > 0 && is_nan(data[actual_size - 1]))
                actual_size--;
 
        // Need to allocate space
        if (v.size() < actual_size) {
 
            v.resize(actual_size);
 
            if (v.size() < actual_size) {
                TH_IO_ERROR("io::read_csv", filename, IoError::FormatError);
                algebra::vec_error(v);
                return;
            }
        }
 
        for (size_t i = 0; i < actual_size; i++)
            v[i] = data[i];
 
        for (size_t i = actual_size; i < v.size(); i++)
            v[i] = nan();
    }
 
 
    template<typename Type, unsigned int N, unsigned int M>
    inline void write_csv(
        const std::string& filename, const mat<Type, N, M>& A,
        const std::string& delimiter = ", ", unsigned int precision = 8) {
 
        std::ofstream file (filename);
 
        if (!file.is_open()) {
            TH_IO_ERROR("io::write_csv", filename, IoError::FileNotFound);
            return;
        }
 
        for (size_t i = 0; i < A.rows(); i++) {
            for (size_t j = 0; j < A.cols(); j++) {
                
                file << std::setprecision(precision) << A(i, j);
 
                if (j != A.cols() - 1)
                    file << delimiter;
                else
                    file << std::endl;
            }
        }
    }
 
 
    template<unsigned int N, unsigned int K>
    inline void read_csv(const std::string& filename, mat<real, N, K>& A) {
 
        std::ifstream file (filename);
        std::string line;
 
        if (!file.is_open()) {
            TH_IO_ERROR("io::read_csv", filename, IoError::FileNotFound);
            return;
        }
 
        std::vector<std::vector<real>> rows;
 
        // Read first line to check for header
        if (!std::getline(file, line))
            return;
 
        std::vector<std::string> first_row = parse_csv(line);
 
        // Check if first line is a header
        bool has_header = false;
 
        for (const auto& cell : first_row) {
 
            if (!io::is_number(cell)) {
                has_header = true;
                break;
            }
        }
 
        // If first line is not a header, process it as data
        if (!has_header) {
 
            std::vector<real> row;
 
            for (auto cell : first_row) {
 
                std::replace(cell.begin(), cell.end(), ',', '.');
 
                try {
                    row.emplace_back(std::stod(cell));
                } catch (const std::exception& e) {
                    row.emplace_back(nan());
                }
            }
 
            if (!row.empty())
                rows.emplace_back(row);
        }
 
        // Read remaining lines
        while (std::getline(file, line)) {
 
            // Skip empty lines
            if (line.empty())
                continue;
 
            std::vector<std::string> cells = parse_csv(line);
            std::vector<real> row;
 
            for (auto cell : cells) {
 
                std::replace(cell.begin(), cell.end(), ',', '.');
 
                try {
                    row.emplace_back(std::stod(cell));
                } catch (const std::exception& e) {
                    row.emplace_back(nan());
                }
            }
 
            if (!row.empty()) {
                rows.emplace_back(row);
            }
        }
 
        A.resize(rows.size(), rows[0].size());
 
        if (A.rows() < rows[0].size() || A.cols() < rows.size()) {
            TH_IO_ERROR("io::read_csv", filename, IoError::FormatError);
            algebra::mat_error(A);
            return;
        }
 
        // Fill matrix with parsed data
        for (size_t i = 0; i < min(rows.size(), A.rows()); ++i) {
 
            for (size_t j = 0; j < min(rows[i].size(), A.cols()); ++j)
                A(i, j) = rows[i][j];
 
            // Pad remaining columns with NaN
            for (size_t j = rows[i].size(); j < A.cols(); ++j)
                A(i, j) = nan();
        }
 
        // Pad remaining rows with NaN
        for (size_t i = rows.size(); i < A.rows(); ++i)
            for (size_t j = 0; j < A.cols(); ++j)
                A(i, j) = nan();
    }
 
 
    inline void write_csv(
        const std::string& filename, const data_table& table,
        const std::string& delimiter = ", ", unsigned int precision = 8) {
 
        std::ofstream file (filename);
 
        if (!file.is_open()) {
            TH_IO_ERROR("io::write_csv", filename, IoError::FileNotFound);
            return;
        }
 
        // Write header
        bool first = true;
        for (const std::string& name : table.header()) {
 
            if (!first)
                file << delimiter;
 
            file << quote_csv(name);
            first = false;
        }
        file << std::endl;
 
        // Write data rows
        size_t max_rows = table.rows();
        for (size_t i = 0; i < max_rows; ++i) {
 
            first = true;
            for (const auto& col : table.data()) {
 
                if (!first)
                    file << delimiter;
 
                if (i < col.size())
                    file << std::setprecision(precision) << col[i];
                else
                    file << nan();
 
                first = false;
            }
            file << std::endl;
        }
 
    }
 
 
    inline void read_csv(const std::string& filename, data_table& table) {
 
        std::ifstream file (filename);
        std::string line;
 
        if (!file.is_open()) {
            TH_IO_ERROR("io::read_csv", filename, IoError::FileNotFound);
            return;
        }
 
        // Read header
        if (!std::getline(file, line))
            return;
 
        std::vector<std::string> first_row = parse_csv(line);
        
        if (first_row.empty())
            return;
        
        std::vector<std::string> column_names;
        size_t num_cols = first_row.size();
        std::vector<vec<real>> columns (num_cols);
 
        // Check if first line is a header
        bool has_header = false;
 
        for (const auto& cell : first_row) {
 
            if (!io::is_number(cell)) {
                has_header = true;
                break;
            }
        }
 
        // If first line is not a header, process it as data
        if (!has_header) {
 
            for (size_t j = 0; j < first_row.size(); ++j) {
 
                std::string cell = first_row[j];
                std::replace(cell.begin(), cell.end(), ',', '.');
 
                try {
                    columns[j].append(std::stod(cell));
                } catch (const std::exception& e) {
                    columns[j].append(nan());
                }
            }
        
            // Generate default column names
            for (size_t j = 0; j < num_cols; ++j) {
                column_names.emplace_back("col" + std::to_string(j));
            }
        
        } else {
        
            for (const auto& name : first_row)
                column_names.emplace_back(io::unquote(io::trim(name)));
        }
        first_row.clear();
 
        // Read data rows
        while (std::getline(file, line)) {
 
            if (line.empty())
                continue;
 
            std::vector<std::string> cells = parse_csv(line);
 
            for (size_t j = 0; j < num_cols; ++j) {
 
                if (j < cells.size()) {
                    std::string cell = cells[j];
                    std::replace(cell.begin(), cell.end(), ',', '.');
 
                    try {
                        real val = std::stod(cell);
                        columns[j].append(val);
                    } catch (const std::exception& e) {
                        columns[j].append(nan());
                    }
                } else {
                    columns[j].append(nan());
                }
            }
        }
        
        for (size_t j = 0; j < num_cols; ++j)
            table.insert(column_names[j], columns[j]);
    }
 
 
    inline void write_csv(
        const std::string& filename, const histogram& hist,
        bool normalized = false, bool lower_extreme = false,
        const std::string& delimiter = ", ", unsigned int precision = 8) {
        
        std::ofstream file (filename);
        if (!file.is_open()) {
            TH_IO_ERROR("io::write_csv", filename, IoError::FileNotFound);
            return;
        }
 
        const auto bin_counts = hist.bins();
 
        // Can't write histogram without bins
        if (!bin_counts.size()) {
            TH_IO_ERROR("io::write_csv", filename, IoError::FormatError);
            return;
        }
 
        const real bin_dx = (hist.range()[1] - hist.range()[0]) / bin_counts.size();
        real norm_factor = normalized ? hist.number() * bin_dx : 1.0;
        if (abs(norm_factor) <= MACH_EPSILON)
            norm_factor = 1.0;
 
        // Keep track of the coordinate of the current bin, starting from the lowest bin edge or center.
        real bin_value = lower_extreme ? hist.range()[0] : (hist.range()[0] + 0.5 * bin_dx);
 
        // Write header with histogram statistics
        file << "bins, counts, number, average, tss, min, max" << std::endl;
        if (!bin_counts.size())
            return;
        
        file << std::setprecision(precision) << bin_value << delimiter;
        file << std::setprecision(precision) << (bin_counts[0] / norm_factor) << delimiter;
        file << hist.number() << delimiter;
        file << hist.mean() << delimiter;
        file << hist.tss() << delimiter;
        file << hist.min() << delimiter;
        file << hist.max() << std::endl;
 
        for (size_t i = 1; i < bin_counts.size(); i++) {
        
            bin_value += bin_dx;
            file << std::setprecision(precision) << bin_value << delimiter;
            file << std::setprecision(precision) << (bin_counts[i] / norm_factor) << std::endl;
        }
    }
 
 
    inline void read_csv(const std::string& filename, histogram& hist, bool lower_extreme = false) {
 
        std::ifstream file (filename);
        if (!file.is_open()) {
            TH_IO_ERROR("io::read_csv", filename, IoError::FileNotFound);
            return;
        }
 
        std::string line;
        std::vector<std::string> cells;
 
        // Read header
        std::getline(file, line);
        cells = parse_csv(line);
 
        int bin_index = -1, count_index = -1, number_index = -1, average_index = -1;
        int tss_index = -1, min_index = -1, max_index = -1;
 
        // Find column indices for the expected headers
        for (size_t i = 0; i < cells.size(); i++) {
 
            if (cells[i] == "bins") bin_index = i;
            else if (cells[i] == "counts") count_index = i;
            else if (cells[i] == "number") number_index = i;
            else if (cells[i] == "average") average_index = i;
            else if (cells[i] == "tss") tss_index = i;
            else if (cells[i] == "min") min_index = i;
            else if (cells[i] == "max") max_index = i;
        }
 
        if (bin_index == -1 || count_index == -1 || number_index == -1 ||
            average_index == -1 || tss_index == -1 || min_index == -1 || max_index == -1) {
 
            TH_IO_ERROR("io::read_csv", filename, IoError::FormatError);
            return;
        }
 
        // Find the maximum required column index
        int min_size = bin_index;
        if (min_size < count_index) min_size = count_index;
        if (min_size < number_index) min_size = number_index;
        if (min_size < average_index) min_size = average_index;
        if (min_size < tss_index) min_size = tss_index;
        if (min_size < min_index) min_size = min_index;
        if (min_size < max_index) min_size = max_index;
        min_size++;
 
        // Read first data line for statistics
        std::getline(file, line);
        cells = parse_csv(line);
 
        if (cells.size() < size_t(min_size)) {
            TH_IO_ERROR("io::read_csv", filename, IoError::FormatError);
            return;
        }
 
        vec<real> counts, bins;
        size_t N;
        real run_average, run_tss, value_min, value_max;
 
        try {
 
            counts = {std::stod(cells[count_index])};
            bins = {std::stod(cells[bin_index])};
 
            N = std::stod(cells[number_index]);
            run_average = std::stod(cells[average_index]);
            run_tss = std::stod(cells[tss_index]);
            value_min = std::stod(cells[min_index]);
            value_max = std::stod(cells[max_index]);
 
        } catch (const std::invalid_argument& e) {
            TH_IO_ERROR("io::read_csv", filename, IoError::FormatError);
            return;
        }
 
        int bins_max_index = (bin_index > count_index ? bin_index : count_index) + 1;
 
        // Read remaining data lines
        while (std::getline(file, line)) {
            
            cells = parse_csv(line);
 
            if (cells.size() < size_t(bins_max_index)) {
                TH_IO_ERROR("io::read_csv", filename, IoError::FormatError);
                return;
            }
 
            try {
                counts.append(cells[count_index] != "" ? std::stod(cells[count_index]) : nan());
                bins.append(cells[bin_index] != "" ? std::stod(cells[bin_index]) : nan());
            } catch (const std::exception& e) {
                TH_IO_ERROR("io::read_csv", filename, IoError::FormatError);
                return;
            }
            
        }
 
        real range_min;
        real range_max;
 
        bool is_normalized = false;
        for (size_t i = 0; i < counts.size(); i++) {
 
            // Check if any bin counts are not integers
            if (counts[i] != floor(counts[i])) {
                is_normalized = true;
                break;
            }
        }
 
        std::vector<unsigned int> bin_counts (counts.size());
        for (size_t i = 0; i < counts.size(); i++) {
            bin_counts[i] = (unsigned int) (is_normalized ? (counts[i] * N) : counts[i]);
        }
 
        const real bin_dx = bins.size() > 1 ? (bins[1] - bins[0]) : 0;
        range_min = lower_extreme ? bins[0] : (bins[0] - 0.5 * bin_dx);
        range_max = lower_extreme ? bins[bins.size() - 1] + bin_dx : bins[bins.size() - 1] + 0.5 * bin_dx;
 
        // Check constant bin spacing
        for (size_t i = 1; i < bins.size(); i++) {
            if (abs((bins[i] - bins[i - 1]) - bin_dx) > 1e-6) {
                TH_IO_ERROR("io::read_csv", filename, IoError::FormatError);
                return;
            }
        }
 
        hist.rebuild(
            bin_counts, vec2({range_min, range_max}),
            N, run_average, run_tss,
            value_min, value_max
        );
 
    }
 
    template<typename Type>
    inline Type read_csv(const std::string& filename) {
        Type A;
        read_csv(filename, A);
        return A;
    }
 
}}
 
#endif