benchmark.h

Go to the documentation of this file.

 
 
#ifndef CHEBYSHEV_BENCHMARK_H
#define CHEBYSHEV_BENCHMARK_H
 
#include <ctime>
#include <iostream>
#include <thread>
#include <future>
#include <memory>
 
#include "./core/random.h"
#include "./core/output.h"
#include "./benchmark/timer.h"
#include "./benchmark/generator.h"
#include "./benchmark/benchmark_structures.h"
 
 
namespace chebyshev {
 
 
namespace benchmark {
 
 
    template<typename InputType, typename Function>
    inline long double runtime(
        Function func, const std::vector<InputType>& input) {
 
        if (input.size() == 0)
            return 0.0;
 
        // Dummy variable
        __volatile__ auto c = func(input[0]);
 
        timer t = timer();
 
        for (unsigned int j = 0; j < input.size(); ++j)
            c += func(input[j]);
 
        return t();
    }
 
 
    struct benchmark_settings {
 
        std::string moduleName;
 
        unsigned int defaultIterations = CHEBYSHEV_BENCHMARK_ITER;
 
        unsigned int defaultRuns = CHEBYSHEV_BENCHMARK_RUNS;
 
        std::vector<std::string> outputFiles {};
 
        std::map<std::string, bool> pickedBenchmarks {};
 
        std::vector<std::string> benchmarkOutputFiles {};
 
        std::vector<std::string> benchmarkColumns = {
            "name", "averageRuntime", "stdevRuntime", "runsPerSecond"
        };
 
        bool multithreading = true;
        
    };
 
 
    class benchmark_context {
 
    private:
 
        std::map <std::string, std::vector<benchmark_result>> benchmarkResults {};
 
        std::mutex benchmarkMutex;
 
        std::vector<std::thread> benchmarkThreads {};
 
        bool wasTerminated {false};
 
    public:
 
        benchmark_settings settings;
 
        std::shared_ptr<output::output_context> output;
 
        std::shared_ptr<random::random_context> random;
 
 
        inline void setup(
            std::string moduleName,
            int argc = 0,
            const char** argv = nullptr) {
 
            // Initialize other modules
            settings = benchmark_settings();
            output = std::make_shared<output::output_context>();
            random = std::make_shared<random::random_context>();
 
            // Initialize list of picked tests
            if(argc && argv)
                for (int i = 1; i < argc; ++i)
                    settings.pickedBenchmarks[argv[i]] = true;
 
            output->info("Starting benchmarks of the " + moduleName + " module ...");
 
            settings.moduleName = moduleName;
            benchmarkResults.clear();
            wasTerminated = false;
        }
 
 
        inline void terminate(bool exit = false) {
 
            // Ensure that all benchmarks have been run
            this->wait_results();
 
            std::lock_guard<std::mutex> lock(benchmarkMutex);
 
            unsigned int totalBenchmarks = 0;
            unsigned int failedBenchmarks = 0;
 
            for (const auto& pair : benchmarkResults) {
                for (const auto& testCase : pair.second) {
                    totalBenchmarks++;
                    failedBenchmarks += testCase.failed ? 1 : 0;
                }
            }
 
 
            // Ensure that an output file is specified
            if(  output->settings.outputToFile &&
                !output->settings.outputFiles.size() &&
                !settings.benchmarkOutputFiles.size() &&
                !settings.outputFiles.size()) {
                
                settings.outputFiles = { settings.moduleName + "_results" };
            }
 
 
            // Print benchmark results
            std::vector<std::string> outputFiles;
            outputFiles  = settings.outputFiles;
            outputFiles.insert(
                outputFiles.end(),
                settings.benchmarkOutputFiles.begin(),
                settings.benchmarkOutputFiles.end()
            );
 
            output->print_results(
                benchmarkResults,
                settings.benchmarkColumns,
                outputFiles
            );
 
 
            // Print overall test results
            double percent = totalBenchmarks > 0 ? (failedBenchmarks / (double) totalBenchmarks) * 100 : 0;
            output->info("Finished benchmarks of the " + settings.moduleName);
            output->info(
                std::to_string(totalBenchmarks) + " total benchmarks, " +
                std::to_string(failedBenchmarks) + " failed (" + std::to_string(percent).substr(0, 4) + "%)"
            );
 
            if (totalBenchmarks == 0)
                output->warn("No benchmarks were executed!");
 
            if(exit) {
                output->terminate();
                std::exit(failedBenchmarks);
            }
 
            wasTerminated = true;
        }
 
 
        benchmark_context(
            std::string moduleName, int argc = 0,
            const char** argv = nullptr) {
 
            setup(moduleName, argc, argv);
        }
 
 
        ~benchmark_context() {
 
            if (!wasTerminated)
                terminate();
        }
 
 
        benchmark_context(const benchmark_context& other) {
 
            std::lock_guard<std::mutex> lock(benchmarkMutex);
            benchmarkResults = other.benchmarkResults;
            settings = other.settings;
            output = other.output;
            random = other.random;
        }
 
 
        inline benchmark_context& operator=(const benchmark_context& other) {
 
            std::lock_guard<std::mutex> lock(benchmarkMutex);
            benchmarkResults = other.benchmarkResults;
            settings = other.settings;
            output = other.output;
            random = other.random;
 
            return *this;
        }
 
 
        template<typename InputType = double, typename Function>
        inline void benchmark(
            const std::string& name,
            Function func,
            const std::vector<InputType>& input,
            unsigned int runs = 0,
            bool quiet = false) {
 
            if (runs == 0)
                runs = settings.defaultRuns;
 
            output->debug("Running benchmark: " + name);
 
            auto task = [this, name, func, input, runs, quiet]() {
 
                // Whether the benchmark failed because of an exception
                bool failed = false;
 
                // Running average
                prec_t averageRuntime;
 
                // Running total sum of squares
                prec_t sumSquares;
 
                // Total runtime
                prec_t totalRuntime;
 
                try {
 
                    // Use Welford's algorithm to compute
                    // the average and the variance
                    totalRuntime = runtime(func, input);
                    averageRuntime = totalRuntime / input.size();
                    sumSquares = 0.0;
 
                    for (unsigned int i = 1; i < runs; ++i) {
                        
                        // Compute the runtime for a single run
                        // and update the running estimates
                        const prec_t currentRun = runtime(func, input);
                        const prec_t currentAverage = currentRun / input.size();
                        totalRuntime += currentRun;
 
                        const prec_t tmp = averageRuntime;
                        averageRuntime = tmp + (currentAverage - tmp) / (i + 1);
                        sumSquares += (currentAverage - tmp)
                            * (currentAverage - averageRuntime);
                    }
 
                } catch(...) {
 
                    // Catch any exception and mark the benchmark as failed
                    failed = true;
                }
 
                benchmark_result res {};
                res.name = name;
                res.runs = runs;
                res.iterations = input.size();
                res.totalRuntime = totalRuntime;
                res.averageRuntime = averageRuntime;
                res.runsPerSecond = 1000.0 / res.averageRuntime;
                res.failed = failed;
                res.quiet = quiet;
 
                if (runs > 1)
                    res.stdevRuntime = std::sqrt(sumSquares / (runs - 1));
 
                std::lock_guard<std::mutex> lock(benchmarkMutex);
                benchmarkResults[name].push_back(res);
            };
 
            settings.multithreading ?
                benchmarkThreads.emplace_back(task) : task();
        }
 
 
        template <typename InputType = double, typename Function>
        inline void benchmark(
            const std::string& name,
            Function func,
            benchmark_options<InputType> opt) {
 
            if (opt.runs == 0)
                opt.runs = settings.defaultRuns;
 
            output->debug("Running benchmark: " + name);
 
            // Generate input set
            random::random_source rnd (opt.seed);
 
            if (opt.seed == 0)
                rnd = random->get_rnd();
 
            std::vector<InputType> input (opt.iterations);
 
            for (unsigned int i = 0; i < opt.iterations; ++i)
                input[i] = opt.inputGenerator(rnd);
 
            uint64_t seed = rnd.get_seed();
 
            // Package task for multi-threaded execution
            auto task = [this, input, name, func, opt, seed]() {
 
                // Whether the benchmark failed because of an exception
                bool failed = false;
 
                // Running average
                prec_t averageRuntime;
 
                // Running total sum of squares
                prec_t sumSquares;
 
                // Total runtime
                prec_t totalRuntime;
 
                try {
 
                    // Use Welford's algorithm to compute
                    // the average and the variance
                    totalRuntime = runtime(func, input);
                    averageRuntime = totalRuntime / input.size();
                    sumSquares = 0.0;
 
                    for (unsigned int i = 1; i < opt.runs; ++i) {
                        
                        // Compute the runtime for a single run
                        // and update the running estimates
                        const prec_t currentRun = runtime(func, input);
                        const prec_t currentAverage = currentRun / input.size();
                        totalRuntime += currentRun;
 
                        const prec_t tmp = averageRuntime;
                        averageRuntime = tmp + (currentAverage - tmp) / (i + 1);
                        sumSquares += (currentAverage - tmp)
                            * (currentAverage - averageRuntime);
                    }
 
                } catch(...) {
 
                    // Catch any exception and mark the benchmark as failed
                    failed = true;
                }
 
                benchmark_result res {};
                res.name = name;
                res.runs = opt.runs;
                res.iterations = input.size();
                res.totalRuntime = totalRuntime;
                res.averageRuntime = averageRuntime;
                res.runsPerSecond = 1000.0 / res.averageRuntime;
                res.failed = failed;
                res.quiet = opt.quiet;
                res.seed = seed;
 
                if (opt.runs > 1)
                    res.stdevRuntime = std::sqrt(sumSquares / (opt.runs - 1));
 
                std::lock_guard<std::mutex> lock(benchmarkMutex);
                benchmarkResults[name].push_back(res);
            };
 
            settings.multithreading ?
                benchmarkThreads.emplace_back(task) : task();
        }
 
 
        template<typename InputType = double, typename Function>
        inline void benchmark(
            const std::string& name,
            Function func,
            InputGenerator<InputType> inputGenerator,
            unsigned int runs = 0,
            unsigned int iterations = 0,
            bool quiet = false) {
 
            if (runs == 0)
                runs = settings.defaultRuns;
 
            if (iterations == 0)
                iterations = settings.defaultIterations;
 
            benchmark_options<InputType> opt;
            opt.runs = runs;
            opt.iterations = iterations;
            opt.inputGenerator = inputGenerator;
            opt.quiet = quiet;
 
            benchmark(name, func, opt);
        }
 
 
        inline void wait_results() {
 
            for (auto& t : benchmarkThreads)
                if (t.joinable())
                    t.join();
 
            benchmarkThreads.clear();
        }
 
 
        inline std::vector<benchmark_result> get_benchmark(const std::string& name) {
 
            this->wait_results();
            return benchmarkResults[name];
        }
 
 
        inline benchmark_result get_benchmark(const std::string& name, unsigned int i) {
            
            this->wait_results();
            return benchmarkResults[name].at(i);
        }
 
    };
 
 
    benchmark_context make_context(const std::string& moduleName,
            int argc = 0, const char** argv = nullptr) {
 
        return benchmark_context(moduleName, argc, argv);
    }
 
}}
 
#endif