Chebyshev
Unit testing for scientific software
prec.h
Go to the documentation of this file.
1 
5 
6 #ifndef CHEBYSHEV_PREC_H
7 #define CHEBYSHEV_PREC_H
8 
9 #include <string>
10 #include <vector>
11 #include <map>
12 #include <fstream>
13 #include <iostream>
14 #include <ctime>
15 
16 #include "./prec/prec_structures.h"
17 #include "./prec/fail.h"
18 #include "./prec/estimator.h"
19 #include "./core/output.h"
20 #include "./core/random.h"
21 
22 
23 namespace chebyshev {
24 
31  namespace prec {
32 
33 
35  struct prec_settings {
36 
38  std::string moduleName = "unknown";
39 
41  bool quiet = false;
42 
44  bool outputToFile = true;
45 
48 
51 
54 
56  std::vector<std::string> outputFiles {};
57 
59  std::vector<std::string> estimateColumns = {
60  "name", "meanErr", "rmsErr", "maxErr", "failed"
61  };
62 
65  std::vector<std::string> estimateOutputFiles {};
66 
68  std::vector<std::string> equationColumns = {
69  "name", "difference", "tolerance", "failed"
70  };
71 
74  std::vector<std::string> equationOutputFiles {};
75 
79  std::map<std::string, bool> pickedTests {};
80 
81  } settings;
82 
83 
85  struct prec_results {
86 
88  unsigned int totalTests = 0;
89 
91  unsigned int failedTests = 0;
92 
94  std::map<std::string, std::vector<estimate_result>> estimateResults {};
95 
97  std::map<std::string, std::vector<equation_result>> equationResults {};
98 
99  } results;
100 
101 
108  inline void setup(
109  std::string moduleName,
110  int argc = 0,
111  const char** argv = nullptr) {
112 
113 
114  // Initialize list of picked tests
115  if(argc && argv)
116  for (int i = 1; i < argc; ++i)
117  settings.pickedTests[argv[i]] = true;
118 
119  std::cout << "Starting precision testing of the "
120  << moduleName << " module ..." << std::endl;
121 
122  settings.moduleName = moduleName;
123  results.failedTests = 0;
124  results.totalTests = 0;
125 
126  random::setup();
127  output::setup();
128  }
129 
130 
135  inline void terminate(bool exit = true) {
136 
137  output::settings.quiet = settings.quiet;
138 
139  // Output to file is true but no specific files are specified, add default output file.
140  if( settings.outputToFile &&
141  !output::settings.outputFiles.size() &&
142  !settings.estimateOutputFiles.size() &&
143  !settings.equationOutputFiles.size() &&
144  !settings.outputFiles.size()) {
145 
146  settings.outputFiles = { settings.moduleName + "_results" };
147  }
148 
149  std::vector<std::string> outputFiles;
150 
151  // Print estimate results
152  outputFiles = settings.outputFiles;
153  outputFiles.insert(outputFiles.end(), settings.estimateOutputFiles.begin(), settings.estimateOutputFiles.end());
154 
155  output::print_results(results.estimateResults, settings.estimateColumns, outputFiles);
156 
157  // Print equation results
158  outputFiles = settings.outputFiles;
159  outputFiles.insert(outputFiles.end(), settings.equationOutputFiles.begin(), settings.equationOutputFiles.end());
160 
161  output::print_results(results.equationResults, settings.equationColumns, outputFiles);
162 
163  std::cout << "Finished testing " << settings.moduleName << '\n';
164  std::cout << results.totalTests << " total tests, "
165  << results.failedTests << " failed (" << std::setprecision(3) <<
166  (results.failedTests / (double) results.totalTests) * 100 << "%)"
167  << '\n';
168 
169  // Discard previous results
170  results = prec_results();
171 
172  if(exit) {
174  std::exit(results.failedTests);
175  }
176  }
177 
178 
187  template<typename R, typename ...Args,
188  typename Function1 = std::function<R(Args...)>,
189  typename Function2 = Function1>
190 
191  inline void estimate(
192  const std::string& name,
193  Function1 funcApprox,
194  Function2 funcExpected,
196 
197  // Skip the test case if any tests have been picked
198  // and this one was not picked.
199  if(settings.pickedTests.size())
200  if(settings.pickedTests.find(name) == settings.pickedTests.end())
201  return;
202 
203  // Use the estimator to estimate error integrals.
204  auto res = opt.estimator(funcApprox, funcExpected, opt);
205 
206  res.name = name;
207  res.domain = opt.domain;
208  res.tolerance = opt.tolerance;
209  res.quiet = opt.quiet;
210  res.iterations = opt.iterations;
211 
212  // Use the fail function to determine whether the test failed.
213  res.failed = opt.fail(res);
214 
215  results.totalTests++;
216  if(res.failed)
217  results.failedTests++;
218 
219  results.estimateResults[name].push_back(res);
220  }
221 
222 
236  template<typename R, typename ...Args,
237  typename Function1 = std::function<R(Args...)>,
238  typename Function2 = Function1>
239 
240  inline void estimate(
241  const std::string& name,
242  Function1 funcApprox,
243  Function2 funcExpected,
244  std::vector<interval> domain,
245  long double tolerance, unsigned int iterations,
246  FailFunction fail,
247  Estimator<R, Args...> estimator,
248  bool quiet = false) {
249 
250  estimate_options<R, Args...> opt {};
251  opt.domain = domain;
252  opt.tolerance = tolerance;
253  opt.iterations = iterations;
254  opt.fail = fail;
255  opt.estimator = estimator;
256  opt.quiet = quiet;
257 
258  estimate(name, funcApprox, funcExpected, opt);
259  }
260 
261 
276  inline void estimate(
277  const std::string& name,
278  EndoFunction<double> funcApprox,
279  EndoFunction<double> funcExpected,
280  interval domain,
281  long double tolerance = settings.defaultTolerance,
282  unsigned int iterations = settings.defaultIterations,
284  Estimator<double, double> estimator = estimator::quadrature1D<double>(),
285  bool quiet = false) {
286 
288  opt.domain = { domain };
289  opt.tolerance = tolerance;
290  opt.iterations = iterations;
291  opt.fail = fail;
292  opt.estimator = estimator;
293  opt.quiet = quiet;
294 
295  estimate(name, funcApprox, funcExpected, opt);
296  }
297 
305  namespace property {
306 
313  template<typename Type, typename Identity = EndoFunction<Type>>
314  inline void identity(
315  const std::string& name,
316  Identity id,
317  const estimate_options<Type, Type>& opt) {
318 
319  // Apply the identity function
320  EndoFunction<Type> funcApprox = [&](Type x) -> Type {
321  return id(x);
322  };
323 
324  // And compare it to the identity
325  EndoFunction<Type> funcExpected = [](Type x) -> Type {
326  return x;
327  };
328 
329  estimate(name, funcApprox, funcExpected, opt);
330  }
331 
339  template<typename Type, typename Involution = EndoFunction<Type>>
340  inline void involution(
341  const std::string& name,
342  Involution invol,
343  const estimate_options<Type, Type>& opt) {
344 
345  // Apply the involution two times
346  EndoFunction<Type> funcApprox = [&](Type x) -> Type {
347  return invol(invol(x));
348  };
349 
350  // And compare it to the identity
351  EndoFunction<Type> funcExpected = [](Type x) -> Type {
352  return x;
353  };
354 
355  estimate(name, funcApprox, funcExpected, opt);
356  }
357 
358 
366  template<typename Type, typename Involution = EndoFunction<Type>>
367  inline void idempotence(
368  const std::string& name,
369  Involution idem,
370  const estimate_options<Type, Type>& opt) {
371 
372  // Apply the idem two times
373  EndoFunction<Type> funcApprox = [&](Type x) -> Type {
374  return idem(idem(x));
375  };
376 
377  // And compare it to the identity
378  EndoFunction<Type> funcExpected = [&](Type x) -> Type {
379  return idem(x);
380  };
381 
382  estimate(name, funcApprox, funcExpected, opt);
383  }
384 
385 
397  template<typename InputType, typename OutputType = InputType,
398  typename Homogeneous = std::function<OutputType(InputType)>>
399  inline void homogeneous(
400  const std::string& name,
401  Homogeneous hom,
403  OutputType zero_element = OutputType(0.0)) {
404 
405  // Apply the homogeneous function
406  std::function<OutputType(InputType)> funcApprox =
407  [&](InputType x) -> OutputType {
408  return hom(x);
409  };
410 
411  // And compare it to the zero element
412  std::function<OutputType(InputType)> funcExpected =
413  [&](InputType x) -> OutputType {
414  return zero_element;
415  };
416 
417  estimate(name, funcApprox, funcExpected, opt);
418  }
419  }
420 
421 
429  template<typename T = double>
430  inline void equals(
431  const std::string& name,
432  const T& evaluated, const T& expected,
434 
435  // Skip the test case if any tests have been picked
436  // and this one was not picked.
437  if(settings.pickedTests.size())
438  if(settings.pickedTests.find(name) == settings.pickedTests.end())
439  return;
440 
441  equation_result res {};
442 
443  long double diff = opt.distance(evaluated, expected);
444 
445  // Mark the test as failed if the
446  // distance between the two values
447  // is bigger than the tolerance.
448  res.failed = (diff > opt.tolerance);
449 
450  res.name = name;
451  res.difference = diff;
452  res.tolerance = opt.tolerance;
453  res.quiet = opt.quiet;
454 
455  results.totalTests++;
456  if(res.failed)
457  results.failedTests++;
458 
459  // Register the result of the equation by name
460  results.equationResults[name].push_back(res);
461  }
462 
463 
473  template<typename T = double>
474  inline void equals(
475  const std::string& name,
476  const T& evaluated, const T& expected,
477  long double tolerance,
478  DistanceFunction<T> distance,
479  bool quiet = false) {
480 
481  equation_options<T> opt {};
482  opt.tolerance = tolerance;
483  opt.distance = distance;
484  opt.quiet = quiet;
485 
486  equals(name, evaluated, expected, opt);
487  }
488 
489 
498  inline void equals(
499  const std::string& name,
500  long double evaluated, long double expected,
501  long double tolerance = settings.defaultTolerance,
502  bool quiet = false) {
503 
504  // Skip the test case if any tests have been picked
505  // and this one was not picked.
506  if(settings.pickedTests.size())
507  if(settings.pickedTests.find(name) == settings.pickedTests.end())
508  return;
509 
510  equation_result res {};
511 
512  long double diff = distance::abs_distance(evaluated, expected);
513 
514  // Mark the test as failed if the
515  // distance between the two values
516  // is bigger than the tolerance.
517  res.failed = (diff > tolerance);
518 
519  res.name = name;
520  res.difference = diff;
521  res.tolerance = tolerance;
522  res.quiet = quiet;
523 
524  res.evaluated = evaluated;
525  res.expected = expected;
526 
527  results.totalTests++;
528  if(res.failed)
529  results.failedTests++;
530 
531  // Register the result of the equation by name
532  results.equationResults[name].push_back(res);
533  }
534 
535 
543  template<typename T>
544  inline void equals(
545  const std::string& name,
546  std::vector<std::array<T, 2>> values,
547  long double tolerance = settings.defaultTolerance,
548  bool quiet = false) {
549 
550  // Skip the test case if any tests have been picked
551  // and this one was not picked.
552  if(settings.pickedTests.size())
553  if(settings.pickedTests.find(name) == settings.pickedTests.end())
554  return;
555 
556  for (const auto& v : values)
557  equals(name, v[0], v[1], tolerance, quiet);
558  }
559  }
560 }
561 
562 #endif
#define CHEBYSHEV_PREC_ITER
Default number of function evaluations in precision testing.
Definition: common.h:12
#define CHEBYSHEV_PREC_TOLERANCE
Default tolerance in precision testing.
Definition: common.h:17
Default precision estimators.
Default fail functions.
void print_results(const std::map< std::string, std::vector< ResultType >> &results, const std::vector< std::string > &fields, const std::vector< std::string > &filenames)
Print the test results to standard output and output files with their given formats,...
Definition: output.h:907
void setup()
Setup printing to the output stream with default options.
Definition: output.h:539
void terminate()
Terminate the output module by closing all output files and resetting its settings.
Definition: output.h:598
FloatType abs_distance(FloatType a, FloatType b)
Absolute distance between two real values.
Definition: distance.h:19
auto fail_on_max_err()
Default fail function which marks the test as failed if the maximum error on the domain is bigger tha...
Definition: fail.h:34
void involution(const std::string &name, Involution invol, const estimate_options< Type, Type > &opt)
Precision testing of an endofunction which is an involution.
Definition: prec.h:340
void identity(const std::string &name, Identity id, const estimate_options< Type, Type > &opt)
Precision testing of an endofunction which is equivalent to the identity.
Definition: prec.h:314
void homogeneous(const std::string &name, Homogeneous hom, const estimate_options< InputType, OutputType > &opt, OutputType zero_element=OutputType(0.0))
Precision testing of an function which is homogeneous over the domain.
Definition: prec.h:399
void idempotence(const std::string &name, Involution idem, const estimate_options< Type, Type > &opt)
Precision testing of an endofunction which is idempotent.
Definition: prec.h:367
std::function< long double(Type, Type)> DistanceFunction
Distance function between two elements.
Definition: prec_structures.h:71
void estimate(const std::string &name, Function1 funcApprox, Function2 funcExpected, estimate_options< R, Args... > opt)
Estimate error integrals over a function with respect to an exact function, with the given options.
Definition: prec.h:191
std::function< bool(const estimate_result &)> FailFunction
A function which determines whether an estimation failed.
Definition: prec_structures.h:66
typename estimate_options< R, Args... >::Estimator_t Estimator
Generic precision estimator function signature.
Definition: prec_structures.h:158
void setup(std::string moduleName, int argc=0, const char **argv=nullptr)
Setup the precision testing environment.
Definition: prec.h:108
void equals(const std::string &name, const T &evaluated, const T &expected, equation_options< T > opt=equation_options< T >())
Test an equivalence up to a tolerance, with the given options (e.g.
Definition: prec.h:430
void terminate(bool exit=true)
Terminate the precision testing environment, printing the results to standard output and output files...
Definition: prec.h:135
std::string string(size_t length)
Generate a random string made of human-readable ASCII characters.
Definition: random.h:102
void setup(uint64_t seed=0)
Initialize the random module.
Definition: random.h:32
General namespace of the framework.
Definition: benchmark_structures.h:16
std::function< Type(Type)> EndoFunction
An endofunction is a function which has the same type of input and output (e.g.
Definition: common.h:50
The output module, with formatting capabilities.
Structures for precision testing.
The pseudorandom number generation and sampling module.
std::string moduleName
Name of the module currently being benchmarked.
Definition: benchmark.h:38
bool outputToFile
Whether to output results to a file.
Definition: benchmark.h:47
unsigned int defaultIterations
Default number of iterations.
Definition: benchmark.h:41
std::vector< std::string > outputFiles
The files to write all benchmark results to.
Definition: benchmark.h:50
bool quiet
Whether to print benchmark results to standard output.
Definition: benchmark.h:35
Structure holding options for equivalence evaluation.
Definition: prec_structures.h:195
long double tolerance
Tolerance on the absolute difference.
Definition: prec_structures.h:198
A structure holding the result of an evaluation.
Definition: prec_structures.h:163
bool failed
Whether the test failed.
Definition: prec_structures.h:185
A structure holding the options for precision estimation.
Definition: prec_structures.h:77
FailFunction fail
The function to determine whether the test failed (defaults to fail::fail_on_max_err).
Definition: prec_structures.h:105
unsigned int iterations
Number of function evaluations to use.
Definition: prec_structures.h:101
std::vector< interval > domain
The domain of estimation.
Definition: prec_structures.h:86
Estimator_t estimator
The precision estimator to use (defaults to a dummy estimator)
Definition: prec_structures.h:90
long double tolerance
The tolerance to use to determine whether the test failed.
Definition: prec_structures.h:98
bool quiet
Whether to show the test result or not.
Definition: prec_structures.h:110
An interval on the real numbers.
Definition: interval.h:16
of the precision testing module.
Definition: prec.h:85
std::map< std::string, std::vector< estimate_result > > estimateResults
Results of error estimation.
Definition: prec.h:94
unsigned int failedTests
Number of failed tests.
Definition: prec.h:91
unsigned int totalTests
Total number of tests run.
Definition: prec.h:88
std::map< std::string, std::vector< equation_result > > equationResults
Results of equation evaluation.
Definition: prec.h:97
of the precision testing module.
Definition: prec.h:35
std::map< std::string, bool > pickedTests
Target tests marked for execution, can be picked by passing test case names by command line.
Definition: prec.h:79
std::string moduleName
Name of the module being tested.
Definition: prec.h:38
bool quiet
Print to standard output or not.
Definition: prec.h:41
std::vector< std::string > estimateOutputFiles
The files to write estimate results to (if empty, all results are output to a generic file).
Definition: prec.h:65
std::vector< std::string > equationOutputFiles
The files to write equation results to (if empty, all results are output to a generic file).
Definition: prec.h:74
FailFunction defaultFailFunction
Default fail function.
Definition: prec.h:50
std::vector< std::string > outputFiles
The files to write all precision testing results to.
Definition: prec.h:56
unsigned int defaultIterations
Default number of iterations for integral quadrature.
Definition: prec.h:47
bool outputToFile
Output to file?
Definition: prec.h:44
std::vector< std::string > estimateColumns
Default columns to print for precision estimates.
Definition: prec.h:59
long double defaultTolerance
Default tolerance on max absolute error.
Definition: prec.h:53
std::vector< std::string > equationColumns
Default columns to print for equations.
Definition: prec.h:68