libwreport  3.40
testrunner.h
1 #ifndef WREPORT_TESTSRUNNER_H
2 #define WREPORT_TESTSRUNNER_H
3 
4 #include <functional>
5 #include <memory>
6 #include <string>
7 #include <vector>
8 
9 namespace wreport {
10 
11 namespace term {
12 struct Terminal;
13 }
14 
15 namespace tests {
16 
17 struct TestFailed;
18 struct TestStack;
19 struct TestCase;
20 struct TestMethod;
21 
26 {
28  std::string test_case;
29 
31  std::string test_method;
32 
34  std::string error_message;
35 
37  std::shared_ptr<TestStack> error_stack;
38 
40  std::string exception_typeid;
41 
43  bool skipped = false;
44 
46  std::string skipped_reason;
47 
49  unsigned long long elapsed_ns = 0;
50 
51  TestMethodResult(const std::string& test_case_,
52  const std::string& test_method_)
53  : test_case(test_case_), test_method(test_method_)
54  {
55  }
56 
57  void set_failed(TestFailed& e);
58 
59  void set_exception(std::exception& e)
60  {
61  error_message = e.what();
62  if (error_message.empty())
64  "test threw an exception with an empty error message";
65  exception_typeid = typeid(e).name();
66  }
67 
68  void set_unknown_exception() { error_message = "unknown exception caught"; }
69 
70  void set_setup_exception(std::exception& e)
71  {
72  error_message = "[setup failed: ";
73  error_message += e.what();
74  error_message += "]";
75  }
76 
77  void set_teardown_exception(std::exception& e)
78  {
79  error_message = "[teardown failed: ";
80  error_message += e.what();
81  error_message += "]";
82  }
83 
84  bool is_success() const { return error_message.empty(); }
85 
86  void print_failure_details(FILE* out) const;
87 };
88 
93 {
95  std::string test_case;
97  std::vector<TestMethodResult> methods;
99  std::string fail_setup;
102  std::string fail_teardown;
104  bool skipped = false;
105 
106  explicit TestCaseResult(const std::string& test_case_)
107  : test_case(test_case_)
108  {
109  }
110 
111  void set_setup_failed()
112  {
113  fail_setup = "test case setup method threw an unknown exception";
114  }
115 
116  void set_setup_failed(std::exception& e)
117  {
118  fail_setup = "test case setup method threw an exception: ";
119  fail_setup += e.what();
120  }
121 
122  void set_teardown_failed()
123  {
124  fail_teardown = "test case teardown method threw an unknown exception";
125  }
126 
127  void set_teardown_failed(std::exception& e)
128  {
129  fail_teardown = "test case teardown method threw an exception: ";
130  fail_teardown += e.what();
131  }
132 
133  void add_test_method(TestMethodResult&& e)
134  {
135  methods.emplace_back(std::move(e));
136  }
137 
138  bool is_success() const
139  {
140  if (!fail_setup.empty() || !fail_teardown.empty())
141  return false;
142  for (const auto& m : methods)
143  if (!m.is_success())
144  return false;
145  return true;
146  }
147 
148  unsigned long long elapsed_ns() const;
149 };
150 
158 {
159  virtual ~TestController() {}
160 
167  virtual bool test_case_begin(const TestCase&, const TestCaseResult&)
168  {
169  return true;
170  }
171 
175  virtual void test_case_end(const TestCase&, const TestCaseResult&) {}
176 
183  virtual bool test_method_begin(const TestMethod&, const TestMethodResult&)
184  {
185  return true;
186  }
187 
191  virtual void test_method_end(const TestMethod&, const TestMethodResult&) {}
192 };
193 
199 {
201  std::string allowlist = std::string();
202 
204  std::string blocklist = std::string();
205 
206  bool test_method_should_run(const std::string& fullname) const;
207 };
208 
216 {
217  wreport::term::Terminal& output;
218 
220 
221  bool test_case_begin(const TestCase& test_case,
222  const TestCaseResult& test_case_result) override;
223  void test_case_end(const TestCase& test_case,
224  const TestCaseResult& test_case_result) override;
225  bool test_method_begin(const TestMethod& test_method,
226  const TestMethodResult& test_method_result) override;
227  void test_method_end(const TestMethod& test_method,
228  const TestMethodResult& test_method_result) override;
229 };
230 
238 {
239  wreport::term::Terminal& output;
240 
242 
243  bool test_case_begin(const TestCase& test_case,
244  const TestCaseResult& test_case_result) override;
245  void test_case_end(const TestCase& test_case,
246  const TestCaseResult& test_case_result) override;
247  bool test_method_begin(const TestMethod& test_method,
248  const TestMethodResult& test_method_result) override;
249  void test_method_end(const TestMethod& test_method,
250  const TestMethodResult& test_method_result) override;
251 };
252 
260 {
262  std::vector<TestCase*> entries = std::vector<TestCase*>();
263 
270  void register_test_case(TestCase& test_case);
271 
279  std::function<void(const TestCase&, const TestMethod&)>);
280 
284  std::vector<TestCaseResult> run_tests(TestController& controller);
285 
287  static TestRegistry& get();
288 };
289 
291 {
292  const std::vector<TestCaseResult>& results;
293  unsigned methods_ok = 0;
294  unsigned methods_failed = 0;
295  unsigned methods_skipped = 0;
296  unsigned test_cases_ok = 0;
297  unsigned test_cases_failed = 0;
298  bool success = false;
299  bool skipped = false;
300 
301  TestResultStats(const std::vector<TestCaseResult>& results);
302 
303  void print_results(wreport::term::Terminal& out);
304  void print_stats(wreport::term::Terminal& out);
305  void print_summary(wreport::term::Terminal& out);
306 };
307 
308 } // namespace tests
309 } // namespace wreport
310 #endif
String functions.
Definition: benchmark.h:13
Definition: term.h:18
Test controller that filters tests via a blocklist/allowlist system containing glob patterns on testc...
Definition: testrunner.h:199
std::string blocklist
Any method matching this glob expression will not be run.
Definition: testrunner.h:204
std::string allowlist
Any method not matching this glob expression will not be run.
Definition: testrunner.h:201
Simple default implementation of TestController.
Definition: testrunner.h:216
void test_method_end(const TestMethod &test_method, const TestMethodResult &test_method_result) override
Called after running a test method.
bool test_method_begin(const TestMethod &test_method, const TestMethodResult &test_method_result) override
Called before running a test method.
void test_case_end(const TestCase &test_case, const TestCaseResult &test_case_result) override
Called after running a test case.
bool test_case_begin(const TestCase &test_case, const TestCaseResult &test_case_result) override
Called before running a test case.
Result of running a whole test case.
Definition: testrunner.h:93
std::string fail_setup
Set to a non-empty string if the setup method of the test case failed.
Definition: testrunner.h:99
std::string fail_teardown
Set to a non-empty string if the teardown method of the test case failed.
Definition: testrunner.h:102
std::vector< TestMethodResult > methods
Outcome of all the methods that have been run.
Definition: testrunner.h:97
bool skipped
Set to true if this test case has been skipped.
Definition: testrunner.h:104
std::string test_case
Name of the test case.
Definition: testrunner.h:95
Test case collecting several test methods, and self-registering with the singleton instance of TestRe...
Definition: utils/tests.h:642
Abstract interface for the objects that supervise test execution.
Definition: testrunner.h:158
virtual bool test_method_begin(const TestMethod &, const TestMethodResult &)
Called before running a test method.
Definition: testrunner.h:183
virtual bool test_case_begin(const TestCase &, const TestCaseResult &)
Called before running a test case.
Definition: testrunner.h:167
virtual void test_method_end(const TestMethod &, const TestMethodResult &)
Called after running a test method.
Definition: testrunner.h:191
virtual void test_case_end(const TestCase &, const TestCaseResult &)
Called after running a test case.
Definition: testrunner.h:175
Exception thrown when a test assertion fails, normally by Location::fail_test.
Definition: utils/tests.h:107
Result of running a test method.
Definition: testrunner.h:26
std::string skipped_reason
If the test has been skipped, this is an optional reason.
Definition: testrunner.h:46
std::shared_ptr< TestStack > error_stack
Stack frame of where the error happened.
Definition: testrunner.h:37
std::string test_case
Name of the test case.
Definition: testrunner.h:28
bool skipped
True if the test has been skipped.
Definition: testrunner.h:43
std::string error_message
If non-empty, the test failed with this error.
Definition: testrunner.h:34
unsigned long long elapsed_ns
Time in nanoseconds it took the test to run.
Definition: testrunner.h:49
std::string exception_typeid
If non-empty, the test threw an exception and this is its type ID.
Definition: testrunner.h:40
std::string test_method
Name of the test method.
Definition: testrunner.h:31
Test method information.
Definition: utils/tests.h:615
Test registry.
Definition: testrunner.h:260
void iterate_test_methods(std::function< void(const TestCase &, const TestMethod &)>)
Iterate on all test methods known by this registry.
std::vector< TestCase * > entries
All known test cases.
Definition: testrunner.h:262
void register_test_case(TestCase &test_case)
Register a new test case.
std::vector< TestCaseResult > run_tests(TestController &controller)
Run all the registered tests using the given controller.
static TestRegistry & get()
Get the singleton instance of TestRegistry.
Definition: testrunner.h:291
Verbose implementation of TestController.
Definition: testrunner.h:238
void test_method_end(const TestMethod &test_method, const TestMethodResult &test_method_result) override
Called after running a test method.
bool test_case_begin(const TestCase &test_case, const TestCaseResult &test_case_result) override
Called before running a test case.
void test_case_end(const TestCase &test_case, const TestCaseResult &test_case_result) override
Called after running a test case.
bool test_method_begin(const TestMethod &test_method, const TestMethodResult &test_method_result) override
Called before running a test method.