libwreport 3.40
testrunner.h
1#ifndef WREPORT_TESTSRUNNER_H
2#define WREPORT_TESTSRUNNER_H
3
4#include <functional>
5#include <memory>
6#include <string>
7#include <vector>
8
9namespace wreport {
10
11namespace term {
12struct Terminal;
13}
14
15namespace tests {
16
17struct TestFailed;
18struct TestStack;
19struct TestCase;
20struct TestMethod;
21
26{
28 std::string test_case;
29
31 std::string test_method;
32
34 std::string error_message;
35
37 std::shared_ptr<TestStack> error_stack;
38
40 std::string exception_typeid;
41
43 bool skipped = false;
44
46 std::string skipped_reason;
47
49 unsigned long long elapsed_ns = 0;
50
51 TestMethodResult(const std::string& test_case_,
52 const std::string& test_method_)
53 : test_case(test_case_), test_method(test_method_)
54 {
55 }
56
57 void set_failed(TestFailed& e);
58
59 void set_exception(std::exception& e)
60 {
61 error_message = e.what();
62 if (error_message.empty())
64 "test threw an exception with an empty error message";
65 exception_typeid = typeid(e).name();
66 }
67
68 void set_unknown_exception() { error_message = "unknown exception caught"; }
69
70 void set_setup_exception(std::exception& e)
71 {
72 error_message = "[setup failed: ";
73 error_message += e.what();
74 error_message += "]";
75 }
76
77 void set_teardown_exception(std::exception& e)
78 {
79 error_message = "[teardown failed: ";
80 error_message += e.what();
81 error_message += "]";
82 }
83
84 bool is_success() const { return error_message.empty(); }
85
86 void print_failure_details(FILE* out) const;
87};
88
93{
95 std::string test_case;
97 std::vector<TestMethodResult> methods;
99 std::string fail_setup;
102 std::string fail_teardown;
104 bool skipped = false;
105
106 explicit TestCaseResult(const std::string& test_case_)
107 : test_case(test_case_)
108 {
109 }
110
111 void set_setup_failed()
112 {
113 fail_setup = "test case setup method threw an unknown exception";
114 }
115
116 void set_setup_failed(std::exception& e)
117 {
118 fail_setup = "test case setup method threw an exception: ";
119 fail_setup += e.what();
120 }
121
122 void set_teardown_failed()
123 {
124 fail_teardown = "test case teardown method threw an unknown exception";
125 }
126
127 void set_teardown_failed(std::exception& e)
128 {
129 fail_teardown = "test case teardown method threw an exception: ";
130 fail_teardown += e.what();
131 }
132
133 void add_test_method(TestMethodResult&& e)
134 {
135 methods.emplace_back(std::move(e));
136 }
137
138 bool is_success() const
139 {
140 if (!fail_setup.empty() || !fail_teardown.empty())
141 return false;
142 for (const auto& m : methods)
143 if (!m.is_success())
144 return false;
145 return true;
146 }
147
148 unsigned long long elapsed_ns() const;
149};
150
158{
159 virtual ~TestController() {}
160
167 virtual bool test_case_begin(const TestCase&, const TestCaseResult&)
168 {
169 return true;
170 }
171
175 virtual void test_case_end(const TestCase&, const TestCaseResult&) {}
176
183 virtual bool test_method_begin(const TestMethod&, const TestMethodResult&)
184 {
185 return true;
186 }
187
191 virtual void test_method_end(const TestMethod&, const TestMethodResult&) {}
192};
193
199{
201 std::string allowlist = std::string();
202
204 std::string blocklist = std::string();
205
206 bool test_method_should_run(const std::string& fullname) const;
207};
208
216{
218
220
221 bool test_case_begin(const TestCase& test_case,
222 const TestCaseResult& test_case_result) override;
223 void test_case_end(const TestCase& test_case,
224 const TestCaseResult& test_case_result) override;
225 bool test_method_begin(const TestMethod& test_method,
226 const TestMethodResult& test_method_result) override;
227 void test_method_end(const TestMethod& test_method,
228 const TestMethodResult& test_method_result) override;
229};
230
238{
240
242
243 bool test_case_begin(const TestCase& test_case,
244 const TestCaseResult& test_case_result) override;
245 void test_case_end(const TestCase& test_case,
246 const TestCaseResult& test_case_result) override;
247 bool test_method_begin(const TestMethod& test_method,
248 const TestMethodResult& test_method_result) override;
249 void test_method_end(const TestMethod& test_method,
250 const TestMethodResult& test_method_result) override;
251};
252
260{
262 std::vector<TestCase*> entries = std::vector<TestCase*>();
263
270 void register_test_case(TestCase& test_case);
271
279 std::function<void(const TestCase&, const TestMethod&)>);
280
284 std::vector<TestCaseResult> run_tests(TestController& controller);
285
287 static TestRegistry& get();
288};
289
291{
292 const std::vector<TestCaseResult>& results;
293 unsigned methods_ok = 0;
294 unsigned methods_failed = 0;
295 unsigned methods_skipped = 0;
296 unsigned test_cases_ok = 0;
297 unsigned test_cases_failed = 0;
298 bool success = false;
299 bool skipped = false;
300
301 TestResultStats(const std::vector<TestCaseResult>& results);
302
303 void print_results(wreport::term::Terminal& out);
304 void print_stats(wreport::term::Terminal& out);
305 void print_summary(wreport::term::Terminal& out);
306};
307
308} // namespace tests
309} // namespace wreport
310#endif
String functions.
Definition benchmark.h:13
Definition term.h:18
Test controller that filters tests via a blocklist/allowlist system containing glob patterns on testc...
Definition testrunner.h:199
std::string blocklist
Any method matching this glob expression will not be run.
Definition testrunner.h:204
std::string allowlist
Any method not matching this glob expression will not be run.
Definition testrunner.h:201
Simple default implementation of TestController.
Definition testrunner.h:216
void test_method_end(const TestMethod &test_method, const TestMethodResult &test_method_result) override
Called after running a test method.
bool test_method_begin(const TestMethod &test_method, const TestMethodResult &test_method_result) override
Called before running a test method.
void test_case_end(const TestCase &test_case, const TestCaseResult &test_case_result) override
Called after running a test case.
bool test_case_begin(const TestCase &test_case, const TestCaseResult &test_case_result) override
Called before running a test case.
Result of running a whole test case.
Definition testrunner.h:93
std::string fail_setup
Set to a non-empty string if the setup method of the test case failed.
Definition testrunner.h:99
std::string fail_teardown
Set to a non-empty string if the teardown method of the test case failed.
Definition testrunner.h:102
std::vector< TestMethodResult > methods
Outcome of all the methods that have been run.
Definition testrunner.h:97
bool skipped
Set to true if this test case has been skipped.
Definition testrunner.h:104
std::string test_case
Name of the test case.
Definition testrunner.h:95
Test case collecting several test methods, and self-registering with the singleton instance of TestRe...
Definition utils/tests.h:642
Abstract interface for the objects that supervise test execution.
Definition testrunner.h:158
virtual bool test_method_begin(const TestMethod &, const TestMethodResult &)
Called before running a test method.
Definition testrunner.h:183
virtual bool test_case_begin(const TestCase &, const TestCaseResult &)
Called before running a test case.
Definition testrunner.h:167
virtual void test_method_end(const TestMethod &, const TestMethodResult &)
Called after running a test method.
Definition testrunner.h:191
virtual void test_case_end(const TestCase &, const TestCaseResult &)
Called after running a test case.
Definition testrunner.h:175
Exception thrown when a test assertion fails, normally by Location::fail_test.
Definition utils/tests.h:107
Result of running a test method.
Definition testrunner.h:26
std::string skipped_reason
If the test has been skipped, this is an optional reason.
Definition testrunner.h:46
std::shared_ptr< TestStack > error_stack
Stack frame of where the error happened.
Definition testrunner.h:37
std::string test_case
Name of the test case.
Definition testrunner.h:28
bool skipped
True if the test has been skipped.
Definition testrunner.h:43
std::string error_message
If non-empty, the test failed with this error.
Definition testrunner.h:34
unsigned long long elapsed_ns
Time in nanoseconds it took the test to run.
Definition testrunner.h:49
std::string exception_typeid
If non-empty, the test threw an exception and this is its type ID.
Definition testrunner.h:40
std::string test_method
Name of the test method.
Definition testrunner.h:31
Test method information.
Definition utils/tests.h:615
Test registry.
Definition testrunner.h:260
void iterate_test_methods(std::function< void(const TestCase &, const TestMethod &)>)
Iterate on all test methods known by this registry.
std::vector< TestCaseResult > run_tests(TestController &controller)
Run all the registered tests using the given controller.
std::vector< TestCase * > entries
All known test cases.
Definition testrunner.h:262
void register_test_case(TestCase &test_case)
Register a new test case.
static TestRegistry & get()
Get the singleton instance of TestRegistry.
Definition testrunner.h:291
Verbose implementation of TestController.
Definition testrunner.h:238
void test_method_end(const TestMethod &test_method, const TestMethodResult &test_method_result) override
Called after running a test method.
bool test_case_begin(const TestCase &test_case, const TestCaseResult &test_case_result) override
Called before running a test case.
void test_case_end(const TestCase &test_case, const TestCaseResult &test_case_result) override
Called after running a test case.
bool test_method_begin(const TestMethod &test_method, const TestMethodResult &test_method_result) override
Called before running a test method.