libwreport 3.38
testrunner.h
1#ifndef WREPORT_TESTSRUNNER_H
2#define WREPORT_TESTSRUNNER_H
3
4#include <string>
5#include <vector>
6#include <functional>
7#include <memory>
8
9namespace wreport {
10
11namespace term {
12struct Terminal;
13}
14
15namespace tests {
16
17struct TestFailed;
18struct TestStack;
19struct TestCase;
20struct TestMethod;
21
22
26struct TestMethodResult
27{
29 std::string test_case;
30
32 std::string test_method;
33
35 std::string error_message;
36
38 std::shared_ptr<TestStack> error_stack;
39
41 std::string exception_typeid;
42
44 bool skipped = false;
45
47 std::string skipped_reason;
48
50 unsigned long long elapsed_ns = 0;
51
52
53 TestMethodResult(const std::string& test_case_, const std::string& test_method_)
54 : test_case(test_case_), test_method(test_method_) {}
55
56 void set_failed(TestFailed& e);
57
58 void set_exception(std::exception& e)
59 {
60 error_message = e.what();
61 if (error_message.empty())
62 error_message = "test threw an exception with an empty error message";
63 exception_typeid = typeid(e).name();
64 }
65
66 void set_unknown_exception()
67 {
68 error_message = "unknown exception caught";
69 }
70
71 void set_setup_exception(std::exception& e)
72 {
73 error_message = "[setup failed: ";
74 error_message += e.what();
75 error_message += "]";
76 }
77
78 void set_teardown_exception(std::exception& e)
79 {
80 error_message = "[teardown failed: ";
81 error_message += e.what();
82 error_message += "]";
83 }
84
85 bool is_success() const
86 {
87 return error_message.empty();
88 }
89
90 void print_failure_details(FILE* out) const;
91};
92
96struct TestCaseResult
97{
99 std::string test_case;
101 std::vector<TestMethodResult> methods;
103 std::string fail_setup;
106 std::string fail_teardown;
108 bool skipped = false;
109
110 explicit TestCaseResult(const std::string& test_case_) : test_case(test_case_) {}
111
112 void set_setup_failed()
113 {
114 fail_setup = "test case setup method threw an unknown exception";
115 }
116
117 void set_setup_failed(std::exception& e)
118 {
119 fail_setup = "test case setup method threw an exception: ";
120 fail_setup += e.what();
121 }
122
123 void set_teardown_failed()
124 {
125 fail_teardown = "test case teardown method threw an unknown exception";
126 }
127
128 void set_teardown_failed(std::exception& e)
129 {
130 fail_teardown = "test case teardown method threw an exception: ";
131 fail_teardown += e.what();
132 }
133
134 void add_test_method(TestMethodResult&& e)
135 {
136 methods.emplace_back(std::move(e));
137 }
138
139 bool is_success() const
140 {
141 if (!fail_setup.empty() || !fail_teardown.empty()) return false;
142 for (const auto& m: methods)
143 if (!m.is_success())
144 return false;
145 return true;
146 }
147
148 unsigned long long elapsed_ns() const;
149};
150
151
159{
160 virtual ~TestController() {}
161
167 virtual bool test_case_begin(const TestCase&, const TestCaseResult&) { return true; }
168
172 virtual void test_case_end(const TestCase&, const TestCaseResult&) {}
173
179 virtual bool test_method_begin(const TestMethod&, const TestMethodResult&) { return true; }
180
184 virtual void test_method_end(const TestMethod&, const TestMethodResult&) {}
185};
186
192{
194 std::string allowlist = std::string();
195
197 std::string blocklist = std::string();
198
199 bool test_method_should_run(const std::string& fullname) const;
200};
201
202
209struct SimpleTestController : public FilteringTestController
210{
212
213 SimpleTestController(wreport::term::Terminal& output);
214
215 bool test_case_begin(const TestCase& test_case, const TestCaseResult& test_case_result) override;
216 void test_case_end(const TestCase& test_case, const TestCaseResult& test_case_result) override;
217 bool test_method_begin(const TestMethod& test_method, const TestMethodResult& test_method_result) override;
218 void test_method_end(const TestMethod& test_method, const TestMethodResult& test_method_result) override;
219};
220
221
228struct VerboseTestController : public FilteringTestController
229{
231
232 VerboseTestController(wreport::term::Terminal& output);
233
234 bool test_case_begin(const TestCase& test_case, const TestCaseResult& test_case_result) override;
235 void test_case_end(const TestCase& test_case, const TestCaseResult& test_case_result) override;
236 bool test_method_begin(const TestMethod& test_method, const TestMethodResult& test_method_result) override;
237 void test_method_end(const TestMethod& test_method, const TestMethodResult& test_method_result) override;
238};
239
240
248{
250 std::vector<TestCase*> entries = std::vector<TestCase*>();
251
258 void register_test_case(TestCase& test_case);
259
266 void iterate_test_methods(std::function<void(const TestCase&, const TestMethod&)>);
267
271 std::vector<TestCaseResult> run_tests(TestController& controller);
272
274 static TestRegistry& get();
275};
276
277
278struct TestResultStats
279{
280 const std::vector<TestCaseResult>& results;
281 unsigned methods_ok = 0;
282 unsigned methods_failed = 0;
283 unsigned methods_skipped = 0;
284 unsigned test_cases_ok = 0;
285 unsigned test_cases_failed = 0;
286 bool success = false;
287
288 TestResultStats(const std::vector<TestCaseResult>& results);
289
290 void print_results(wreport::term::Terminal& out);
291 void print_stats(wreport::term::Terminal& out);
292 void print_summary(wreport::term::Terminal& out);
293};
294
295}
296}
297#endif
String functions.
Definition benchmark.h:13
Definition term.h:18
Test controller that filters tests via a blocklist/allowlist system containing glob patterns on testc...
Definition testrunner.h:192
std::string blocklist
Any method matching this glob expression will not be run.
Definition testrunner.h:197
std::string allowlist
Any method not matching this glob expression will not be run.
Definition testrunner.h:194
void test_method_end(const TestMethod &test_method, const TestMethodResult &test_method_result) override
Called after running a test method.
bool test_method_begin(const TestMethod &test_method, const TestMethodResult &test_method_result) override
Called before running a test method.
void test_case_end(const TestCase &test_case, const TestCaseResult &test_case_result) override
Called after running a test case.
bool test_case_begin(const TestCase &test_case, const TestCaseResult &test_case_result) override
Called before running a test case.
Result of running a whole test case.
Definition testrunner.h:97
std::string fail_setup
Set to a non-empty string if the setup method of the test case failed.
Definition testrunner.h:103
std::string fail_teardown
Set to a non-empty string if the teardown method of the test case failed.
Definition testrunner.h:106
std::vector< TestMethodResult > methods
Outcome of all the methods that have been run.
Definition testrunner.h:101
bool skipped
Set to true if this test case has been skipped.
Definition testrunner.h:108
std::string test_case
Name of the test case.
Definition testrunner.h:99
Test case collecting several test methods, and self-registering with the singleton instance of TestRe...
Definition utils/tests.h:542
Abstract interface for the objects that supervise test execution.
Definition testrunner.h:159
virtual bool test_method_begin(const TestMethod &, const TestMethodResult &)
Called before running a test method.
Definition testrunner.h:179
virtual bool test_case_begin(const TestCase &, const TestCaseResult &)
Called before running a test case.
Definition testrunner.h:167
virtual void test_method_end(const TestMethod &, const TestMethodResult &)
Called after running a test method.
Definition testrunner.h:184
virtual void test_case_end(const TestCase &, const TestCaseResult &)
Called after running a test case.
Definition testrunner.h:172
Exception thrown when a test assertion fails, normally by Location::fail_test.
Definition utils/tests.h:105
Result of running a test method.
Definition testrunner.h:27
std::string skipped_reason
If the test has been skipped, this is an optional reason.
Definition testrunner.h:47
std::shared_ptr< TestStack > error_stack
Stack frame of where the error happened.
Definition testrunner.h:38
std::string test_case
Name of the test case.
Definition testrunner.h:29
bool skipped
True if the test has been skipped.
Definition testrunner.h:44
std::string error_message
If non-empty, the test failed with this error.
Definition testrunner.h:35
unsigned long long elapsed_ns
Time in nanoseconds it took the test to run.
Definition testrunner.h:50
std::string exception_typeid
If non-empty, the test threw an exception and this is its type ID.
Definition testrunner.h:41
std::string test_method
Name of the test method.
Definition testrunner.h:32
Test method information.
Definition utils/tests.h:515
Test registry.
Definition testrunner.h:248
void iterate_test_methods(std::function< void(const TestCase &, const TestMethod &)>)
Iterate on all test methods known by this registry.
std::vector< TestCaseResult > run_tests(TestController &controller)
Run all the registered tests using the given controller.
std::vector< TestCase * > entries
All known test cases.
Definition testrunner.h:250
void register_test_case(TestCase &test_case)
Register a new test case.
static TestRegistry & get()
Get the singleton instance of TestRegistry.
Definition utils/tests.h:90
void test_method_end(const TestMethod &test_method, const TestMethodResult &test_method_result) override
Called after running a test method.
bool test_case_begin(const TestCase &test_case, const TestCaseResult &test_case_result) override
Called before running a test case.
void test_case_end(const TestCase &test_case, const TestCaseResult &test_case_result) override
Called after running a test case.
bool test_method_begin(const TestMethod &test_method, const TestMethodResult &test_method_result) override
Called before running a test method.