unit-tests: Ignore tests not test_runner
[strongswan.git] / src / libstrongswan / tests / test_runner.c
1 /*
2 * Copyright (C) 2013 Tobias Brunner
3 * Hochschule fuer Technik Rapperswil
4 * Copyright (C) 2013 Martin Willi
5 * Copyright (C) 2013 revosec AG
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License as published by the
9 * Free Software Foundation; either version 2 of the License, or (at your
10 * option) any later version. See <http://www.fsf.org/copyleft/gpl.txt>.
11 *
12 * This program is distributed in the hope that it will be useful, but
13 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
14 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 * for more details.
16 */
17
18 #include "test_runner.h"
19
20 #include <library.h>
21 #include <plugins/plugin_feature.h>
22 #include <collections/array.h>
23 #include <utils/test.h>
24
25 #include <dirent.h>
26 #include <unistd.h>
27 #include <limits.h>
28
29 /**
30 * Get a tty color escape character for stderr
31 */
32 #define TTY(color) tty_escape_get(2, TTY_FG_##color)
33
34 /**
35 * Initialize the lookup table for testable functions (defined in libstrongswan)
36 */
37 static void testable_functions_create() __attribute__ ((constructor(1000)));
38 static void testable_functions_create()
39 {
40 testable_functions = hashtable_create(hashtable_hash_str,
41 hashtable_equals_str, 8);
42 }
43
44 /**
45 * Destroy the lookup table for testable functions
46 */
47 static void testable_functions_destroy() __attribute__ ((destructor(1000)));
48 static void testable_functions_destroy()
49 {
50 testable_functions->destroy(testable_functions);
51 /* if leak detective is enabled plugins are not actually unloaded, which
52 * means their destructor is called AFTER this one when the process
53 * terminates, even though the priority says differently, make sure this
54 * does not crash */
55 testable_functions = NULL;
56 }
57
58 /**
59 * Load all available test suites
60 */
61 static array_t *load_suites(test_configuration_t configs[],
62 test_runner_init_t init)
63 {
64 array_t *suites;
65 bool old = FALSE;
66 int i;
67
68 library_init(NULL, "test-runner");
69
70 test_setup_handler();
71
72 if (init && !init(TRUE))
73 {
74 library_deinit();
75 return NULL;
76 }
77 lib->plugins->status(lib->plugins, LEVEL_CTRL);
78
79 if (lib->leak_detective)
80 {
81 old = lib->leak_detective->set_state(lib->leak_detective, FALSE);
82 }
83
84 suites = array_create(0, 0);
85
86 for (i = 0; configs[i].suite; i++)
87 {
88 if (configs[i].feature.type == 0 ||
89 lib->plugins->has_feature(lib->plugins, configs[i].feature))
90 {
91 array_insert(suites, -1, configs[i].suite());
92 }
93 }
94
95 if (lib->leak_detective)
96 {
97 lib->leak_detective->set_state(lib->leak_detective, old);
98 }
99
100 if (init)
101 {
102 init(FALSE);
103 }
104 library_deinit();
105
106 return suites;
107 }
108
109 /**
110 * Unload and destroy test suites and associated data
111 */
112 static void unload_suites(array_t *suites)
113 {
114 test_suite_t *suite;
115 test_case_t *tcase;
116
117 while (array_remove(suites, 0, &suite))
118 {
119 while (array_remove(suite->tcases, 0, &tcase))
120 {
121 array_destroy(tcase->functions);
122 array_destroy(tcase->fixtures);
123 }
124 free(suite);
125 }
126 array_destroy(suites);
127 }
128
129 /**
130 * Run a single test function, return FALSE on failure
131 */
132 static bool run_test(test_function_t *tfun, int i)
133 {
134 if (test_restore_point())
135 {
136 tfun->cb(i);
137 return TRUE;
138 }
139 return FALSE;
140 }
141
142 /**
143 * Invoke fixture setup/teardown
144 */
145 static bool call_fixture(test_case_t *tcase, bool up)
146 {
147 enumerator_t *enumerator;
148 test_fixture_t *fixture;
149 bool failure = FALSE;
150
151 enumerator = array_create_enumerator(tcase->fixtures);
152 while (enumerator->enumerate(enumerator, &fixture))
153 {
154 if (test_restore_point())
155 {
156 if (up)
157 {
158 fixture->setup();
159 }
160 else
161 {
162 fixture->teardown();
163 }
164 }
165 else
166 {
167 failure = TRUE;
168 break;
169 }
170 }
171 enumerator->destroy(enumerator);
172
173 return !failure;
174 }
175
176 /**
177 * Test initialization, initializes libstrongswan for the next run
178 */
179 static bool pre_test(test_runner_init_t init)
180 {
181 library_init(NULL, "test-runner");
182
183 /* use non-blocking RNG to generate keys fast */
184 lib->settings->set_default_str(lib->settings,
185 "libstrongswan.plugins.random.random",
186 lib->settings->get_str(lib->settings,
187 "libstrongswan.plugins.random.urandom", "/dev/urandom"));
188
189 if (lib->leak_detective)
190 {
191 /* disable leak reports during testing */
192 lib->leak_detective->set_report_cb(lib->leak_detective,
193 NULL, NULL, NULL);
194 }
195 if (init && !init(TRUE))
196 {
197 library_deinit();
198 return FALSE;
199 }
200 dbg_default_set_level(LEVEL_SILENT);
201 return TRUE;
202 }
203
204 /**
205 * Failure description
206 */
207 typedef struct {
208 char *name;
209 char msg[512 - sizeof(char*) - 2 * sizeof(int)];
210 const char *file;
211 int line;
212 int i;
213 backtrace_t *bt;
214 } failure_t;
215
216 /**
217 * Data passed to leak report callbacks
218 */
219 typedef struct {
220 array_t *failures;
221 char *name;
222 int i;
223 int leaks;
224 } report_data_t;
225
226 /**
227 * Leak report callback, build failures from leaks
228 */
229 static void report_leaks(report_data_t *data, int count, size_t bytes,
230 backtrace_t *bt, bool detailed)
231 {
232 failure_t failure = {
233 .name = data->name,
234 .i = data->i,
235 .bt = bt->clone(bt),
236 };
237
238 snprintf(failure.msg, sizeof(failure.msg),
239 "Leak detected: %d allocations using %zu bytes", count, bytes);
240
241 array_insert(data->failures, -1, &failure);
242 }
243
244 /**
245 * Leak summary callback, check if any leaks found
246 */
247 static void sum_leaks(report_data_t *data, int count, size_t bytes,
248 int whitelisted)
249 {
250 data->leaks = count;
251 }
252
253 /**
254 * Do library cleanup and optionally check for memory leaks
255 */
256 static bool post_test(test_runner_init_t init, bool check_leaks,
257 array_t *failures, char *name, int i)
258 {
259 report_data_t data = {
260 .failures = failures,
261 .name = name,
262 .i = i,
263 };
264
265 if (init)
266 {
267 init(FALSE);
268 }
269 if (check_leaks && lib->leak_detective)
270 {
271 lib->leak_detective->set_report_cb(lib->leak_detective,
272 (leak_detective_report_cb_t)report_leaks,
273 (leak_detective_summary_cb_t)sum_leaks, &data);
274 }
275 library_deinit();
276
277 return data.leaks != 0;
278 }
279
280 /**
281 * Collect failure information, add failure_t to array
282 */
283 static void collect_failure_info(array_t *failures, char *name, int i)
284 {
285 failure_t failure = {
286 .name = name,
287 .i = i,
288 .bt = test_failure_backtrace(),
289 };
290
291 failure.line = test_failure_get(failure.msg, sizeof(failure.msg),
292 &failure.file);
293
294 array_insert(failures, -1, &failure);
295 }
296
297 /**
298 * Print array of collected failure_t to stderr
299 */
300 static void print_failures(array_t *failures)
301 {
302 failure_t failure;
303
304 backtrace_init();
305
306 while (array_remove(failures, 0, &failure))
307 {
308 fprintf(stderr, " %sFailure in '%s': %s (",
309 TTY(RED), failure.name, failure.msg);
310 if (failure.line)
311 {
312 fprintf(stderr, "%s:%d, ", failure.file, failure.line);
313 }
314 fprintf(stderr, "i = %d)%s\n", failure.i, TTY(DEF));
315 if (failure.bt)
316 {
317 failure.bt->log(failure.bt, stderr, TRUE);
318 failure.bt->destroy(failure.bt);
319 }
320 }
321
322 backtrace_deinit();
323 }
324
325 /**
326 * Run a single test case with fixtures
327 */
328 static bool run_case(test_case_t *tcase, test_runner_init_t init)
329 {
330 enumerator_t *enumerator;
331 test_function_t *tfun;
332 int passed = 0;
333 array_t *failures;
334
335 failures = array_create(sizeof(failure_t), 0);
336
337 fprintf(stderr, " Running case '%s': ", tcase->name);
338 fflush(stderr);
339
340 enumerator = array_create_enumerator(tcase->functions);
341 while (enumerator->enumerate(enumerator, &tfun))
342 {
343 int i, rounds = 0;
344
345 for (i = tfun->start; i < tfun->end; i++)
346 {
347 if (pre_test(init))
348 {
349 bool ok = FALSE, leaks = FALSE;
350
351 test_setup_timeout(tcase->timeout);
352
353 if (call_fixture(tcase, TRUE))
354 {
355 if (run_test(tfun, i))
356 {
357 if (call_fixture(tcase, FALSE))
358 {
359 ok = TRUE;
360 }
361 }
362 else
363 {
364 call_fixture(tcase, FALSE);
365 }
366
367 }
368 leaks = post_test(init, ok, failures, tfun->name, i);
369
370 test_setup_timeout(0);
371
372 if (ok)
373 {
374 if (!leaks)
375 {
376 rounds++;
377 fprintf(stderr, "%s+%s", TTY(GREEN), TTY(DEF));
378 }
379 }
380 else
381 {
382 collect_failure_info(failures, tfun->name, i);
383 }
384 if (!ok || leaks)
385 {
386 fprintf(stderr, "%s-%s", TTY(RED), TTY(DEF));
387 }
388 }
389 else
390 {
391 fprintf(stderr, "!");
392 }
393 }
394 fflush(stderr);
395 if (rounds == tfun->end - tfun->start)
396 {
397 passed++;
398 }
399 }
400 enumerator->destroy(enumerator);
401
402 fprintf(stderr, "\n");
403
404 print_failures(failures);
405 array_destroy(failures);
406
407 return passed == array_count(tcase->functions);
408 }
409
410 /**
411 * Run a single test suite
412 */
413 static bool run_suite(test_suite_t *suite, test_runner_init_t init)
414 {
415 enumerator_t *enumerator;
416 test_case_t *tcase;
417 int passed = 0;
418
419 fprintf(stderr, " Running suite '%s':\n", suite->name);
420
421 enumerator = array_create_enumerator(suite->tcases);
422 while (enumerator->enumerate(enumerator, &tcase))
423 {
424 if (run_case(tcase, init))
425 {
426 passed++;
427 }
428 }
429 enumerator->destroy(enumerator);
430
431 if (passed == array_count(suite->tcases))
432 {
433 fprintf(stderr, " %sPassed all %u '%s' test cases%s\n",
434 TTY(GREEN), array_count(suite->tcases), suite->name, TTY(DEF));
435 return TRUE;
436 }
437 fprintf(stderr, " %sPassed %u/%u '%s' test cases%s\n",
438 TTY(RED), passed, array_count(suite->tcases), suite->name, TTY(DEF));
439 return FALSE;
440 }
441
442 /**
443 * See header.
444 */
445 int test_runner_run(const char *name, test_configuration_t configs[],
446 test_runner_init_t init)
447 {
448 array_t *suites;
449 test_suite_t *suite;
450 enumerator_t *enumerator;
451 int passed = 0, result;
452
453 /* redirect all output to stderr (to redirect make's stdout to /dev/null) */
454 dup2(2, 1);
455
456 suites = load_suites(configs, init);
457 if (!suites)
458 {
459 return EXIT_FAILURE;
460 }
461
462 fprintf(stderr, "Running %u '%s' test suites:\n", array_count(suites), name);
463
464 enumerator = array_create_enumerator(suites);
465 while (enumerator->enumerate(enumerator, &suite))
466 {
467 if (run_suite(suite, init))
468 {
469 passed++;
470 }
471 }
472 enumerator->destroy(enumerator);
473
474 if (passed == array_count(suites))
475 {
476 fprintf(stderr, "%sPassed all %u '%s' suites%s\n",
477 TTY(GREEN), array_count(suites), name, TTY(DEF));
478 result = EXIT_SUCCESS;
479 }
480 else
481 {
482 fprintf(stderr, "%sPassed %u of %u '%s' suites%s\n",
483 TTY(RED), passed, array_count(suites), name, TTY(DEF));
484 result = EXIT_FAILURE;
485 }
486
487 unload_suites(suites);
488
489 return result;
490 }