unit-tests: Catch timeouts during test runner deinit function
[strongswan.git] / src / libstrongswan / tests / test_runner.c
1 /*
2 * Copyright (C) 2013 Tobias Brunner
3 * Hochschule fuer Technik Rapperswil
4 * Copyright (C) 2013 Martin Willi
5 * Copyright (C) 2013 revosec AG
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License as published by the
9 * Free Software Foundation; either version 2 of the License, or (at your
10 * option) any later version. See <http://www.fsf.org/copyleft/gpl.txt>.
11 *
12 * This program is distributed in the hope that it will be useful, but
13 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
14 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 * for more details.
16 */
17
18 #include "test_runner.h"
19
20 #include <library.h>
21 #include <plugins/plugin_feature.h>
22 #include <collections/array.h>
23 #include <utils/test.h>
24
25 #include <stdlib.h>
26 #include <dirent.h>
27 #include <unistd.h>
28 #include <limits.h>
29
30 /**
31 * Get a tty color escape character for stderr
32 */
33 #define TTY(color) tty_escape_get(2, TTY_FG_##color)
34
35 /**
36 * Initialize the lookup table for testable functions (defined in
37 * libstrongswan). We don't use the constructor attribute as the order can't
38 * really be defined (clang does not support it and gcc does not adhere to it in
39 * the monolithic build). The function here is a weak symbol in libstrongswan.
40 */
41 void testable_functions_create()
42 {
43 if (!testable_functions)
44 {
45 testable_functions = hashtable_create(hashtable_hash_str,
46 hashtable_equals_str, 8);
47 }
48 }
49
50 /**
51 * Destroy the lookup table for testable functions
52 */
53 static void testable_functions_destroy() __attribute__ ((destructor));
54 static void testable_functions_destroy()
55 {
56 DESTROY_IF(testable_functions);
57 /* if leak detective is enabled plugins are not actually unloaded, which
58 * means their destructor is called AFTER this one when the process
59 * terminates, make sure this does not crash */
60 testable_functions = NULL;
61 }
62
63 /**
64 * Destroy a single test suite and associated data
65 */
66 static void destroy_suite(test_suite_t *suite)
67 {
68 test_case_t *tcase;
69
70 while (array_remove(suite->tcases, 0, &tcase))
71 {
72 array_destroy(tcase->functions);
73 array_destroy(tcase->fixtures);
74 }
75 free(suite);
76 }
77
78 /**
79 * Removes and destroys test suites that are not selected.
80 */
81 static void filter_suites(array_t *loaded)
82 {
83 enumerator_t *enumerator, *names;
84 hashtable_t *selected;
85 test_suite_t *suite;
86 char *suites, *name;
87
88 suites = getenv("TESTS_SUITES");
89 if (!suites)
90 {
91 return;
92 }
93 selected = hashtable_create(hashtable_hash_str, hashtable_equals_str, 8);
94 names = enumerator_create_token(suites, ",", " ");
95 while (names->enumerate(names, &name))
96 {
97 selected->put(selected, name, name);
98 }
99 enumerator = array_create_enumerator(loaded);
100 while (enumerator->enumerate(enumerator, &suite))
101 {
102 if (!selected->get(selected, suite->name))
103 {
104 array_remove_at(loaded, enumerator);
105 destroy_suite(suite);
106 }
107 }
108 enumerator->destroy(enumerator);
109 selected->destroy(selected);
110 names->destroy(names);
111 }
112
113 /**
114 * Load all available test suites, or optionally only selected ones.
115 */
116 static array_t *load_suites(test_configuration_t configs[],
117 test_runner_init_t init)
118 {
119 array_t *suites;
120 bool old = FALSE;
121 int i;
122
123 library_init(NULL, "test-runner");
124
125 test_setup_handler();
126
127 if (init && !init(TRUE))
128 {
129 library_deinit();
130 return NULL;
131 }
132 lib->plugins->status(lib->plugins, LEVEL_CTRL);
133
134 if (lib->leak_detective)
135 {
136 old = lib->leak_detective->set_state(lib->leak_detective, FALSE);
137 }
138
139 suites = array_create(0, 0);
140
141 for (i = 0; configs[i].suite; i++)
142 {
143 if (configs[i].feature.type == 0 ||
144 lib->plugins->has_feature(lib->plugins, configs[i].feature))
145 {
146 array_insert(suites, -1, configs[i].suite());
147 }
148 }
149 filter_suites(suites);
150
151 if (lib->leak_detective)
152 {
153 lib->leak_detective->set_state(lib->leak_detective, old);
154 }
155
156 if (init)
157 {
158 init(FALSE);
159 }
160 library_deinit();
161
162 return suites;
163 }
164
165 /**
166 * Unload and destroy test suites and associated data
167 */
168 static void unload_suites(array_t *suites)
169 {
170 test_suite_t *suite;
171
172 while (array_remove(suites, 0, &suite))
173 {
174 destroy_suite(suite);
175 }
176 array_destroy(suites);
177 }
178
179 /**
180 * Run a single test function, return FALSE on failure
181 */
182 static bool run_test(test_function_t *tfun, int i)
183 {
184 if (test_restore_point())
185 {
186 tfun->cb(i);
187 return TRUE;
188 }
189 return FALSE;
190 }
191
192 /**
193 * Invoke fixture setup/teardown
194 */
195 static bool call_fixture(test_case_t *tcase, bool up)
196 {
197 enumerator_t *enumerator;
198 test_fixture_t *fixture;
199 bool failure = FALSE;
200
201 enumerator = array_create_enumerator(tcase->fixtures);
202 while (enumerator->enumerate(enumerator, &fixture))
203 {
204 if (test_restore_point())
205 {
206 if (up)
207 {
208 fixture->setup();
209 }
210 else
211 {
212 fixture->teardown();
213 }
214 }
215 else
216 {
217 failure = TRUE;
218 break;
219 }
220 }
221 enumerator->destroy(enumerator);
222
223 return !failure;
224 }
225
226 /**
227 * Test initialization, initializes libstrongswan for the next run
228 */
229 static bool pre_test(test_runner_init_t init)
230 {
231 level_t level = LEVEL_SILENT;
232 char *verbosity;
233
234 library_init(NULL, "test-runner");
235
236 /* use non-blocking RNG to generate keys fast */
237 lib->settings->set_default_str(lib->settings,
238 "libstrongswan.plugins.random.random",
239 lib->settings->get_str(lib->settings,
240 "libstrongswan.plugins.random.urandom", "/dev/urandom"));
241 /* same for the gcrypt plugin */
242 lib->settings->set_default_str(lib->settings,
243 "libstrongswan.plugins.gcrypt.quick_random", "yes");
244
245 if (lib->leak_detective)
246 {
247 /* disable leak reports during testing */
248 lib->leak_detective->set_report_cb(lib->leak_detective,
249 NULL, NULL, NULL);
250 }
251 if (init && !init(TRUE))
252 {
253 library_deinit();
254 return FALSE;
255 }
256 verbosity = getenv("TESTS_VERBOSITY");
257 if (verbosity)
258 {
259 level = atoi(verbosity);
260 }
261 dbg_default_set_level(level);
262 return TRUE;
263 }
264
265 /**
266 * Failure description
267 */
268 typedef struct {
269 char *name;
270 char msg[512 - sizeof(char*) - 2 * sizeof(int)];
271 const char *file;
272 int line;
273 int i;
274 backtrace_t *bt;
275 } failure_t;
276
277 /**
278 * Data passed to leak report callbacks
279 */
280 typedef struct {
281 array_t *failures;
282 char *name;
283 int i;
284 int leaks;
285 } report_data_t;
286
287 /**
288 * Leak report callback, build failures from leaks
289 */
290 static void report_leaks(report_data_t *data, int count, size_t bytes,
291 backtrace_t *bt, bool detailed)
292 {
293 failure_t failure = {
294 .name = data->name,
295 .i = data->i,
296 .bt = bt->clone(bt),
297 };
298
299 snprintf(failure.msg, sizeof(failure.msg),
300 "Leak detected: %d allocations using %zu bytes", count, bytes);
301
302 array_insert(data->failures, -1, &failure);
303 }
304
305 /**
306 * Leak summary callback, check if any leaks found
307 */
308 static void sum_leaks(report_data_t *data, int count, size_t bytes,
309 int whitelisted)
310 {
311 data->leaks = count;
312 }
313
314 /**
315 * Do library cleanup and optionally check for memory leaks
316 */
317 static bool post_test(test_runner_init_t init, bool check_leaks,
318 array_t *failures, char *name, int i, int *leaks)
319 {
320 report_data_t data = {
321 .failures = failures,
322 .name = name,
323 .i = i,
324 };
325
326 if (init)
327 {
328 if (test_restore_point())
329 {
330 init(FALSE);
331 }
332 else
333 {
334 library_deinit();
335 return FALSE;
336 }
337 }
338 if (check_leaks && lib->leak_detective)
339 {
340 lib->leak_detective->set_report_cb(lib->leak_detective,
341 (leak_detective_report_cb_t)report_leaks,
342 (leak_detective_summary_cb_t)sum_leaks, &data);
343 }
344 library_deinit();
345
346 *leaks = data.leaks;
347 return TRUE;
348 }
349
350 /**
351 * Collect failure information, add failure_t to array
352 */
353 static void collect_failure_info(array_t *failures, char *name, int i)
354 {
355 failure_t failure = {
356 .name = name,
357 .i = i,
358 .bt = test_failure_backtrace(),
359 };
360
361 failure.line = test_failure_get(failure.msg, sizeof(failure.msg),
362 &failure.file);
363
364 array_insert(failures, -1, &failure);
365 }
366
367 /**
368 * Print array of collected failure_t to stderr
369 */
370 static void print_failures(array_t *failures)
371 {
372 failure_t failure;
373
374 backtrace_init();
375
376 while (array_remove(failures, 0, &failure))
377 {
378 fprintf(stderr, " %sFailure in '%s': %s (",
379 TTY(RED), failure.name, failure.msg);
380 if (failure.line)
381 {
382 fprintf(stderr, "%s:%d, ", failure.file, failure.line);
383 }
384 fprintf(stderr, "i = %d)%s\n", failure.i, TTY(DEF));
385 if (failure.bt)
386 {
387 failure.bt->log(failure.bt, stderr, TRUE);
388 failure.bt->destroy(failure.bt);
389 }
390 }
391
392 backtrace_deinit();
393 }
394
395 /**
396 * Run a single test case with fixtures
397 */
398 static bool run_case(test_case_t *tcase, test_runner_init_t init)
399 {
400 enumerator_t *enumerator;
401 test_function_t *tfun;
402 int passed = 0;
403 array_t *failures;
404
405 failures = array_create(sizeof(failure_t), 0);
406
407 fprintf(stderr, " Running case '%s': ", tcase->name);
408 fflush(stderr);
409
410 enumerator = array_create_enumerator(tcase->functions);
411 while (enumerator->enumerate(enumerator, &tfun))
412 {
413 int i, rounds = 0;
414
415 for (i = tfun->start; i < tfun->end; i++)
416 {
417 if (pre_test(init))
418 {
419 bool ok = FALSE;
420 int leaks = 0;
421
422 test_setup_timeout(tcase->timeout);
423
424 if (call_fixture(tcase, TRUE))
425 {
426 if (run_test(tfun, i))
427 {
428 if (call_fixture(tcase, FALSE))
429 {
430 ok = TRUE;
431 }
432 }
433 else
434 {
435 call_fixture(tcase, FALSE);
436 }
437 }
438 if (!post_test(init, ok, failures, tfun->name, i, &leaks))
439 {
440 ok = FALSE;
441 }
442
443 test_setup_timeout(0);
444
445 if (ok)
446 {
447 if (!leaks)
448 {
449 rounds++;
450 fprintf(stderr, "%s+%s", TTY(GREEN), TTY(DEF));
451 }
452 }
453 else
454 {
455 collect_failure_info(failures, tfun->name, i);
456 }
457 if (!ok || leaks)
458 {
459 fprintf(stderr, "%s-%s", TTY(RED), TTY(DEF));
460 }
461 }
462 else
463 {
464 fprintf(stderr, "!");
465 }
466 }
467 fflush(stderr);
468 if (rounds == tfun->end - tfun->start)
469 {
470 passed++;
471 }
472 }
473 enumerator->destroy(enumerator);
474
475 fprintf(stderr, "\n");
476
477 print_failures(failures);
478 array_destroy(failures);
479
480 return passed == array_count(tcase->functions);
481 }
482
483 /**
484 * Run a single test suite
485 */
486 static bool run_suite(test_suite_t *suite, test_runner_init_t init)
487 {
488 enumerator_t *enumerator;
489 test_case_t *tcase;
490 int passed = 0;
491
492 fprintf(stderr, " Running suite '%s':\n", suite->name);
493
494 enumerator = array_create_enumerator(suite->tcases);
495 while (enumerator->enumerate(enumerator, &tcase))
496 {
497 if (run_case(tcase, init))
498 {
499 passed++;
500 }
501 }
502 enumerator->destroy(enumerator);
503
504 if (passed == array_count(suite->tcases))
505 {
506 fprintf(stderr, " %sPassed all %u '%s' test cases%s\n",
507 TTY(GREEN), array_count(suite->tcases), suite->name, TTY(DEF));
508 return TRUE;
509 }
510 fprintf(stderr, " %sPassed %u/%u '%s' test cases%s\n",
511 TTY(RED), passed, array_count(suite->tcases), suite->name, TTY(DEF));
512 return FALSE;
513 }
514
515 /**
516 * See header.
517 */
518 int test_runner_run(const char *name, test_configuration_t configs[],
519 test_runner_init_t init)
520 {
521 array_t *suites;
522 test_suite_t *suite;
523 enumerator_t *enumerator;
524 int passed = 0, result;
525
526 /* redirect all output to stderr (to redirect make's stdout to /dev/null) */
527 dup2(2, 1);
528
529 suites = load_suites(configs, init);
530 if (!suites)
531 {
532 return EXIT_FAILURE;
533 }
534
535 fprintf(stderr, "Running %u '%s' test suites:\n", array_count(suites), name);
536
537 enumerator = array_create_enumerator(suites);
538 while (enumerator->enumerate(enumerator, &suite))
539 {
540 if (run_suite(suite, init))
541 {
542 passed++;
543 }
544 }
545 enumerator->destroy(enumerator);
546
547 if (passed == array_count(suites))
548 {
549 fprintf(stderr, "%sPassed all %u '%s' suites%s\n",
550 TTY(GREEN), array_count(suites), name, TTY(DEF));
551 result = EXIT_SUCCESS;
552 }
553 else
554 {
555 fprintf(stderr, "%sPassed %u of %u '%s' suites%s\n",
556 TTY(RED), passed, array_count(suites), name, TTY(DEF));
557 result = EXIT_FAILURE;
558 }
559
560 unload_suites(suites);
561
562 return result;
563 }