unit-tests: Prevent a failing worker thread to go wild after it fails
[strongswan.git] / src / libstrongswan / tests / test_runner.c
1 /*
2 * Copyright (C) 2013 Tobias Brunner
3 * Hochschule fuer Technik Rapperswil
4 * Copyright (C) 2013 Martin Willi
5 * Copyright (C) 2013 revosec AG
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License as published by the
9 * Free Software Foundation; either version 2 of the License, or (at your
10 * option) any later version. See <http://www.fsf.org/copyleft/gpl.txt>.
11 *
12 * This program is distributed in the hope that it will be useful, but
13 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
14 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 * for more details.
16 */
17
18 #include "test_runner.h"
19
20 #include <library.h>
21 #include <plugins/plugin_feature.h>
22 #include <collections/array.h>
23 #include <utils/test.h>
24
25 #include <stdlib.h>
26 #include <dirent.h>
27 #include <unistd.h>
28 #include <limits.h>
29
30 /**
31 * Get a tty color escape character for stderr
32 */
33 #define TTY(color) tty_escape_get(2, TTY_FG_##color)
34
35 /**
36 * Initialize the lookup table for testable functions (defined in
37 * libstrongswan). We don't use the constructor attribute as the order can't
38 * really be defined (clang does not support it and gcc does not adhere to it in
39 * the monolithic build). The function here is a weak symbol in libstrongswan.
40 */
41 void testable_functions_create()
42 {
43 if (!testable_functions)
44 {
45 testable_functions = hashtable_create(hashtable_hash_str,
46 hashtable_equals_str, 8);
47 }
48 }
49
50 /**
51 * Destroy the lookup table for testable functions
52 */
53 static void testable_functions_destroy() __attribute__ ((destructor));
54 static void testable_functions_destroy()
55 {
56 DESTROY_IF(testable_functions);
57 /* if leak detective is enabled plugins are not actually unloaded, which
58 * means their destructor is called AFTER this one when the process
59 * terminates, make sure this does not crash */
60 testable_functions = NULL;
61 }
62
63 /**
64 * Destroy a single test suite and associated data
65 */
66 static void destroy_suite(test_suite_t *suite)
67 {
68 test_case_t *tcase;
69
70 while (array_remove(suite->tcases, 0, &tcase))
71 {
72 array_destroy(tcase->functions);
73 array_destroy(tcase->fixtures);
74 }
75 free(suite);
76 }
77
78 /**
79 * Removes and destroys test suites that are not selected.
80 */
81 static void filter_suites(array_t *loaded)
82 {
83 enumerator_t *enumerator, *names;
84 hashtable_t *selected;
85 test_suite_t *suite;
86 char *suites, *name;
87
88 suites = getenv("TESTS_SUITES");
89 if (!suites)
90 {
91 return;
92 }
93 selected = hashtable_create(hashtable_hash_str, hashtable_equals_str, 8);
94 names = enumerator_create_token(suites, ",", " ");
95 while (names->enumerate(names, &name))
96 {
97 selected->put(selected, name, name);
98 }
99 enumerator = array_create_enumerator(loaded);
100 while (enumerator->enumerate(enumerator, &suite))
101 {
102 if (!selected->get(selected, suite->name))
103 {
104 array_remove_at(loaded, enumerator);
105 destroy_suite(suite);
106 }
107 }
108 enumerator->destroy(enumerator);
109 selected->destroy(selected);
110 names->destroy(names);
111 }
112
113 /**
114 * Load all available test suites, or optionally only selected ones.
115 */
116 static array_t *load_suites(test_configuration_t configs[],
117 test_runner_init_t init)
118 {
119 array_t *suites;
120 bool old = FALSE;
121 int i;
122
123 library_init(NULL, "test-runner");
124
125 test_setup_handler();
126
127 if (init && !init(TRUE))
128 {
129 library_deinit();
130 return NULL;
131 }
132 lib->plugins->status(lib->plugins, LEVEL_CTRL);
133
134 if (lib->leak_detective)
135 {
136 old = lib->leak_detective->set_state(lib->leak_detective, FALSE);
137 }
138
139 suites = array_create(0, 0);
140
141 for (i = 0; configs[i].suite; i++)
142 {
143 if (configs[i].feature.type == 0 ||
144 lib->plugins->has_feature(lib->plugins, configs[i].feature))
145 {
146 array_insert(suites, -1, configs[i].suite());
147 }
148 }
149 filter_suites(suites);
150
151 if (lib->leak_detective)
152 {
153 lib->leak_detective->set_state(lib->leak_detective, old);
154 }
155
156 if (init)
157 {
158 init(FALSE);
159 }
160 library_deinit();
161
162 return suites;
163 }
164
165 /**
166 * Unload and destroy test suites and associated data
167 */
168 static void unload_suites(array_t *suites)
169 {
170 test_suite_t *suite;
171
172 while (array_remove(suites, 0, &suite))
173 {
174 destroy_suite(suite);
175 }
176 array_destroy(suites);
177 }
178
179 /**
180 * Run a single test function, return FALSE on failure
181 */
182 static bool run_test(test_function_t *tfun, int i)
183 {
184 if (test_restore_point())
185 {
186 tfun->cb(i);
187 return TRUE;
188 }
189 return FALSE;
190 }
191
192 /**
193 * Invoke fixture setup/teardown
194 */
195 static bool call_fixture(test_case_t *tcase, bool up)
196 {
197 enumerator_t *enumerator;
198 test_fixture_t *fixture;
199 bool failure = FALSE;
200
201 enumerator = array_create_enumerator(tcase->fixtures);
202 while (enumerator->enumerate(enumerator, &fixture))
203 {
204 if (test_restore_point())
205 {
206 if (up)
207 {
208 fixture->setup();
209 }
210 else
211 {
212 fixture->teardown();
213 }
214 }
215 else
216 {
217 failure = TRUE;
218 break;
219 }
220 }
221 enumerator->destroy(enumerator);
222
223 return !failure;
224 }
225
226 /**
227 * Test initialization, initializes libstrongswan for the next run
228 */
229 static bool pre_test(test_runner_init_t init)
230 {
231 level_t level = LEVEL_SILENT;
232 char *verbosity;
233
234 library_init(NULL, "test-runner");
235
236 /* use non-blocking RNG to generate keys fast */
237 lib->settings->set_default_str(lib->settings,
238 "libstrongswan.plugins.random.random",
239 lib->settings->get_str(lib->settings,
240 "libstrongswan.plugins.random.urandom", "/dev/urandom"));
241 /* same for the gcrypt plugin */
242 lib->settings->set_default_str(lib->settings,
243 "libstrongswan.plugins.gcrypt.quick_random", "yes");
244
245 if (lib->leak_detective)
246 {
247 /* disable leak reports during testing */
248 lib->leak_detective->set_report_cb(lib->leak_detective,
249 NULL, NULL, NULL);
250 }
251 if (init && !init(TRUE))
252 {
253 library_deinit();
254 return FALSE;
255 }
256 verbosity = getenv("TESTS_VERBOSITY");
257 if (verbosity)
258 {
259 level = atoi(verbosity);
260 }
261 dbg_default_set_level(level);
262 return TRUE;
263 }
264
265 /**
266 * Failure description
267 */
268 typedef struct {
269 char *name;
270 char msg[512 - sizeof(char*) - 2 * sizeof(int)];
271 const char *file;
272 int line;
273 int i;
274 backtrace_t *bt;
275 } failure_t;
276
277 /**
278 * Data passed to leak report callbacks
279 */
280 typedef struct {
281 array_t *failures;
282 char *name;
283 int i;
284 int leaks;
285 } report_data_t;
286
287 /**
288 * Leak report callback, build failures from leaks
289 */
290 static void report_leaks(report_data_t *data, int count, size_t bytes,
291 backtrace_t *bt, bool detailed)
292 {
293 failure_t failure = {
294 .name = data->name,
295 .i = data->i,
296 .bt = bt->clone(bt),
297 };
298
299 snprintf(failure.msg, sizeof(failure.msg),
300 "Leak detected: %d allocations using %zu bytes", count, bytes);
301
302 array_insert(data->failures, -1, &failure);
303 }
304
305 /**
306 * Leak summary callback, check if any leaks found
307 */
308 static void sum_leaks(report_data_t *data, int count, size_t bytes,
309 int whitelisted)
310 {
311 data->leaks = count;
312 }
313
314 /**
315 * Do library cleanup and optionally check for memory leaks
316 */
317 static bool post_test(test_runner_init_t init, bool check_leaks,
318 array_t *failures, char *name, int i)
319 {
320 report_data_t data = {
321 .failures = failures,
322 .name = name,
323 .i = i,
324 };
325
326 if (init)
327 {
328 init(FALSE);
329 }
330 if (check_leaks && lib->leak_detective)
331 {
332 lib->leak_detective->set_report_cb(lib->leak_detective,
333 (leak_detective_report_cb_t)report_leaks,
334 (leak_detective_summary_cb_t)sum_leaks, &data);
335 }
336 library_deinit();
337
338 return data.leaks != 0;
339 }
340
341 /**
342 * Collect failure information, add failure_t to array
343 */
344 static void collect_failure_info(array_t *failures, char *name, int i)
345 {
346 failure_t failure = {
347 .name = name,
348 .i = i,
349 .bt = test_failure_backtrace(),
350 };
351
352 failure.line = test_failure_get(failure.msg, sizeof(failure.msg),
353 &failure.file);
354
355 array_insert(failures, -1, &failure);
356 }
357
358 /**
359 * Print array of collected failure_t to stderr
360 */
361 static void print_failures(array_t *failures)
362 {
363 failure_t failure;
364
365 backtrace_init();
366
367 while (array_remove(failures, 0, &failure))
368 {
369 fprintf(stderr, " %sFailure in '%s': %s (",
370 TTY(RED), failure.name, failure.msg);
371 if (failure.line)
372 {
373 fprintf(stderr, "%s:%d, ", failure.file, failure.line);
374 }
375 fprintf(stderr, "i = %d)%s\n", failure.i, TTY(DEF));
376 if (failure.bt)
377 {
378 failure.bt->log(failure.bt, stderr, TRUE);
379 failure.bt->destroy(failure.bt);
380 }
381 }
382
383 backtrace_deinit();
384 }
385
386 /**
387 * Run a single test case with fixtures
388 */
389 static bool run_case(test_case_t *tcase, test_runner_init_t init)
390 {
391 enumerator_t *enumerator;
392 test_function_t *tfun;
393 int passed = 0;
394 array_t *failures;
395
396 failures = array_create(sizeof(failure_t), 0);
397
398 fprintf(stderr, " Running case '%s': ", tcase->name);
399 fflush(stderr);
400
401 enumerator = array_create_enumerator(tcase->functions);
402 while (enumerator->enumerate(enumerator, &tfun))
403 {
404 int i, rounds = 0;
405
406 for (i = tfun->start; i < tfun->end; i++)
407 {
408 if (pre_test(init))
409 {
410 bool ok = FALSE, leaks = FALSE;
411
412 test_setup_timeout(tcase->timeout);
413
414 if (call_fixture(tcase, TRUE))
415 {
416 if (run_test(tfun, i))
417 {
418 if (call_fixture(tcase, FALSE))
419 {
420 ok = TRUE;
421 }
422 }
423 else
424 {
425 call_fixture(tcase, FALSE);
426 }
427
428 }
429 leaks = post_test(init, ok, failures, tfun->name, i);
430
431 test_setup_timeout(0);
432
433 if (ok)
434 {
435 if (!leaks)
436 {
437 rounds++;
438 fprintf(stderr, "%s+%s", TTY(GREEN), TTY(DEF));
439 }
440 }
441 else
442 {
443 collect_failure_info(failures, tfun->name, i);
444 }
445 if (!ok || leaks)
446 {
447 fprintf(stderr, "%s-%s", TTY(RED), TTY(DEF));
448 }
449 }
450 else
451 {
452 fprintf(stderr, "!");
453 }
454 }
455 fflush(stderr);
456 if (rounds == tfun->end - tfun->start)
457 {
458 passed++;
459 }
460 }
461 enumerator->destroy(enumerator);
462
463 fprintf(stderr, "\n");
464
465 print_failures(failures);
466 array_destroy(failures);
467
468 return passed == array_count(tcase->functions);
469 }
470
471 /**
472 * Run a single test suite
473 */
474 static bool run_suite(test_suite_t *suite, test_runner_init_t init)
475 {
476 enumerator_t *enumerator;
477 test_case_t *tcase;
478 int passed = 0;
479
480 fprintf(stderr, " Running suite '%s':\n", suite->name);
481
482 enumerator = array_create_enumerator(suite->tcases);
483 while (enumerator->enumerate(enumerator, &tcase))
484 {
485 if (run_case(tcase, init))
486 {
487 passed++;
488 }
489 }
490 enumerator->destroy(enumerator);
491
492 if (passed == array_count(suite->tcases))
493 {
494 fprintf(stderr, " %sPassed all %u '%s' test cases%s\n",
495 TTY(GREEN), array_count(suite->tcases), suite->name, TTY(DEF));
496 return TRUE;
497 }
498 fprintf(stderr, " %sPassed %u/%u '%s' test cases%s\n",
499 TTY(RED), passed, array_count(suite->tcases), suite->name, TTY(DEF));
500 return FALSE;
501 }
502
503 /**
504 * See header.
505 */
506 int test_runner_run(const char *name, test_configuration_t configs[],
507 test_runner_init_t init)
508 {
509 array_t *suites;
510 test_suite_t *suite;
511 enumerator_t *enumerator;
512 int passed = 0, result;
513
514 /* redirect all output to stderr (to redirect make's stdout to /dev/null) */
515 dup2(2, 1);
516
517 suites = load_suites(configs, init);
518 if (!suites)
519 {
520 return EXIT_FAILURE;
521 }
522
523 fprintf(stderr, "Running %u '%s' test suites:\n", array_count(suites), name);
524
525 enumerator = array_create_enumerator(suites);
526 while (enumerator->enumerate(enumerator, &suite))
527 {
528 if (run_suite(suite, init))
529 {
530 passed++;
531 }
532 }
533 enumerator->destroy(enumerator);
534
535 if (passed == array_count(suites))
536 {
537 fprintf(stderr, "%sPassed all %u '%s' suites%s\n",
538 TTY(GREEN), array_count(suites), name, TTY(DEF));
539 result = EXIT_SUCCESS;
540 }
541 else
542 {
543 fprintf(stderr, "%sPassed %u of %u '%s' suites%s\n",
544 TTY(RED), passed, array_count(suites), name, TTY(DEF));
545 result = EXIT_FAILURE;
546 }
547
548 unload_suites(suites);
549
550 return result;
551 }