unit-tests: Add option to exclude specific test suites
[strongswan.git] / src / libstrongswan / tests / test_runner.c
1 /*
2 * Copyright (C) 2013 Tobias Brunner
3 * Hochschule fuer Technik Rapperswil
4 * Copyright (C) 2013 Martin Willi
5 * Copyright (C) 2013 revosec AG
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License as published by the
9 * Free Software Foundation; either version 2 of the License, or (at your
10 * option) any later version. See <http://www.fsf.org/copyleft/gpl.txt>.
11 *
12 * This program is distributed in the hope that it will be useful, but
13 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
14 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 * for more details.
16 */
17
18 #include "test_runner.h"
19
20 #include <library.h>
21 #include <threading/thread.h>
22 #include <plugins/plugin_feature.h>
23 #include <collections/array.h>
24 #include <utils/test.h>
25
26 #include <stdlib.h>
27 #include <dirent.h>
28 #include <unistd.h>
29 #include <limits.h>
30 #include <stdlib.h>
31
32 /**
33 * Get a tty color escape character for stderr
34 */
35 #define TTY(color) tty_escape_get(2, TTY_FG_##color)
36
37 /**
38 * A global symbol indicating libtest linkage
39 */
40 #ifdef WIN32
41 __declspec(dllexport)
42 #endif
43 bool test_runner_available = TRUE;
44
45 /**
46 * Destroy a single test suite and associated data
47 */
48 static void destroy_suite(test_suite_t *suite)
49 {
50 test_case_t *tcase;
51
52 while (array_remove(suite->tcases, 0, &tcase))
53 {
54 array_destroy(tcase->functions);
55 array_destroy(tcase->fixtures);
56 }
57 free(suite);
58 }
59
60 /**
61 * Filter loaded test suites, either remove suites listed (exclude=TRUE), or all
62 * that are not listed (exclude=FALSE).
63 */
64 static void apply_filter(array_t *loaded, char *filter, bool exclude)
65 {
66 enumerator_t *enumerator, *names;
67 hashtable_t *listed;
68 test_suite_t *suite;
69 char *name;
70
71 listed = hashtable_create(hashtable_hash_str, hashtable_equals_str, 8);
72 names = enumerator_create_token(filter, ",", " ");
73 while (names->enumerate(names, &name))
74 {
75 listed->put(listed, name, name);
76 }
77 enumerator = array_create_enumerator(loaded);
78 while (enumerator->enumerate(enumerator, &suite))
79 {
80 if ((exclude && listed->get(listed, suite->name)) ||
81 (!exclude && !listed->get(listed, suite->name)))
82 {
83 array_remove_at(loaded, enumerator);
84 destroy_suite(suite);
85 }
86 }
87 enumerator->destroy(enumerator);
88 listed->destroy(listed);
89 names->destroy(names);
90 }
91
92 /**
93 * Removes and destroys test suites that are not selected or
94 * explicitly excluded.
95 */
96 static void filter_suites(array_t *loaded)
97 {
98 char *filter;
99
100 filter = getenv("TESTS_SUITES");
101 if (filter)
102 {
103 apply_filter(loaded, filter, FALSE);
104 }
105 filter = getenv("TESTS_SUITES_EXCLUDE");
106 if (filter)
107 {
108 apply_filter(loaded, filter, TRUE);
109 }
110 }
111
112 /**
113 * Load all available test suites, or optionally only selected ones.
114 */
115 static array_t *load_suites(test_configuration_t configs[],
116 test_runner_init_t init, char *cfg)
117 {
118 array_t *suites;
119 bool old = FALSE;
120 int i;
121
122 library_init(cfg, "test-runner");
123
124 test_setup_handler();
125
126 if (init && !init(TRUE))
127 {
128 library_deinit();
129 return NULL;
130 }
131 lib->plugins->status(lib->plugins, LEVEL_CTRL);
132
133 if (lib->leak_detective)
134 {
135 old = lib->leak_detective->set_state(lib->leak_detective, FALSE);
136 }
137
138 suites = array_create(0, 0);
139
140 for (i = 0; configs[i].suite; i++)
141 {
142 if (configs[i].feature.type == 0 ||
143 lib->plugins->has_feature(lib->plugins, configs[i].feature))
144 {
145 array_insert(suites, -1, configs[i].suite());
146 }
147 }
148 filter_suites(suites);
149
150 if (lib->leak_detective)
151 {
152 lib->leak_detective->set_state(lib->leak_detective, old);
153 }
154
155 if (init)
156 {
157 init(FALSE);
158 }
159 library_deinit();
160
161 return suites;
162 }
163
164 /**
165 * Unload and destroy test suites and associated data
166 */
167 static void unload_suites(array_t *suites)
168 {
169 test_suite_t *suite;
170
171 while (array_remove(suites, 0, &suite))
172 {
173 destroy_suite(suite);
174 }
175 array_destroy(suites);
176 }
177
178 /**
179 * Run a single test function, return FALSE on failure
180 */
181 static bool run_test(test_function_t *tfun, int i)
182 {
183 if (test_restore_point())
184 {
185 tfun->cb(i);
186 return TRUE;
187 }
188 return FALSE;
189 }
190
191 /**
192 * Invoke fixture setup/teardown
193 */
194 static bool call_fixture(test_case_t *tcase, bool up)
195 {
196 enumerator_t *enumerator;
197 test_fixture_t *fixture;
198 bool failure = FALSE;
199
200 enumerator = array_create_enumerator(tcase->fixtures);
201 while (enumerator->enumerate(enumerator, &fixture))
202 {
203 if (test_restore_point())
204 {
205 if (up)
206 {
207 if (fixture->setup)
208 {
209 fixture->setup();
210 }
211 }
212 else
213 {
214 if (fixture->teardown)
215 {
216 fixture->teardown();
217 }
218 }
219 }
220 else
221 {
222 failure = TRUE;
223 break;
224 }
225 }
226 enumerator->destroy(enumerator);
227
228 return !failure;
229 }
230
231 /**
232 * Test initialization, initializes libstrongswan for the next run
233 */
234 static bool pre_test(test_runner_init_t init, char *cfg)
235 {
236 level_t level = LEVEL_SILENT;
237 char *verbosity;
238
239 library_init(cfg, "test-runner");
240
241 /* use non-blocking RNG to generate keys fast */
242 lib->settings->set_default_str(lib->settings,
243 "libstrongswan.plugins.random.random",
244 lib->settings->get_str(lib->settings,
245 "libstrongswan.plugins.random.urandom", "/dev/urandom"));
246 /* same for the gcrypt plugin */
247 lib->settings->set_default_str(lib->settings,
248 "libstrongswan.plugins.gcrypt.quick_random", "yes");
249
250 if (lib->leak_detective)
251 {
252 /* disable leak reports during testing */
253 lib->leak_detective->set_report_cb(lib->leak_detective,
254 NULL, NULL, NULL);
255 }
256 if (init && !init(TRUE))
257 {
258 library_deinit();
259 return FALSE;
260 }
261 verbosity = getenv("TESTS_VERBOSITY");
262 if (verbosity)
263 {
264 level = atoi(verbosity);
265 }
266 dbg_default_set_level(level);
267 return TRUE;
268 }
269
270 /**
271 * Failure description
272 */
273 typedef struct {
274 char *name;
275 char msg[512 - sizeof(char*) - 2 * sizeof(int)];
276 const char *file;
277 int line;
278 int i;
279 backtrace_t *bt;
280 } failure_t;
281
282 /**
283 * Data passed to leak report callbacks
284 */
285 typedef struct {
286 array_t *failures;
287 char *name;
288 int i;
289 int leaks;
290 } report_data_t;
291
292 /**
293 * Leak report callback, build failures from leaks
294 */
295 static void report_leaks(report_data_t *data, int count, size_t bytes,
296 backtrace_t *bt, bool detailed)
297 {
298 failure_t failure = {
299 .name = data->name,
300 .i = data->i,
301 .bt = bt->clone(bt),
302 };
303
304 snprintf(failure.msg, sizeof(failure.msg),
305 "Leak detected: %d allocations using %zu bytes", count, bytes);
306
307 array_insert(data->failures, -1, &failure);
308 }
309
310 /**
311 * Leak summary callback, check if any leaks found
312 */
313 static void sum_leaks(report_data_t *data, int count, size_t bytes,
314 int whitelisted)
315 {
316 data->leaks = count;
317 }
318
319 /**
320 * Do library cleanup and optionally check for memory leaks
321 */
322 static bool post_test(test_runner_init_t init, bool check_leaks,
323 array_t *failures, char *name, int i, int *leaks)
324 {
325 report_data_t data = {
326 .failures = failures,
327 .name = name,
328 .i = i,
329 };
330
331 if (init)
332 {
333 if (test_restore_point())
334 {
335 init(FALSE);
336 }
337 else
338 {
339 library_deinit();
340 return FALSE;
341 }
342 }
343 if (check_leaks && lib->leak_detective)
344 {
345 lib->leak_detective->set_report_cb(lib->leak_detective,
346 (leak_detective_report_cb_t)report_leaks,
347 (leak_detective_summary_cb_t)sum_leaks, &data);
348 }
349 library_deinit();
350
351 *leaks = data.leaks;
352 return TRUE;
353 }
354
355 /**
356 * Collect failure information, add failure_t to array
357 */
358 static void collect_failure_info(array_t *failures, char *name, int i)
359 {
360 failure_t failure = {
361 .name = name,
362 .i = i,
363 .bt = test_failure_backtrace(),
364 };
365
366 failure.line = test_failure_get(failure.msg, sizeof(failure.msg),
367 &failure.file);
368
369 array_insert(failures, -1, &failure);
370 }
371
372 /**
373 * Print array of collected failure_t to stderr
374 */
375 static void print_failures(array_t *failures)
376 {
377 failure_t failure;
378
379 threads_init();
380 backtrace_init();
381
382 while (array_remove(failures, 0, &failure))
383 {
384 fprintf(stderr, " %sFailure in '%s': %s (",
385 TTY(RED), failure.name, failure.msg);
386 if (failure.line)
387 {
388 fprintf(stderr, "%s:%d, ", failure.file, failure.line);
389 }
390 fprintf(stderr, "i = %d)%s\n", failure.i, TTY(DEF));
391 if (failure.bt)
392 {
393 failure.bt->log(failure.bt, stderr, TRUE);
394 failure.bt->destroy(failure.bt);
395 }
396 }
397
398 backtrace_deinit();
399 threads_deinit();
400 }
401
402 /**
403 * Run a single test case with fixtures
404 */
405 static bool run_case(test_case_t *tcase, test_runner_init_t init, char *cfg)
406 {
407 enumerator_t *enumerator;
408 test_function_t *tfun;
409 int passed = 0;
410 array_t *failures;
411
412 failures = array_create(sizeof(failure_t), 0);
413
414 fprintf(stderr, " Running case '%s': ", tcase->name);
415 fflush(stderr);
416
417 enumerator = array_create_enumerator(tcase->functions);
418 while (enumerator->enumerate(enumerator, &tfun))
419 {
420 int i, rounds = 0;
421
422 for (i = tfun->start; i < tfun->end; i++)
423 {
424 if (pre_test(init, cfg))
425 {
426 bool ok = FALSE;
427 int leaks = 0;
428
429 test_setup_timeout(tcase->timeout);
430
431 if (call_fixture(tcase, TRUE))
432 {
433 if (run_test(tfun, i))
434 {
435 if (call_fixture(tcase, FALSE))
436 {
437 ok = TRUE;
438 }
439 }
440 else
441 {
442 call_fixture(tcase, FALSE);
443 }
444 }
445 if (!post_test(init, ok, failures, tfun->name, i, &leaks))
446 {
447 ok = FALSE;
448 }
449
450 test_setup_timeout(0);
451
452 if (ok)
453 {
454 if (!leaks)
455 {
456 rounds++;
457 fprintf(stderr, "%s+%s", TTY(GREEN), TTY(DEF));
458 }
459 }
460 else
461 {
462 collect_failure_info(failures, tfun->name, i);
463 }
464 if (!ok || leaks)
465 {
466 fprintf(stderr, "%s-%s", TTY(RED), TTY(DEF));
467 }
468 }
469 else
470 {
471 fprintf(stderr, "!");
472 }
473 }
474 fflush(stderr);
475 if (rounds == tfun->end - tfun->start)
476 {
477 passed++;
478 }
479 }
480 enumerator->destroy(enumerator);
481
482 fprintf(stderr, "\n");
483
484 print_failures(failures);
485 array_destroy(failures);
486
487 return passed == array_count(tcase->functions);
488 }
489
490 /**
491 * Run a single test suite
492 */
493 static bool run_suite(test_suite_t *suite, test_runner_init_t init, char *cfg)
494 {
495 enumerator_t *enumerator;
496 test_case_t *tcase;
497 int passed = 0;
498
499 fprintf(stderr, " Running suite '%s':\n", suite->name);
500
501 enumerator = array_create_enumerator(suite->tcases);
502 while (enumerator->enumerate(enumerator, &tcase))
503 {
504 if (run_case(tcase, init, cfg))
505 {
506 passed++;
507 }
508 }
509 enumerator->destroy(enumerator);
510
511 if (passed == array_count(suite->tcases))
512 {
513 fprintf(stderr, " %sPassed all %u '%s' test cases%s\n",
514 TTY(GREEN), array_count(suite->tcases), suite->name, TTY(DEF));
515 return TRUE;
516 }
517 fprintf(stderr, " %sPassed %u/%u '%s' test cases%s\n",
518 TTY(RED), passed, array_count(suite->tcases), suite->name, TTY(DEF));
519 return FALSE;
520 }
521
522 /**
523 * See header.
524 */
525 int test_runner_run(const char *name, test_configuration_t configs[],
526 test_runner_init_t init)
527 {
528 array_t *suites;
529 test_suite_t *suite;
530 enumerator_t *enumerator;
531 int passed = 0, result;
532 char *cfg;
533
534 /* redirect all output to stderr (to redirect make's stdout to /dev/null) */
535 dup2(2, 1);
536
537 cfg = getenv("TESTS_STRONGSWAN_CONF");
538
539 suites = load_suites(configs, init, cfg);
540 if (!suites)
541 {
542 return EXIT_FAILURE;
543 }
544
545 fprintf(stderr, "Running %u '%s' test suites:\n", array_count(suites), name);
546
547 enumerator = array_create_enumerator(suites);
548 while (enumerator->enumerate(enumerator, &suite))
549 {
550 if (run_suite(suite, init, cfg))
551 {
552 passed++;
553 }
554 }
555 enumerator->destroy(enumerator);
556
557 if (passed == array_count(suites))
558 {
559 fprintf(stderr, "%sPassed all %u '%s' suites%s\n",
560 TTY(GREEN), array_count(suites), name, TTY(DEF));
561 result = EXIT_SUCCESS;
562 }
563 else
564 {
565 fprintf(stderr, "%sPassed %u of %u '%s' suites%s\n",
566 TTY(RED), passed, array_count(suites), name, TTY(DEF));
567 result = EXIT_FAILURE;
568 }
569
570 unload_suites(suites);
571
572 return result;
573 }