unit-tests: Set test verbosity just after test suite loading
[strongswan.git] / src / libstrongswan / tests / test_runner.c
1 /*
2 * Copyright (C) 2013 Tobias Brunner
3 * Hochschule fuer Technik Rapperswil
4 * Copyright (C) 2013 Martin Willi
5 * Copyright (C) 2013 revosec AG
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License as published by the
9 * Free Software Foundation; either version 2 of the License, or (at your
10 * option) any later version. See <http://www.fsf.org/copyleft/gpl.txt>.
11 *
12 * This program is distributed in the hope that it will be useful, but
13 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
14 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 * for more details.
16 */
17
18 #include "test_runner.h"
19
20 #include <library.h>
21 #include <threading/thread.h>
22 #include <plugins/plugin_feature.h>
23 #include <collections/array.h>
24 #include <utils/test.h>
25
26 #include <stdlib.h>
27 #include <dirent.h>
28 #include <unistd.h>
29 #include <limits.h>
30 #include <stdlib.h>
31
32 /**
33 * Get a tty color escape character for stderr
34 */
35 #define TTY(color) tty_escape_get(2, TTY_FG_##color)
36
37 /**
38 * A global symbol indicating libtest linkage
39 */
40 #ifdef WIN32
41 __declspec(dllexport)
42 #endif
43 bool test_runner_available = TRUE;
44
45 /**
46 * Destroy a single test suite and associated data
47 */
48 static void destroy_suite(test_suite_t *suite)
49 {
50 test_case_t *tcase;
51
52 while (array_remove(suite->tcases, 0, &tcase))
53 {
54 array_destroy(tcase->functions);
55 array_destroy(tcase->fixtures);
56 }
57 free(suite);
58 }
59
60 /**
61 * Filter loaded test suites, either remove suites listed (exclude=TRUE), or all
62 * that are not listed (exclude=FALSE).
63 */
64 static void apply_filter(array_t *loaded, char *filter, bool exclude)
65 {
66 enumerator_t *enumerator, *names;
67 hashtable_t *listed;
68 test_suite_t *suite;
69 char *name;
70
71 listed = hashtable_create(hashtable_hash_str, hashtable_equals_str, 8);
72 names = enumerator_create_token(filter, ",", " ");
73 while (names->enumerate(names, &name))
74 {
75 listed->put(listed, name, name);
76 }
77 enumerator = array_create_enumerator(loaded);
78 while (enumerator->enumerate(enumerator, &suite))
79 {
80 if ((exclude && listed->get(listed, suite->name)) ||
81 (!exclude && !listed->get(listed, suite->name)))
82 {
83 array_remove_at(loaded, enumerator);
84 destroy_suite(suite);
85 }
86 }
87 enumerator->destroy(enumerator);
88 listed->destroy(listed);
89 names->destroy(names);
90 }
91
92 /**
93 * Removes and destroys test suites that are not selected or
94 * explicitly excluded.
95 */
96 static void filter_suites(array_t *loaded)
97 {
98 char *filter;
99
100 filter = getenv("TESTS_SUITES");
101 if (filter)
102 {
103 apply_filter(loaded, filter, FALSE);
104 }
105 filter = getenv("TESTS_SUITES_EXCLUDE");
106 if (filter)
107 {
108 apply_filter(loaded, filter, TRUE);
109 }
110 }
111
112 /**
113 * Load all available test suites, or optionally only selected ones.
114 */
115 static array_t *load_suites(test_configuration_t configs[],
116 test_runner_init_t init, char *cfg)
117 {
118 array_t *suites;
119 bool old = FALSE;
120 int i;
121
122 library_init(cfg, "test-runner");
123
124 test_setup_handler();
125
126 if (init && !init(TRUE))
127 {
128 library_deinit();
129 return NULL;
130 }
131 lib->plugins->status(lib->plugins, LEVEL_CTRL);
132
133 if (lib->leak_detective)
134 {
135 old = lib->leak_detective->set_state(lib->leak_detective, FALSE);
136 }
137
138 suites = array_create(0, 0);
139
140 for (i = 0; configs[i].suite; i++)
141 {
142 if (configs[i].feature.type == 0 ||
143 lib->plugins->has_feature(lib->plugins, configs[i].feature))
144 {
145 array_insert(suites, -1, configs[i].suite());
146 }
147 }
148 filter_suites(suites);
149
150 if (lib->leak_detective)
151 {
152 lib->leak_detective->set_state(lib->leak_detective, old);
153 }
154
155 if (init)
156 {
157 init(FALSE);
158 }
159 library_deinit();
160
161 return suites;
162 }
163
164 /**
165 * Unload and destroy test suites and associated data
166 */
167 static void unload_suites(array_t *suites)
168 {
169 test_suite_t *suite;
170
171 while (array_remove(suites, 0, &suite))
172 {
173 destroy_suite(suite);
174 }
175 array_destroy(suites);
176 }
177
178 /**
179 * Run a single test function, return FALSE on failure
180 */
181 static bool run_test(test_function_t *tfun, int i)
182 {
183 if (test_restore_point())
184 {
185 tfun->cb(i);
186 return TRUE;
187 }
188 thread_cleanup_popall();
189 return FALSE;
190 }
191
192 /**
193 * Invoke fixture setup/teardown
194 */
195 static bool call_fixture(test_case_t *tcase, bool up)
196 {
197 enumerator_t *enumerator;
198 test_fixture_t *fixture;
199 bool failure = FALSE;
200
201 enumerator = array_create_enumerator(tcase->fixtures);
202 while (enumerator->enumerate(enumerator, &fixture))
203 {
204 if (test_restore_point())
205 {
206 if (up)
207 {
208 if (fixture->setup)
209 {
210 fixture->setup();
211 }
212 }
213 else
214 {
215 if (fixture->teardown)
216 {
217 fixture->teardown();
218 }
219 }
220 }
221 else
222 {
223 thread_cleanup_popall();
224 failure = TRUE;
225 break;
226 }
227 }
228 enumerator->destroy(enumerator);
229
230 return !failure;
231 }
232
233 /**
234 * Test initialization, initializes libstrongswan for the next run
235 */
236 static bool pre_test(test_runner_init_t init, char *cfg)
237 {
238 library_init(cfg, "test-runner");
239
240 /* use non-blocking RNG to generate keys fast */
241 lib->settings->set_default_str(lib->settings,
242 "libstrongswan.plugins.random.random",
243 lib->settings->get_str(lib->settings,
244 "libstrongswan.plugins.random.urandom", "/dev/urandom"));
245 /* same for the gcrypt plugin */
246 lib->settings->set_default_str(lib->settings,
247 "libstrongswan.plugins.gcrypt.quick_random", "yes");
248
249 if (lib->leak_detective)
250 {
251 /* disable leak reports during testing */
252 lib->leak_detective->set_report_cb(lib->leak_detective,
253 NULL, NULL, NULL);
254 }
255 if (init && !init(TRUE))
256 {
257 library_deinit();
258 return FALSE;
259 }
260 return TRUE;
261 }
262
263 /**
264 * Failure description
265 */
266 typedef struct {
267 char *name;
268 char msg[512 - sizeof(char*) - 2 * sizeof(int)];
269 const char *file;
270 int line;
271 int i;
272 backtrace_t *bt;
273 } failure_t;
274
275 /**
276 * Data passed to leak report callbacks
277 */
278 typedef struct {
279 array_t *failures;
280 char *name;
281 int i;
282 int leaks;
283 } report_data_t;
284
285 /**
286 * Leak report callback, build failures from leaks
287 */
288 static void report_leaks(report_data_t *data, int count, size_t bytes,
289 backtrace_t *bt, bool detailed)
290 {
291 failure_t failure = {
292 .name = data->name,
293 .i = data->i,
294 .bt = bt->clone(bt),
295 };
296
297 snprintf(failure.msg, sizeof(failure.msg),
298 "Leak detected: %d allocations using %zu bytes", count, bytes);
299
300 array_insert(data->failures, -1, &failure);
301 }
302
303 /**
304 * Leak summary callback, check if any leaks found
305 */
306 static void sum_leaks(report_data_t *data, int count, size_t bytes,
307 int whitelisted)
308 {
309 data->leaks = count;
310 }
311
312 /**
313 * Do library cleanup and optionally check for memory leaks
314 */
315 static bool post_test(test_runner_init_t init, bool check_leaks,
316 array_t *failures, char *name, int i, int *leaks)
317 {
318 report_data_t data = {
319 .failures = failures,
320 .name = name,
321 .i = i,
322 };
323
324 if (init)
325 {
326 if (test_restore_point())
327 {
328 init(FALSE);
329 }
330 else
331 {
332 thread_cleanup_popall();
333 library_deinit();
334 return FALSE;
335 }
336 }
337 if (check_leaks && lib->leak_detective)
338 {
339 lib->leak_detective->set_report_cb(lib->leak_detective,
340 (leak_detective_report_cb_t)report_leaks,
341 (leak_detective_summary_cb_t)sum_leaks, &data);
342 }
343 library_deinit();
344
345 *leaks = data.leaks;
346 return TRUE;
347 }
348
349 /**
350 * Collect failure information, add failure_t to array
351 */
352 static void collect_failure_info(array_t *failures, char *name, int i)
353 {
354 failure_t failure = {
355 .name = name,
356 .i = i,
357 .bt = test_failure_backtrace(),
358 };
359
360 failure.line = test_failure_get(failure.msg, sizeof(failure.msg),
361 &failure.file);
362
363 array_insert(failures, -1, &failure);
364 }
365
366 /**
367 * Print array of collected failure_t to stderr
368 */
369 static void print_failures(array_t *failures)
370 {
371 failure_t failure;
372
373 threads_init();
374 backtrace_init();
375
376 while (array_remove(failures, 0, &failure))
377 {
378 fprintf(stderr, " %sFailure in '%s': %s (",
379 TTY(RED), failure.name, failure.msg);
380 if (failure.line)
381 {
382 fprintf(stderr, "%s:%d, ", failure.file, failure.line);
383 }
384 fprintf(stderr, "i = %d)%s\n", failure.i, TTY(DEF));
385 if (failure.bt)
386 {
387 failure.bt->log(failure.bt, stderr, TRUE);
388 failure.bt->destroy(failure.bt);
389 }
390 }
391
392 backtrace_deinit();
393 threads_deinit();
394 }
395
396 /**
397 * Run a single test case with fixtures
398 */
399 static bool run_case(test_case_t *tcase, test_runner_init_t init, char *cfg)
400 {
401 enumerator_t *enumerator;
402 test_function_t *tfun;
403 int passed = 0;
404 array_t *failures;
405
406 failures = array_create(sizeof(failure_t), 0);
407
408 fprintf(stderr, " Running case '%s': ", tcase->name);
409 fflush(stderr);
410
411 enumerator = array_create_enumerator(tcase->functions);
412 while (enumerator->enumerate(enumerator, &tfun))
413 {
414 int i, rounds = 0;
415
416 for (i = tfun->start; i < tfun->end; i++)
417 {
418 if (pre_test(init, cfg))
419 {
420 bool ok = FALSE;
421 int leaks = 0;
422
423 test_setup_timeout(tcase->timeout);
424
425 if (call_fixture(tcase, TRUE))
426 {
427 if (run_test(tfun, i))
428 {
429 if (call_fixture(tcase, FALSE))
430 {
431 ok = TRUE;
432 }
433 }
434 else
435 {
436 call_fixture(tcase, FALSE);
437 }
438 }
439 if (!post_test(init, ok, failures, tfun->name, i, &leaks))
440 {
441 ok = FALSE;
442 }
443
444 test_setup_timeout(0);
445
446 if (ok)
447 {
448 if (!leaks)
449 {
450 rounds++;
451 fprintf(stderr, "%s+%s", TTY(GREEN), TTY(DEF));
452 }
453 }
454 else
455 {
456 collect_failure_info(failures, tfun->name, i);
457 }
458 if (!ok || leaks)
459 {
460 fprintf(stderr, "%s-%s", TTY(RED), TTY(DEF));
461 }
462 }
463 else
464 {
465 fprintf(stderr, "!");
466 }
467 }
468 fflush(stderr);
469 if (rounds == tfun->end - tfun->start)
470 {
471 passed++;
472 }
473 }
474 enumerator->destroy(enumerator);
475
476 fprintf(stderr, "\n");
477
478 print_failures(failures);
479 array_destroy(failures);
480
481 return passed == array_count(tcase->functions);
482 }
483
484 /**
485 * Run a single test suite
486 */
487 static bool run_suite(test_suite_t *suite, test_runner_init_t init, char *cfg)
488 {
489 enumerator_t *enumerator;
490 test_case_t *tcase;
491 int passed = 0;
492
493 fprintf(stderr, " Running suite '%s':\n", suite->name);
494
495 enumerator = array_create_enumerator(suite->tcases);
496 while (enumerator->enumerate(enumerator, &tcase))
497 {
498 if (run_case(tcase, init, cfg))
499 {
500 passed++;
501 }
502 }
503 enumerator->destroy(enumerator);
504
505 if (passed == array_count(suite->tcases))
506 {
507 fprintf(stderr, " %sPassed all %u '%s' test cases%s\n",
508 TTY(GREEN), array_count(suite->tcases), suite->name, TTY(DEF));
509 return TRUE;
510 }
511 fprintf(stderr, " %sPassed %u/%u '%s' test cases%s\n",
512 TTY(RED), passed, array_count(suite->tcases), suite->name, TTY(DEF));
513 return FALSE;
514 }
515
516 /**
517 * See header.
518 */
519 int test_runner_run(const char *name, test_configuration_t configs[],
520 test_runner_init_t init)
521 {
522 array_t *suites;
523 test_suite_t *suite;
524 enumerator_t *enumerator;
525 int passed = 0, result;
526 level_t level = LEVEL_SILENT;
527 char *cfg, *verbosity;
528
529 /* redirect all output to stderr (to redirect make's stdout to /dev/null) */
530 dup2(2, 1);
531
532 cfg = getenv("TESTS_STRONGSWAN_CONF");
533
534 suites = load_suites(configs, init, cfg);
535 if (!suites)
536 {
537 return EXIT_FAILURE;
538 }
539
540 verbosity = getenv("TESTS_VERBOSITY");
541 if (verbosity)
542 {
543 level = atoi(verbosity);
544 }
545 dbg_default_set_level(level);
546
547 fprintf(stderr, "Running %u '%s' test suites:\n", array_count(suites), name);
548
549 enumerator = array_create_enumerator(suites);
550 while (enumerator->enumerate(enumerator, &suite))
551 {
552 if (run_suite(suite, init, cfg))
553 {
554 passed++;
555 }
556 }
557 enumerator->destroy(enumerator);
558
559 if (passed == array_count(suites))
560 {
561 fprintf(stderr, "%sPassed all %u '%s' suites%s\n",
562 TTY(GREEN), array_count(suites), name, TTY(DEF));
563 result = EXIT_SUCCESS;
564 }
565 else
566 {
567 fprintf(stderr, "%sPassed %u of %u '%s' suites%s\n",
568 TTY(RED), passed, array_count(suites), name, TTY(DEF));
569 result = EXIT_FAILURE;
570 }
571
572 unload_suites(suites);
573
574 return result;
575 }