8f2e9855e8318d5dcde9b1c962ec8c25587d9648
[strongswan.git] / src / libstrongswan / tests / test_runner.c
1 /*
2 * Copyright (C) 2013 Tobias Brunner
3 * Hochschule fuer Technik Rapperswil
4 * Copyright (C) 2013 Martin Willi
5 * Copyright (C) 2013 revosec AG
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License as published by the
9 * Free Software Foundation; either version 2 of the License, or (at your
10 * option) any later version. See <http://www.fsf.org/copyleft/gpl.txt>.
11 *
12 * This program is distributed in the hope that it will be useful, but
13 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
14 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 * for more details.
16 */
17
18 #include "test_runner.h"
19
20 #include <library.h>
21 #include <threading/thread.h>
22 #include <plugins/plugin_feature.h>
23 #include <collections/array.h>
24 #include <utils/test.h>
25
26 #include <stdlib.h>
27 #include <dirent.h>
28 #include <unistd.h>
29 #include <limits.h>
30 #include <stdlib.h>
31
32 /**
33 * Get a tty color escape character for stderr
34 */
35 #define TTY(color) tty_escape_get(2, TTY_FG_##color)
36
37 /**
38 * A global symbol indicating libtest linkage
39 */
40 #ifdef WIN32
41 __declspec(dllexport)
42 #endif
43 bool test_runner_available = TRUE;
44
45 /**
46 * Destroy a single test suite and associated data
47 */
48 static void destroy_suite(test_suite_t *suite)
49 {
50 test_case_t *tcase;
51
52 while (array_remove(suite->tcases, 0, &tcase))
53 {
54 array_destroy(tcase->functions);
55 array_destroy(tcase->fixtures);
56 }
57 free(suite);
58 }
59
60 /**
61 * Removes and destroys test suites that are not selected.
62 */
63 static void filter_suites(array_t *loaded)
64 {
65 enumerator_t *enumerator, *names;
66 hashtable_t *selected;
67 test_suite_t *suite;
68 char *suites, *name;
69
70 suites = getenv("TESTS_SUITES");
71 if (!suites)
72 {
73 return;
74 }
75 selected = hashtable_create(hashtable_hash_str, hashtable_equals_str, 8);
76 names = enumerator_create_token(suites, ",", " ");
77 while (names->enumerate(names, &name))
78 {
79 selected->put(selected, name, name);
80 }
81 enumerator = array_create_enumerator(loaded);
82 while (enumerator->enumerate(enumerator, &suite))
83 {
84 if (!selected->get(selected, suite->name))
85 {
86 array_remove_at(loaded, enumerator);
87 destroy_suite(suite);
88 }
89 }
90 enumerator->destroy(enumerator);
91 selected->destroy(selected);
92 names->destroy(names);
93 }
94
95 /**
96 * Load all available test suites, or optionally only selected ones.
97 */
98 static array_t *load_suites(test_configuration_t configs[],
99 test_runner_init_t init, char *cfg)
100 {
101 array_t *suites;
102 bool old = FALSE;
103 int i;
104
105 library_init(cfg, "test-runner");
106
107 test_setup_handler();
108
109 if (init && !init(TRUE))
110 {
111 library_deinit();
112 return NULL;
113 }
114 lib->plugins->status(lib->plugins, LEVEL_CTRL);
115
116 if (lib->leak_detective)
117 {
118 old = lib->leak_detective->set_state(lib->leak_detective, FALSE);
119 }
120
121 suites = array_create(0, 0);
122
123 for (i = 0; configs[i].suite; i++)
124 {
125 if (configs[i].feature.type == 0 ||
126 lib->plugins->has_feature(lib->plugins, configs[i].feature))
127 {
128 array_insert(suites, -1, configs[i].suite());
129 }
130 }
131 filter_suites(suites);
132
133 if (lib->leak_detective)
134 {
135 lib->leak_detective->set_state(lib->leak_detective, old);
136 }
137
138 if (init)
139 {
140 init(FALSE);
141 }
142 library_deinit();
143
144 return suites;
145 }
146
147 /**
148 * Unload and destroy test suites and associated data
149 */
150 static void unload_suites(array_t *suites)
151 {
152 test_suite_t *suite;
153
154 while (array_remove(suites, 0, &suite))
155 {
156 destroy_suite(suite);
157 }
158 array_destroy(suites);
159 }
160
161 /**
162 * Run a single test function, return FALSE on failure
163 */
164 static bool run_test(test_function_t *tfun, int i)
165 {
166 if (test_restore_point())
167 {
168 tfun->cb(i);
169 return TRUE;
170 }
171 return FALSE;
172 }
173
174 /**
175 * Invoke fixture setup/teardown
176 */
177 static bool call_fixture(test_case_t *tcase, bool up)
178 {
179 enumerator_t *enumerator;
180 test_fixture_t *fixture;
181 bool failure = FALSE;
182
183 enumerator = array_create_enumerator(tcase->fixtures);
184 while (enumerator->enumerate(enumerator, &fixture))
185 {
186 if (test_restore_point())
187 {
188 if (up)
189 {
190 if (fixture->setup)
191 {
192 fixture->setup();
193 }
194 }
195 else
196 {
197 if (fixture->teardown)
198 {
199 fixture->teardown();
200 }
201 }
202 }
203 else
204 {
205 failure = TRUE;
206 break;
207 }
208 }
209 enumerator->destroy(enumerator);
210
211 return !failure;
212 }
213
214 /**
215 * Test initialization, initializes libstrongswan for the next run
216 */
217 static bool pre_test(test_runner_init_t init, char *cfg)
218 {
219 level_t level = LEVEL_SILENT;
220 char *verbosity;
221
222 library_init(cfg, "test-runner");
223
224 /* use non-blocking RNG to generate keys fast */
225 lib->settings->set_default_str(lib->settings,
226 "libstrongswan.plugins.random.random",
227 lib->settings->get_str(lib->settings,
228 "libstrongswan.plugins.random.urandom", "/dev/urandom"));
229 /* same for the gcrypt plugin */
230 lib->settings->set_default_str(lib->settings,
231 "libstrongswan.plugins.gcrypt.quick_random", "yes");
232
233 if (lib->leak_detective)
234 {
235 /* disable leak reports during testing */
236 lib->leak_detective->set_report_cb(lib->leak_detective,
237 NULL, NULL, NULL);
238 }
239 if (init && !init(TRUE))
240 {
241 library_deinit();
242 return FALSE;
243 }
244 verbosity = getenv("TESTS_VERBOSITY");
245 if (verbosity)
246 {
247 level = atoi(verbosity);
248 }
249 dbg_default_set_level(level);
250 return TRUE;
251 }
252
253 /**
254 * Failure description
255 */
256 typedef struct {
257 char *name;
258 char msg[512 - sizeof(char*) - 2 * sizeof(int)];
259 const char *file;
260 int line;
261 int i;
262 backtrace_t *bt;
263 } failure_t;
264
265 /**
266 * Data passed to leak report callbacks
267 */
268 typedef struct {
269 array_t *failures;
270 char *name;
271 int i;
272 int leaks;
273 } report_data_t;
274
275 /**
276 * Leak report callback, build failures from leaks
277 */
278 static void report_leaks(report_data_t *data, int count, size_t bytes,
279 backtrace_t *bt, bool detailed)
280 {
281 failure_t failure = {
282 .name = data->name,
283 .i = data->i,
284 .bt = bt->clone(bt),
285 };
286
287 snprintf(failure.msg, sizeof(failure.msg),
288 "Leak detected: %d allocations using %zu bytes", count, bytes);
289
290 array_insert(data->failures, -1, &failure);
291 }
292
293 /**
294 * Leak summary callback, check if any leaks found
295 */
296 static void sum_leaks(report_data_t *data, int count, size_t bytes,
297 int whitelisted)
298 {
299 data->leaks = count;
300 }
301
302 /**
303 * Do library cleanup and optionally check for memory leaks
304 */
305 static bool post_test(test_runner_init_t init, bool check_leaks,
306 array_t *failures, char *name, int i, int *leaks)
307 {
308 report_data_t data = {
309 .failures = failures,
310 .name = name,
311 .i = i,
312 };
313
314 if (init)
315 {
316 if (test_restore_point())
317 {
318 init(FALSE);
319 }
320 else
321 {
322 library_deinit();
323 return FALSE;
324 }
325 }
326 if (check_leaks && lib->leak_detective)
327 {
328 lib->leak_detective->set_report_cb(lib->leak_detective,
329 (leak_detective_report_cb_t)report_leaks,
330 (leak_detective_summary_cb_t)sum_leaks, &data);
331 }
332 library_deinit();
333
334 *leaks = data.leaks;
335 return TRUE;
336 }
337
338 /**
339 * Collect failure information, add failure_t to array
340 */
341 static void collect_failure_info(array_t *failures, char *name, int i)
342 {
343 failure_t failure = {
344 .name = name,
345 .i = i,
346 .bt = test_failure_backtrace(),
347 };
348
349 failure.line = test_failure_get(failure.msg, sizeof(failure.msg),
350 &failure.file);
351
352 array_insert(failures, -1, &failure);
353 }
354
355 /**
356 * Print array of collected failure_t to stderr
357 */
358 static void print_failures(array_t *failures)
359 {
360 failure_t failure;
361
362 threads_init();
363 backtrace_init();
364
365 while (array_remove(failures, 0, &failure))
366 {
367 fprintf(stderr, " %sFailure in '%s': %s (",
368 TTY(RED), failure.name, failure.msg);
369 if (failure.line)
370 {
371 fprintf(stderr, "%s:%d, ", failure.file, failure.line);
372 }
373 fprintf(stderr, "i = %d)%s\n", failure.i, TTY(DEF));
374 if (failure.bt)
375 {
376 failure.bt->log(failure.bt, stderr, TRUE);
377 failure.bt->destroy(failure.bt);
378 }
379 }
380
381 backtrace_deinit();
382 threads_deinit();
383 }
384
385 /**
386 * Run a single test case with fixtures
387 */
388 static bool run_case(test_case_t *tcase, test_runner_init_t init, char *cfg)
389 {
390 enumerator_t *enumerator;
391 test_function_t *tfun;
392 int passed = 0;
393 array_t *failures;
394
395 failures = array_create(sizeof(failure_t), 0);
396
397 fprintf(stderr, " Running case '%s': ", tcase->name);
398 fflush(stderr);
399
400 enumerator = array_create_enumerator(tcase->functions);
401 while (enumerator->enumerate(enumerator, &tfun))
402 {
403 int i, rounds = 0;
404
405 for (i = tfun->start; i < tfun->end; i++)
406 {
407 if (pre_test(init, cfg))
408 {
409 bool ok = FALSE;
410 int leaks = 0;
411
412 test_setup_timeout(tcase->timeout);
413
414 if (call_fixture(tcase, TRUE))
415 {
416 if (run_test(tfun, i))
417 {
418 if (call_fixture(tcase, FALSE))
419 {
420 ok = TRUE;
421 }
422 }
423 else
424 {
425 call_fixture(tcase, FALSE);
426 }
427 }
428 if (!post_test(init, ok, failures, tfun->name, i, &leaks))
429 {
430 ok = FALSE;
431 }
432
433 test_setup_timeout(0);
434
435 if (ok)
436 {
437 if (!leaks)
438 {
439 rounds++;
440 fprintf(stderr, "%s+%s", TTY(GREEN), TTY(DEF));
441 }
442 }
443 else
444 {
445 collect_failure_info(failures, tfun->name, i);
446 }
447 if (!ok || leaks)
448 {
449 fprintf(stderr, "%s-%s", TTY(RED), TTY(DEF));
450 }
451 }
452 else
453 {
454 fprintf(stderr, "!");
455 }
456 }
457 fflush(stderr);
458 if (rounds == tfun->end - tfun->start)
459 {
460 passed++;
461 }
462 }
463 enumerator->destroy(enumerator);
464
465 fprintf(stderr, "\n");
466
467 print_failures(failures);
468 array_destroy(failures);
469
470 return passed == array_count(tcase->functions);
471 }
472
473 /**
474 * Run a single test suite
475 */
476 static bool run_suite(test_suite_t *suite, test_runner_init_t init, char *cfg)
477 {
478 enumerator_t *enumerator;
479 test_case_t *tcase;
480 int passed = 0;
481
482 fprintf(stderr, " Running suite '%s':\n", suite->name);
483
484 enumerator = array_create_enumerator(suite->tcases);
485 while (enumerator->enumerate(enumerator, &tcase))
486 {
487 if (run_case(tcase, init, cfg))
488 {
489 passed++;
490 }
491 }
492 enumerator->destroy(enumerator);
493
494 if (passed == array_count(suite->tcases))
495 {
496 fprintf(stderr, " %sPassed all %u '%s' test cases%s\n",
497 TTY(GREEN), array_count(suite->tcases), suite->name, TTY(DEF));
498 return TRUE;
499 }
500 fprintf(stderr, " %sPassed %u/%u '%s' test cases%s\n",
501 TTY(RED), passed, array_count(suite->tcases), suite->name, TTY(DEF));
502 return FALSE;
503 }
504
505 /**
506 * See header.
507 */
508 int test_runner_run(const char *name, test_configuration_t configs[],
509 test_runner_init_t init)
510 {
511 array_t *suites;
512 test_suite_t *suite;
513 enumerator_t *enumerator;
514 int passed = 0, result;
515 char *cfg;
516
517 /* redirect all output to stderr (to redirect make's stdout to /dev/null) */
518 dup2(2, 1);
519
520 cfg = getenv("TESTS_STRONGSWAN_CONF");
521
522 suites = load_suites(configs, init, cfg);
523 if (!suites)
524 {
525 return EXIT_FAILURE;
526 }
527
528 fprintf(stderr, "Running %u '%s' test suites:\n", array_count(suites), name);
529
530 enumerator = array_create_enumerator(suites);
531 while (enumerator->enumerate(enumerator, &suite))
532 {
533 if (run_suite(suite, init, cfg))
534 {
535 passed++;
536 }
537 }
538 enumerator->destroy(enumerator);
539
540 if (passed == array_count(suites))
541 {
542 fprintf(stderr, "%sPassed all %u '%s' suites%s\n",
543 TTY(GREEN), array_count(suites), name, TTY(DEF));
544 result = EXIT_SUCCESS;
545 }
546 else
547 {
548 fprintf(stderr, "%sPassed %u of %u '%s' suites%s\n",
549 TTY(RED), passed, array_count(suites), name, TTY(DEF));
550 result = EXIT_FAILURE;
551 }
552
553 unload_suites(suites);
554
555 return result;
556 }