crypto-factory: Remove obsolete transform testing functions
[strongswan.git] / src / libstrongswan / tests / test_runner.c
1 /*
2 * Copyright (C) 2013 Tobias Brunner
3 * Hochschule fuer Technik Rapperswil
4 * Copyright (C) 2013 Martin Willi
5 * Copyright (C) 2013 revosec AG
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License as published by the
9 * Free Software Foundation; either version 2 of the License, or (at your
10 * option) any later version. See <http://www.fsf.org/copyleft/gpl.txt>.
11 *
12 * This program is distributed in the hope that it will be useful, but
13 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
14 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 * for more details.
16 */
17
18 #include "test_runner.h"
19
20 #include <library.h>
21 #include <threading/thread.h>
22 #include <plugins/plugin_feature.h>
23 #include <collections/array.h>
24 #include <utils/test.h>
25
26 #include <stdlib.h>
27 #include <dirent.h>
28 #include <unistd.h>
29 #include <limits.h>
30 #include <stdlib.h>
31
32 /**
33 * Get a tty color escape character for stderr
34 */
35 #define TTY(color) tty_escape_get(2, TTY_FG_##color)
36
37 /**
38 * A global symbol indicating libtest linkage
39 */
40 #ifdef WIN32
41 __declspec(dllexport)
42 #endif
43 bool test_runner_available = TRUE;
44
45 /**
46 * Destroy a single test suite and associated data
47 */
48 static void destroy_suite(test_suite_t *suite)
49 {
50 test_case_t *tcase;
51
52 while (array_remove(suite->tcases, 0, &tcase))
53 {
54 array_destroy(tcase->functions);
55 array_destroy(tcase->fixtures);
56 }
57 free(suite);
58 }
59
60 /**
61 * Filter loaded test suites, either remove suites listed (exclude=TRUE), or all
62 * that are not listed (exclude=FALSE).
63 */
64 static void apply_filter(array_t *loaded, char *filter, bool exclude)
65 {
66 enumerator_t *enumerator, *names;
67 hashtable_t *listed;
68 test_suite_t *suite;
69 char *name;
70
71 listed = hashtable_create(hashtable_hash_str, hashtable_equals_str, 8);
72 names = enumerator_create_token(filter, ",", " ");
73 while (names->enumerate(names, &name))
74 {
75 listed->put(listed, name, name);
76 }
77 enumerator = array_create_enumerator(loaded);
78 while (enumerator->enumerate(enumerator, &suite))
79 {
80 if ((exclude && listed->get(listed, suite->name)) ||
81 (!exclude && !listed->get(listed, suite->name)))
82 {
83 array_remove_at(loaded, enumerator);
84 destroy_suite(suite);
85 }
86 }
87 enumerator->destroy(enumerator);
88 listed->destroy(listed);
89 names->destroy(names);
90 }
91
92 /**
93 * Removes and destroys test suites that are not selected or
94 * explicitly excluded.
95 */
96 static void filter_suites(array_t *loaded)
97 {
98 char *filter;
99
100 filter = getenv("TESTS_SUITES");
101 if (filter)
102 {
103 apply_filter(loaded, filter, FALSE);
104 }
105 filter = getenv("TESTS_SUITES_EXCLUDE");
106 if (filter)
107 {
108 apply_filter(loaded, filter, TRUE);
109 }
110 }
111
112 /**
113 * Load all available test suites, or optionally only selected ones.
114 */
115 static array_t *load_suites(test_configuration_t configs[],
116 test_runner_init_t init, char *cfg)
117 {
118 array_t *suites;
119 bool old = FALSE;
120 int i;
121
122 library_init(cfg, "test-runner");
123
124 test_setup_handler();
125
126 if (init && !init(TRUE))
127 {
128 library_deinit();
129 return NULL;
130 }
131 lib->plugins->status(lib->plugins, LEVEL_CTRL);
132
133 if (lib->leak_detective)
134 {
135 old = lib->leak_detective->set_state(lib->leak_detective, FALSE);
136 }
137
138 suites = array_create(0, 0);
139
140 for (i = 0; configs[i].suite; i++)
141 {
142 if (configs[i].feature.type == 0 ||
143 lib->plugins->has_feature(lib->plugins, configs[i].feature))
144 {
145 array_insert(suites, -1, configs[i].suite());
146 }
147 }
148 filter_suites(suites);
149
150 if (lib->leak_detective)
151 {
152 lib->leak_detective->set_state(lib->leak_detective, old);
153 }
154
155 if (init)
156 {
157 init(FALSE);
158 }
159 library_deinit();
160
161 return suites;
162 }
163
164 /**
165 * Unload and destroy test suites and associated data
166 */
167 static void unload_suites(array_t *suites)
168 {
169 test_suite_t *suite;
170
171 while (array_remove(suites, 0, &suite))
172 {
173 destroy_suite(suite);
174 }
175 array_destroy(suites);
176 }
177
178 /**
179 * Run a single test function, return FALSE on failure
180 */
181 static bool run_test(test_function_t *tfun, int i)
182 {
183 if (test_restore_point())
184 {
185 tfun->cb(i);
186 return TRUE;
187 }
188 thread_cleanup_popall();
189 return FALSE;
190 }
191
192 /**
193 * Invoke fixture setup/teardown
194 */
195 static bool call_fixture(test_case_t *tcase, bool up)
196 {
197 enumerator_t *enumerator;
198 test_fixture_t *fixture;
199 bool failure = FALSE;
200
201 enumerator = array_create_enumerator(tcase->fixtures);
202 while (enumerator->enumerate(enumerator, &fixture))
203 {
204 if (test_restore_point())
205 {
206 if (up)
207 {
208 if (fixture->setup)
209 {
210 fixture->setup();
211 }
212 }
213 else
214 {
215 if (fixture->teardown)
216 {
217 fixture->teardown();
218 }
219 }
220 }
221 else
222 {
223 thread_cleanup_popall();
224 failure = TRUE;
225 break;
226 }
227 }
228 enumerator->destroy(enumerator);
229
230 return !failure;
231 }
232
233 /**
234 * Test initialization, initializes libstrongswan for the next run
235 */
236 static bool pre_test(test_runner_init_t init, char *cfg)
237 {
238 level_t level = LEVEL_SILENT;
239 char *verbosity;
240
241 library_init(cfg, "test-runner");
242
243 /* use non-blocking RNG to generate keys fast */
244 lib->settings->set_default_str(lib->settings,
245 "libstrongswan.plugins.random.random",
246 lib->settings->get_str(lib->settings,
247 "libstrongswan.plugins.random.urandom", "/dev/urandom"));
248 /* same for the gcrypt plugin */
249 lib->settings->set_default_str(lib->settings,
250 "libstrongswan.plugins.gcrypt.quick_random", "yes");
251
252 if (lib->leak_detective)
253 {
254 /* disable leak reports during testing */
255 lib->leak_detective->set_report_cb(lib->leak_detective,
256 NULL, NULL, NULL);
257 }
258 if (init && !init(TRUE))
259 {
260 library_deinit();
261 return FALSE;
262 }
263 verbosity = getenv("TESTS_VERBOSITY");
264 if (verbosity)
265 {
266 level = atoi(verbosity);
267 }
268 dbg_default_set_level(level);
269 return TRUE;
270 }
271
272 /**
273 * Failure description
274 */
275 typedef struct {
276 char *name;
277 char msg[512 - sizeof(char*) - 2 * sizeof(int)];
278 const char *file;
279 int line;
280 int i;
281 backtrace_t *bt;
282 } failure_t;
283
284 /**
285 * Data passed to leak report callbacks
286 */
287 typedef struct {
288 array_t *failures;
289 char *name;
290 int i;
291 int leaks;
292 } report_data_t;
293
294 /**
295 * Leak report callback, build failures from leaks
296 */
297 static void report_leaks(report_data_t *data, int count, size_t bytes,
298 backtrace_t *bt, bool detailed)
299 {
300 failure_t failure = {
301 .name = data->name,
302 .i = data->i,
303 .bt = bt->clone(bt),
304 };
305
306 snprintf(failure.msg, sizeof(failure.msg),
307 "Leak detected: %d allocations using %zu bytes", count, bytes);
308
309 array_insert(data->failures, -1, &failure);
310 }
311
312 /**
313 * Leak summary callback, check if any leaks found
314 */
315 static void sum_leaks(report_data_t *data, int count, size_t bytes,
316 int whitelisted)
317 {
318 data->leaks = count;
319 }
320
321 /**
322 * Do library cleanup and optionally check for memory leaks
323 */
324 static bool post_test(test_runner_init_t init, bool check_leaks,
325 array_t *failures, char *name, int i, int *leaks)
326 {
327 report_data_t data = {
328 .failures = failures,
329 .name = name,
330 .i = i,
331 };
332
333 if (init)
334 {
335 if (test_restore_point())
336 {
337 init(FALSE);
338 }
339 else
340 {
341 thread_cleanup_popall();
342 library_deinit();
343 return FALSE;
344 }
345 }
346 if (check_leaks && lib->leak_detective)
347 {
348 lib->leak_detective->set_report_cb(lib->leak_detective,
349 (leak_detective_report_cb_t)report_leaks,
350 (leak_detective_summary_cb_t)sum_leaks, &data);
351 }
352 library_deinit();
353
354 *leaks = data.leaks;
355 return TRUE;
356 }
357
358 /**
359 * Collect failure information, add failure_t to array
360 */
361 static void collect_failure_info(array_t *failures, char *name, int i)
362 {
363 failure_t failure = {
364 .name = name,
365 .i = i,
366 .bt = test_failure_backtrace(),
367 };
368
369 failure.line = test_failure_get(failure.msg, sizeof(failure.msg),
370 &failure.file);
371
372 array_insert(failures, -1, &failure);
373 }
374
375 /**
376 * Print array of collected failure_t to stderr
377 */
378 static void print_failures(array_t *failures)
379 {
380 failure_t failure;
381
382 threads_init();
383 backtrace_init();
384
385 while (array_remove(failures, 0, &failure))
386 {
387 fprintf(stderr, " %sFailure in '%s': %s (",
388 TTY(RED), failure.name, failure.msg);
389 if (failure.line)
390 {
391 fprintf(stderr, "%s:%d, ", failure.file, failure.line);
392 }
393 fprintf(stderr, "i = %d)%s\n", failure.i, TTY(DEF));
394 if (failure.bt)
395 {
396 failure.bt->log(failure.bt, stderr, TRUE);
397 failure.bt->destroy(failure.bt);
398 }
399 }
400
401 backtrace_deinit();
402 threads_deinit();
403 }
404
405 /**
406 * Run a single test case with fixtures
407 */
408 static bool run_case(test_case_t *tcase, test_runner_init_t init, char *cfg)
409 {
410 enumerator_t *enumerator;
411 test_function_t *tfun;
412 int passed = 0;
413 array_t *failures;
414
415 failures = array_create(sizeof(failure_t), 0);
416
417 fprintf(stderr, " Running case '%s': ", tcase->name);
418 fflush(stderr);
419
420 enumerator = array_create_enumerator(tcase->functions);
421 while (enumerator->enumerate(enumerator, &tfun))
422 {
423 int i, rounds = 0;
424
425 for (i = tfun->start; i < tfun->end; i++)
426 {
427 if (pre_test(init, cfg))
428 {
429 bool ok = FALSE;
430 int leaks = 0;
431
432 test_setup_timeout(tcase->timeout);
433
434 if (call_fixture(tcase, TRUE))
435 {
436 if (run_test(tfun, i))
437 {
438 if (call_fixture(tcase, FALSE))
439 {
440 ok = TRUE;
441 }
442 }
443 else
444 {
445 call_fixture(tcase, FALSE);
446 }
447 }
448 if (!post_test(init, ok, failures, tfun->name, i, &leaks))
449 {
450 ok = FALSE;
451 }
452
453 test_setup_timeout(0);
454
455 if (ok)
456 {
457 if (!leaks)
458 {
459 rounds++;
460 fprintf(stderr, "%s+%s", TTY(GREEN), TTY(DEF));
461 }
462 }
463 else
464 {
465 collect_failure_info(failures, tfun->name, i);
466 }
467 if (!ok || leaks)
468 {
469 fprintf(stderr, "%s-%s", TTY(RED), TTY(DEF));
470 }
471 }
472 else
473 {
474 fprintf(stderr, "!");
475 }
476 }
477 fflush(stderr);
478 if (rounds == tfun->end - tfun->start)
479 {
480 passed++;
481 }
482 }
483 enumerator->destroy(enumerator);
484
485 fprintf(stderr, "\n");
486
487 print_failures(failures);
488 array_destroy(failures);
489
490 return passed == array_count(tcase->functions);
491 }
492
493 /**
494 * Run a single test suite
495 */
496 static bool run_suite(test_suite_t *suite, test_runner_init_t init, char *cfg)
497 {
498 enumerator_t *enumerator;
499 test_case_t *tcase;
500 int passed = 0;
501
502 fprintf(stderr, " Running suite '%s':\n", suite->name);
503
504 enumerator = array_create_enumerator(suite->tcases);
505 while (enumerator->enumerate(enumerator, &tcase))
506 {
507 if (run_case(tcase, init, cfg))
508 {
509 passed++;
510 }
511 }
512 enumerator->destroy(enumerator);
513
514 if (passed == array_count(suite->tcases))
515 {
516 fprintf(stderr, " %sPassed all %u '%s' test cases%s\n",
517 TTY(GREEN), array_count(suite->tcases), suite->name, TTY(DEF));
518 return TRUE;
519 }
520 fprintf(stderr, " %sPassed %u/%u '%s' test cases%s\n",
521 TTY(RED), passed, array_count(suite->tcases), suite->name, TTY(DEF));
522 return FALSE;
523 }
524
525 /**
526 * See header.
527 */
528 int test_runner_run(const char *name, test_configuration_t configs[],
529 test_runner_init_t init)
530 {
531 array_t *suites;
532 test_suite_t *suite;
533 enumerator_t *enumerator;
534 int passed = 0, result;
535 char *cfg;
536
537 /* redirect all output to stderr (to redirect make's stdout to /dev/null) */
538 dup2(2, 1);
539
540 cfg = getenv("TESTS_STRONGSWAN_CONF");
541
542 suites = load_suites(configs, init, cfg);
543 if (!suites)
544 {
545 return EXIT_FAILURE;
546 }
547
548 fprintf(stderr, "Running %u '%s' test suites:\n", array_count(suites), name);
549
550 enumerator = array_create_enumerator(suites);
551 while (enumerator->enumerate(enumerator, &suite))
552 {
553 if (run_suite(suite, init, cfg))
554 {
555 passed++;
556 }
557 }
558 enumerator->destroy(enumerator);
559
560 if (passed == array_count(suites))
561 {
562 fprintf(stderr, "%sPassed all %u '%s' suites%s\n",
563 TTY(GREEN), array_count(suites), name, TTY(DEF));
564 result = EXIT_SUCCESS;
565 }
566 else
567 {
568 fprintf(stderr, "%sPassed %u of %u '%s' suites%s\n",
569 TTY(RED), passed, array_count(suites), name, TTY(DEF));
570 result = EXIT_FAILURE;
571 }
572
573 unload_suites(suites);
574
575 return result;
576 }