unit-tests: Generate weak keys with gcrypt plugin (but quickly)
[strongswan.git] / src / libstrongswan / tests / test_runner.c
1 /*
2 * Copyright (C) 2013 Tobias Brunner
3 * Hochschule fuer Technik Rapperswil
4 * Copyright (C) 2013 Martin Willi
5 * Copyright (C) 2013 revosec AG
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License as published by the
9 * Free Software Foundation; either version 2 of the License, or (at your
10 * option) any later version. See <http://www.fsf.org/copyleft/gpl.txt>.
11 *
12 * This program is distributed in the hope that it will be useful, but
13 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
14 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 * for more details.
16 */
17
18 #include "test_runner.h"
19
20 #include <library.h>
21 #include <plugins/plugin_feature.h>
22 #include <collections/array.h>
23 #include <utils/test.h>
24
25 #include <dirent.h>
26 #include <unistd.h>
27 #include <limits.h>
28
29 /**
30 * Get a tty color escape character for stderr
31 */
32 #define TTY(color) tty_escape_get(2, TTY_FG_##color)
33
34 /**
35 * Initialize the lookup table for testable functions (defined in libstrongswan)
36 */
37 static void testable_functions_create() __attribute__ ((constructor(1000)));
38 static void testable_functions_create()
39 {
40 testable_functions = hashtable_create(hashtable_hash_str,
41 hashtable_equals_str, 8);
42 }
43
44 /**
45 * Destroy the lookup table for testable functions
46 */
47 static void testable_functions_destroy() __attribute__ ((destructor(1000)));
48 static void testable_functions_destroy()
49 {
50 testable_functions->destroy(testable_functions);
51 /* if leak detective is enabled plugins are not actually unloaded, which
52 * means their destructor is called AFTER this one when the process
53 * terminates, even though the priority says differently, make sure this
54 * does not crash */
55 testable_functions = NULL;
56 }
57
58 /**
59 * Load all available test suites
60 */
61 static array_t *load_suites(test_configuration_t configs[],
62 test_runner_init_t init)
63 {
64 array_t *suites;
65 bool old = FALSE;
66 int i;
67
68 library_init(NULL, "test-runner");
69
70 test_setup_handler();
71
72 if (init && !init(TRUE))
73 {
74 library_deinit();
75 return NULL;
76 }
77 lib->plugins->status(lib->plugins, LEVEL_CTRL);
78
79 if (lib->leak_detective)
80 {
81 old = lib->leak_detective->set_state(lib->leak_detective, FALSE);
82 }
83
84 suites = array_create(0, 0);
85
86 for (i = 0; configs[i].suite; i++)
87 {
88 if (configs[i].feature.type == 0 ||
89 lib->plugins->has_feature(lib->plugins, configs[i].feature))
90 {
91 array_insert(suites, -1, configs[i].suite());
92 }
93 }
94
95 if (lib->leak_detective)
96 {
97 lib->leak_detective->set_state(lib->leak_detective, old);
98 }
99
100 if (init)
101 {
102 init(FALSE);
103 }
104 library_deinit();
105
106 return suites;
107 }
108
109 /**
110 * Unload and destroy test suites and associated data
111 */
112 static void unload_suites(array_t *suites)
113 {
114 test_suite_t *suite;
115 test_case_t *tcase;
116
117 while (array_remove(suites, 0, &suite))
118 {
119 while (array_remove(suite->tcases, 0, &tcase))
120 {
121 array_destroy(tcase->functions);
122 array_destroy(tcase->fixtures);
123 }
124 free(suite);
125 }
126 array_destroy(suites);
127 }
128
129 /**
130 * Run a single test function, return FALSE on failure
131 */
132 static bool run_test(test_function_t *tfun, int i)
133 {
134 if (test_restore_point())
135 {
136 tfun->cb(i);
137 return TRUE;
138 }
139 return FALSE;
140 }
141
142 /**
143 * Invoke fixture setup/teardown
144 */
145 static bool call_fixture(test_case_t *tcase, bool up)
146 {
147 enumerator_t *enumerator;
148 test_fixture_t *fixture;
149 bool failure = FALSE;
150
151 enumerator = array_create_enumerator(tcase->fixtures);
152 while (enumerator->enumerate(enumerator, &fixture))
153 {
154 if (test_restore_point())
155 {
156 if (up)
157 {
158 fixture->setup();
159 }
160 else
161 {
162 fixture->teardown();
163 }
164 }
165 else
166 {
167 failure = TRUE;
168 break;
169 }
170 }
171 enumerator->destroy(enumerator);
172
173 return !failure;
174 }
175
176 /**
177 * Test initialization, initializes libstrongswan for the next run
178 */
179 static bool pre_test(test_runner_init_t init)
180 {
181 library_init(NULL, "test-runner");
182
183 /* use non-blocking RNG to generate keys fast */
184 lib->settings->set_default_str(lib->settings,
185 "libstrongswan.plugins.random.random",
186 lib->settings->get_str(lib->settings,
187 "libstrongswan.plugins.random.urandom", "/dev/urandom"));
188 /* same for the gcrypt plugin */
189 lib->settings->set_default_str(lib->settings,
190 "libstrongswan.plugins.gcrypt.quick_random", "yes");
191
192 if (lib->leak_detective)
193 {
194 /* disable leak reports during testing */
195 lib->leak_detective->set_report_cb(lib->leak_detective,
196 NULL, NULL, NULL);
197 }
198 if (init && !init(TRUE))
199 {
200 library_deinit();
201 return FALSE;
202 }
203 dbg_default_set_level(LEVEL_SILENT);
204 return TRUE;
205 }
206
207 /**
208 * Failure description
209 */
210 typedef struct {
211 char *name;
212 char msg[512 - sizeof(char*) - 2 * sizeof(int)];
213 const char *file;
214 int line;
215 int i;
216 backtrace_t *bt;
217 } failure_t;
218
219 /**
220 * Data passed to leak report callbacks
221 */
222 typedef struct {
223 array_t *failures;
224 char *name;
225 int i;
226 int leaks;
227 } report_data_t;
228
229 /**
230 * Leak report callback, build failures from leaks
231 */
232 static void report_leaks(report_data_t *data, int count, size_t bytes,
233 backtrace_t *bt, bool detailed)
234 {
235 failure_t failure = {
236 .name = data->name,
237 .i = data->i,
238 .bt = bt->clone(bt),
239 };
240
241 snprintf(failure.msg, sizeof(failure.msg),
242 "Leak detected: %d allocations using %zu bytes", count, bytes);
243
244 array_insert(data->failures, -1, &failure);
245 }
246
247 /**
248 * Leak summary callback, check if any leaks found
249 */
250 static void sum_leaks(report_data_t *data, int count, size_t bytes,
251 int whitelisted)
252 {
253 data->leaks = count;
254 }
255
256 /**
257 * Do library cleanup and optionally check for memory leaks
258 */
259 static bool post_test(test_runner_init_t init, bool check_leaks,
260 array_t *failures, char *name, int i)
261 {
262 report_data_t data = {
263 .failures = failures,
264 .name = name,
265 .i = i,
266 };
267
268 if (init)
269 {
270 init(FALSE);
271 }
272 if (check_leaks && lib->leak_detective)
273 {
274 lib->leak_detective->set_report_cb(lib->leak_detective,
275 (leak_detective_report_cb_t)report_leaks,
276 (leak_detective_summary_cb_t)sum_leaks, &data);
277 }
278 library_deinit();
279
280 return data.leaks != 0;
281 }
282
283 /**
284 * Collect failure information, add failure_t to array
285 */
286 static void collect_failure_info(array_t *failures, char *name, int i)
287 {
288 failure_t failure = {
289 .name = name,
290 .i = i,
291 .bt = test_failure_backtrace(),
292 };
293
294 failure.line = test_failure_get(failure.msg, sizeof(failure.msg),
295 &failure.file);
296
297 array_insert(failures, -1, &failure);
298 }
299
300 /**
301 * Print array of collected failure_t to stderr
302 */
303 static void print_failures(array_t *failures)
304 {
305 failure_t failure;
306
307 backtrace_init();
308
309 while (array_remove(failures, 0, &failure))
310 {
311 fprintf(stderr, " %sFailure in '%s': %s (",
312 TTY(RED), failure.name, failure.msg);
313 if (failure.line)
314 {
315 fprintf(stderr, "%s:%d, ", failure.file, failure.line);
316 }
317 fprintf(stderr, "i = %d)%s\n", failure.i, TTY(DEF));
318 if (failure.bt)
319 {
320 failure.bt->log(failure.bt, stderr, TRUE);
321 failure.bt->destroy(failure.bt);
322 }
323 }
324
325 backtrace_deinit();
326 }
327
328 /**
329 * Run a single test case with fixtures
330 */
331 static bool run_case(test_case_t *tcase, test_runner_init_t init)
332 {
333 enumerator_t *enumerator;
334 test_function_t *tfun;
335 int passed = 0;
336 array_t *failures;
337
338 failures = array_create(sizeof(failure_t), 0);
339
340 fprintf(stderr, " Running case '%s': ", tcase->name);
341 fflush(stderr);
342
343 enumerator = array_create_enumerator(tcase->functions);
344 while (enumerator->enumerate(enumerator, &tfun))
345 {
346 int i, rounds = 0;
347
348 for (i = tfun->start; i < tfun->end; i++)
349 {
350 if (pre_test(init))
351 {
352 bool ok = FALSE, leaks = FALSE;
353
354 test_setup_timeout(tcase->timeout);
355
356 if (call_fixture(tcase, TRUE))
357 {
358 if (run_test(tfun, i))
359 {
360 if (call_fixture(tcase, FALSE))
361 {
362 ok = TRUE;
363 }
364 }
365 else
366 {
367 call_fixture(tcase, FALSE);
368 }
369
370 }
371 leaks = post_test(init, ok, failures, tfun->name, i);
372
373 test_setup_timeout(0);
374
375 if (ok)
376 {
377 if (!leaks)
378 {
379 rounds++;
380 fprintf(stderr, "%s+%s", TTY(GREEN), TTY(DEF));
381 }
382 }
383 else
384 {
385 collect_failure_info(failures, tfun->name, i);
386 }
387 if (!ok || leaks)
388 {
389 fprintf(stderr, "%s-%s", TTY(RED), TTY(DEF));
390 }
391 }
392 else
393 {
394 fprintf(stderr, "!");
395 }
396 }
397 fflush(stderr);
398 if (rounds == tfun->end - tfun->start)
399 {
400 passed++;
401 }
402 }
403 enumerator->destroy(enumerator);
404
405 fprintf(stderr, "\n");
406
407 print_failures(failures);
408 array_destroy(failures);
409
410 return passed == array_count(tcase->functions);
411 }
412
413 /**
414 * Run a single test suite
415 */
416 static bool run_suite(test_suite_t *suite, test_runner_init_t init)
417 {
418 enumerator_t *enumerator;
419 test_case_t *tcase;
420 int passed = 0;
421
422 fprintf(stderr, " Running suite '%s':\n", suite->name);
423
424 enumerator = array_create_enumerator(suite->tcases);
425 while (enumerator->enumerate(enumerator, &tcase))
426 {
427 if (run_case(tcase, init))
428 {
429 passed++;
430 }
431 }
432 enumerator->destroy(enumerator);
433
434 if (passed == array_count(suite->tcases))
435 {
436 fprintf(stderr, " %sPassed all %u '%s' test cases%s\n",
437 TTY(GREEN), array_count(suite->tcases), suite->name, TTY(DEF));
438 return TRUE;
439 }
440 fprintf(stderr, " %sPassed %u/%u '%s' test cases%s\n",
441 TTY(RED), passed, array_count(suite->tcases), suite->name, TTY(DEF));
442 return FALSE;
443 }
444
445 /**
446 * See header.
447 */
448 int test_runner_run(const char *name, test_configuration_t configs[],
449 test_runner_init_t init)
450 {
451 array_t *suites;
452 test_suite_t *suite;
453 enumerator_t *enumerator;
454 int passed = 0, result;
455
456 /* redirect all output to stderr (to redirect make's stdout to /dev/null) */
457 dup2(2, 1);
458
459 suites = load_suites(configs, init);
460 if (!suites)
461 {
462 return EXIT_FAILURE;
463 }
464
465 fprintf(stderr, "Running %u '%s' test suites:\n", array_count(suites), name);
466
467 enumerator = array_create_enumerator(suites);
468 while (enumerator->enumerate(enumerator, &suite))
469 {
470 if (run_suite(suite, init))
471 {
472 passed++;
473 }
474 }
475 enumerator->destroy(enumerator);
476
477 if (passed == array_count(suites))
478 {
479 fprintf(stderr, "%sPassed all %u '%s' suites%s\n",
480 TTY(GREEN), array_count(suites), name, TTY(DEF));
481 result = EXIT_SUCCESS;
482 }
483 else
484 {
485 fprintf(stderr, "%sPassed %u of %u '%s' suites%s\n",
486 TTY(RED), passed, array_count(suites), name, TTY(DEF));
487 result = EXIT_FAILURE;
488 }
489
490 unload_suites(suites);
491
492 return result;
493 }