unit-tests: Add test suite for streams and services
[strongswan.git] / src / libstrongswan / tests / test_runner.c
1 /*
2 * Copyright (C) 2013 Tobias Brunner
3 * Hochschule fuer Technik Rapperswil
4 * Copyright (C) 2013 Martin Willi
5 * Copyright (C) 2013 revosec AG
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License as published by the
9 * Free Software Foundation; either version 2 of the License, or (at your
10 * option) any later version. See <http://www.fsf.org/copyleft/gpl.txt>.
11 *
12 * This program is distributed in the hope that it will be useful, but
13 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
14 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 * for more details.
16 */
17
18 #include "test_runner.h"
19
20 #include <library.h>
21 #include <plugins/plugin_feature.h>
22 #include <collections/array.h>
23
24 #include <dirent.h>
25 #include <unistd.h>
26 #include <limits.h>
27
28 /**
29 * Get a tty color escape character for stderr
30 */
31 #define TTY(color) tty_escape_get(2, TTY_FG_##color)
32
33 /**
34 * Load plugins from builddir
35 */
36 static bool load_plugins()
37 {
38 enumerator_t *enumerator;
39 char *name, path[PATH_MAX], dir[64];
40
41 enumerator = enumerator_create_token(PLUGINS, " ", "");
42 while (enumerator->enumerate(enumerator, &name))
43 {
44 snprintf(dir, sizeof(dir), "%s", name);
45 translate(dir, "-", "_");
46 snprintf(path, sizeof(path), "%s/%s/.libs", PLUGINDIR, dir);
47 lib->plugins->add_path(lib->plugins, path);
48 }
49 enumerator->destroy(enumerator);
50
51 return lib->plugins->load(lib->plugins, PLUGINS);
52 }
53
54 /**
55 * Check if a specific feature is available, return falg if so
56 */
57 static int check_feature(plugin_feature_t feature, int flag)
58 {
59 if (lib->plugins->has_feature(lib->plugins, feature))
60 {
61 return flag;
62 }
63 return 0;
64 }
65
66 /**
67 * Load all available test suites
68 */
69 static array_t *load_suites()
70 {
71 array_t *suites;
72 enum {
73 OTEST_RSA = (1<<0),
74 OTEST_ECDSA = (1<<1),
75 } otest = 0;
76
77 library_init(NULL);
78
79 test_setup_handler();
80
81 if (!load_plugins())
82 {
83 library_deinit();
84 return NULL;
85 }
86 lib->plugins->status(lib->plugins, LEVEL_CTRL);
87
88 /* we have to build the test suite array without leak detective, so
89 * separate plugin checks and suite creation */
90 otest |= check_feature(PLUGIN_DEPENDS(PRIVKEY_GEN, KEY_RSA), OTEST_RSA);
91 otest |= check_feature(PLUGIN_DEPENDS(PRIVKEY_GEN, KEY_ECDSA), OTEST_ECDSA);
92
93 library_deinit();
94
95 suites = array_create(0, 0);
96
97 array_insert(suites, -1, bio_reader_suite_create());
98 array_insert(suites, -1, bio_writer_suite_create());
99 array_insert(suites, -1, chunk_suite_create());
100 array_insert(suites, -1, enum_suite_create());
101 array_insert(suites, -1, enumerator_suite_create());
102 array_insert(suites, -1, linked_list_suite_create());
103 array_insert(suites, -1, linked_list_enumerator_suite_create());
104 array_insert(suites, -1, hashtable_suite_create());
105 array_insert(suites, -1, array_suite_create());
106 array_insert(suites, -1, identification_suite_create());
107 array_insert(suites, -1, threading_suite_create());
108 array_insert(suites, -1, watcher_suite_create());
109 array_insert(suites, -1, stream_suite_create());
110 array_insert(suites, -1, utils_suite_create());
111 array_insert(suites, -1, host_suite_create());
112 array_insert(suites, -1, vectors_suite_create());
113 array_insert(suites, -1, pen_suite_create());
114 array_insert(suites, -1, asn1_suite_create());
115 array_insert(suites, -1, printf_suite_create());
116 if (otest & OTEST_RSA)
117 {
118 array_insert(suites, -1, rsa_suite_create());
119 }
120 if (otest & OTEST_ECDSA)
121 {
122 array_insert(suites, -1, ecdsa_suite_create());
123 }
124
125 return suites;
126 }
127
128 /**
129 * Unload and destroy test suites and associated data
130 */
131 static void unload_suites(array_t *suites)
132 {
133 test_suite_t *suite;
134 test_case_t *tcase;
135
136 while (array_remove(suites, 0, &suite))
137 {
138 while (array_remove(suite->tcases, 0, &tcase))
139 {
140 array_destroy(tcase->functions);
141 array_destroy(tcase->fixtures);
142 }
143 free(suite);
144 }
145 array_destroy(suites);
146 }
147
148 /**
149 * Run a single test function, return FALSE on failure
150 */
151 static bool run_test(test_function_t *tfun, int i)
152 {
153 if (test_restore_point())
154 {
155 tfun->cb(i);
156 return TRUE;
157 }
158 return FALSE;
159 }
160
161 /**
162 * Invoke fixture setup/teardown
163 */
164 static bool call_fixture(test_case_t *tcase, bool up)
165 {
166 enumerator_t *enumerator;
167 test_fixture_t *fixture;
168 bool failure = FALSE;
169
170 enumerator = array_create_enumerator(tcase->fixtures);
171 while (enumerator->enumerate(enumerator, &fixture))
172 {
173 if (test_restore_point())
174 {
175 if (up)
176 {
177 fixture->setup();
178 }
179 else
180 {
181 fixture->teardown();
182 }
183 }
184 else
185 {
186 failure = TRUE;
187 break;
188 }
189 }
190 enumerator->destroy(enumerator);
191
192 return !failure;
193 }
194
195 /**
196 * Test initialization, initializes libstrongswan for the next run
197 */
198 static bool pre_test()
199 {
200 library_init(NULL);
201
202 /* use non-blocking RNG to generate keys fast */
203 lib->settings->set_default_str(lib->settings,
204 "libstrongswan.plugins.random.random",
205 lib->settings->get_str(lib->settings,
206 "libstrongswan.plugins.random.urandom", "/dev/urandom"));
207
208 if (lib->leak_detective)
209 {
210 /* disable leak reports during testing */
211 lib->leak_detective->set_report_cb(lib->leak_detective,
212 NULL, NULL, NULL);
213 }
214 if (!load_plugins())
215 {
216 library_deinit();
217 return FALSE;
218 }
219
220 dbg_default_set_level(LEVEL_SILENT);
221 return TRUE;
222 }
223
224 /**
225 * Failure description
226 */
227 typedef struct {
228 char *name;
229 char msg[512 - sizeof(char*) - 2 * sizeof(int)];
230 const char *file;
231 int line;
232 int i;
233 backtrace_t *bt;
234 } failure_t;
235
236 /**
237 * Data passed to leak report callbacks
238 */
239 typedef struct {
240 array_t *failures;
241 char *name;
242 int i;
243 int leaks;
244 } report_data_t;
245
246 /**
247 * Leak report callback, build failures from leaks
248 */
249 static void report_leaks(report_data_t *data, int count, size_t bytes,
250 backtrace_t *bt, bool detailed)
251 {
252 failure_t failure = {
253 .name = data->name,
254 .i = data->i,
255 .bt = bt->clone(bt),
256 };
257
258 snprintf(failure.msg, sizeof(failure.msg),
259 "Leak detected: %d allocations using %zu bytes", count, bytes);
260
261 array_insert(data->failures, -1, &failure);
262 }
263
264 /**
265 * Leak summary callback, check if any leaks found
266 */
267 static void sum_leaks(report_data_t *data, int count, size_t bytes,
268 int whitelisted)
269 {
270 data->leaks = count;
271 }
272
273 /**
274 * Do library cleanup and optionally check for memory leaks
275 */
276 static bool post_test(bool check_leaks, array_t *failures, char *name, int i)
277 {
278 report_data_t data = {
279 .failures = failures,
280 .name = name,
281 .i = i,
282 };
283
284 lib->processor->set_threads(lib->processor, 0);
285 lib->processor->cancel(lib->processor);
286 lib->plugins->unload(lib->plugins);
287
288 if (check_leaks && lib->leak_detective)
289 {
290 lib->leak_detective->set_report_cb(lib->leak_detective,
291 (leak_detective_report_cb_t)report_leaks,
292 (leak_detective_summary_cb_t)sum_leaks, &data);
293 }
294 library_deinit();
295
296 return data.leaks != 0;
297 }
298
299 /**
300 * Collect failure information, add failure_t to array
301 */
302 static void collect_failure_info(array_t *failures, char *name, int i)
303 {
304 failure_t failure = {
305 .name = name,
306 .i = i,
307 .bt = test_failure_backtrace(),
308 };
309
310 failure.line = test_failure_get(failure.msg, sizeof(failure.msg),
311 &failure.file);
312
313 array_insert(failures, -1, &failure);
314 }
315
316 /**
317 * Print array of collected failure_t to stderr
318 */
319 static void print_failures(array_t *failures)
320 {
321 failure_t failure;
322
323 backtrace_init();
324
325 while (array_remove(failures, 0, &failure))
326 {
327 fprintf(stderr, " %sFailure in '%s': %s (",
328 TTY(RED), failure.name, failure.msg);
329 if (failure.line)
330 {
331 fprintf(stderr, "%s:%d, ", failure.file, failure.line);
332 }
333 fprintf(stderr, "i = %d)%s\n", failure.i, TTY(DEF));
334 if (failure.bt)
335 {
336 failure.bt->log(failure.bt, stderr, TRUE);
337 failure.bt->destroy(failure.bt);
338 }
339 }
340
341 backtrace_deinit();
342 }
343
344 /**
345 * Run a single test case with fixtures
346 */
347 static bool run_case(test_case_t *tcase)
348 {
349 enumerator_t *enumerator;
350 test_function_t *tfun;
351 int passed = 0;
352 array_t *failures;
353
354 failures = array_create(sizeof(failure_t), 0);
355
356 fprintf(stderr, " Running case '%s': ", tcase->name);
357 fflush(stderr);
358
359 enumerator = array_create_enumerator(tcase->functions);
360 while (enumerator->enumerate(enumerator, &tfun))
361 {
362 int i, rounds = 0;
363
364 for (i = tfun->start; i < tfun->end; i++)
365 {
366 if (pre_test())
367 {
368 bool ok = FALSE, leaks = FALSE;
369
370 test_setup_timeout(tcase->timeout);
371
372 if (call_fixture(tcase, TRUE))
373 {
374 if (run_test(tfun, i))
375 {
376 if (call_fixture(tcase, FALSE))
377 {
378 ok = TRUE;
379 }
380 }
381 else
382 {
383 call_fixture(tcase, FALSE);
384 }
385
386 }
387 leaks = post_test(ok, failures, tfun->name, i);
388
389 test_setup_timeout(0);
390
391 if (ok)
392 {
393 if (!leaks)
394 {
395 rounds++;
396 fprintf(stderr, "%s+%s", TTY(GREEN), TTY(DEF));
397 }
398 }
399 else
400 {
401 collect_failure_info(failures, tfun->name, i);
402 }
403 if (!ok || leaks)
404 {
405 fprintf(stderr, "%s-%s", TTY(RED), TTY(DEF));
406 }
407 }
408 else
409 {
410 fprintf(stderr, "!");
411 }
412 }
413 fflush(stderr);
414 if (rounds == tfun->end - tfun->start)
415 {
416 passed++;
417 }
418 }
419 enumerator->destroy(enumerator);
420
421 fprintf(stderr, "\n");
422
423 print_failures(failures);
424 array_destroy(failures);
425
426 return passed == array_count(tcase->functions);
427 }
428
429 /**
430 * Run a single test suite
431 */
432 static bool run_suite(test_suite_t *suite)
433 {
434 enumerator_t *enumerator;
435 test_case_t *tcase;
436 int passed = 0;
437
438 fprintf(stderr, " Running suite '%s':\n", suite->name);
439
440 enumerator = array_create_enumerator(suite->tcases);
441 while (enumerator->enumerate(enumerator, &tcase))
442 {
443 if (run_case(tcase))
444 {
445 passed++;
446 }
447 }
448 enumerator->destroy(enumerator);
449
450 if (passed == array_count(suite->tcases))
451 {
452 fprintf(stderr, " %sPassed all %u '%s' test cases%s\n",
453 TTY(GREEN), array_count(suite->tcases), suite->name, TTY(DEF));
454 return TRUE;
455 }
456 fprintf(stderr, " %sPassed %u/%u '%s' test cases%s\n",
457 TTY(RED), passed, array_count(suite->tcases), suite->name, TTY(DEF));
458 return FALSE;
459 }
460
461 int main(int argc, char *argv[])
462 {
463 array_t *suites;
464 test_suite_t *suite;
465 enumerator_t *enumerator;
466 int passed = 0, result;
467
468 /* redirect all output to stderr (to redirect make's stdout to /dev/null) */
469 dup2(2, 1);
470
471 suites = load_suites();
472 if (!suites)
473 {
474 return EXIT_FAILURE;
475 }
476
477 fprintf(stderr, "Running %u test suites:\n", array_count(suites));
478
479 enumerator = array_create_enumerator(suites);
480 while (enumerator->enumerate(enumerator, &suite))
481 {
482 if (run_suite(suite))
483 {
484 passed++;
485 }
486 }
487 enumerator->destroy(enumerator);
488
489 if (passed == array_count(suites))
490 {
491 fprintf(stderr, "%sPassed all %u suites%s\n",
492 TTY(GREEN), array_count(suites), TTY(DEF));
493 result = EXIT_SUCCESS;
494 }
495 else
496 {
497 fprintf(stderr, "%sPassed %u of %u suites%s\n",
498 TTY(RED), passed, array_count(suites), TTY(DEF));
499 result = EXIT_FAILURE;
500 }
501
502 unload_suites(suites);
503
504 return result;
505 }