src/libstrongswan/tests/test_runner.c

   1 /*
   2  * Copyright (C) 2013 Tobias Brunner
   3  * Hochschule fuer Technik Rapperswil
   4  * Copyright (C) 2013 Martin Willi
   5  * Copyright (C) 2013 revosec AG
   6  *
   7  * This program is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License as published by the
   9  * Free Software Foundation; either version 2 of the License, or (at your
  10  * option) any later version.  See <http://www.fsf.org/copyleft/gpl.txt>.
  11  *
  12  * This program is distributed in the hope that it will be useful, but
  13  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  14  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15  * for more details.
  16  */
  17
  18 #include "test_runner.h"
  19
  20 #include <library.h>
  21 #include <plugins/plugin_feature.h>
  22 #include <collections/array.h>
  23 #include <utils/test.h>
  24
  25 #include <stdlib.h>
  26 #include <dirent.h>
  27 #include <unistd.h>
  28 #include <limits.h>
  29
  30 /**
  31  * Get a tty color escape character for stderr
  32  */
  33 #define TTY(color) tty_escape_get(2, TTY_FG_##color)
  34
  35 /**
  36  * Initialize the lookup table for testable functions (defined in
  37  * libstrongswan).  We don't use the constructor attribute as the order can't
  38  * really be defined (clang does not support it and gcc does not adhere to it in
  39  * the monolithic build).  The function here is a weak symbol in libstrongswan.
  40  */
  41 void testable_functions_create()
  42 {
  43         if (!testable_functions)
  44         {
  45                 testable_functions = hashtable_create(hashtable_hash_str,
  46                                                                                           hashtable_equals_str, 8);
  47         }
  48 }
  49
  50 /**
  51  * Destroy the lookup table for testable functions
  52  */
  53 static void testable_functions_destroy() __attribute__ ((destructor));
  54 static void testable_functions_destroy()
  55 {
  56         DESTROY_IF(testable_functions);
  57         /* if leak detective is enabled plugins are not actually unloaded, which
  58          * means their destructor is called AFTER this one when the process
  59          * terminates, make sure this does not crash */
  60         testable_functions = NULL;
  61 }
  62
  63 /**
  64  * Destroy a single test suite and associated data
  65  */
  66 static void destroy_suite(test_suite_t *suite)
  67 {
  68         test_case_t *tcase;
  69
  70         while (array_remove(suite->tcases, 0, &tcase))
  71         {
  72                 array_destroy(tcase->functions);
  73                 array_destroy(tcase->fixtures);
  74         }
  75         free(suite);
  76 }
  77
  78 /**
  79  * Removes and destroys test suites that are not selected.
  80  */
  81 static void filter_suites(array_t *loaded)
  82 {
  83         enumerator_t *enumerator, *names;
  84         hashtable_t *selected;
  85         test_suite_t *suite;
  86         char *suites, *name;
  87
  88         suites = getenv("TESTS_SUITES");
  89         if (!suites)
  90         {
  91                 return;
  92         }
  93         selected = hashtable_create(hashtable_hash_str, hashtable_equals_str, 8);
  94         names = enumerator_create_token(suites, ",", " ");
  95         while (names->enumerate(names, &name))
  96         {
  97                 selected->put(selected, name, name);
  98         }
  99         enumerator = array_create_enumerator(loaded);
 100         while (enumerator->enumerate(enumerator, &suite))
 101         {
 102                 if (!selected->get(selected, suite->name))
 103                 {
 104                         array_remove_at(loaded, enumerator);
 105                         destroy_suite(suite);
 106                 }
 107         }
 108         enumerator->destroy(enumerator);
 109         selected->destroy(selected);
 110         names->destroy(names);
 111 }
 112
 113 /**
 114  * Load all available test suites, or optionally only selected ones.
 115  */
 116 static array_t *load_suites(test_configuration_t configs[],
 117                                                         test_runner_init_t init)
 118 {
 119         array_t *suites;
 120         bool old = FALSE;
 121         int i;
 122
 123         library_init(NULL, "test-runner");
 124
 125         test_setup_handler();
 126
 127         if (init && !init(TRUE))
 128         {
 129                 library_deinit();
 130                 return NULL;
 131         }
 132         lib->plugins->status(lib->plugins, LEVEL_CTRL);
 133
 134         if (lib->leak_detective)
 135         {
 136                 old = lib->leak_detective->set_state(lib->leak_detective, FALSE);
 137         }
 138
 139         suites = array_create(0, 0);
 140
 141         for (i = 0; configs[i].suite; i++)
 142         {
 143                 if (configs[i].feature.type == 0 ||
 144                         lib->plugins->has_feature(lib->plugins, configs[i].feature))
 145                 {
 146                         array_insert(suites, -1, configs[i].suite());
 147                 }
 148         }
 149         filter_suites(suites);
 150
 151         if (lib->leak_detective)
 152         {
 153                 lib->leak_detective->set_state(lib->leak_detective, old);
 154         }
 155
 156         if (init)
 157         {
 158                 init(FALSE);
 159         }
 160         library_deinit();
 161
 162         return suites;
 163 }
 164
 165 /**
 166  * Unload and destroy test suites and associated data
 167  */
 168 static void unload_suites(array_t *suites)
 169 {
 170         test_suite_t *suite;
 171
 172         while (array_remove(suites, 0, &suite))
 173         {
 174                 destroy_suite(suite);
 175         }
 176         array_destroy(suites);
 177 }
 178
 179 /**
 180  * Run a single test function, return FALSE on failure
 181  */
 182 static bool run_test(test_function_t *tfun, int i)
 183 {
 184         if (test_restore_point())
 185         {
 186                 tfun->cb(i);
 187                 return TRUE;
 188         }
 189         return FALSE;
 190 }
 191
 192 /**
 193  * Invoke fixture setup/teardown
 194  */
 195 static bool call_fixture(test_case_t *tcase, bool up)
 196 {
 197         enumerator_t *enumerator;
 198         test_fixture_t *fixture;
 199         bool failure = FALSE;
 200
 201         enumerator = array_create_enumerator(tcase->fixtures);
 202         while (enumerator->enumerate(enumerator, &fixture))
 203         {
 204                 if (test_restore_point())
 205                 {
 206                         if (up)
 207                         {
 208                                 fixture->setup();
 209                         }
 210                         else
 211                         {
 212                                 fixture->teardown();
 213                         }
 214                 }
 215                 else
 216                 {
 217                         failure = TRUE;
 218                         break;
 219                 }
 220         }
 221         enumerator->destroy(enumerator);
 222
 223         return !failure;
 224 }
 225
 226 /**
 227  * Test initialization, initializes libstrongswan for the next run
 228  */
 229 static bool pre_test(test_runner_init_t init)
 230 {
 231         level_t level = LEVEL_SILENT;
 232         char *verbosity;
 233
 234         library_init(NULL, "test-runner");
 235
 236         /* use non-blocking RNG to generate keys fast */
 237         lib->settings->set_default_str(lib->settings,
 238                         "libstrongswan.plugins.random.random",
 239                         lib->settings->get_str(lib->settings,
 240                                 "libstrongswan.plugins.random.urandom", "/dev/urandom"));
 241         /* same for the gcrypt plugin */
 242         lib->settings->set_default_str(lib->settings,
 243                         "libstrongswan.plugins.gcrypt.quick_random", "yes");
 244
 245         if (lib->leak_detective)
 246         {
 247                 /* disable leak reports during testing */
 248                 lib->leak_detective->set_report_cb(lib->leak_detective,
 249                                                                                    NULL, NULL, NULL);
 250         }
 251         if (init && !init(TRUE))
 252         {
 253                 library_deinit();
 254                 return FALSE;
 255         }
 256         verbosity = getenv("TESTS_VERBOSITY");
 257         if (verbosity)
 258         {
 259                 level = atoi(verbosity);
 260         }
 261         dbg_default_set_level(level);
 262         return TRUE;
 263 }
 264
 265 /**
 266  * Failure description
 267  */
 268 typedef struct {
 269         char *name;
 270         char msg[512 - sizeof(char*) - 2 * sizeof(int)];
 271         const char *file;
 272         int line;
 273         int i;
 274         backtrace_t *bt;
 275 } failure_t;
 276
 277 /**
 278  * Data passed to leak report callbacks
 279  */
 280 typedef struct {
 281         array_t *failures;
 282         char *name;
 283         int i;
 284         int leaks;
 285 } report_data_t;
 286
 287 /**
 288  * Leak report callback, build failures from leaks
 289  */
 290 static void report_leaks(report_data_t *data, int count, size_t bytes,
 291                                                  backtrace_t *bt, bool detailed)
 292 {
 293         failure_t failure = {
 294                 .name = data->name,
 295                 .i = data->i,
 296                 .bt = bt->clone(bt),
 297         };
 298
 299         snprintf(failure.msg, sizeof(failure.msg),
 300                          "Leak detected: %d allocations using %zu bytes", count, bytes);
 301
 302         array_insert(data->failures, -1, &failure);
 303 }
 304
 305 /**
 306  * Leak summary callback, check if any leaks found
 307  */
 308 static void sum_leaks(report_data_t *data, int count, size_t bytes,
 309                                           int whitelisted)
 310 {
 311         data->leaks = count;
 312 }
 313
 314 /**
 315  * Do library cleanup and optionally check for memory leaks
 316  */
 317 static bool post_test(test_runner_init_t init, bool check_leaks,
 318                                           array_t *failures, char *name, int i, int *leaks)
 319 {
 320         report_data_t data = {
 321                 .failures = failures,
 322                 .name = name,
 323                 .i = i,
 324         };
 325
 326         if (init)
 327         {
 328                 if (test_restore_point())
 329                 {
 330                         init(FALSE);
 331                 }
 332                 else
 333                 {
 334                         library_deinit();
 335                         return FALSE;
 336                 }
 337         }
 338         if (check_leaks && lib->leak_detective)
 339         {
 340                 lib->leak_detective->set_report_cb(lib->leak_detective,
 341                                                                 (leak_detective_report_cb_t)report_leaks,
 342                                                                 (leak_detective_summary_cb_t)sum_leaks, &data);
 343         }
 344         library_deinit();
 345
 346         *leaks = data.leaks;
 347         return TRUE;
 348 }
 349
 350 /**
 351  * Collect failure information, add failure_t to array
 352  */
 353 static void collect_failure_info(array_t *failures, char *name, int i)
 354 {
 355         failure_t failure = {
 356                 .name = name,
 357                 .i = i,
 358                 .bt = test_failure_backtrace(),
 359         };
 360
 361         failure.line = test_failure_get(failure.msg, sizeof(failure.msg),
 362                                                                         &failure.file);
 363
 364         array_insert(failures, -1, &failure);
 365 }
 366
 367 /**
 368  * Print array of collected failure_t to stderr
 369  */
 370 static void print_failures(array_t *failures)
 371 {
 372         failure_t failure;
 373
 374         backtrace_init();
 375
 376         while (array_remove(failures, 0, &failure))
 377         {
 378                 fprintf(stderr, "      %sFailure in '%s': %s (",
 379                                 TTY(RED), failure.name, failure.msg);
 380                 if (failure.line)
 381                 {
 382                         fprintf(stderr, "%s:%d, ", failure.file, failure.line);
 383                 }
 384                 fprintf(stderr, "i = %d)%s\n", failure.i, TTY(DEF));
 385                 if (failure.bt)
 386                 {
 387                         failure.bt->log(failure.bt, stderr, TRUE);
 388                         failure.bt->destroy(failure.bt);
 389                 }
 390         }
 391
 392         backtrace_deinit();
 393 }
 394
 395 /**
 396  * Run a single test case with fixtures
 397  */
 398 static bool run_case(test_case_t *tcase, test_runner_init_t init)
 399 {
 400         enumerator_t *enumerator;
 401         test_function_t *tfun;
 402         int passed = 0;
 403         array_t *failures;
 404
 405         failures = array_create(sizeof(failure_t), 0);
 406
 407         fprintf(stderr, "    Running case '%s': ", tcase->name);
 408         fflush(stderr);
 409
 410         enumerator = array_create_enumerator(tcase->functions);
 411         while (enumerator->enumerate(enumerator, &tfun))
 412         {
 413                 int i, rounds = 0;
 414
 415                 for (i = tfun->start; i < tfun->end; i++)
 416                 {
 417                         if (pre_test(init))
 418                         {
 419                                 bool ok = FALSE;
 420                                 int leaks = 0;
 421
 422                                 test_setup_timeout(tcase->timeout);
 423
 424                                 if (call_fixture(tcase, TRUE))
 425                                 {
 426                                         if (run_test(tfun, i))
 427                                         {
 428                                                 if (call_fixture(tcase, FALSE))
 429                                                 {
 430                                                         ok = TRUE;
 431                                                 }
 432                                         }
 433                                         else
 434                                         {
 435                                                 call_fixture(tcase, FALSE);
 436                                         }
 437                                 }
 438                                 if (!post_test(init, ok, failures, tfun->name, i, &leaks))
 439                                 {
 440                                         ok = FALSE;
 441                                 }
 442
 443                                 test_setup_timeout(0);
 444
 445                                 if (ok)
 446                                 {
 447                                         if (!leaks)
 448                                         {
 449                                                 rounds++;
 450                                                 fprintf(stderr, "%s+%s", TTY(GREEN), TTY(DEF));
 451                                         }
 452                                 }
 453                                 else
 454                                 {
 455                                         collect_failure_info(failures, tfun->name, i);
 456                                 }
 457                                 if (!ok || leaks)
 458                                 {
 459                                         fprintf(stderr, "%s-%s", TTY(RED), TTY(DEF));
 460                                 }
 461                         }
 462                         else
 463                         {
 464                                 fprintf(stderr, "!");
 465                         }
 466                 }
 467                 fflush(stderr);
 468                 if (rounds == tfun->end - tfun->start)
 469                 {
 470                         passed++;
 471                 }
 472         }
 473         enumerator->destroy(enumerator);
 474
 475         fprintf(stderr, "\n");
 476
 477         print_failures(failures);
 478         array_destroy(failures);
 479
 480         return passed == array_count(tcase->functions);
 481 }
 482
 483 /**
 484  * Run a single test suite
 485  */
 486 static bool run_suite(test_suite_t *suite, test_runner_init_t init)
 487 {
 488         enumerator_t *enumerator;
 489         test_case_t *tcase;
 490         int passed = 0;
 491
 492         fprintf(stderr, "  Running suite '%s':\n", suite->name);
 493
 494         enumerator = array_create_enumerator(suite->tcases);
 495         while (enumerator->enumerate(enumerator, &tcase))
 496         {
 497                 if (run_case(tcase, init))
 498                 {
 499                         passed++;
 500                 }
 501         }
 502         enumerator->destroy(enumerator);
 503
 504         if (passed == array_count(suite->tcases))
 505         {
 506                 fprintf(stderr, "  %sPassed all %u '%s' test cases%s\n",
 507                                 TTY(GREEN), array_count(suite->tcases), suite->name, TTY(DEF));
 508                 return TRUE;
 509         }
 510         fprintf(stderr, "  %sPassed %u/%u '%s' test cases%s\n",
 511                         TTY(RED), passed, array_count(suite->tcases), suite->name, TTY(DEF));
 512         return FALSE;
 513 }
 514
 515 /**
 516  * See header.
 517  */
 518 int test_runner_run(const char *name, test_configuration_t configs[],
 519                                         test_runner_init_t init)
 520 {
 521         array_t *suites;
 522         test_suite_t *suite;
 523         enumerator_t *enumerator;
 524         int passed = 0, result;
 525
 526         /* redirect all output to stderr (to redirect make's stdout to /dev/null) */
 527         dup2(2, 1);
 528
 529         suites = load_suites(configs, init);
 530         if (!suites)
 531         {
 532                 return EXIT_FAILURE;
 533         }
 534
 535         fprintf(stderr, "Running %u '%s' test suites:\n", array_count(suites), name);
 536
 537         enumerator = array_create_enumerator(suites);
 538         while (enumerator->enumerate(enumerator, &suite))
 539         {
 540                 if (run_suite(suite, init))
 541                 {
 542                         passed++;
 543                 }
 544         }
 545         enumerator->destroy(enumerator);
 546
 547         if (passed == array_count(suites))
 548         {
 549                 fprintf(stderr, "%sPassed all %u '%s' suites%s\n",
 550                                 TTY(GREEN), array_count(suites), name, TTY(DEF));
 551                 result = EXIT_SUCCESS;
 552         }
 553         else
 554         {
 555                 fprintf(stderr, "%sPassed %u of %u '%s' suites%s\n",
 556                                 TTY(RED), passed, array_count(suites), name, TTY(DEF));
 557                 result = EXIT_FAILURE;
 558         }
 559
 560         unload_suites(suites);
 561
 562         return result;
 563 }