testing/do-tests

   1 #!/bin/bash
   2 # Automatically execute the strongSwan test cases
   3 #
   4 # Copyright (C) 2004  Eric Marchionni, Patrik Rayo
   5 # Zuercher Hochschule Winterthur
   6 #
   7 # This program is free software; you can redistribute it and/or modify it
   8 # under the terms of the GNU General Public License as published by the
   9 # Free Software Foundation; either version 2 of the License, or (at your
  10 # option) any later version.  See <http://www.fsf.org/copyleft/gpl.txt>.
  11 #
  12 # This program is distributed in the hope that it will be useful, but
  13 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  14 # or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 # for more details.
  16
  17 DIR=$(dirname `readlink -f $0`)
  18 . $DIR/testing.conf
  19 . $DIR/scripts/function.sh
  20 SSHCONF="-F $DIR/ssh_config"
  21
  22 [ -d $DIR/hosts ] || die "Directory 'hosts' not found"
  23 [ -d $DIR/tests ] || die "Directory 'tests' not found"
  24 [ -d $BUILDDIR ] ||
  25         die "Directory '$BUILDDIR' does not exist, please run make-testing first"
  26 running_any $STRONGSWANHOSTS || die "Please start test environment before running $0"
  27
  28 ln -sfT $DIR $TESTDIR/testing
  29
  30 ##############################################################################
  31 # take care of new path and file variables
  32 #
  33
  34 [ -d $TESTRESULTSDIR ] || mkdir $TESTRESULTSDIR
  35
  36 TESTDATE=`date +%Y%m%d-%H%M-%S`
  37
  38 TODAYDIR=$TESTRESULTSDIR/$TESTDATE
  39 mkdir $TODAYDIR
  40 TESTRESULTSHTML=$TODAYDIR/all.html
  41 INDEX=$TODAYDIR/index.html
  42 DEFAULTTESTSDIR=$TESTDIR/testing/tests
  43
  44 SOURCEIP_ROUTING_TABLE=220
  45
  46 testnumber="0"
  47 failed_cnt="0"
  48 passed_cnt="0"
  49 subdir_cnt="0"
  50
  51 ##############################################################################
  52 # parse optional arguments
  53 #
  54 while getopts "vt" opt
  55 do
  56         case "$opt" in
  57         v)
  58                 verbose=YES
  59                 timestamps=YES
  60                 ;;
  61         t)
  62                 timestamps=YES
  63                 ;;
  64         esac
  65 done
  66 shift $((OPTIND-1))
  67
  68
  69 function print_time()
  70 {
  71         [ "$timestamps" == "YES" ] && echo "$(date +%T.%N) ~ "
  72 }
  73
  74 ##############################################################################
  75 # copy default tests to $BUILDDIR
  76 #
  77
  78 TESTSDIR=$BUILDDIR/tests
  79 [ -d $TESTSDIR ] || mkdir $TESTSDIR
  80
  81 ##############################################################################
  82 # assign IP for each host to hostname
  83 #
  84
  85 for host in $STRONGSWANHOSTS
  86 do
  87     eval ipv4_${host}="`echo $HOSTNAMEIPV4 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $1 }' | awk '{ print $1 }'`"
  88     eval ipv6_${host}="`echo $HOSTNAMEIPV6 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $1 }' | awk '{ print $1 }'`"
  89
  90     case $host in
  91     moon)
  92         eval ipv4_moon1="`echo $HOSTNAMEIPV4 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $2 }' | awk '{ print $1 }'`"
  93         eval ipv6_moon1="`echo $HOSTNAMEIPV6 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $2 }' | awk '{ print $1 }'`"
  94         ;;
  95     sun)
  96         eval ipv4_sun1="`echo $HOSTNAMEIPV4 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $2 }' | awk '{ print $1 }'`"
  97         eval ipv6_sun1="`echo $HOSTNAMEIPV6 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $2 }' | awk '{ print $1 }'`"
  98         ;;
  99     alice)
 100         eval ipv4_alice1="`echo $HOSTNAMEIPV4 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $2 }' | awk '{ print $1 }'`"
 101         eval ipv6_alice1="`echo $HOSTNAMEIPV6 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $2 }' | awk '{ print $1 }'`"
 102         ;;
 103     venus)
 104         ;;
 105     bob)
 106         ;;
 107     carol)
 108         eval ipv4_carol1="`echo $HOSTNAMEIPV4 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $2 }' | awk '{ print $1 }'`"
 109         eval ipv6_carol1="`echo $HOSTNAMEIPV6 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $2 }' | awk '{ print $1 }'`"
 110          ;;
 111     dave)
 112         eval ipv4_dave1="`echo $HOSTNAMEIPV4 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $2 }' | awk '{ print $1 }'`"
 113         eval ipv6_dave1="`echo $HOSTNAMEIPV6 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $2 }' | awk '{ print $1 }'`"
 114         ;;
 115     winnetou)
 116         ;;
 117     esac
 118 done
 119
 120
 121 ##############################################################################
 122 # open ssh sessions
 123 #
 124 for host in $STRONGSWANHOSTS
 125 do
 126     ssh $SSHCONF -N root@`eval echo \\\$ipv4_$host` >/dev/null 2>&1 &
 127     eval ssh_pid_$host="`echo $!`"
 128     do_on_exit kill `eval echo \\\$ssh_pid_$host`
 129 done
 130
 131 ##############################################################################
 132 # determine actual software versions
 133 #
 134
 135 [ -f $SHAREDDIR/.strongswan-version ] && SWANVERSION=`cat $SHAREDDIR/.strongswan-version`
 136 KERNELVERSION=`ssh $SSHCONF root@\$ipv4_winnetou uname -r 2>/dev/null`
 137
 138 # check if tcpdump supports --immediate-mode
 139 ssh $SSHCONF root@$ipv4_winnetou tcpdump --immediate-mode -c 1 >/dev/null 2>&1
 140 if [ $? -eq 0 ]
 141 then
 142         TCPDUMP_IM=--immediate-mode
 143 fi
 144
 145 ##############################################################################
 146 # create header for the results html file
 147 #
 148
 149 ENVIRONMENT_HEADER=$(cat <<@EOF
 150   <table border="0" cellspacing="2" cellpadding="2">
 151     <tr valign="top">
 152       <td><b>Host</b></td>
 153       <td colspan="3">`uname -a`</td>
 154     </tr>
 155     <tr valign="top">
 156       <td><b>Guest kernel</b></td>
 157       <td colspan="3">$KERNELVERSION</td>
 158     </tr>
 159     <tr valign="top">
 160       <td><b>strongSwan</b></td>
 161       <td colspan="3">$SWANVERSION</td>
 162     </tr>
 163     <tr valign="top">
 164       <td><b>Date</b></td>
 165       <td colspan="3">$TESTDATE</td>
 166     </tr>
 167     <tr>
 168       <td width="100">&nbsp;</td>
 169       <td width="300">&nbsp;</td>
 170       <td width=" 80">&nbsp;</td>
 171       <td >&nbsp;</td>
 172     </tr>
 173 @EOF
 174 )
 175
 176 cat > $INDEX <<@EOF
 177 <html>
 178 <head>
 179   <title>strongSwan KVM Tests</title>
 180 </head>
 181 <body>
 182   <h2>strongSwan KVM Tests</h2>
 183   $ENVIRONMENT_HEADER
 184 @EOF
 185
 186 cat > $TESTRESULTSHTML <<@EOF
 187 <html>
 188 <head>
 189   <title>strongSwan KVM Tests - All Tests</title>
 190 </head>
 191 <body>
 192   <div><a href="index.html">strongSwan KVM Tests</a> / All Tests</div>
 193   <h2>All Tests</h2>
 194   $ENVIRONMENT_HEADER
 195     <tr align="left">
 196       <th>Number</th>
 197       <th>Test</th>
 198       <th align="right">Time [s]</th>
 199       <th>Result</th>
 200     </tr>
 201 @EOF
 202
 203 echo "Guest kernel : $KERNELVERSION"
 204 echo "strongSwan   : $SWANVERSION"
 205 echo "Date         : $TESTDATE"
 206 echo
 207
 208 ##############################################################################
 209 # trap CTRL-C to properly terminate a long run
 210 #
 211
 212 function abort_tests()
 213 {
 214         echo -n "...aborting..." > /dev/tty
 215         aborted=YES
 216 }
 217 trap abort_tests INT
 218
 219 ##############################################################################
 220 # enter specific test directory
 221 #
 222 if [ $# -gt 0 ]
 223 then
 224     TESTS=$(printf "%s\n" $* | sort -u)
 225 else
 226     TESTS=$(ls $DEFAULTTESTSDIR)
 227 fi
 228
 229 for SUBDIR in $TESTS
 230 do
 231     SUBTESTS="`basename $SUBDIR`"
 232
 233         if [ $SUBTESTS = $SUBDIR ]
 234         then
 235                 SUBTESTS="`ls $DEFAULTTESTSDIR/$SUBDIR`"
 236         else
 237                 if [[ $SUBTESTS == *'*'* ]]
 238                 then
 239                         SUBTESTS="`basename -a $DEFAULTTESTSDIR/$SUBDIR`"
 240                 fi
 241                 SUBDIR="`dirname $SUBDIR`"
 242         fi
 243
 244     if [ ! -d $TODAYDIR/$SUBDIR ]
 245     then
 246         mkdir $TODAYDIR/$SUBDIR
 247         if [ $testnumber == 0 ]
 248         then
 249             FIRST="<b>Category</b>"
 250         else
 251             FIRST="&nbsp;"
 252         fi
 253
 254         if [ $subdir_cnt != 0 ]
 255         then
 256             echo "      <td align=\"right\">$subdir_cnt</td>" >> $INDEX
 257             echo "      <td>&nbsp;</td>" >> $INDEX
 258             echo "    </tr>" >> $INDEX
 259             subdir_cnt="0"
 260         fi
 261         echo "    <tr>" >> $INDEX
 262     echo "      <td>$FIRST</td>">> $INDEX
 263     echo "      <td><a href=\"$SUBDIR/index.html\">$SUBDIR</a></td>" >> $INDEX
 264
 265         SUBTESTSINDEX=$TODAYDIR/$SUBDIR/index.html
 266         cat > $SUBTESTSINDEX <<@EOF
 267 <html>
 268 <head>
 269   <title>strongSwan $SUBDIR Tests</title>
 270 </head>
 271 <body>
 272   <div><a href="../index.html">strongSwan KVM Tests</a> / $SUBDIR</div>
 273   <h2>strongSwan $SUBDIR Tests</h2>
 274   <table border="0" cellspacing="2" cellpadding="2">
 275     <tr valign="top">
 276       <td><b>Guest kernel</b></td>
 277       <td colspan="3">$KERNELVERSION</td>
 278     </tr>
 279     <tr valign="top">
 280       <td><b>strongSwan</b></td>
 281       <td colspan="3">$SWANVERSION</td>
 282     </tr>
 283     <tr valign="top">
 284       <td><b>Date</b></td>
 285       <td colspan="3">$TESTDATE</td>
 286     </tr>
 287     <tr>
 288       <td width="100">&nbsp;</td>
 289       <td width="300">&nbsp;</td>
 290       <td width=" 50">&nbsp;</td>
 291       <td >&nbsp;</td>
 292     </tr>
 293     <tr align="left">
 294        <th>Number</th>
 295        <th>Test</th>
 296        <th colspan="2">Result</th>
 297     </tr>
 298 @EOF
 299     fi
 300
 301     for name in $SUBTESTS
 302     do
 303         let "testnumber += 1"
 304         let "subdir_cnt += 1"
 305         testname=$SUBDIR/$name
 306         log_action " $testnumber $testname:"
 307
 308         teststart=$(date +%s)
 309
 310         if [ ! -d $DEFAULTTESTSDIR/${testname} ]
 311         then
 312             echo "is missing..skipped"
 313             continue
 314         fi
 315
 316         [ -f $DEFAULTTESTSDIR/${testname}/description.txt ] || die "!! File 'description.txt' is missing"
 317         [ -f $DEFAULTTESTSDIR/${testname}/test.conf ]       || die "!! File 'test.conf' is missing"
 318         [ -f $DEFAULTTESTSDIR/${testname}/pretest.dat ]     || die "!! File 'pretest.dat' is missing"
 319         [ -f $DEFAULTTESTSDIR/${testname}/posttest.dat ]    || die "!! File 'posttest.dat' is missing"
 320         [ -f $DEFAULTTESTSDIR/${testname}/evaltest.dat ]    || die "!! File 'evaltest.dat' is missing"
 321
 322         TESTRESULTDIR=$TODAYDIR/$testname
 323         mkdir -p $TESTRESULTDIR
 324         CONSOLE_LOG=$TESTRESULTDIR/console.log
 325         touch $CONSOLE_LOG
 326
 327         TESTDIR=$TESTSDIR/${testname}
 328
 329         ##########################################################################
 330         # copy test specific configurations to hosts and clear log files
 331         #
 332
 333         DBDIR=/etc/db.d
 334
 335         $DIR/scripts/load-testconfig $testname
 336         unset RADIUSHOSTS
 337         unset DBHOSTS
 338         unset IPV6
 339         unset SWANCTL
 340         source $TESTDIR/test.conf
 341
 342
 343         ##########################################################################
 344         # run tcpdump in the background
 345         #
 346
 347         if [ "$TCPDUMPHOSTS" != "" ]
 348         then
 349             echo -e "TCPDUMP\n" >> $CONSOLE_LOG 2>&1
 350
 351             for host_iface in $TCPDUMPHOSTS
 352             do
 353                 host=`echo $host_iface | awk -F ":" '{print $1}'`
 354                 iface=`echo $host_iface | awk -F ":" '{if ($2 != "") { print $2 } else { printf("eth0") }}'`
 355                 tcpdump_cmd="tcpdump -l $TCPDUMP_IM -i $iface not port ssh and not port domain >/tmp/tcpdump.log 2>/tmp/tcpdump.err.log &"
 356                 echo "$(print_time)${host}# $tcpdump_cmd" >> $CONSOLE_LOG
 357                 ssh $SSHCONF root@`eval echo \\\$ipv4_$host '$tcpdump_cmd'`
 358                 eval TDUP_${host}="true"
 359             done
 360         fi
 361
 362         ##########################################################################
 363         # create database directory in RAM
 364         #
 365
 366         for host in $DBHOSTS
 367         do
 368                 eval HOSTLOGIN=root@\$ipv4_${host}
 369             ssh $SSHCONF $HOSTLOGIN "mkdir -p $DBDIR; mount -t ramfs -o size=5m ramfs $DBDIR" >/dev/null 2>&1
 370             ssh $SSHCONF $HOSTLOGIN "chgrp www-data $DBDIR; chmod g+w $DBDIR" >/dev/null 2>&1
 371         done
 372
 373         ##########################################################################
 374         # flush conntrack table on all hosts
 375         #
 376
 377         for host in $STRONGSWANHOSTS
 378         do
 379                 ssh $SSHCONF root@`eval echo \\\$ipv4_$host` 'conntrack -F' >/dev/null 2>&1
 380         done
 381
 382         ##########################################################################
 383         # remove leak detective log on all hosts
 384         #
 385
 386         export LEAK_DETECTIVE_LOG=/var/log/leak-detective.log
 387         for host in $STRONGSWANHOSTS
 388         do
 389                 ssh $SSHCONF root@`eval echo \\\$ipv4_$host` 'rm -f $LEAK_DETECTIVE_LOG' >/dev/null 2>&1
 390         done
 391
 392         ##########################################################################
 393         # flush IPsec state on all hosts
 394         #
 395
 396         for host in $STRONGSWANHOSTS
 397         do
 398                 ssh $SSHCONF root@`eval echo \\\$ipv4_$host` 'ip xfrm state flush; ip xfrm policy flush' >/dev/null 2>&1
 399         done
 400
 401         ##########################################################################
 402         # execute pre-test commands
 403         #
 404
 405         echo -n "pre.."
 406         echo -e "\nPRE-TEST\n" >> $CONSOLE_LOG 2>&1
 407
 408         eval `awk -F "::" '{
 409             if ($1 !~ /^#.*/ && $2 != "")
 410             {
 411                 printf("echo \"$(print_time)%s# %s\"; ", $1, $2)
 412                 printf("ssh \044SSHCONF root@\044ipv4_%s \"%s\"; ", $1, $2)
 413                 printf("echo;\n")
 414             }
 415         }' $TESTDIR/pretest.dat` >> $CONSOLE_LOG 2>&1
 416
 417
 418         ##########################################################################
 419         # stop tcpdump
 420         #
 421
 422         function stop_tcpdump {
 423             # wait for packets to get processed, but don't wait longer than 1s
 424             eval ssh $SSHCONF root@\$ipv4_${1} "\"i=100; while [ \\\$i -gt 0 ]; do pkill -USR1 tcpdump; tail -1 /tmp/tcpdump.err.log | perl -n -e '/(\\d+).*?(\\d+)/; exit (\\\$1 == \\\$2)' || break; sleep 0.01; i=\\\$((\\\$i-1)); done;\""
 425             echo "$(print_time)${1}# killall tcpdump" >> $CONSOLE_LOG
 426             eval ssh $SSHCONF root@\$ipv4_${1} "\"killall tcpdump; while true; do killall -q -0 tcpdump || break; sleep 0.01; done;\""
 427             eval TDUP_${1}="false"
 428             echo "" >> $CONSOLE_LOG
 429         }
 430
 431
 432         ##########################################################################
 433         # get and evaluate test results
 434         #
 435
 436         echo -n "test.."
 437         echo -e "\nTEST\n" >> $CONSOLE_LOG 2>&1
 438
 439         STATUS="passed"
 440
 441         eval `awk -F "::" '{
 442                 host=$1
 443                 command=$2
 444                 pattern=$3
 445                 hit=$4
 446                 if (host ~ /^#.*/ || command == "")
 447                 {
 448                         next
 449                 }
 450                 printf("cmd_err=\044(tempfile -p test -s err); ")
 451                 printf("cmd_out=\044(tempfile -p test -s out); ")
 452                 if (command == "tcpdump")
 453                 {
 454                         printf("if [ \044TDUP_%s == \"true\" ]; then stop_tcpdump %s; fi; \n", host, host)
 455                         printf("ssh \044SSHCONF root@\044ipv4_%s cat /tmp/tcpdump.log > \044cmd_out; ", host)
 456                 }
 457                 else
 458                 {
 459                         printf("ssh \044SSHCONF root@\044ipv4_%s %s >\044cmd_out 2>\044cmd_err; ",  host, command)
 460                 }
 461                 printf("cmd_res=\044(cat \044cmd_out | grep \"%s\"); ", pattern)
 462                 printf("cmd_exit=\044?; ")
 463                 printf("cmd_fail=0; ")
 464                 if (hit ~ /^[0-9]+$/)
 465                 {
 466                         printf("if [ \044(echo \"\044cmd_res\" | wc -l) -ne %d ] ", hit)
 467                 }
 468                 else
 469                 {
 470                         printf("if [ \044cmd_exit -eq 0 -a \"%s\" = \"NO\"  ] ", hit)
 471                         printf("|| [ \044cmd_exit -ne 0 -a \"%s\" = \"YES\" ] ", hit)
 472                 }
 473                 printf("; then STATUS=\"failed\"; cmd_fail=1; fi; \n")
 474
 475                 printf("if [ \044cmd_fail -ne 0 ]; then echo \"~~~~~~~ FAIL ~~~~~~~\"; fi; \n")
 476                 if (command == "tcpdump")
 477                 {
 478                         printf("echo \"$(print_time)%s# cat /tmp/tcpdump.log | grep \047%s\047  [%s]\"; ", host, pattern, hit)
 479                 }
 480                 else
 481                 {
 482                         printf("echo \"$(print_time)%s# %s | grep \047%s\047  [%s]\"; ", host, command, pattern, hit)
 483                 }
 484                 printf("if [ -n \"\044cmd_res\" ]; then echo \"\044cmd_res\"; fi; \n")
 485                 printf("cat \044cmd_err; \n")
 486                 printf("if [ \044cmd_fail -ne 0 ]; then \n")
 487                 printf("if [ -s \044cmd_out ]; then echo \"~~ output ~~~~~~~~~~\"; \n")
 488                 printf("if [ \"\044verbose\" == \"YES\" ]; then cat \044cmd_out;\n")
 489                 printf("else cat \044cmd_out | head; fi; fi; \n")
 490                 printf("echo \"~~~~~~~~~~~~~~~~~~~~\"; fi; \n")
 491                 printf("rm -f -- \044cmd_out \044cmd_err; \n")
 492                 printf("echo; ")
 493         }' $TESTDIR/evaltest.dat` >> $CONSOLE_LOG 2>&1
 494
 495
 496         ##########################################################################
 497         # log statusall and listall output
 498         # get copies of ipsec.conf, ipsec.secrets
 499         # create index.html for the given test case
 500
 501         cat > $TESTRESULTDIR/index.html <<@EOF
 502 <html>
 503 <head>
 504   <title>Test $testname</title>
 505 </head>
 506 <body>
 507 <table border="0" cellpadding="0" cellspacing="0" width="600">
 508   <tr><td>
 509     <div><a href="../../index.html">strongSwan KVM Tests</a> / <a href="../index.html">$SUBDIR</a> / $name</div>
 510     <h2>Test $testname</h2>
 511     <h3>Description</h3>
 512 @EOF
 513
 514         cat $TESTDIR/description.txt >> $TESTRESULTDIR/index.html
 515
 516         cat >> $TESTRESULTDIR/index.html <<@EOF
 517     <ul>
 518       <li><a href="console.log">console.log</a></li>
 519     </ul>
 520     <img src="../../images/$DIAGRAM" alt="$VIRTHOSTS">
 521 @EOF
 522
 523         IPTABLES_CMD_V4="echo -e '=== filter table ==='; iptables  -v -n -L; echo -e '\n=== nat table ==='; iptables  -v -n -t nat -L; echo -e '\n=== mangle table ==='; iptables  -v -n -t mangle -L"
 524         IPTABLES_CMD_V6="echo -e '=== filter table ==='; ip6tables -v -n -L; echo -e '\n=== nat table ==='; ip6tables -v -n -t nat -L; echo -e '\n=== mangle table ==='; ip6tables -v -n -t mangle -L"
 525
 526         if [ -n "$IPV6" ]
 527         then
 528             IPROUTE_CMD="ip -6 route list table $SOURCEIP_ROUTING_TABLE"
 529             IPROUTE_DSP=$IPROUTE_CMD
 530             IPTABLES_CMD="$IPTABLES_CMD_V6"
 531             IPTABLES_DSP="ip6tables -L"
 532             IPTABLES_SAVE_CMD="ip6tables-save"
 533             IPTABLES_SAVE_DSP="ip6tables-save"
 534         else
 535             IPROUTE_CMD="ip route list table $SOURCEIP_ROUTING_TABLE"
 536             IPROUTE_DSP=$IPROUTE_CMD
 537             IPTABLES_CMD="$IPTABLES_CMD_V4"
 538             IPTABLES_DSP="iptables -L"
 539             IPTABLES_SAVE_CMD="iptables-save"
 540             IPTABLES_SAVE_DSP="iptables-save"
 541         fi
 542
 543         if [ $name = "net2net-ip4-in-ip6-ikev2" -o $name = "net2net-ip6-in-ip4-ikev2" ]
 544         then
 545             IPROUTE_CMD="ip route list table $SOURCEIP_ROUTING_TABLE; echo; ip -6 route list table $SOURCEIP_ROUTING_TABLE"
 546             IPROUTE_DSP="ip (-6) route list table $SOURCEIP_ROUTING_TABLE"
 547             IPTABLES_CMD="$IPTABLES_CMD_V4; echo; $IPTABLES_CMD_V6"
 548             IPTABLES_DSP="iptables -L ; ip6tables -L"
 549             IPTABLES_SAVE_CMD="iptables-save; echo; ip6tables-save"
 550             IPTABLES_SAVE_DSP="iptables-save ; ip6tables-save"
 551         fi
 552
 553         for host in $DBHOSTS
 554         do
 555             eval HOSTLOGIN=root@\$ipv4_${host}
 556
 557                 scp $SSHCONF $HOSTLOGIN:/etc/db.d/ipsec.sql \
 558                     $TESTRESULTDIR/${host}.ipsec.sql  > /dev/null 2>&1
 559         done
 560
 561         for host in $IPSECHOSTS
 562         do
 563             eval HOSTLOGIN=root@\$ipv4_${host}
 564
 565                 scp $SSHCONF $HOSTLOGIN:/etc/strongswan.conf \
 566                     $TESTRESULTDIR/${host}.strongswan.conf  > /dev/null 2>&1
 567                 if [  -n "$SWANCTL" ]
 568                 then
 569                         scp $SSHCONF $HOSTLOGIN:/etc/swanctl/swanctl.conf \
 570                         $TESTRESULTDIR/${host}.swanctl.conf  > /dev/null 2>&1
 571
 572                         for subsys in conns algs certs pools authorities sas pols
 573                         do
 574                                 ssh $SSHCONF $HOSTLOGIN swanctl --list-$subsys \
 575                                         > $TESTRESULTDIR/${host}.swanctl.$subsys 2>/dev/null
 576                         done
 577
 578                         ssh $SSHCONF $HOSTLOGIN swanctl --stats \
 579                                 > $TESTRESULTDIR/${host}.swanctl.stats 2>/dev/null
 580
 581                         echo "" >> $TESTRESULTDIR/${host}.swanctl.sas
 582                         cat $TESTRESULTDIR/${host}.swanctl.pols >> \
 583                                 $TESTRESULTDIR/${host}.swanctl.sas
 584                         cat $TESTRESULTDIR/${host}.swanctl.algs >> \
 585                                 $TESTRESULTDIR/${host}.swanctl.stats
 586                 else
 587                     for file in ipsec.conf ipsec.secrets
 588                     do
 589                                 scp $SSHCONF $HOSTLOGIN:/etc/$file \
 590                             $TESTRESULTDIR/${host}.$file  > /dev/null 2>&1
 591                     done
 592
 593                     for command in statusall listall
 594                     do
 595                                 ssh $SSHCONF $HOSTLOGIN ipsec $command \
 596                             > $TESTRESULTDIR/${host}.$command 2>/dev/null
 597                     done
 598                 fi
 599
 600                 if (! [ -f $TESTRESULTDIR/${host}.ipsec.sql ] ) then
 601                         scp $SSHCONF $HOSTLOGIN:/etc/ipsec.d/ipsec.sql \
 602                                 $TESTRESULTDIR/${host}.ipsec.sql  > /dev/null 2>&1
 603                 fi
 604
 605             ssh $SSHCONF $HOSTLOGIN ip -s xfrm policy \
 606                     > $TESTRESULTDIR/${host}.ip.policy 2>/dev/null
 607             ssh $SSHCONF $HOSTLOGIN ip -s xfrm state \
 608                     > $TESTRESULTDIR/${host}.ip.state 2>/dev/null
 609             ssh $SSHCONF $HOSTLOGIN $IPROUTE_CMD \
 610                     > $TESTRESULTDIR/${host}.ip.route 2>/dev/null
 611             ssh $SSHCONF $HOSTLOGIN $IPTABLES_CMD \
 612                     > $TESTRESULTDIR/${host}.iptables 2>/dev/null
 613                 ssh $SSHCONF $HOSTLOGIN $IPTABLES_SAVE_CMD \
 614                     > $TESTRESULTDIR/${host}.iptables-save 2>/dev/null
 615             chmod a+r $TESTRESULTDIR/*
 616
 617                 if [ -n "$SWANCTL" ]
 618                 then
 619                     cat >> $TESTRESULTDIR/index.html <<@EOF
 620     <h3>$host</h3>
 621       <table border="0" cellspacing="0" width="600">
 622       <tr>
 623         <td valign="top">
 624           <ul>
 625             <li><a href="$host.swanctl.conf">swanctl.conf</a></li>
 626             <li><a href="$host.swanctl.conns">swanctl --list-conns</a></li>
 627             <li><a href="$host.swanctl.certs">swanctl --list-certs</a></li>
 628             <li><a href="$host.strongswan.conf">strongswan.conf</a></li>
 629             <li><a href="$host.ipsec.sql">ipsec.sql</a></li>
 630           </ul>
 631         </td>
 632         <td valign="top">
 633           <ul>
 634             <li><a href="$host.swanctl.sas">swanctl --list-sas|--list-pols</a></li>
 635             <li><a href="$host.swanctl.pools">swanctl --list-pools</a></li>
 636             <li><a href="$host.swanctl.authorities">swanctl --list-authorities</a></li>
 637             <li><a href="$host.swanctl.stats">swanctl --stats|--list-algs</a></li>
 638             <li><a href="$host.auth.log">auth.log</a></li>
 639             <li><a href="$host.daemon.log">daemon.log</a></li>
 640           </ul>
 641       </td>
 642         <td valign="top">
 643           <ul>
 644             <li><a href="$host.ip.policy">ip -s xfrm policy</a></li>
 645             <li><a href="$host.ip.state">ip -s xfrm state</a></li>
 646             <li><a href="$host.ip.route">$IPROUTE_DSP</a></li>
 647             <li><a href="$host.iptables">$IPTABLES_DSP</a></li>
 648             <li><a href="$host.iptables-save">$IPTABLES_SAVE_DSP</a></li>
 649           </ul>
 650           &nbsp;
 651       </td>
 652     </tr>
 653     </table>
 654 @EOF
 655
 656                 else
 657                     cat >> $TESTRESULTDIR/index.html <<@EOF
 658     <h3>$host</h3>
 659       <table border="0" cellspacing="0" width="600">
 660       <tr>
 661         <td valign="top">
 662           <ul>
 663             <li><a href="$host.ipsec.conf">ipsec.conf</a></li>
 664             <li><a href="$host.ipsec.secrets">ipsec.secrets</a></li>
 665             <li><a href="$host.ipsec.sql">ipsec.sql</a></li>
 666             <li><a href="$host.strongswan.conf">strongswan.conf</a></li>
 667           </ul>
 668         </td>
 669         <td valign="top">
 670           <ul>
 671             <li><a href="$host.statusall">ipsec statusall</a></li>
 672             <li><a href="$host.listall">ipsec listall</a></li>
 673             <li><a href="$host.auth.log">auth.log</a></li>
 674             <li><a href="$host.daemon.log">daemon.log</a></li>
 675           </ul>
 676       </td>
 677         <td valign="top">
 678           <ul>
 679             <li><a href="$host.ip.policy">ip -s xfrm policy</a></li>
 680             <li><a href="$host.ip.state">ip -s xfrm state</a></li>
 681             <li><a href="$host.ip.route">$IPROUTE_DSP</a></li>
 682             <li><a href="$host.iptables">$IPTABLES_DSP</a></li>
 683             <li><a href="$host.iptables-save">$IPTABLES_SAVE_DSP</a></li>
 684           </ul>
 685       </td>
 686     </tr>
 687     </table>
 688 @EOF
 689                 fi
 690         done
 691
 692         for host in $RADIUSHOSTS
 693         do
 694             eval HOSTLOGIN=root@\$ipv4_${host}
 695
 696                 RADIUS_DIR=/etc/freeradius/3.0
 697                 RADIUS_EAP_FILE=mods-enabled/eap
 698                 RADIUS_EAP_NAME=eap
 699                 if [ "$BASEIMGSUITE" == "jessie" ]
 700                 then
 701                         RADIUS_DIR=/etc/freeradius
 702                         RADIUS_EAP_FILE=eap.conf
 703                         RADIUS_EAP_NAME=eap.conf
 704                 fi
 705
 706                 for file in clients.conf radiusd.conf proxy.conf users sites-enabled/default sites-enabled/inner-tunnel $RADIUS_EAP_FILE
 707             do
 708                 scp $SSHCONF $HOSTLOGIN:$RADIUS_DIR/$file \
 709                     $TESTRESULTDIR/${host}.$(basename $file) > /dev/null 2>&1
 710             done
 711
 712             scp $SSHCONF $HOSTLOGIN:/var/log/freeradius/radius.log \
 713                 $TESTRESULTDIR/${host}.radius.log  > /dev/null 2>&1
 714
 715             chmod a+r $TESTRESULTDIR/*
 716             cat >> $TESTRESULTDIR/index.html <<@EOF
 717     <h3>$host</h3>
 718       <table border="0" cellspacing="0" width="600">
 719       <tr>
 720         <td valign="top">
 721           <ul>
 722             <li><a href="$host.clients.conf">clients.conf</a></li>
 723             <li><a href="$host.radiusd.conf">radiusd.conf</a></li>
 724             <li><a href="$host.$RADIUS_EAP_NAME">$RADIUS_EAP_NAME</a></li>
 725           </ul>
 726         </td>
 727         <td valign="top">
 728           <ul>
 729             <li><a href="$host.default">sites-enabled/default</a></li>
 730             <li><a href="$host.inner-tunnel">sites-enabled/inner-tunnel</a></li>
 731             <li><a href="$host.radius.log">radius.log</a></li>
 732           </ul>
 733       </td>
 734         <td valign="top">
 735           <ul>
 736             <li><a href="$host.proxy.conf">proxy.conf</a></li>
 737             <li><a href="$host.users">users</a></li>
 738           </ul>
 739       </td>
 740     </tr>
 741     </table>
 742 @EOF
 743
 744         done
 745
 746         cat >> $TESTRESULTDIR/index.html <<@EOF
 747         <h3>tcpdump</h3>
 748         <ul>
 749 @EOF
 750
 751         for host in $TCPDUMPHOSTS
 752         do
 753             cat >> $TESTRESULTDIR/index.html <<@EOF
 754             <li><a href="$host.tcpdump.log">$host tcpdump.log</a></li>
 755 @EOF
 756         done
 757
 758         cat >> $TESTRESULTDIR/index.html <<@EOF
 759         </ul>
 760 @EOF
 761
 762         cat >> $TESTRESULTDIR/index.html <<@EOF
 763   </td></tr>
 764 </table>
 765 </body>
 766 </html>
 767 @EOF
 768
 769
 770         ##########################################################################
 771         # execute post-test commands
 772         #
 773
 774         echo -n "post"
 775         echo -e "\nPOST-TEST\n" >> $CONSOLE_LOG 2>&1
 776
 777         eval `awk -F "::" '{
 778             if ($1 !~ /^#.*/ && $2 != "")
 779             {
 780                 printf("echo \"$(print_time)%s# %s\"; ", $1, $2)
 781                 printf("ssh \044SSHCONF root@\044ipv4_%s \"%s\"; ", $1, $2)
 782                 printf("echo;\n")
 783             }
 784         }' $TESTDIR/posttest.dat` >> $CONSOLE_LOG 2>&1
 785
 786         ##########################################################################
 787         # check that IPsec state was cleaned up properly
 788         #
 789
 790         for host in $IPSECHOSTS
 791         do
 792                 eval HOSTLOGIN=root@\$ipv4_${host}
 793                 IPSECSTATE=`ssh $SSHCONF $HOSTLOGIN 'ip xfrm state'`
 794                 # ignore IPv4/v6 states created with IPComp SAs
 795                 IPSECSTATEISSUE=`echo "$IPSECSTATE" | grep 'proto.*spi' | grep -v 'proto 4'`
 796                 IPSECPOLICY=`ssh $SSHCONF $HOSTLOGIN 'ip xfrm policy'`
 797                 if [ -n "$IPSECSTATEISSUE" -o -n "$IPSECPOLICY" ]
 798                 then
 799                         echo -e "\n$host# ip xfrm state [NO]" >> $CONSOLE_LOG
 800                         echo "$IPSECSTATE" >> $CONSOLE_LOG
 801                         echo -e "\n$host# ip xfrm policy [NO]" >> $CONSOLE_LOG
 802                         echo "$IPSECPOLICY" >> $CONSOLE_LOG
 803                         STATUS="failed"
 804                 fi
 805         done
 806
 807
 808         ##########################################################################
 809         # make sure there were no leaks
 810         #
 811
 812         for host in $STRONGSWANHOSTS
 813         do
 814                 eval HOSTLOGIN=root@\$ipv4_${host}
 815                 LEAKS=`ssh $SSHCONF $HOSTLOGIN 'cat $LEAK_DETECTIVE_LOG 2>/dev/null | grep -v "No leaks detected.*"'`
 816                 if [ -n "$LEAKS" ]
 817                 then
 818                         echo -e "\n$host# cat $LEAK_DETECTIVE_LOG [NO]" >> $CONSOLE_LOG
 819                         echo "$LEAKS" >> $CONSOLE_LOG
 820                         echo "<<< $host $LEAK_DETECTIVE_LOG >>>" >> $CONSOLE_LOG
 821                         STATUS="failed"
 822                 fi
 823         done
 824
 825
 826         ##########################################################################
 827         # get a copy of /var/log/auth.log
 828         #
 829
 830         for host in $IPSECHOSTS
 831         do
 832             eval HOSTLOGIN=root@\$ipv4_${host}
 833             ssh $SSHCONF $HOSTLOGIN "grep -s -E 'charon|last message repeated|imcv|pt-tls-client' \
 834                 /var/log/auth.log" >> $TESTRESULTDIR/${host}.auth.log
 835         done
 836
 837
 838         ##########################################################################
 839         # get a copy of /var/log/daemon.log
 840         #
 841
 842         for host in $IPSECHOSTS
 843         do
 844             eval HOSTLOGIN=root@\$ipv4_${host}
 845             ssh $SSHCONF $HOSTLOGIN "grep -s -E 'systemd|swanctl|charon|last message repeated|imcv' \
 846                 /var/log/daemon.log" >> $TESTRESULTDIR/${host}.daemon.log
 847         done
 848
 849
 850         ##########################################################################
 851         # stop tcpdump if necessary
 852         #
 853
 854         for host in $TCPDUMPHOSTS
 855         do
 856             if [ "`eval echo \\\$TDUP_${host}`" = "true" ]
 857             then
 858                         stop_tcpdump $host
 859             fi
 860             eval HOSTLOGIN=root@\$ipv4_${host}
 861                 scp $SSHCONF $HOSTLOGIN:/tmp/tcpdump.log \
 862                         $TESTRESULTDIR/${host}.tcpdump.log > /dev/null 2>&1
 863         done
 864
 865         ##########################################################################
 866         # remove database directory if needed
 867         #
 868
 869         for host in $DBHOSTS
 870         do
 871                 eval HOSTLOGIN=root@\$ipv4_${host}
 872             ssh $SSHCONF $HOSTLOGIN "umount $DBDIR; rm -r $DBDIR" > /dev/null 2>&1
 873         done
 874
 875         ##########################################################################
 876         # copy default host config back if necessary
 877         #
 878
 879         $DIR/scripts/restore-defaults $testname
 880
 881
 882         ##########################################################################
 883         # set counters
 884         #
 885
 886         if [ $STATUS = "failed" ]
 887         then
 888             let "failed_cnt += 1"
 889         else
 890             let "passed_cnt += 1"
 891         fi
 892
 893
 894         ##########################################################################
 895         # write test status to html file
 896         #
 897         testend=$(date +%s)
 898         let "testend -= teststart"
 899         let "timetotal += testend"
 900
 901         if [ $STATUS = "passed" ]
 902         then
 903                 COLOR="green"
 904                 log_status 0
 905         else
 906                 COLOR="red"
 907                 log_status 1
 908         fi
 909
 910         cat >> $TESTRESULTSHTML << @EOF
 911   <tr>
 912     <td>$testnumber</td>
 913     <td><a href="$testname/index.html">$testname</a></td>
 914     <td align="right">$testend</td>
 915     <td><a href="$testname/console.log"><font color="$COLOR">$STATUS</font></a></td>
 916   </tr>
 917 @EOF
 918         cat >> $SUBTESTSINDEX << @EOF
 919   <tr>
 920     <td>$testnumber</td>
 921     <td><a href="$name/index.html">$name</a></td>
 922     <td><a href="$name/console.log"><font color="$COLOR">$STATUS</font></a></td>
 923     <td>&nbsp;</td>
 924   </tr>
 925 @EOF
 926
 927
 928         ##########################################################################
 929         # remove any charon.pid files that still may exist
 930         #
 931
 932         for host in $IPSECHOSTS
 933         do
 934             eval HOSTLOGIN=root@\$ipv4_${host}
 935             ssh $SSHCONF $HOSTLOGIN 'if [ -f /var/run/charon.pid ]; then rm /var/run/charon.pid; echo "    removed charon.pid on `hostname`"; fi'
 936         done
 937
 938         if [ -n "$aborted" ]
 939         then
 940             break 2
 941         fi
 942
 943     done
 944
 945 done
 946
 947
 948 ##############################################################################
 949 # finish the results html file
 950 #
 951
 952 cat >> $TESTRESULTSHTML << @EOF
 953     <tr>
 954       <td>&nbsp;</td><td>&nbsp;</td><td>&nbsp;</td><td>&nbsp;</td>
 955     </tr>
 956     <tr>
 957       <td><b>Passed</b></td><td><b><font color="green">$passed_cnt</font></b></td><td>&nbsp;</td><td>&nbsp;</td>
 958     </tr>
 959     <tr>
 960       <td><b>Failed</b></td><td><b><font color="red">$failed_cnt</font></b></td><td>&nbsp;</td><td>&nbsp;</td>
 961     </tr>
 962     <tr>
 963       <td><b>Time [s]</b></td><td><b><font color="blue">$timetotal</font></b></td><td>&nbsp;</td><td>&nbsp;</td>
 964     </tr>
 965   </table>
 966 </body>
 967 </html>
 968 @EOF
 969
 970 if [ $subdir_cnt != 0 ]
 971 then
 972 cat >> $INDEX << @EOF
 973       <td align="right">$subdir_cnt</td>
 974       <td>&nbsp;</td>
 975     </tr>
 976 @EOF
 977 fi
 978
 979 let "all_cnt = $passed_cnt + $failed_cnt"
 980
 981 cat >> $INDEX << @EOF
 982     <tr>
 983       <td>&nbsp;</td>
 984       <td><a href="all.html"><b>all</b></a></td>
 985       <td align="right"><b>$all_cnt</b></td>
 986       <td>&nbsp;</td>
 987     </tr>
 988     <tr>
 989       <td><b>Failed</b></td>
 990       <td>&nbsp;</td>
 991       <td align="right"><b><font color="red">$failed_cnt</font></b></td>
 992       <td>&nbsp;</td>
 993     </tr>
 994   </table>
 995 </body>
 996 </html>
 997 @EOF
 998
 999 echo
1000 echo_ok     "Passed : $passed_cnt"
1001 echo_failed "Failed : $failed_cnt"
1002
1003 echo
1004 echo "The results are available in $TODAYDIR"
1005 echo "or via the link http://$ipv4_winnetou/testresults/$TESTDATE"
1006
1007 ENDDATE=`date +%Y%m%d-%H%M-%S`
1008 echo
1009 echo "Finished : $ENDDATE"