1#!/bin/sh 2# 3# Copyright (c) 2006 - 2007 Kungliga Tekniska Högskolan 4# (Royal Institute of Technology, Stockholm, Sweden). 5# All rights reserved. 6# 7# Redistribution and use in source and binary forms, with or without 8# modification, are permitted provided that the following conditions 9# are met: 10# 11# 1. Redistributions of source code must retain the above copyright 12# notice, this list of conditions and the following disclaimer. 13# 14# 2. Redistributions in binary form must reproduce the above copyright 15# notice, this list of conditions and the following disclaimer in the 16# documentation and/or other materials provided with the distribution. 17# 18# 3. Neither the name of the Institute nor the names of its contributors 19# may be used to endorse or promote products derived from this software 20# without specific prior written permission. 21# 22# THIS SOFTWARE IS PROVIDED BY THE INSTITUTE AND CONTRIBUTORS ``AS IS'' AND 23# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25# ARE DISCLAIMED. IN NO EVENT SHALL THE INSTITUTE OR CONTRIBUTORS BE LIABLE 26# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32# SUCH DAMAGE. 33 34top_builddir="@top_builddir@" 35env_setup="@env_setup@" 36objdir="@objdir@" 37 38db_type=@db_type@ 39 40. ${env_setup} 41 42# If there is no useful db support compiled in, disable test 43${have_db} || exit 77 44 45 46# Don't run this test in AFS, since it lacks support for AF_UNIX 47expr "X`/bin/pwd || pwd`" : "X/afs/.*" > /dev/null 2>/dev/null && exit 77 48 49R=TEST.H5L.SE 50 51port=@port@ 52 53cache="FILE:${objdir}/cache.krb5" 54keytabfile=${objdir}/iprop.keytab 55keytab="FILE:${keytabfile}" 56 57kdc="${kdc} --addresses=localhost -P $port" 58kadmin="${kadmin} -r $R" 59kinit="${kinit} -c $cache ${afs_no_afslog}" 60 61slave_ver_from_master_old= 62slave_ver_from_master_new= 63slave_ver_old= 64slave_ver_new= 65get_iprop_ver () { 66 min_change=${1:-1} 67 slave_ver_from_master_new=`grep '^iprop/' iprop-stats | head -1 | awk '{print $3}'` 68 slave_ver_new=`grep 'up-to-date with version:' iprop-slave-status | awk '{print $4}'` 69 if [ -z "$slave_ver_from_master_new" -o -z "$slave_ver_new" ]; then 70 return 1 71 fi 72 if [ x"$slave_ver_from_master_new" != x"$slave_ver_new" ]; then 73 return 1 74 fi 75 if [ x"$slave_ver_from_master_old" != x ]; then 76 change=`expr "$slave_ver_from_master_new" - "$slave_ver_from_master_old"` 77 if [ "$change" -lt "$min_change" ]; then 78 return 1 79 fi 80 fi 81 slave_ver_from_master_old=$slave_ver_from_master_new 82 slave_ver_old=$slave_ver_new 83 return 0 84} 85 86waitsec=65 87sleeptime=2 88wait_for () { 89 msg=$1 90 shift 91 t=0 92 while ! "$@"; do 93 sleep $sleeptime; 94 t=`expr $t + $sleeptime` 95 if [ $t -gt $waitsec ]; then 96 echo "Waited too long for $msg" 97 exit 1 98 fi 99 done 100 return 0 101} 102 103check_pidfile_is_dead () { 104 if test ! -f lt-${1}.pid -a ! -f ${1}.pid; then 105 return 0 106 fi 107 _pid=`cat lt-${1}.pid ${1}.pid 2>/dev/null` 108 if [ -z "$_pid" ]; then 109 return 0 110 fi 111 if kill -0 $_pid 2>/dev/null; then 112 return 1 113 fi 114 return 0 115} 116 117wait_for_slave () { 118 wait_for "iprop versions to change and/or slave to catch up" get_iprop_ver "$@" 119} 120 121wait_for_master_down () { 122 wait_for "master to exit" check_pidfile_is_dead ipropd-master 123} 124 125wait_for_slave_down () { 126 wait_for "slave to exit" check_pidfile_is_dead ipropd-slave 127} 128 129KRB5_CONFIG="${objdir}/krb5.conf" 130export KRB5_CONFIG 131 132rm -f ${keytabfile} 133rm -f current-db* 134rm -f current*.log 135rm -f out-* 136rm -f mkey.file* 137rm -f messages.log 138 139> messages.log 140 141echo Creating database 142${kadmin} -l \ 143 init \ 144 --realm-max-ticket-life=1day \ 145 --realm-max-renewable-life=1month \ 146 ${R} || exit 1 147 148${kadmin} -l add -p foo --use-defaults user@${R} || exit 1 149 150${kadmin} -l add --random-key --use-defaults iprop/localhost@${R} || exit 1 151${kadmin} -l ext -k ${keytab} iprop/localhost@${R} || exit 1 152${kadmin} -l add --random-key --use-defaults iprop/slave.test.h5l.se@${R} || exit 1 153${kadmin} -l ext -k ${keytab} iprop/slave.test.h5l.se@${R} || exit 1 154 155echo foo > ${objdir}/foopassword 156 157echo "Test log recovery" 158${kadmin} -l add --random-key --use-defaults recovtest@${R} || exit 1 159# Test theory: save the log, make a change and save the record it 160# produced, restore the log, append to it the saved record, then add dummy 161# record. 162 163# Save the log 164cp current.log current.log.tmp 165ls -l current.log.tmp | awk '{print $5}' > tmp 166read sz < tmp 167# Make a change 168${kadmin} -l mod -a requires-pre-auth recovtest@${R} || exit 1 169${kadmin} -l get recovtest@${R} | grep 'Attributes: requires-pre-auth$' > /dev/null || exit 1 170# Save the resulting log record 171ls -l current.log | awk '{print $5}' > tmp 172read nsz < tmp 173rm tmp 174dd bs=1 if=current.log skip=$sz of=current.log.tmp.saved-record count=`expr $nsz - $sz` 2>/dev/null 175# Undo the change 176${kadmin} -l mod -a -requires-pre-auth recovtest@${R} || exit 1 177${kadmin} -l get recovtest@${R} | grep 'Attributes:.$' > /dev/null || exit 1 178# Restore the log 179cp current.log current.log.save 180mv current.log.tmp current.log 181# Append the saved record 182cat current.log.tmp.saved-record >> current.log 183rm current.log.tmp.saved-record 184# Check that we still see the principal as modified after another write forcing 185# log recovery. 186${kadmin} -l add --random-key --use-defaults dummy@${R} || exit 1 187${kadmin} -l del dummy@${R} || exit 1 188${kadmin} -l get recovtest@${R} | grep 'Attributes: requires-pre-auth$' > /dev/null || exit 1 189 190# -- foo 191ipds= 192ipdm= 193kdcpid= 194 195> iprop-stats 196rm -f iprop-slave-status 197 198ipropd_slave="${ipropd_slave} --status-file=iprop-slave-status" 199 200trap "echo 'killing ipropd s + m + kdc'; kill -9 \${ipdm} \${ipds} \${kdcpid} >/dev/null 2>/dev/null; tail messages.log ; tail iprop-stats; exit 1;" EXIT 201 202echo Starting kdc ; > messages.log 203${kdc} --detach --testing || { echo "kdc failed to start"; exit 1; } 204kdcpid=`getpid kdc` 205 206echo "starting master" ; > messages.log 207env ${HEIM_MALLOC_DEBUG} \ 208${ipropd_master} --hostname=localhost -k ${keytab} \ 209 --database=${objdir}/current-db --detach || 210 { echo "ipropd-master failed to start"; exit 1; } 211ipdm=`getpid ipropd-master` 212 213echo "starting slave" ; > messages.log 214env ${HEIM_MALLOC_DEBUG} \ 215KRB5_CONFIG="${objdir}/krb5-slave.conf" \ 216${ipropd_slave} --hostname=slave.test.h5l.se -k ${keytab} --detach localhost || 217 { echo "ipropd-slave failed to start"; exit 1; } 218ipds=`getpid ipropd-slave` 219sh ${wait_kdc} ipropd-slave messages.log 'slave status change: up-to-date' || exit 1 220get_iprop_ver || exit 1 221 222echo "checking slave is up" 223${EGREP} 'iprop/slave.test.h5l.se@TEST.H5L.SE.*Up' iprop-stats >/dev/null || exit 1 224${EGREP} 'up-to-date with version' iprop-slave-status >/dev/null || { echo "slave to up to date" ; cat iprop-slave-status ; exit 1; } 225 226# ----------------- checking: pushing lives changes 227 228slave_get() { KRB5_CONFIG="${objdir}/krb5-slave.conf" ${kadmin} -l get "$@"; } 229slave_check_exists() { 230 # Creation with a random key is not atomic, there are at present 231 # 3 log entries to create a random key principal, the entry is 232 # "invalid" for the first two of these. We wait for the entry to 233 # exist and not be invalid 234 # 235 attrs=`slave_get -o attributes "$@" 2>/dev/null` || return 1 236 echo $attrs | egrep 'Attributes:' | egrep -v invalid >/dev/null || return 1 237 get_iprop_ver 0 238} 239 240echo "Add host" 241${kadmin} -l add --random-key --use-defaults host/foo@${R} || exit 1 242wait_for "Slave sees new host" slave_check_exists "host/foo@${R}" 243 244echo "Rollover host keys" 245${kadmin} -l cpw -r --keepold host/foo@${R} || exit 1 246${kadmin} -l cpw -r --keepold host/foo@${R} || exit 1 247${kadmin} -l cpw -r --keepold host/foo@${R} || exit 1 248wait_for_slave 3 249slave_get host/foo@${R} | \ 250 ${EGREP} Keytypes: | cut -d: -f2 | tr ' ' ' 251' | sed 's/^.*[[]\(.*\)[]].*$/\1/' | grep '[0-9]' | sort -nu | tr -d ' 252' | ${EGREP} 1234 > /dev/null || exit 1 253 254echo "Delete 3DES keys" 255${kadmin} -l del_enctype host/foo@${R} des3-cbc-sha1 256wait_for_slave 257KRB5_CONFIG="${objdir}/krb5-slave.conf" \ 258${kadmin} -l get host/foo@${R} | \ 259 ${EGREP} Keytypes: | cut -d: -f2 | tr ' ' ' 260' | sed 's/^.*[[]\(.*\)[]].*$/\1/' | grep '[0-9]' | sort -nu | tr -d ' 261' | ${EGREP} 1234 > /dev/null || exit 1 262KRB5_CONFIG="${objdir}/krb5-slave.conf" \ 263${kadmin} -l get host/foo@${R} | \ 264 ${EGREP} 'Keytypes:.*des3-cbc-sha1' > /dev/null && exit 1 265 266echo "Change policy host" 267${kadmin} -l modify --policy=default host/foo@${R} || exit 1 268wait_for_slave 269KRB5_CONFIG="${objdir}/krb5-slave.conf" \ 270${kadmin} -l get host/foo@${R} > /dev/null 2>/dev/null || exit 1 271 272echo "Rename host" 273${kadmin} -l rename host/foo@${R} host/bar@${R} || exit 1 274wait_for_slave 275KRB5_CONFIG="${objdir}/krb5-slave.conf" \ 276${kadmin} -l get host/foo@${R} > /dev/null 2>/dev/null && exit 1 277KRB5_CONFIG="${objdir}/krb5-slave.conf" \ 278${kadmin} -l get host/bar@${R} > /dev/null || exit 1 279 280echo "Delete host" 281${kadmin} -l delete host/bar@${R} || exit 1 282wait_for_slave 283KRB5_CONFIG="${objdir}/krb5-slave.conf" \ 284${kadmin} -l get host/bar@${R} > /dev/null 2>/dev/null && exit 1 285 286# See note below in LMDB sanity checking 287echo "Re-add host" 288${kadmin} -l add --random-key --use-defaults host/foo@${R} || exit 1 289${kadmin} -l add --random-key --use-defaults host/bar@${R} || exit 1 290wait_for "Slave sees re-added host" slave_check_exists "host/bar@${R}" 291 292echo "kill slave and remove log and database" 293> iprop-stats 294sh ${leaks_kill} ipropd-slave $ipds || exit 1 295rm -f iprop-slave-status 296 297wait_for_slave_down 298${EGREP} 'iprop/slave.test.h5l.se@TEST.H5L.SE.*Down' iprop-stats >/dev/null || exit 1 299 300# ----------------- checking: slave is missing changes while down 301 302rm current.slave.log current-db.slave* || exit 1 303 304echo "doing changes while slave is down" 305${kadmin} -l cpw --random-password user@${R} > /dev/null || exit 1 306${kadmin} -l cpw --random-password user@${R} > /dev/null || exit 1 307 308echo "Making a copy of the master log file" 309cp ${objdir}/current.log ${objdir}/current.log.tmp 310 311# ----------------- checking: checking that master and slaves resyncs 312 313echo "starting slave again" ; > messages.log 314> iprop-stats 315env ${HEIM_MALLOC_DEBUG} \ 316KRB5_CONFIG="${objdir}/krb5-slave.conf" \ 317${ipropd_slave} --hostname=slave.test.h5l.se -k ${keytab} --detach localhost || 318 { echo "ipropd-slave failed to start"; exit 1; } 319ipds=`getpid ipropd-slave` 320 321echo "checking slave is up again" 322wait_for "slave to start and connect to master" \ 323 ${EGREP} 'iprop/slave.test.h5l.se@TEST.H5L.SE.*Up' iprop-stats >/dev/null 324wait_for_slave 2 325${EGREP} 'up-to-date with version' iprop-slave-status >/dev/null || { echo "slave not up to date" ; cat iprop-slave-status ; exit 1; } 326echo "checking for replay problems" 327${EGREP} 'Entry already exists in database' messages.log && exit 1 328 329echo "compare versions on master and slave logs (no lock)" 330KRB5_CONFIG=${objdir}/krb5-slave.conf \ 331${iprop_log} last-version -n > slave-last.tmp 332${iprop_log} last-version -n > master-last.tmp 333cmp master-last.tmp slave-last.tmp || exit 1 334 335echo "kill slave and remove log and database" 336sh ${leaks_kill} ipropd-slave $ipds || exit 1 337wait_for_slave_down 338 339rm current.slave.log current-db.slave* || exit 1 340> iprop-stats 341rm -f iprop-slave-status 342echo "starting slave" ; > messages.log 343env ${HEIM_MALLOC_DEBUG} \ 344KRB5_CONFIG="${objdir}/krb5-slave.conf" \ 345${ipropd_slave} --hostname=slave.test.h5l.se -k ${keytab} --detach localhost || 346 { echo "ipropd-slave failed to start"; exit 1; } 347ipds=`getpid ipropd-slave` 348wait_for_slave 0 349 350echo "checking slave is up again" 351wait_for "slave to start and connect to master" \ 352 ${EGREP} 'iprop/slave.test.h5l.se@TEST.H5L.SE.*Up' iprop-stats >/dev/null 353${EGREP} 'up-to-date with version' iprop-slave-status >/dev/null || { echo "slave not up to date" ; cat iprop-slave-status ; exit 1; } 354echo "checking for replay problems" 355${EGREP} 'Entry already exists in database' messages.log && exit 1 356 357# ----------------- checking: checking live truncation of master log 358 359${kadmin} -l cpw --random-password user@${R} > /dev/null || exit 1 360wait_for_slave 361 362echo "live truncate on master log" 363${iprop_log} truncate -K 5 || exit 1 364wait_for_slave 0 365 366echo "Killing master and slave" 367sh ${leaks_kill} ipropd-master $ipdm || exit 1 368sh ${leaks_kill} ipropd-slave $ipds || exit 1 369 370rm -f iprop-slave-status 371 372wait_for_slave_down 373wait_for_master_down 374 375echo "compare versions on master and slave logs" 376KRB5_CONFIG=${objdir}/krb5-slave.conf \ 377${iprop_log} last-version > slave-last.tmp 378${iprop_log} last-version > master-last.tmp 379cmp master-last.tmp slave-last.tmp || exit 1 380 381# ----------------- checking: master going backward 382> iprop-stats 383> messages.log 384 385echo "Going back to old version of the master log file" 386cp ${objdir}/current.log.tmp ${objdir}/current.log 387 388echo "starting master" ; > messages.log 389env ${HEIM_MALLOC_DEBUG} \ 390${ipropd_master} --hostname=localhost -k ${keytab} \ 391 --database=${objdir}/current-db --detach || 392 { echo "ipropd-master failed to start"; exit 1; } 393ipdm=`getpid ipropd-master` 394 395echo "starting slave" ; > messages.log 396env ${HEIM_MALLOC_DEBUG} \ 397KRB5_CONFIG="${objdir}/krb5-slave.conf" \ 398${ipropd_slave} --hostname=slave.test.h5l.se -k ${keytab} --detach localhost || 399 { echo "ipropd-slave failed to start"; exit 1; } 400ipds=`getpid ipropd-slave` 401wait_for_slave -1 402 403echo "checking slave is up again" 404wait_for "slave to start and connect to master" \ 405 ${EGREP} 'iprop/slave.test.h5l.se@TEST.H5L.SE.*Up' iprop-stats >/dev/null 406${EGREP} 'up-to-date with version' iprop-slave-status >/dev/null || { echo "slave to up to date" ; cat iprop-slave-status ; exit 1; } 407echo "checking for replay problems" 408${EGREP} 'Entry already exists in database' messages.log && exit 1 409 410echo "pushing one change" 411${kadmin} -l cpw --random-password user@${R} > /dev/null || exit 1 412wait_for_slave 413 414echo "Killing master" 415sh ${leaks_kill} ipropd-master $ipdm || exit 1 416 417wait_for_master_down 418 419wait_for "slave to disconnect" \ 420 ${EGREP} 'disconnected' iprop-slave-status >/dev/null 421 422if ! tail -30 messages.log | grep 'disconnected for server' > /dev/null; then 423 echo "client didnt disconnect" 424 exit 1 425fi 426 427echo "probing for slave pid" 428kill -0 ${ipds} || { echo "slave no longer there"; exit 1; } 429 430> messages.log 431 432echo "Staring master again" ; > messages.log 433env ${HEIM_MALLOC_DEBUG} \ 434${ipropd_master} --hostname=localhost -k ${keytab} \ 435 --database=${objdir}/current-db --detach || 436 { echo "ipropd-master failed to start"; exit 1; } 437ipdm=`getpid ipropd-master` 438 439echo "probing for slave pid" 440kill -0 ${ipds} || { echo "slave no longer there"; exit 1; } 441 442 443echo "pushing one change" 444${kadmin} -l cpw --random-password user@${R} > /dev/null || exit 1 445wait_for_slave 446 447echo "shutting down all services" 448 449leaked=false 450sh ${leaks_kill} kdc $kdcpid || leaked=true 451sh ${leaks_kill} ipropd-master $ipdm || leaked=true 452sh ${leaks_kill} ipropd-slave $ipds || leaked=true 453rm -f iprop-slave-status 454trap "" EXIT 455$leaked && exit 1 456 457echo "compare versions on master and slave logs" 458KRB5_CONFIG=${objdir}/krb5-slave.conf \ 459${iprop_log} last-version > slave-last.tmp 460${iprop_log} last-version > master-last.tmp 461cmp master-last.tmp slave-last.tmp || exit 1 462 463if [ "$db_type" = lmdb ] && type mdb_stat > /dev/null 2>&1; then 464 # Sanity check that we have the same number of principals at the HDB 465 # and LMDB levels. 466 # 467 # We should also do this for the sqlite backend, but that would 468 # require a sqlite3(1) shell that is capable of opening our HDB 469 # files. 470 echo "checking that principals in DB == entries in LMDB" 471 # Add one to match lmdb overhead 472 princs=`(echo; ${kadmin} -l list '*') | wc -l` 473 entries=`mdb_stat -n current-db.mdb | grep 'Entries:' | awk '{print $2}'` 474 [ "$princs" -eq "$entries" ] || exit 1 475fi 476 477exit 0 478