1#!/bin/sh 2# 3# Copyright (c) 2006 - 2007 Kungliga Tekniska Högskolan 4# (Royal Institute of Technology, Stockholm, Sweden). 5# All rights reserved. 6# 7# Redistribution and use in source and binary forms, with or without 8# modification, are permitted provided that the following conditions 9# are met: 10# 11# 1. Redistributions of source code must retain the above copyright 12# notice, this list of conditions and the following disclaimer. 13# 14# 2. Redistributions in binary form must reproduce the above copyright 15# notice, this list of conditions and the following disclaimer in the 16# documentation and/or other materials provided with the distribution. 17# 18# 3. Neither the name of the Institute nor the names of its contributors 19# may be used to endorse or promote products derived from this software 20# without specific prior written permission. 21# 22# THIS SOFTWARE IS PROVIDED BY THE INSTITUTE AND CONTRIBUTORS ``AS IS'' AND 23# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25# ARE DISCLAIMED. IN NO EVENT SHALL THE INSTITUTE OR CONTRIBUTORS BE LIABLE 26# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32# SUCH DAMAGE. 33 34top_builddir="@top_builddir@" 35env_setup="@env_setup@" 36objdir="@objdir@" 37 38db_type=@db_type@ 39 40. ${env_setup} 41 42# If there is no useful db support compiled in, disable test 43${have_db} || exit 77 44 45 46# Don't run this test in AFS, since it lacks support for AF_UNIX 47expr "X`/bin/pwd || pwd`" : "X/afs/.*" > /dev/null 2>/dev/null && exit 77 48 49R=TEST.H5L.SE 50 51port=@port@ 52 53cache="FILE:${objdir}/cache.krb5" 54keytabfile=${objdir}/iprop.keytab 55keytab="FILE:${keytabfile}" 56 57kdc="${kdc} --addresses=localhost -P $port" 58kadmin="${kadmin} -r $R" 59kinit="${kinit} -c $cache ${afs_no_afslog}" 60 61slave_ver_from_master_old= 62slave_ver_from_master_new= 63slave_ver_old= 64slave_ver_new= 65get_iprop_ver () { 66 min_change=${1:-1} 67 slave_ver_from_master_new=`grep '^iprop/' iprop-stats | head -1 | awk '{print $3}'` 68 slave_ver_new=`grep 'up-to-date with version:' iprop-slave-status | awk '{print $4}'` 69 if [ -z "$slave_ver_from_master_new" -o -z "$slave_ver_new" ]; then 70 return 1 71 fi 72 if [ x"$slave_ver_from_master_new" != x"$slave_ver_new" ]; then 73 return 1 74 fi 75 if [ x"$slave_ver_from_master_old" != x ]; then 76 change=`expr "$slave_ver_from_master_new" - "$slave_ver_from_master_old"` 77 if [ "$change" -lt "$min_change" ]; then 78 return 1 79 fi 80 fi 81 slave_ver_from_master_old=$slave_ver_from_master_new 82 slave_ver_old=$slave_ver_new 83 return 0 84} 85 86waitsec=65 87sleeptime=2 88wait_for () { 89 msg=$1 90 shift 91 t=0 92 while ! "$@"; do 93 sleep $sleeptime; 94 t=`expr $t + $sleeptime` 95 if [ $t -gt $waitsec ]; then 96 echo "Waited too long for $msg" 97 exit 1 98 fi 99 done 100 return 0 101} 102 103check_pidfile_is_dead () { 104 if test ! -f lt-${1}.pid -a ! -f ${1}.pid; then 105 return 0 106 fi 107 _pid=`cat lt-${1}.pid ${1}.pid 2>/dev/null` 108 if [ -z "$_pid" ]; then 109 return 0 110 fi 111 if kill -0 $_pid 2>/dev/null; then 112 return 1 113 fi 114 return 0 115} 116 117wait_for_slave () { 118 wait_for "iprop versions to change and/or slave to catch up" get_iprop_ver "$@" 119} 120 121wait_for_master_down () { 122 wait_for "master to exit" check_pidfile_is_dead ipropd-master 123} 124 125wait_for_slave_down () { 126 wait_for "slave to exit" check_pidfile_is_dead ipropd-slave 127} 128 129KRB5_CONFIG="${objdir}/krb5.conf" 130export KRB5_CONFIG 131 132rm -f ${keytabfile} 133rm -f current-db* 134rm -f current*.log 135rm -f out-* 136rm -f mkey.file* 137rm -f messages.log 138 139> messages.log 140 141echo Creating database 142${kadmin} -l \ 143 init \ 144 --realm-max-ticket-life=1day \ 145 --realm-max-renewable-life=1month \ 146 ${R} || exit 1 147 148${kadmin} -l add -p foo --use-defaults user@${R} || exit 1 149 150${kadmin} -l add --random-key --use-defaults iprop/localhost@${R} || exit 1 151${kadmin} -l ext -k ${keytab} iprop/localhost@${R} || exit 1 152${kadmin} -l add --random-key --use-defaults iprop/slave.test.h5l.se@${R} || exit 1 153${kadmin} -l ext -k ${keytab} iprop/slave.test.h5l.se@${R} || exit 1 154 155echo foo > ${objdir}/foopassword 156 157echo "Test log recovery" 158${kadmin} -l add --random-key --use-defaults recovtest@${R} || exit 1 159# Test theory: save the log, make a change and save the record it 160# produced, restore the log, append to it the saved record, then get 161 162# Save the log 163cp current.log current.log.tmp 164ls -l current.log.tmp | awk '{print $5}' > tmp 165read sz < tmp 166# Make a change 167${kadmin} -l mod -a requires-pre-auth recovtest@${R} || exit 1 168${kadmin} -l get recovtest@${R} | grep 'Attributes: requires-pre-auth$' > /dev/null || exit 1 169# Save the resulting log record 170ls -l current.log | awk '{print $5}' > tmp 171read nsz < tmp 172rm tmp 173dd bs=1 if=current.log skip=$sz of=current.log.tmp.saved-record count=`expr $nsz - $sz` 2>/dev/null 174# Undo the change 175${kadmin} -l mod -a -requires-pre-auth recovtest@${R} || exit 1 176${kadmin} -l get recovtest@${R} | grep 'Attributes:.$' > /dev/null || exit 1 177# Restore the log 178cp current.log current.log.save 179mv current.log.tmp current.log 180# Append the saved record 181cat current.log.tmp.saved-record >> current.log 182rm current.log.tmp.saved-record 183# Check that we still see the principal as modified 184${kadmin} -l get recovtest@${R} | grep 'Attributes: requires-pre-auth$' > /dev/null || exit 1 185 186# -- foo 187ipds= 188ipdm= 189kdcpid= 190 191> iprop-stats 192rm -f iprop-slave-status 193 194ipropd_slave="${ipropd_slave} --status-file=iprop-slave-status" 195 196trap "echo 'killing ipropd s + m + kdc'; kill -9 \${ipdm} \${ipds} \${kdcpid} >/dev/null 2>/dev/null; tail messages.log ; tail iprop-stats; exit 1;" EXIT 197 198echo Starting kdc ; > messages.log 199${kdc} --detach --testing || { echo "kdc failed to start"; exit 1; } 200kdcpid=`getpid kdc` 201 202echo "starting master" ; > messages.log 203env ${HEIM_MALLOC_DEBUG} \ 204${ipropd_master} --hostname=localhost -k ${keytab} \ 205 --database=${objdir}/current-db --detach || 206 { echo "ipropd-master failed to start"; exit 1; } 207ipdm=`getpid ipropd-master` 208 209echo "starting slave" ; > messages.log 210env ${HEIM_MALLOC_DEBUG} \ 211KRB5_CONFIG="${objdir}/krb5-slave.conf" \ 212${ipropd_slave} --hostname=slave.test.h5l.se -k ${keytab} --detach localhost || 213 { echo "ipropd-slave failed to start"; exit 1; } 214ipds=`getpid ipropd-slave` 215sh ${wait_kdc} ipropd-slave messages.log 'slave status change: up-to-date' || exit 1 216get_iprop_ver || exit 1 217 218echo "checking slave is up" 219${EGREP} 'iprop/slave.test.h5l.se@TEST.H5L.SE.*Up' iprop-stats >/dev/null || exit 1 220${EGREP} 'up-to-date with version' iprop-slave-status >/dev/null || { echo "slave to up to date" ; cat iprop-slave-status ; exit 1; } 221 222# ----------------- checking: pushing lives changes 223 224slave_get() { KRB5_CONFIG="${objdir}/krb5-slave.conf" ${kadmin} -l get "$@"; } 225slave_check_exists() { 226 # Creation with a random key is not atomic, there are at present 227 # 3 log entries to create a random key principal, the entry is 228 # "invalid" for the first two of these. We wait for the entry to 229 # exist and not be invalid 230 # 231 attrs=`slave_get -o attributes "$@" 2>/dev/null` || return 1 232 echo $attrs | egrep 'Attributes:' | egrep -v invalid >/dev/null || return 1 233 get_iprop_ver 0 234} 235 236echo "Add host" 237${kadmin} -l add --random-key --use-defaults host/foo@${R} || exit 1 238wait_for "Slave sees new host" slave_check_exists "host/foo@${R}" 239 240echo "Rollover host keys" 241${kadmin} -l cpw -r --keepold host/foo@${R} || exit 1 242${kadmin} -l cpw -r --keepold host/foo@${R} || exit 1 243${kadmin} -l cpw -r --keepold host/foo@${R} || exit 1 244wait_for_slave 3 245slave_get host/foo@${R} | \ 246 ${EGREP} Keytypes: | cut -d: -f2 | tr ' ' ' 247' | sed 's/^.*[[]\(.*\)[]].*$/\1/' | grep '[0-9]' | sort -nu | tr -d ' 248' | ${EGREP} 1234 > /dev/null || exit 1 249 250echo "Delete 3DES keys" 251${kadmin} -l del_enctype host/foo@${R} des3-cbc-sha1 252wait_for_slave 253KRB5_CONFIG="${objdir}/krb5-slave.conf" \ 254${kadmin} -l get host/foo@${R} | \ 255 ${EGREP} Keytypes: | cut -d: -f2 | tr ' ' ' 256' | sed 's/^.*[[]\(.*\)[]].*$/\1/' | grep '[0-9]' | sort -nu | tr -d ' 257' | ${EGREP} 1234 > /dev/null || exit 1 258KRB5_CONFIG="${objdir}/krb5-slave.conf" \ 259${kadmin} -l get host/foo@${R} | \ 260 ${EGREP} 'Keytypes:.*des3-cbc-sha1' > /dev/null && exit 1 261 262echo "Change policy host" 263${kadmin} -l modify --policy=default host/foo@${R} || exit 1 264wait_for_slave 265KRB5_CONFIG="${objdir}/krb5-slave.conf" \ 266${kadmin} -l get host/foo@${R} > /dev/null 2>/dev/null || exit 1 267 268echo "Rename host" 269${kadmin} -l rename host/foo@${R} host/bar@${R} || exit 1 270wait_for_slave 271KRB5_CONFIG="${objdir}/krb5-slave.conf" \ 272${kadmin} -l get host/foo@${R} > /dev/null 2>/dev/null && exit 1 273KRB5_CONFIG="${objdir}/krb5-slave.conf" \ 274${kadmin} -l get host/bar@${R} > /dev/null || exit 1 275 276echo "Delete host" 277${kadmin} -l delete host/bar@${R} || exit 1 278wait_for_slave 279KRB5_CONFIG="${objdir}/krb5-slave.conf" \ 280${kadmin} -l get host/bar@${R} > /dev/null 2>/dev/null && exit 1 281 282# See note below in LMDB sanity checking 283echo "Re-add host" 284${kadmin} -l add --random-key --use-defaults host/foo@${R} || exit 1 285${kadmin} -l add --random-key --use-defaults host/bar@${R} || exit 1 286wait_for "Slave sees re-added host" slave_check_exists "host/bar@${R}" 287 288echo "kill slave and remove log and database" 289> iprop-stats 290sh ${leaks_kill} ipropd-slave $ipds || exit 1 291rm -f iprop-slave-status 292 293wait_for_slave_down 294${EGREP} 'iprop/slave.test.h5l.se@TEST.H5L.SE.*Down' iprop-stats >/dev/null || exit 1 295 296# ----------------- checking: slave is missing changes while down 297 298rm current.slave.log current-db.slave* || exit 1 299 300echo "doing changes while slave is down" 301${kadmin} -l cpw --random-password user@${R} > /dev/null || exit 1 302${kadmin} -l cpw --random-password user@${R} > /dev/null || exit 1 303 304echo "Making a copy of the master log file" 305cp ${objdir}/current.log ${objdir}/current.log.tmp 306 307# ----------------- checking: checking that master and slaves resyncs 308 309echo "starting slave again" ; > messages.log 310> iprop-stats 311env ${HEIM_MALLOC_DEBUG} \ 312KRB5_CONFIG="${objdir}/krb5-slave.conf" \ 313${ipropd_slave} --hostname=slave.test.h5l.se -k ${keytab} --detach localhost || 314 { echo "ipropd-slave failed to start"; exit 1; } 315ipds=`getpid ipropd-slave` 316 317echo "checking slave is up again" 318wait_for "slave to start and connect to master" \ 319 ${EGREP} 'iprop/slave.test.h5l.se@TEST.H5L.SE.*Up' iprop-stats >/dev/null 320wait_for_slave 2 321${EGREP} 'up-to-date with version' iprop-slave-status >/dev/null || { echo "slave not up to date" ; cat iprop-slave-status ; exit 1; } 322echo "checking for replay problems" 323${EGREP} 'Entry already exists in database' messages.log && exit 1 324 325echo "compare versions on master and slave logs (no lock)" 326KRB5_CONFIG=${objdir}/krb5-slave.conf \ 327${iprop_log} last-version -n > slave-last.tmp 328${iprop_log} last-version -n > master-last.tmp 329cmp master-last.tmp slave-last.tmp || exit 1 330 331echo "kill slave and remove log and database" 332sh ${leaks_kill} ipropd-slave $ipds || exit 1 333wait_for_slave_down 334 335rm current.slave.log current-db.slave* || exit 1 336> iprop-stats 337rm -f iprop-slave-status 338echo "starting slave" ; > messages.log 339env ${HEIM_MALLOC_DEBUG} \ 340KRB5_CONFIG="${objdir}/krb5-slave.conf" \ 341${ipropd_slave} --hostname=slave.test.h5l.se -k ${keytab} --detach localhost || 342 { echo "ipropd-slave failed to start"; exit 1; } 343ipds=`getpid ipropd-slave` 344wait_for_slave 0 345 346echo "checking slave is up again" 347wait_for "slave to start and connect to master" \ 348 ${EGREP} 'iprop/slave.test.h5l.se@TEST.H5L.SE.*Up' iprop-stats >/dev/null 349${EGREP} 'up-to-date with version' iprop-slave-status >/dev/null || { echo "slave not up to date" ; cat iprop-slave-status ; exit 1; } 350echo "checking for replay problems" 351${EGREP} 'Entry already exists in database' messages.log && exit 1 352 353# ----------------- checking: checking live truncation of master log 354 355${kadmin} -l cpw --random-password user@${R} > /dev/null || exit 1 356wait_for_slave 357 358echo "live truncate on master log" 359${iprop_log} truncate -K 5 || exit 1 360wait_for_slave 0 361 362echo "Killing master and slave" 363sh ${leaks_kill} ipropd-master $ipdm || exit 1 364sh ${leaks_kill} ipropd-slave $ipds || exit 1 365 366rm -f iprop-slave-status 367 368wait_for_slave_down 369wait_for_master_down 370 371echo "compare versions on master and slave logs" 372KRB5_CONFIG=${objdir}/krb5-slave.conf \ 373${iprop_log} last-version > slave-last.tmp 374${iprop_log} last-version > master-last.tmp 375cmp master-last.tmp slave-last.tmp || exit 1 376 377# ----------------- checking: master going backward 378> iprop-stats 379> messages.log 380 381echo "Going back to old version of the master log file" 382cp ${objdir}/current.log.tmp ${objdir}/current.log 383 384echo "starting master" ; > messages.log 385env ${HEIM_MALLOC_DEBUG} \ 386${ipropd_master} --hostname=localhost -k ${keytab} \ 387 --database=${objdir}/current-db --detach || 388 { echo "ipropd-master failed to start"; exit 1; } 389ipdm=`getpid ipropd-master` 390 391echo "starting slave" ; > messages.log 392env ${HEIM_MALLOC_DEBUG} \ 393KRB5_CONFIG="${objdir}/krb5-slave.conf" \ 394${ipropd_slave} --hostname=slave.test.h5l.se -k ${keytab} --detach localhost || 395 { echo "ipropd-slave failed to start"; exit 1; } 396ipds=`getpid ipropd-slave` 397wait_for_slave -1 398 399echo "checking slave is up again" 400wait_for "slave to start and connect to master" \ 401 ${EGREP} 'iprop/slave.test.h5l.se@TEST.H5L.SE.*Up' iprop-stats >/dev/null 402${EGREP} 'up-to-date with version' iprop-slave-status >/dev/null || { echo "slave to up to date" ; cat iprop-slave-status ; exit 1; } 403echo "checking for replay problems" 404${EGREP} 'Entry already exists in database' messages.log && exit 1 405 406echo "pushing one change" 407${kadmin} -l cpw --random-password user@${R} > /dev/null || exit 1 408wait_for_slave 409 410echo "Killing master" 411sh ${leaks_kill} ipropd-master $ipdm || exit 1 412 413wait_for_master_down 414 415wait_for "slave to disconnect" \ 416 ${EGREP} 'disconnected' iprop-slave-status >/dev/null 417 418if ! tail -30 messages.log | grep 'disconnected for server' > /dev/null; then 419 echo "client didnt disconnect" 420 exit 1 421fi 422 423echo "probing for slave pid" 424kill -0 ${ipds} || { echo "slave no longer there"; exit 1; } 425 426> messages.log 427 428echo "Staring master again" ; > messages.log 429env ${HEIM_MALLOC_DEBUG} \ 430${ipropd_master} --hostname=localhost -k ${keytab} \ 431 --database=${objdir}/current-db --detach || 432 { echo "ipropd-master failed to start"; exit 1; } 433ipdm=`getpid ipropd-master` 434 435echo "probing for slave pid" 436kill -0 ${ipds} || { echo "slave no longer there"; exit 1; } 437 438 439echo "pushing one change" 440${kadmin} -l cpw --random-password user@${R} > /dev/null || exit 1 441wait_for_slave 442 443echo "shutting down all services" 444 445leaked=false 446sh ${leaks_kill} kdc $kdcpid || leaked=true 447sh ${leaks_kill} ipropd-master $ipdm || leaked=true 448sh ${leaks_kill} ipropd-slave $ipds || leaked=true 449rm -f iprop-slave-status 450trap "" EXIT 451$leaked && exit 1 452 453echo "compare versions on master and slave logs" 454KRB5_CONFIG=${objdir}/krb5-slave.conf \ 455${iprop_log} last-version > slave-last.tmp 456${iprop_log} last-version > master-last.tmp 457cmp master-last.tmp slave-last.tmp || exit 1 458 459if [ "$db_type" = lmdb ] && type mdb_stat > /dev/null 2>&1; then 460 # Sanity check that we have the same number of principals at the HDB 461 # and LMDB levels. 462 # 463 # We should also do this for the sqlite backend, but that would 464 # require a sqlite3(1) shell that is capable of opening our HDB 465 # files. 466 echo "checking that principals in DB == entries in LMDB" 467 # Add one to match lmdb overhead 468 princs=`(echo; ${kadmin} -l list '*') | wc -l` 469 entries=`mdb_stat -n current-db.mdb | grep 'Entries:' | awk '{print $2}'` 470 [ "$princs" -eq "$entries" ] || exit 1 471fi 472 473exit 0 474