1#!/bin/sh 2# 3# Copyright (c) 2006 - 2007 Kungliga Tekniska Högskolan 4# (Royal Institute of Technology, Stockholm, Sweden). 5# All rights reserved. 6# 7# Redistribution and use in source and binary forms, with or without 8# modification, are permitted provided that the following conditions 9# are met: 10# 11# 1. Redistributions of source code must retain the above copyright 12# notice, this list of conditions and the following disclaimer. 13# 14# 2. Redistributions in binary form must reproduce the above copyright 15# notice, this list of conditions and the following disclaimer in the 16# documentation and/or other materials provided with the distribution. 17# 18# 3. Neither the name of the Institute nor the names of its contributors 19# may be used to endorse or promote products derived from this software 20# without specific prior written permission. 21# 22# THIS SOFTWARE IS PROVIDED BY THE INSTITUTE AND CONTRIBUTORS ``AS IS'' AND 23# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25# ARE DISCLAIMED. IN NO EVENT SHALL THE INSTITUTE OR CONTRIBUTORS BE LIABLE 26# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32# SUCH DAMAGE. 33 34top_builddir="@top_builddir@" 35env_setup="@env_setup@" 36objdir="@objdir@" 37 38db_type=@db_type@ 39 40. ${env_setup} 41 42# If there is no useful db support compile in, disable test 43${have_db} || exit 77 44 45 46# Dont run this test in AFS, since it lacks support for AF_UNIX 47expr "X`/bin/pwd || pwd`" : "X/afs/.*" > /dev/null 2>/dev/null && exit 77 48 49R=TEST.H5L.SE 50 51port=@port@ 52 53cache="FILE:${objdir}/cache.krb5" 54keytabfile=${objdir}/iprop.keytab 55keytab="FILE:${keytabfile}" 56 57kdc="${kdc} --addresses=localhost -P $port" 58kadmin="${kadmin} -r $R" 59kinit="${kinit} -c $cache ${afs_no_afslog}" 60 61slave_ver_from_master_old= 62slave_ver_from_master_new= 63slave_ver_old= 64slave_ver_new= 65get_iprop_ver () { 66 min_change=${1:-1} 67 slave_ver_from_master_new=`grep '^iprop/' iprop-stats | head -1 | awk '{print $3}'` 68 slave_ver_new=`grep 'up-to-date with version:' iprop-slave-status | awk '{print $4}'` 69 if [ -z "$slave_ver_from_master_new" -o -z "$slave_ver_new" ]; then 70 return 1 71 fi 72 if [ x"$slave_ver_from_master_new" != x"$slave_ver_new" ]; then 73 return 1 74 fi 75 if [ x"$slave_ver_from_master_old" != x ]; then 76 change=`expr "$slave_ver_from_master_new" - "$slave_ver_from_master_old"` 77 if [ "$change" -lt "$min_change" ]; then 78 return 1 79 fi 80 fi 81 slave_ver_from_master_old=$slave_ver_from_master_new 82 slave_ver_old=$slave_ver_new 83 return 0 84} 85 86waitsec=65 87sleeptime=2 88wait_for () { 89 msg=$1 90 shift 91 t=0 92 while ! "$@"; do 93 sleep $sleeptime; 94 t=`expr $t + $sleeptime` 95 if [ $t -gt $waitsec ]; then 96 echo "Waited too long for $msg" 97 exit 1 98 fi 99 done 100 return 0 101} 102 103check_pidfile_is_dead () { 104 if test ! -f lt-${1}.pid -a ! -f ${1}.pid; then 105 return 0 106 fi 107 _pid=`cat lt-${1}.pid ${1}.pid 2>/dev/null` 108 if [ -z "$_pid" ]; then 109 return 0 110 fi 111 if kill -0 $_pid 2>/dev/null; then 112 return 1 113 fi 114 return 0 115} 116 117wait_for_slave () { 118 wait_for "iprop versions to change and/or slave to catch up" get_iprop_ver "$@" 119} 120 121wait_for_master_down () { 122 wait_for "master to exit" check_pidfile_is_dead ipropd-master 123} 124 125wait_for_slave_down () { 126 wait_for "slave to exit" check_pidfile_is_dead ipropd-slave 127} 128 129KRB5_CONFIG="${objdir}/krb5.conf" 130export KRB5_CONFIG 131 132rm -f ${keytabfile} 133rm -f current-db* 134rm -f current*.log 135rm -f out-* 136rm -f mkey.file* 137rm -f messages.log 138 139> messages.log 140 141echo Creating database 142${kadmin} -l \ 143 init \ 144 --realm-max-ticket-life=1day \ 145 --realm-max-renewable-life=1month \ 146 ${R} || exit 1 147 148${kadmin} -l add -p foo --use-defaults user@${R} || exit 1 149 150${kadmin} -l add --random-key --use-defaults iprop/localhost@${R} || exit 1 151${kadmin} -l ext -k ${keytab} iprop/localhost@${R} || exit 1 152${kadmin} -l add --random-key --use-defaults iprop/slave.test.h5l.se@${R} || exit 1 153${kadmin} -l ext -k ${keytab} iprop/slave.test.h5l.se@${R} || exit 1 154 155echo foo > ${objdir}/foopassword 156 157echo "Test log recovery" 158${kadmin} -l add --random-key --use-defaults recovtest@${R} || exit 1 159# Test theory: save the log, make a change and save the record it 160# produced, restore the log, append to it the saved record, then get 161 162# Save the log 163cp current.log current.log.tmp 164ls -l current.log.tmp | awk '{print $5}' > tmp 165read sz < tmp 166# Make a change 167${kadmin} -l mod -a requires-pre-auth recovtest@${R} || exit 1 168${kadmin} -l get recovtest@${R} | grep 'Attributes: requires-pre-auth$' > /dev/null || exit 1 169# Save the resulting log record 170ls -l current.log | awk '{print $5}' > tmp 171read nsz < tmp 172rm tmp 173dd bs=1 if=current.log skip=$sz of=current.log.tmp.saved-record count=`expr $nsz - $sz` 2>/dev/null 174# Undo the change 175${kadmin} -l mod -a -requires-pre-auth recovtest@${R} || exit 1 176${kadmin} -l get recovtest@${R} | grep 'Attributes:.$' > /dev/null || exit 1 177# Restore the log 178cp current.log current.log.save 179mv current.log.tmp current.log 180# Append the saved record 181cat current.log.tmp.saved-record >> current.log 182rm current.log.tmp.saved-record 183# Check that we still see the principal as modified 184${kadmin} -l get recovtest@${R} | grep 'Attributes: requires-pre-auth$' > /dev/null || exit 1 185 186# -- foo 187ipds= 188ipdm= 189kdcpid= 190 191> iprop-stats 192rm -f iprop-slave-status 193 194ipropd_slave="${ipropd_slave} --status-file=iprop-slave-status" 195 196trap "echo 'killing ipropd s + m + kdc'; kill -9 \${ipdm} \${ipds} \${kdcpid} >/dev/null 2>/dev/null; tail messages.log ; tail iprop-stats; exit 1;" EXIT 197 198echo Starting kdc ; > messages.log 199${kdc} --detach --testing || { echo "kdc failed to start"; exit 1; } 200kdcpid=`getpid kdc` 201 202echo "starting master" ; > messages.log 203env ${HEIM_MALLOC_DEBUG} \ 204${ipropd_master} --hostname=localhost -k ${keytab} \ 205 --database=${objdir}/current-db --detach || 206 { echo "ipropd-master failed to start"; exit 1; } 207ipdm=`getpid ipropd-master` 208 209echo "starting slave" ; > messages.log 210env ${HEIM_MALLOC_DEBUG} \ 211KRB5_CONFIG="${objdir}/krb5-slave.conf" \ 212${ipropd_slave} --hostname=slave.test.h5l.se -k ${keytab} --detach localhost || 213 { echo "ipropd-slave failed to start"; exit 1; } 214ipds=`getpid ipropd-slave` 215sh ${wait_kdc} ipropd-slave messages.log 'slave status change: up-to-date' || exit 1 216get_iprop_ver || exit 1 217 218echo "checking slave is up" 219${EGREP} 'iprop/slave.test.h5l.se@TEST.H5L.SE.*Up' iprop-stats >/dev/null || exit 1 220${EGREP} 'up-to-date with version' iprop-slave-status >/dev/null || { echo "slave to up to date" ; cat iprop-slave-status ; exit 1; } 221 222# ----------------- checking: pushing lives changes 223 224echo "Add host" 225${kadmin} -l add --random-key --use-defaults host/foo@${R} || exit 1 226wait_for_slave 227KRB5_CONFIG="${objdir}/krb5-slave.conf" \ 228${kadmin} -l get host/foo@${R} > /dev/null || exit 1 229 230echo "Rollover host keys" 231${kadmin} -l cpw -r --keepold host/foo@${R} || exit 1 232${kadmin} -l cpw -r --keepold host/foo@${R} || exit 1 233${kadmin} -l cpw -r --keepold host/foo@${R} || exit 1 234wait_for_slave 3 235KRB5_CONFIG="${objdir}/krb5-slave.conf" \ 236${kadmin} -l get host/foo@${R} | \ 237 ${EGREP} Keytypes: | cut -d: -f2 | tr ' ' ' 238' | sed 's/^.*[[]\(.*\)[]].*$/\1/' | grep '[0-9]' | sort -nu | tr -d ' 239' | ${EGREP} 1234 > /dev/null || exit 1 240 241echo "Delete 3DES keys" 242${kadmin} -l del_enctype host/foo@${R} des3-cbc-sha1 243wait_for_slave 244KRB5_CONFIG="${objdir}/krb5-slave.conf" \ 245${kadmin} -l get host/foo@${R} | \ 246 ${EGREP} Keytypes: | cut -d: -f2 | tr ' ' ' 247' | sed 's/^.*[[]\(.*\)[]].*$/\1/' | grep '[0-9]' | sort -nu | tr -d ' 248' | ${EGREP} 1234 > /dev/null || exit 1 249KRB5_CONFIG="${objdir}/krb5-slave.conf" \ 250${kadmin} -l get host/foo@${R} | \ 251 ${EGREP} 'Keytypes:.*des3-cbc-sha1' > /dev/null && exit 1 252 253echo "Change policy host" 254${kadmin} -l modify --policy=default host/foo@${R} || exit 1 255wait_for_slave 256KRB5_CONFIG="${objdir}/krb5-slave.conf" \ 257${kadmin} -l get host/foo@${R} > /dev/null 2>/dev/null || exit 1 258 259echo "Rename host" 260${kadmin} -l rename host/foo@${R} host/bar@${R} || exit 1 261wait_for_slave 262KRB5_CONFIG="${objdir}/krb5-slave.conf" \ 263${kadmin} -l get host/foo@${R} > /dev/null 2>/dev/null && exit 1 264KRB5_CONFIG="${objdir}/krb5-slave.conf" \ 265${kadmin} -l get host/bar@${R} > /dev/null || exit 1 266 267echo "Delete host" 268${kadmin} -l delete host/bar@${R} || exit 1 269wait_for_slave 270KRB5_CONFIG="${objdir}/krb5-slave.conf" \ 271${kadmin} -l get host/bar@${R} > /dev/null 2>/dev/null && exit 1 272 273# See note below in LMDB sanity checking 274echo "Re-add host" 275${kadmin} -l add --random-key --use-defaults host/foo@${R} || exit 1 276${kadmin} -l add --random-key --use-defaults host/bar@${R} || exit 1 277 278echo "kill slave and remove log and database" 279> iprop-stats 280sh ${leaks_kill} ipropd-slave $ipds || exit 1 281rm -f iprop-slave-status 282 283wait_for_slave_down 284${EGREP} 'iprop/slave.test.h5l.se@TEST.H5L.SE.*Down' iprop-stats >/dev/null || exit 1 285 286# ----------------- checking: slave is missing changes while down 287 288rm current.slave.log current-db.slave* || exit 1 289 290echo "doing changes while slave is down" 291${kadmin} -l cpw --random-password user@${R} > /dev/null || exit 1 292${kadmin} -l cpw --random-password user@${R} > /dev/null || exit 1 293 294echo "Makeing a copy of the master log file" 295cp ${objdir}/current.log ${objdir}/current.log.tmp 296 297# ----------------- checking: checking that master and slaves resyncs 298 299echo "starting slave again" ; > messages.log 300> iprop-stats 301env ${HEIM_MALLOC_DEBUG} \ 302KRB5_CONFIG="${objdir}/krb5-slave.conf" \ 303${ipropd_slave} --hostname=slave.test.h5l.se -k ${keytab} --detach localhost || 304 { echo "ipropd-slave failed to start"; exit 1; } 305ipds=`getpid ipropd-slave` 306 307echo "checking slave is up again" 308wait_for "slave to start and connect to master" \ 309 ${EGREP} 'iprop/slave.test.h5l.se@TEST.H5L.SE.*Up' iprop-stats >/dev/null 310wait_for_slave 2 311${EGREP} 'up-to-date with version' iprop-slave-status >/dev/null || { echo "slave not up to date" ; cat iprop-slave-status ; exit 1; } 312echo "checking for replay problems" 313${EGREP} 'Entry already exists in database' messages.log && exit 1 314 315echo "compare versions on master and slave logs (no lock)" 316KRB5_CONFIG=${objdir}/krb5-slave.conf \ 317${iprop_log} last-version -n > slave-last.tmp 318${iprop_log} last-version -n > master-last.tmp 319cmp master-last.tmp slave-last.tmp || exit 1 320 321echo "kill slave and remove log and database" 322sh ${leaks_kill} ipropd-slave $ipds || exit 1 323wait_for_slave_down 324 325rm current.slave.log current-db.slave* || exit 1 326> iprop-stats 327rm -f iprop-slave-status 328echo "starting slave" ; > messages.log 329env ${HEIM_MALLOC_DEBUG} \ 330KRB5_CONFIG="${objdir}/krb5-slave.conf" \ 331${ipropd_slave} --hostname=slave.test.h5l.se -k ${keytab} --detach localhost || 332 { echo "ipropd-slave failed to start"; exit 1; } 333ipds=`getpid ipropd-slave` 334wait_for_slave 0 335 336echo "checking slave is up again" 337wait_for "slave to start and connect to master" \ 338 ${EGREP} 'iprop/slave.test.h5l.se@TEST.H5L.SE.*Up' iprop-stats >/dev/null 339${EGREP} 'up-to-date with version' iprop-slave-status >/dev/null || { echo "slave not up to date" ; cat iprop-slave-status ; exit 1; } 340echo "checking for replay problems" 341${EGREP} 'Entry already exists in database' messages.log && exit 1 342 343# ----------------- checking: checking live truncation of master log 344 345${kadmin} -l cpw --random-password user@${R} > /dev/null || exit 1 346wait_for_slave 347 348echo "live truncate on master log" 349${iprop_log} truncate -K 5 || exit 1 350wait_for_slave 0 351 352echo "Killing master and slave" 353sh ${leaks_kill} ipropd-master $ipdm || exit 1 354sh ${leaks_kill} ipropd-slave $ipds || exit 1 355 356rm -f iprop-slave-status 357 358wait_for_slave_down 359wait_for_master_down 360 361echo "compare versions on master and slave logs" 362KRB5_CONFIG=${objdir}/krb5-slave.conf \ 363${iprop_log} last-version > slave-last.tmp 364${iprop_log} last-version > master-last.tmp 365cmp master-last.tmp slave-last.tmp || exit 1 366 367# ----------------- checking: master going backward 368> iprop-stats 369> messages.log 370 371echo "Going back to old version of the master log file" 372cp ${objdir}/current.log.tmp ${objdir}/current.log 373 374echo "starting master" ; > messages.log 375env ${HEIM_MALLOC_DEBUG} \ 376${ipropd_master} --hostname=localhost -k ${keytab} \ 377 --database=${objdir}/current-db --detach || 378 { echo "ipropd-master failed to start"; exit 1; } 379ipdm=`getpid ipropd-master` 380 381echo "starting slave" ; > messages.log 382env ${HEIM_MALLOC_DEBUG} \ 383KRB5_CONFIG="${objdir}/krb5-slave.conf" \ 384${ipropd_slave} --hostname=slave.test.h5l.se -k ${keytab} --detach localhost || 385 { echo "ipropd-slave failed to start"; exit 1; } 386ipds=`getpid ipropd-slave` 387wait_for_slave -1 388 389echo "checking slave is up again" 390wait_for "slave to start and connect to master" \ 391 ${EGREP} 'iprop/slave.test.h5l.se@TEST.H5L.SE.*Up' iprop-stats >/dev/null 392${EGREP} 'up-to-date with version' iprop-slave-status >/dev/null || { echo "slave to up to date" ; cat iprop-slave-status ; exit 1; } 393echo "checking for replay problems" 394${EGREP} 'Entry already exists in database' messages.log && exit 1 395 396echo "pushing one change" 397${kadmin} -l cpw --random-password user@${R} > /dev/null || exit 1 398wait_for_slave 399 400echo "Killing master" 401sh ${leaks_kill} ipropd-master $ipdm || exit 1 402 403wait_for_master_down 404 405wait_for "slave to disconnect" \ 406 ${EGREP} 'disconnected' iprop-slave-status >/dev/null 407 408if ! tail -30 messages.log | grep 'disconnected for server' > /dev/null; then 409 echo "client didnt disconnect" 410 exit 1 411fi 412 413echo "probing for slave pid" 414kill -0 ${ipds} || { echo "slave no longer there"; exit 1; } 415 416> messages.log 417 418echo "Staring master again" ; > messages.log 419env ${HEIM_MALLOC_DEBUG} \ 420${ipropd_master} --hostname=localhost -k ${keytab} \ 421 --database=${objdir}/current-db --detach || 422 { echo "ipropd-master failed to start"; exit 1; } 423ipdm=`getpid ipropd-master` 424 425echo "probing for slave pid" 426kill -0 ${ipds} || { echo "slave no longer there"; exit 1; } 427 428 429echo "pushing one change" 430${kadmin} -l cpw --random-password user@${R} > /dev/null || exit 1 431wait_for_slave 432 433echo "shutting down all services" 434 435leaked=false 436sh ${leaks_kill} kdc $kdcpid || leaked=true 437sh ${leaks_kill} ipropd-master $ipdm || leaked=true 438sh ${leaks_kill} ipropd-slave $ipds || leaked=true 439rm -f iprop-slave-status 440trap "" EXIT 441$leaked && exit 1 442 443echo "compare versions on master and slave logs" 444KRB5_CONFIG=${objdir}/krb5-slave.conf \ 445${iprop_log} last-version > slave-last.tmp 446${iprop_log} last-version > master-last.tmp 447cmp master-last.tmp slave-last.tmp || exit 1 448 449if [ "$db_type" = lmdb ]; then 450 # Sanity check that we have the same number of principals at the HDB 451 # and LMDB levels. 452 # 453 # We should also do this for the sqlite backend, but that would 454 # require a sqlite3(1) shell that is capable of opening our HDB 455 # files. W 456 echo "checking that principals in DB == entries in LMDB" 457 princs=`${kadmin} -l list '*' | wc -l` 458 entries=`mdb_stat -n current-db.mdb | grep 'Entries:' | awk '{print $2}'` 459 [ "`expr 1 + "$princs"`" -eq "$entries" ] || exit 1 460fi 461 462exit 0 463