1#! /bin/sh 2# $OpenLDAP$ 3## This work is part of OpenLDAP Software <http://www.openldap.org/>. 4## 5## Copyright 1998-2021 The OpenLDAP Foundation. 6## All rights reserved. 7## 8## Redistribution and use in source and binary forms, with or without 9## modification, are permitted only as authorized by the OpenLDAP 10## Public License. 11## 12## A copy of this license is available in the file LICENSE in the 13## top-level directory of the distribution or, alternatively, at 14## <http://www.OpenLDAP.org/license.html>. 15 16# This script tests race conditions related to setting up the syncrepl 17# refresh phase, especially when the provider is itself a consumer 18# refreshing from its provider again. 19 20# The configuration used is a provider->forwarder->consumer chain, where 21# the forwarder is restarted between add/delete of entries on the provider. 22 23echo "Running defines.sh" 24. $SRCDIR/scripts/defines.sh 25 26test "x$INITIATION_RACE_TESTS" = "x" && INITIATION_RACE_TESTS=1 27 28if test $SYNCPROV = syncprovno; then 29 echo "Syncrepl provider overlay not available, test skipped" 30 exit 0 31fi 32 33RETRY="1 +" 34 35PROV_DIR=$TESTDIR/prov 36CONS_DIR=$TESTDIR/cons 37FWD1_DIR=$TESTDIR/fwd1 38FWD2_DIR=$TESTDIR/fwd2 39 40PROV_URI=$URI1 41CONS_URI=$URI2 42FWD1_URI=$URI3 43 44PROV_LOG=$LOG1 45CONS_LOG=$LOG2 46FWD1_LOG=$LOG3 47 48DIRS="$PROV_DIR $CONS_DIR $FWD1_DIR" 49URIS="$PROV_URI $CONS_URI $FWD1_URI" 50 51noObj=32 52nullExclude="" nullOK="" 53test $BACKEND = null && nullExclude="# " nullOK="OK" noObj=0 54 55mkdir -p $TESTDIR 56 57for dir in $DIRS; do 58 mkdir -p $dir $dir/slapd.d $dir/db 59done 60 61KILLPIDS= 62 63$SLAPPASSWD -g -n >$CONFIGPWF 64 65case "$BACKEND" in 66 *) olcDbCheckpoint="# olcDbCheckpoint";; 67esac 68 69echo "Initializing server configurations" 70for dir in $DIRS; do 71 $SLAPADD -F $dir/slapd.d -n 0 <<EOF 72dn: cn=config 73objectClass: olcGlobal 74cn: config 75olcServerID: 1 $PROV_URI 76olcServerID: 2 $CONS_URI 77olcServerID: 3 $FWD1_URI 78 79dn: olcDatabase={0}config,cn=config 80objectClass: olcDatabaseConfig 81olcDatabase: {0}config 82olcRootPW:< file://$CONFIGPWF 83 84EOF 85done 86 87echo "Starting provider slapd on $PROV_URI" 88cd $PROV_DIR 89$SLAPD -F slapd.d -h $PROV_URI -d $LVL >> $PROV_LOG 2>&1 & 90PROV_PID=$! 91if test $WAIT != 0 ; then 92 echo PID $PROV_PID 93 read foo 94fi 95KILLPIDS="$KILLPIDS $PROV_PID" 96cd $TESTWD 97sleep 1 98for i in 1 2 3 4 5; do 99 $LDAPSEARCH -s base -b "" -H $PROV_URI \ 100 'objectclass=*' > /dev/null 2>&1 101 RC=$? 102 test $RC = 0 && break 103 echo "Waiting $i seconds for slapd to start..." 104 sleep $i 105done 106if test $RC != 0 ; then 107 echo "ldapsearch failed ($RC)!" 108 test $KILLSERVERS != no && kill -HUP $KILLPIDS 109 exit $RC 110fi 111 112echo "Starting forward1 slapd on $FWD1_URI" 113cd $FWD1_DIR 114$SLAPD -F slapd.d -h $FWD1_URI -d $LVL >> $FWD1_LOG 2>&1 & 115FWD1_PID=$! 116if test $WAIT != 0 ; then 117 echo PID $FWD1_PID 118 read foo 119fi 120KILLPIDS="$KILLPIDS $FWD1_PID" 121cd $TESTWD 122sleep 1 123for i in 1 2 3 4 5; do 124 $LDAPSEARCH -s base -b "" -H $FWD1_URI \ 125 'objectclass=*' > /dev/null 2>&1 126 RC=$? 127 test $RC = 0 && break 128 echo "Waiting $i seconds for slapd to start..." 129 sleep $i 130done 131if test $RC != 0 ; then 132 echo "ldapsearch failed ($RC)!" 133 test $KILLSERVERS != no && kill -HUP $KILLPIDS 134 exit $RC 135fi 136 137echo "Starting consumer slapd on $CONS_URI" 138cd $CONS_DIR 139$SLAPD -F slapd.d -h $CONS_URI -d $LVL >> $CONS_LOG 2>&1 & 140CONS_PID=$! 141if test $WAIT != 0 ; then 142 echo PID $CONS_PID 143 read foo 144fi 145KILLPIDS="$KILLPIDS $CONS_PID" 146cd $TESTWD 147sleep 1 148for i in 1 2 3 4 5; do 149 $LDAPSEARCH -s base -b "" -H $CONS_URI \ 150 'objectclass=*' > /dev/null 2>&1 151 RC=$? 152 test $RC = 0 && break 153 echo "Waiting $i seconds for slapd to start..." 154 sleep $i 155done 156if test $RC != 0 ; then 157 echo "ldapsearch failed ($RC)!" 158 test $KILLSERVERS != no && kill -HUP $KILLPIDS 159 exit $RC 160fi 161 162for uri in $URIS; do 163 echo "Adding schema on $uri" 164 $LDAPADD -D cn=config -H $uri -y $CONFIGPWF <<EOF > $TESTOUT 2>&1 165include: file://$ABS_SCHEMADIR/core.ldif 166 167include: file://$ABS_SCHEMADIR/cosine.ldif 168 169include: file://$ABS_SCHEMADIR/inetorgperson.ldif 170 171include: file://$ABS_SCHEMADIR/openldap.ldif 172 173include: file://$ABS_SCHEMADIR/nis.ldif 174 175EOF 176 RC=$? 177 if test $RC != 0 ; then 178 echo "ldapadd failed ($RC)!" 179 test $KILLSERVERS != no && kill -HUP $KILLPIDS 180 exit $RC 181 fi 182 183 [ "$BACKENDTYPE" = mod ] || continue 184 185 echo "Adding backend module on $uri..." 186 $LDAPADD -D cn=config -H $uri -y $CONFIGPWF <<EOF >>$TESTOUT 2>&1 187dn: cn=module,cn=config 188objectClass: olcModuleList 189cn: module 190olcModulePath: $TESTWD/../servers/slapd/back-$BACKEND 191olcModuleLoad: back_$BACKEND.la 192 193EOF 194 RC=$? 195 if test $RC != 0 ; then 196 echo "ldapadd failed for backend module ($RC)!" 197 test $KILLSERVERS != no && kill -HUP $KILLPIDS 198 exit $RC 199 fi 200done 201 202syncprov_module='' 203[ "$AC_syncprov" = syncprovmod ] && syncprov_module=" 204dn: cn=module,cn=config 205objectClass: olcModuleList 206cn: module 207olcModulePath: $TESTWD/../servers/slapd/overlays 208olcModuleLoad: syncprov.la" 209 210for uri in $PROV_URI; do 211 echo "Adding database configuration on $uri" 212 $LDAPADD -D cn=config -H $uri -y $CONFIGPWF <<EOF > $TESTOUT 2>&1 213dn: olcDatabase={1}$BACKEND,cn=config 214objectClass: olcDatabaseConfig 215objectClass: olc${BACKEND}Config 216olcDatabase: {1}$BACKEND 217${nullExclude}olcDbDirectory: ./db 218$olcDbCheckpoint: 1024 5 219olcSuffix: $BASEDN 220olcRootDN: $MANAGERDN 221olcRootPW: $PASSWD 222 223$syncprov_module 224 225dn: olcOverlay={0}syncprov,olcDatabase={1}$BACKEND,cn=config 226objectClass: olcOverlayConfig 227objectClass: olcSyncProvConfig 228olcOverlay: {0}syncprov 229olcSpCheckpoint: 1 1 230 231EOF 232 RC=$? 233 if test $RC != 0 ; then 234 echo "ldapadd failed ($RC)!" 235 test $KILLSERVERS != no && kill -HUP $KILLPIDS 236 exit $RC 237 fi 238 239 echo "Populating provider on $uri" 240 $LDAPADD -D "$MANAGERDN" -H $PROV_URI -w $PASSWD <<EOF >> $TESTOUT 2>&1 241dn: $BASEDN 242objectClass: top 243objectClass: organization 244objectClass: dcObject 245dc: example 246o: Example, Inc 247 248EOF 249 RC=$? 250 if test $RC != 0 ; then 251 echo "ldapadd failed ($RC)!" 252 test $KILLSERVERS != no && kill -HUP $KILLPIDS 253 exit $RC 254 fi 255done 256 257for uri in $FWD1_URI; do 258 echo "Adding database configuration on $uri" 259 $LDAPADD -D cn=config -H $uri -y $CONFIGPWF <<EOF > $TESTOUT 2>&1 260dn: olcDatabase={1}$BACKEND,cn=config 261objectClass: olcDatabaseConfig 262objectClass: olc${BACKEND}Config 263olcDatabase: {1}$BACKEND 264${nullExclude}olcDbDirectory: ./db 265$olcDbCheckpoint: 1024 5 266olcSuffix: $BASEDN 267olcRootDN: $MANAGERDN 268olcRootPW: $PASSWD 269olcSyncRepl: rid=1 provider=$PROV_URI searchbase="$BASEDN" 270 binddn="$MANAGERDN" bindmethod=simple credentials=$PASSWD 271 type=refreshAndPersist retry="$RETRY" timeout=1 272 273$syncprov_module 274 275dn: olcOverlay={0}syncprov,olcDatabase={1}$BACKEND,cn=config 276objectClass: olcOverlayConfig 277objectClass: olcSyncProvConfig 278olcOverlay: {0}syncprov 279olcSpCheckpoint: 1 1 280 281EOF 282 RC=$? 283 if test $RC != 0 ; then 284 echo "ldapadd failed ($RC)!" 285 test $KILLSERVERS != no && kill -HUP $KILLPIDS 286 exit $RC 287 fi 288done 289 290for uri in $CONS_URI; do 291 echo "Adding database configuration on $uri" 292 $LDAPADD -D cn=config -H $uri -y $CONFIGPWF <<EOF > $TESTOUT 2>&1 293dn: olcDatabase={1}$BACKEND,cn=config 294objectClass: olcDatabaseConfig 295objectClass: olc${BACKEND}Config 296olcDatabase: {1}$BACKEND 297${nullExclude}olcDbDirectory: ./db 298$olcDbCheckpoint: 1024 5 299olcSuffix: $BASEDN 300olcRootDN: $MANAGERDN 301olcRootPW: $PASSWD 302olcSyncRepl: rid=1 provider=$FWD1_URI searchbase="$BASEDN" 303 binddn="$MANAGERDN" bindmethod=simple credentials=$PASSWD 304 type=refreshAndPersist retry="$RETRY" timeout=1 305 306EOF 307 RC=$? 308 if test $RC != 0 ; then 309 echo "ldapadd failed ($RC)!" 310 test $KILLSERVERS != no && kill -HUP $KILLPIDS 311 exit $RC 312 fi 313done 314 315for uri in $FWD1_URI $CONS_URI; do 316 echo "Using ldapsearch to check that $uri received database..." 317 for i in 1 2 3 4 5; do 318 $LDAPSEARCH -s base -b "$BASEDN" -H $uri \ 319 'objectclass=*' > /dev/null 2>&1 320 RC=$? 321 test $RC = 0 && break 322 echo "Waiting $i seconds for slapd to receive database..." 323 sleep $i 324 done 325 if test $RC != 0 ; then 326 echo "ldapsearch failed ($RC)!" 327 test $KILLSERVERS != no && kill -HUP $KILLPIDS 328 exit $RC 329 fi 330done 331 332RACE_NUM=0 333ERROR=0 334 335nEntries=10 336 337addEnd=1 338delEnd=1 339 340addIdx=1 341delIdx=1 342 343while test $ERROR -eq 0 -a $RACE_NUM -lt $INITIATION_RACE_TESTS ; do 344 RACE_NUM=`expr $RACE_NUM + 1` 345 echo "Running $RACE_NUM of $INITIATION_RACE_TESTS syncrepl initiation race tests..." 346 347 echo "Stopping forwarders for add test" 348 for pid in $FWD1_PID; do 349 kill -HUP $pid 350 wait $pid 351 KILLPIDS=`echo "$KILLPIDS " | sed -e "s/ $pid / /"`; 352 done 353 354 addStart=$addEnd 355 addEnd=`expr $addEnd + $nEntries` 356 357 echo "Using ldapadd to add $nEntries entries on provider" 358 while test $addIdx -lt $addEnd; do 359 $LDAPADD -D "$MANAGERDN" -H $PROV_URI -w $PASSWD <<EOF >> $TESTOUT 2>&1 360dn: ou=$addIdx,$BASEDN 361objectClass: top 362objectClass: organizationalUnit 363ou: $addIdx 364 365EOF 366 RC=$? 367 if test $RC != 0 ; then 368 echo "ldapadd failed for entry $addIdx ($RC)!" 369 test $KILLSERVERS != no && kill -HUP $KILLPIDS 370 exit $RC 371 fi 372 addIdx=`expr $addIdx + 1` 373 done 374 375 echo "Starting forwarders again" 376 cd $FWD1_DIR 377 $SLAPD -F slapd.d -h $FWD1_URI -d $LVL >> $FWD1_LOG 2>&1 & 378 FWD1_PID=$! 379 KILLPIDS="$KILLPIDS $FWD1_PID" 380 cd $TESTWD 381 382 addEnd=`expr $addEnd + $nEntries` 383 384 echo "Using ldapadd to add $nEntries more entries on provider" 385 while test $addIdx -lt $addEnd; do 386 $LDAPADD -D "$MANAGERDN" -H $PROV_URI -w $PASSWD <<EOF >> $TESTOUT 2>&1 387dn: ou=$addIdx,$BASEDN 388objectClass: top 389objectClass: organizationalUnit 390ou: $addIdx 391 392EOF 393 RC=$? 394 if test $RC != 0 ; then 395 echo "ldapadd failed for entry $addIdx ($RC)!" 396 test $KILLSERVERS != no && kill -HUP $KILLPIDS 397 exit $RC 398 fi 399 addIdx=`expr $addIdx + 1` 400 done 401 402 for uri in $FWD1_URI $CONS_URI; do 403 echo "Checking replication to $uri" 404 RC=32 405 i=$addStart 406 while test $i -lt $addEnd; do 407 for j in 1 2 3 4 5; do 408 RESULT=`$LDAPSEARCH -H $uri -s base -b "ou=$i,$BASEDN" 2>&1 \ 409 | awk '/^dn:/ {print "OK"}'` 410 if test "x$RESULT$nullOK" = "xOK" ; then 411 RC=0 412 break 413 fi 414 echo "Waiting $j seconds for $uri to receive entry $i..." 415 sleep $j 416 done 417 if test $RC != 0 ; then 418 echo "ERROR: Entry $i not replicated to $uri! ($RC)!" 419 ERROR=1 420 break 421 fi 422 i=`expr $i + 1` 423 done 424 if test $ERROR != 0; then break; fi 425 done 426 if test $ERROR != 0; then break; fi 427 428 echo "Stopping forwarders for add/delete test" 429 for pid in $FWD1_PID; do 430 kill -HUP $pid 431 wait $pid 432 KILLPIDS=`echo "$KILLPIDS " | sed -e "s/ $pid / /"`; 433 done 434 435 addStart=$addEnd 436 addEnd=`expr $addEnd + $nEntries` 437 438 echo "Using ldapadd to add $nEntries entries on provider" 439 while test $addIdx -lt $addEnd; do 440 $LDAPADD -D "$MANAGERDN" -H $PROV_URI -w $PASSWD <<EOF >> $TESTOUT 2>&1 441dn: ou=$addIdx,$BASEDN 442objectClass: top 443objectClass: organizationalUnit 444ou: $addIdx 445 446EOF 447 RC=$? 448 if test $RC != 0 ; then 449 echo "ldapadd failed for entry $addIdx ($RC)!" 450 test $KILLSERVERS != no && kill -HUP $KILLPIDS 451 exit $RC 452 fi 453 addIdx=`expr $addIdx + 1` 454 done 455 456 delStart=$delEnd 457 delEnd=`expr $delEnd + $nEntries` 458 459 echo "Using ldapdelete to delete $nEntries entries on provider" 460 while test $delIdx -lt $delEnd; do 461 $LDAPDELETE -D "$MANAGERDN" -H $PROV_URI -w $PASSWD "ou=$delIdx,$BASEDN" 462 RC=$? 463 if test $RC != 0 ; then 464 echo "ldapdelete failed ($RC)!" 465 test $KILLSERVERS != no && kill -HUP $KILLPIDS 466 exit 1 467 fi 468 delIdx=`expr $delIdx + 1` 469 done 470 471 echo "Starting forwarders again" 472 cd $FWD1_DIR 473 $SLAPD -F slapd.d -h $FWD1_URI -d $LVL >> $FWD1_LOG 2>&1 & 474 FWD1_PID=$! 475 KILLPIDS="$KILLPIDS $FWD1_PID" 476 cd $TESTWD 477 478 addEnd=`expr $addEnd + $nEntries` 479 delEnd=`expr $delEnd + $nEntries` 480 481 echo "Using ldapadd to add $nEntries more entries on provider" 482 while test $addIdx -lt $addEnd; do 483 $LDAPADD -D "$MANAGERDN" -H $PROV_URI -w $PASSWD <<EOF >> $TESTOUT 2>&1 484dn: ou=$addIdx,$BASEDN 485objectClass: top 486objectClass: organizationalUnit 487ou: $addIdx 488 489EOF 490 RC=$? 491 if test $RC != 0 ; then 492 echo "ldapadd failed for entry $addIdx ($RC)!" 493 test $KILLSERVERS != no && kill -HUP $KILLPIDS 494 exit $RC 495 fi 496 addIdx=`expr $addIdx + 1` 497 done 498 499 echo "Using ldapdelete to delete $nEntries more entries on provider" 500 while test $delIdx -lt $delEnd; do 501 $LDAPDELETE -D "$MANAGERDN" -H $PROV_URI -w $PASSWD "ou=$delIdx,$BASEDN" 502 RC=$? 503 if test $RC != 0 ; then 504 echo "ldapdelete failed ($RC)!" 505 test $KILLSERVERS != no && kill -HUP $KILLPIDS 506 exit 1 507 fi 508 delIdx=`expr $delIdx + 1` 509 done 510 511 for uri in $FWD1_URI $CONS_URI; do 512 echo "Checking replication to $uri" 513 RC=32 514 i=$addStart 515 while test $i -lt $addEnd; do 516 for j in 1 2 3 4 5; do 517 RESULT=`$LDAPSEARCH -H $uri -s base -b "ou=$i,$BASEDN" 2>&1 \ 518 | awk '/^dn:/ {print "OK"}'` 519 if test "x$RESULT$nullOK" = "xOK" ; then 520 RC=0 521 break 522 fi 523 echo "Waiting $j seconds for $uri to receive entry $i..." 524 sleep $j 525 done 526 if test $RC != 0 ; then 527 echo "ERROR: Entry $i not replicated to $uri! ($RC)!" 528 ERROR=1 529 break 530 fi 531 i=`expr $i + 1` 532 done 533 if test $ERROR != 0; then break; fi 534 535 i=$delStart 536 while test $i -lt $delEnd; do 537 for j in 1 2 3 4 5; do 538 $LDAPSEARCH -s base -b "ou=$i,$BASEDN" -H $uri > /dev/null 2>&1 539 RC=$? 540 if test $RC = $noObj; then break; fi 541 echo "Waiting $j seconds for $uri to delete entry $i..." 542 sleep $j 543 done 544 if test $RC != $noObj; then 545 echo "ERROR: Entry $i not removed on $uri! (RC=$RC)" 546 ERROR=1 547 break 548 fi 549 i=`expr $i + 1` 550 done 551 if test $ERROR != 0; then break; fi 552 done 553 if test $ERROR != 0; then break; fi 554 555 echo "Stopping forwarders for delete test" 556 for pid in $FWD1_PID; do 557 kill -HUP $pid 558 wait $pid 559 KILLPIDS=`echo "$KILLPIDS " | sed -e "s/ $pid / /"`; 560 done 561 562 delStart=$delEnd 563 delEnd=`expr $delEnd + $nEntries` 564 565 echo "Using ldapdelete to delete entries on provider" 566 while test $delIdx -lt $delEnd; do 567 $LDAPDELETE -D "$MANAGERDN" -H $PROV_URI -w $PASSWD "ou=$delIdx,$BASEDN" 568 RC=$? 569 if test $RC != 0 ; then 570 echo "ldapdelete failed ($RC)!" 571 test $KILLSERVERS != no && kill -HUP $KILLPIDS 572 exit 1 573 fi 574 delIdx=`expr $delIdx + 1` 575 done 576 577 echo "Starting forwarders again" 578 cd $FWD1_DIR 579 $SLAPD -F slapd.d -h $FWD1_URI -d $LVL >> $FWD1_LOG 2>&1 & 580 FWD1_PID=$! 581 KILLPIDS="$KILLPIDS $FWD1_PID" 582 cd $TESTWD 583 584 delEnd=`expr $delEnd + $nEntries` 585 586 echo "Using ldapdelete to delete $nEntries more entries on provider" 587 while test $delIdx -lt $delEnd; do 588 $LDAPDELETE -D "$MANAGERDN" -H $PROV_URI -w $PASSWD "ou=$delIdx,$BASEDN" 589 RC=$? 590 if test $RC != 0 ; then 591 echo "ldapdelete failed ($RC)!" 592 test $KILLSERVERS != no && kill -HUP $KILLPIDS 593 exit 1 594 fi 595 delIdx=`expr $delIdx + 1` 596 done 597 598 for uri in $FWD1_URI $CONS_URI; do 599 echo "Checking replication to $uri" 600 RC=0 601 i=$delStart 602 while test $i -lt $delEnd; do 603 for j in 1 2 3 4 5; do 604 $LDAPSEARCH -s base -b "ou=$i,$BASEDN" -H $uri > /dev/null 2>&1 605 RC=$? 606 if test $RC = $noObj; then break; fi 607 echo "Waiting $j seconds for $uri to delete entry $i..." 608 sleep $j 609 done 610 if test $RC != $noObj; then 611 echo "ERROR: Entry $i not removed on $uri! (RC=$RC)" 612 ERROR=1 613 break 614 fi 615 i=`expr $i + 1` 616 done 617 if test $ERROR != 0; then break; fi 618 done 619 if test $ERROR != 0; then break; fi 620 621 sleep 1 622 echo "Checking contextCSN" 623 CSN_ERRORS=0 624 CSN1=`$LDAPSEARCH -H $URI1 -b $BASEDN -s base contextCSN | grep contextCSN` 625 CSN2=`$LDAPSEARCH -H $URI2 -b $BASEDN -s base contextCSN | grep contextCSN` 626 CSN3=`$LDAPSEARCH -H $URI3 -b $BASEDN -s base contextCSN | grep contextCSN` 627 628 if test -z "$CSN1" ; then 629 test $BACKEND = null && break 630 echo "ERROR: contextCSN empty on provider" 631 ERROR=1 632 break 633 fi 634 nCSN=`echo "$CSN1" | wc -l` 635 if test "$nCSN" -ne 1 ; then 636 echo "ERROR: Wrong contextCSN count on provider, should be 1" 637 echo "$CSN1" 638 test $KILLSERVERS != no && kill -HUP $KILLPIDS 639 exit 1 640 fi 641 if test -z "$CSN2" -o "$CSN1" != "$CSN2" ; then 642 echo "ERROR: contextCSN mismatch between provider and consumer" 643 echo "contextCSN on provider: $CSN1" 644 echo "contextCSN on consumer: $CSN2" 645 ERROR=1 646 break 647 fi 648 if test -z "$CSN3" -o "$CSN1" != "$CSN3" ; then 649 echo "ERROR: contextCSN mismatch between provider and forward1" 650 echo "contextCSN on provider: $CSN1" 651 echo "contextCSN on forward1: $CSN3" 652 ERROR=1 653 break 654 fi 655done 656 657test $KILLSERVERS != no && kill -HUP $KILLPIDS 658 659if test $ERROR != 0; then 660 echo "Error found after $RACE_NUM of $INITIATION_RACE_TESTS iterations" 661 exit 1 662else 663 echo "No race errors found after $INITIATION_RACE_TESTS iterations" 664fi 665 666echo ">>>>> Test succeeded" 667 668exit 0 669