1#!/bin/sh 2 3# Tests a set of patches from a directory. 4# Copyright (C) 2007, 2008 Free Software Foundation, Inc. 5# Contributed by Sebastian Pop <sebastian.pop@amd.com> 6 7# This program is free software; you can redistribute it and/or modify 8# it under the terms of the GNU General Public License as published by 9# the Free Software Foundation; either version 3 of the License, or 10# (at your option) any later version. 11 12# This program is distributed in the hope that it will be useful, 13# but WITHOUT ANY WARRANTY; without even the implied warranty of 14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15# GNU General Public License for more details. 16 17# You should have received a copy of the GNU General Public License 18# along with this program; if not, write to the Free Software 19# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 20 21cat <<EOF 22 23WARNING: This script should only be fed with patches from known 24 authorized and trusted sources. Don't even think about 25 hooking it up to a raw feed from the gcc-patches list or 26 you'll regret it. 27 28EOF 29 30args=$@ 31 32svnpath=svn://gcc.gnu.org/svn/gcc 33dashj= 34default_standby=1 35standby=$default_standby 36default_watermark=0.60 37watermark=$default_watermark 38savecompilers=false 39nogpg=false 40stop=false 41 42usage() { 43 cat <<EOF 44patch_tester.sh [-j<N>] [-standby N] [-watermark N] [-savecompilers] [-nogpg] 45 [-svnpath URL] [-stop] 46 <source_dir> [patches_dir [state_dir [build_dir]]] 47 48 J is the flag passed to make. Default is empty string. 49 50 STANDBY is the number of minutes between checks for new patches in 51 PATCHES_DIR. Default is ${default_standby} minutes. 52 53 WATERMARK is the 5 minute average system charge under which a new 54 compile can start. Default is ${default_watermark}. 55 56 SAVECOMPILERS copies the compilers in the same directory as the 57 test results for the non patched version. Default is not copy. 58 59 NOGPG can be used to avoid checking the GPG signature of patches. 60 61 URL is the location of the GCC SVN repository. The default is 62 ${svnpath}. 63 64 STOP exits when PATCHES_DIR is empty. 65 66 SOURCE_DIR is the directory containing GCC's toplevel configure. 67 68 PATCHES_DIR is the directory containing the patches to be tested. 69 Default is SOURCE_DIR/patches. 70 71 STATE_DIR is where the tester maintains its internal state. 72 Default is SOURCE_DIR/state. 73 74 BUILD_DIR is the build tree, a temporary directory that this 75 script will delete and recreate. Default is SOURCE_DIR/obj. 76 77EOF 78 exit 1 79} 80 81makedir () { 82 DIRNAME=$1 83 mkdir -p $DIRNAME 84 if [ $? -ne 0 ]; then 85 echo "ERROR: could not make directory $DIRNAME" 86 exit 1 87 fi 88} 89 90while [ $# -ne 0 ]; do 91 case $1 in 92 -j*) 93 dashj=$1; shift 94 ;; 95 -standby) 96 [[ $# > 2 ]] || usage 97 standby=$2; shift; shift 98 ;; 99 -watermark) 100 [[ $# > 2 ]] || usage 101 watermark=$2; shift; shift 102 ;; 103 -savecompilers) 104 savecompilers=true; shift 105 ;; 106 -nogpg) 107 nogpg=true; shift 108 ;; 109 -stop) 110 stop=true; shift 111 ;; 112 -svnpath) 113 svnpath=$2; shift; shift 114 ;; 115 -*) 116 echo "Invalid option: $1" 117 usage 118 ;; 119 *) 120 break 121 ;; 122 esac 123done 124 125test $# -eq 0 && usage 126 127SOURCE=$1 128PATCHES= 129STATE= 130BUILD= 131 132if [[ $# < 2 ]]; then 133 PATCHES=$SOURCE/patches 134else 135 PATCHES=$2 136fi 137if [[ $# < 3 ]]; then 138 STATE=$SOURCE/state 139else 140 STATE=$3 141fi 142if [[ $# < 4 ]]; then 143 BUILD=$SOURCE/obj 144else 145 BUILD=$4 146fi 147 148[ -d $PATCHES ] || makedir $PATCHES 149[ -d $STATE ] || makedir $STATE 150[ -d $STATE/patched ] || makedir $STATE/patched 151[ -d $SOURCE ] || makedir $SOURCE 152[ -f $SOURCE/config.guess ] || { 153 cd $SOURCE 154 svn -q co $svnpath/trunk . 155 if [ $? -ne 0 ]; then 156 echo "ERROR: initial svn checkout failed" 157 exit 1 158 fi 159} 160 161# This can contain required local settings: 162# default_config configure options, always passed 163# default_make make bootstrap options, always passed 164# default_check make check options, always passed 165[ -f $STATE/defaults ] && . $STATE/defaults 166 167VERSION=`svn info $SOURCE | grep "^Revision:" | sed -e "s/^Revision://g" -e "s/ //g"` 168 169exec >> $STATE/tester.log 2>&1 || exit 1 170set -x 171 172TESTING=$STATE/testing 173REPORT=$TESTING/report 174PRISTINE=$TESTING/pristine 175PATCHED=$TESTING/patched 176PATCH= 177TARGET=`$SOURCE/config.guess || exit 1` 178TESTLOGS="gcc/testsuite/gcc/gcc.sum 179gcc/testsuite/gfortran/gfortran.sum 180gcc/testsuite/g++/g++.sum 181gcc/testsuite/objc/objc.sum 182$TARGET/libstdc++-v3/testsuite/libstdc++.sum 183$TARGET/libffi/testsuite/libffi.sum 184$TARGET/libjava/testsuite/libjava.sum 185$TARGET/libgomp/testsuite/libgomp.sum 186$TARGET/libmudflap/testsuite/libmudflap.sum" 187COMPILERS="gcc/cc1 188gcc/cc1obj 189gcc/cc1plus 190gcc/f951 191gcc/jc1 192gcc/gnat1 193gcc/tree1" 194 195now () { 196 echo `TZ=UTC date +"%Y_%m_%d_%H_%M_%S"` 197} 198 199report () { 200 echo "$@" >> $REPORT 201} 202 203freport () { 204 if [ -s $1 ]; then 205 report "(cat $1" 206 cat $1 >> $REPORT 207 report "tac)" 208 fi 209} 210 211cleanup () { 212 cd $SOURCE 213 svn cleanup && svn revert -R . && svn st | cut -d' ' -f5- | xargs rm -v 214} 215 216selfexec () { 217 exec ${CONFIG_SHELL-/bin/sh} $0 $args 218} 219 220update () { 221 svn_branch=`grep "^branch:" $PATCH | sed -e "s/^branch://g" -e "s/ //g"` 222 if [ x$svn_branch = x ]; then 223 svn_branch=trunk 224 fi 225 226 svn_revision=`grep "^revision:" $PATCH | sed -e "s/^revision://g" -e "s/ //g"` 227 if [ x$svn_revision = x ]; then 228 svn_revision=HEAD 229 fi 230 231 cleanup 232 cd $SOURCE 233 case $svn_branch in 234 trunk) 235 if ! svn switch -r $svn_revision $svnpath/trunk &> $TESTING/svn ; then 236 report "failed to update svn sources with" 237 report "svn switch -r $svn_revision $svnpath/trunk" 238 freport $TESTING/svn 239 return 1 240 fi 241 ;; 242 243 ${svnpath}*) 244 if ! svn switch -r $svn_revision $svn_branch &> $TESTING/svn ; then 245 report "failed to update svn sources with" 246 report "svn switch -r $svn_revision $svn_branch" 247 freport $TESTING/svn 248 return 1 249 fi 250 ;; 251 252 *) 253 if ! svn switch -r $svn_revision $svnpath/branches/$svn_branch &> $TESTING/svn ; then 254 report "failed to update svn sources with" 255 report "svn switch -r $svn_revision $svnpath/branches/$svn_branch" 256 freport $TESTING/svn 257 return 1 258 fi 259 ;; 260 esac 261 contrib/gcc_update --touch 262 263 current_version=`svn info $SOURCE | grep "^Revision:" | sed -e "s/^Revision://g" -e "s/ //g"` 264 if [[ $VERSION < $current_version ]]; then 265 if [ -f $SOURCE/contrib/patch_tester.sh ]; then 266 selfexec 267 fi 268 fi 269 270 return 0 271} 272 273apply_patch () { 274 if [ $nogpg = false ]; then 275 if ! gpg --batch --verify $PATCH &> $TESTING/gpgverify ; then 276 report "your patch failed to verify:" 277 freport $TESTING/gpgverify 278 return 1 279 fi 280 fi 281 282 cd $SOURCE 283 if ! patch -p0 < $PATCH &> $TESTING/patching ; then 284 report "your patch failed to apply:" 285 report "(check that the patch was created at the top level)" 286 freport $TESTING/patching 287 return 1 288 fi 289 290 # Just assume indexes for now -- not really great, but svn always 291 # makes them. 292 grep "^Index: " $PATCH | sed -e 's/Index: //' | while read file; do 293 # If the patch resulted in an empty file, delete it. 294 # This is how svn reports deletions. 295 if [ ! -s $file ]; then 296 rm -f $file 297 report "Deleting empty file $file" 298 fi 299 done 300} 301 302save_compilers () { 303 for COMPILER in $COMPILERS ; do 304 if [ -f $BUILD/$COMPILER ]; then 305 cp $BUILD/$COMPILER $PRISTINE 306 fi 307 done 308} 309 310bootntest () { 311 rm -rf $BUILD 312 mkdir $BUILD 313 cd $BUILD 314 315 CONFIG_OPTIONS=`grep "^configure:" $PATCH | sed -e "s/^configure://g"` 316 CONFIG_OPTIONS="$default_config $CONFIG_OPTIONS" 317 if ! eval $SOURCE/configure $CONFIG_OPTIONS &> $1/configure ; then 318 report "configure with `basename $1` version failed with:" 319 freport $1/configure 320 return 1 321 fi 322 323 MAKE_ARGS=`grep "^make:" $PATCH | sed -e "s/^make://g"` 324 MAKE_ARGS="$default_make $MAKE_ARGS" 325 if ! eval make $dashj $MAKE_ARGS &> $1/bootstrap ; then 326 report "bootstrap with `basename $1` version failed with last lines:" 327 tail -30 $1/bootstrap > $1/last_bootstrap 328 freport $1/last_bootstrap 329 report "grep --context=20 Error bootstrap:" 330 grep --context=20 Error $1/bootstrap > $1/bootstrap_error 331 freport $1/bootstrap_error 332 return 1 333 fi 334 335 CHECK_OPTIONS=`grep "^check:" $PATCH | sed -e "s/^check://g"` 336 CHECK_OPTIONS="$default_check $CHECK_OPTIONS" 337 eval make $dashj $CHECK_OPTIONS -k check &> $1/check 338 339 SUITESRUN="`grep 'Summary ===' $1/check | cut -d' ' -f 2 | sort`" 340 if [ x$SUITESRUN = x ]; then 341 report "check with `basename $1` version failed, no testsuites were run" 342 return 1 343 fi 344 345 for LOG in $TESTLOGS ; do 346 if [ -f $BUILD/$LOG ]; then 347 mv $BUILD/$LOG $1 348 mv `echo "$BUILD/$LOG" | sed -e "s/\.sum/\.log/g"` $1 349 fi 350 done 351 352 return 0 353} 354 355bootntest_patched () { 356 cleanup 357 mkdir -p $PATCHED 358 apply_patch && bootntest $PATCHED 359 return $? 360} 361 362# Build the pristine tree with exactly the same options as the patch under test. 363bootntest_pristine () { 364 cleanup 365 current_branch=`svn info $SOURCE | grep "^URL:" | sed -e "s/URL: //g" -e "s,${svnpath},,g"` 366 current_version=`svn info $SOURCE | grep "^Revision:" | sed -e "s/^Revision://g" -e "s/ //g"` 367 PRISTINE=$STATE/$current_branch/$current_version 368 369 if [ -d $PRISTINE ]; then 370 ln -s $PRISTINE $TESTING/pristine 371 return 0 372 else 373 mkdir -p $PRISTINE 374 ln -s $PRISTINE $TESTING/pristine 375 bootntest $PRISTINE 376 RETVAL=$? 377 if [ $RETVAL = 0 -a $savecompilers = true ]; then 378 save_compilers 379 fi 380 return $RETVAL 381 fi 382} 383 384regtest () { 385 touch $1/report 386 touch $1/passes 387 touch $1/failed 388 touch $1/regress 389 390 for LOG in $TESTLOGS ; do 391 NLOG=`basename $LOG` 392 if [ -f $1/$NLOG ]; then 393 awk '/^FAIL: / { print "'$NLOG'",$2; }' $1/$NLOG 394 fi 395 done | sort | uniq > $1/failed 396 397 comm -12 $1/failed $1/passes >> $1/regress 398 NUMREGRESS=`wc -l < $1/regress | tr -d ' '` 399 400 if [ $NUMREGRESS -eq 0 ] ; then 401 for LOG in $TESTLOGS ; do 402 NLOG=`basename $LOG` 403 if [ -f $1/$NLOG ] ; then 404 awk '/^PASS: / { print "'$NLOG'",$2; }' $1/$NLOG 405 fi 406 done | sort | uniq | comm -23 - $1/failed > $1/passes 407 echo "there are no regressions with your patch." >> $1/report 408 else 409 echo "with your patch there are $NUMREGRESS regressions." >> $1/report 410 echo "list of regressions with your patch:" >> $1/report 411 cat $1/regress >> $1/report 412 fi 413} 414 415contrib_compare_tests () { 416 report "comparing logs with contrib/compare_tests:" 417 for LOG in $TESTLOGS ; do 418 NLOG=`basename $LOG` 419 if [ -f $PRISTINE/$NLOG -a -f $PATCHED/$NLOG ]; then 420 $SOURCE/contrib/compare_tests $PRISTINE/$NLOG $PATCHED/$NLOG > $TESTING/compare_$NLOG 421 freport $TESTING/compare_$NLOG 422 fi 423 done 424} 425 426compare_passes () { 427 regtest $PRISTINE 428 cp $PRISTINE/passes $PATCHED 429 regtest $PATCHED 430 freport $PATCHED/report 431 report "FAILs with patched version:" 432 freport $PATCHED/failed 433 report "FAILs with pristine version:" 434 freport $PRISTINE/failed 435 436 # contrib_compare_tests 437} 438 439write_report () { 440 backup_patched=$STATE/patched/`now` 441 report "The files used for the validation of your patch are stored in $backup_patched on the tester machine." 442 443 EMAIL=`grep "^email:" $PATCH | sed -e "s/^email://g" -e "s/ //g"` 444 if [ x$EMAIL != x ]; then 445 mutt -s "[regtest] Results for `basename $PATCH` on $TARGET" -i $REPORT -a $PATCH $EMAIL 446 fi 447 448 mv $TESTING $backup_patched 449} 450 451announce () { 452 EMAIL=`grep "^email:" $PATCH | sed -e "s/^email://g" -e "s/ //g"` 453 if [ x$EMAIL != x ]; then 454 455 START_REPORT=$TESTING/start_report 456 echo "Hi, " >> $START_REPORT 457 echo "I'm the automatic tester running on $TARGET." >> $START_REPORT 458 echo "I just started to look at your patch `basename $PATCH`." >> $START_REPORT 459 echo "Bye, your automatic tester." >> $START_REPORT 460 mutt -s "[regtest] Starting bootstrap for `basename $PATCH` on $TARGET" -i $START_REPORT $EMAIL 461 fi 462} 463 464# After selfexec, $TESTING is already set up. 465if [ -d $TESTING ]; then 466 # The only file in $TESTING is the patch. 467 PATCH=`ls -rt -1 $TESTING | head -1` 468 PATCH=$TESTING/$PATCH 469 if [ -f $PATCH ]; then 470 bootntest_patched && bootntest_pristine && compare_passes 471 write_report 472 fi 473fi 474 475firstpatch=true 476while true; do 477 PATCH=`ls -rt -1 $PATCHES | head -1` 478 if [ x$PATCH = x ]; then 479 if [ $stop = true ]; then 480 if [ $firstpatch = true ]; then 481 echo "No patches ready to test, quitting." 482 exit 1 483 else 484 echo "No more patches to test." 485 exit 0 486 fi 487 fi 488 sleep ${standby}m 489 else 490 firstpatch=false 491 sysload=`uptime | cut -d, -f 5` 492 if [[ $sysload > $watermark ]]; then 493 # Wait a bit when system load is too high. 494 sleep ${standby}m 495 else 496 mkdir -p $TESTING 497 mv $PATCHES/$PATCH $TESTING/ 498 PATCH=$TESTING/$PATCH 499 500 announce 501 update && bootntest_patched && bootntest_pristine && compare_passes 502 write_report 503 fi 504 fi 505done 506