1#!/usr/bin/env bash 2 3readonly BASEDIR=$(readlink -f $(dirname $0))/.. 4cd $BASEDIR 5 6# exit on errors 7set -e 8 9if ! hash nproc 2> /dev/null; then 10 11 function nproc() { 12 echo 8 13 } 14 15fi 16 17function version_lt() { 18 [ $(echo -e "$1\n$2" | sort -V | head -1) != "$1" ] 19} 20 21function array_contains_string() { 22 name="$1[@]" 23 array=("${!name}") 24 25 for element in "${array[@]}"; do 26 if [ "$element" = "$2" ]; then 27 return $(true) 28 fi 29 done 30 31 return $(false) 32} 33 34rc=0 35 36function check_permissions() { 37 echo -n "Checking file permissions..." 38 39 local rc=0 40 41 while read -r perm _res0 _res1 path; do 42 if [ ! -f "$path" ]; then 43 continue 44 fi 45 46 # Skip symlinks 47 if [[ -L $path ]]; then 48 continue 49 fi 50 fname=$(basename -- "$path") 51 52 case ${fname##*.} in 53 c | h | cpp | cc | cxx | hh | hpp | md | html | js | json | svg | Doxyfile | yml | LICENSE | README | conf | in | Makefile | mk | gitignore | go | txt) 54 # These file types should never be executable 55 if [ "$perm" -eq 100755 ]; then 56 echo "ERROR: $path is marked executable but is a code file." 57 rc=1 58 fi 59 ;; 60 *) 61 shebang=$(head -n 1 $path | cut -c1-3) 62 63 # git only tracks the execute bit, so will only ever return 755 or 644 as the permission. 64 if [ "$perm" -eq 100755 ]; then 65 # If the file has execute permission, it should start with a shebang. 66 if [ "$shebang" != "#!/" ]; then 67 echo "ERROR: $path is marked executable but does not start with a shebang." 68 rc=1 69 fi 70 else 71 # If the file doesnot have execute permissions, it should not start with a shebang. 72 if [ "$shebang" = "#!/" ]; then 73 echo "ERROR: $path is not marked executable but starts with a shebang." 74 rc=1 75 fi 76 fi 77 ;; 78 esac 79 80 done <<< "$(git grep -I --name-only --untracked -e . | git ls-files -s)" 81 82 if [ $rc -eq 0 ]; then 83 echo " OK" 84 fi 85 86 return $rc 87} 88 89function check_c_style() { 90 local rc=0 91 92 if hash astyle; then 93 echo -n "Checking coding style..." 94 if [ "$(astyle -V)" \< "Artistic Style Version 3" ]; then 95 echo -n " Your astyle version is too old so skipping coding style checks. Please update astyle to at least 3.0.1 version..." 96 else 97 rm -f astyle.log 98 touch astyle.log 99 # Exclude rte_vhost code imported from DPDK - we want to keep the original code 100 # as-is to enable ongoing work to synch with a generic upstream DPDK vhost library, 101 # rather than making diffs more complicated by a lot of changes to follow SPDK 102 # coding standards. 103 git ls-files '*.[ch]' '*.cpp' '*.cc' '*.cxx' '*.hh' '*.hpp' \ 104 | grep -v rte_vhost | grep -v cpp_headers \ 105 | xargs -P$(nproc) -n10 astyle --options=.astylerc >> astyle.log 106 if grep -q "^Formatted" astyle.log; then 107 echo " errors detected" 108 git diff --ignore-submodules=all 109 sed -i -e 's/ / /g' astyle.log 110 grep --color=auto "^Formatted.*" astyle.log 111 echo "Incorrect code style detected in one or more files." 112 echo "The files have been automatically formatted." 113 echo "Remember to add the files to your commit." 114 rc=1 115 else 116 echo " OK" 117 fi 118 rm -f astyle.log 119 fi 120 else 121 echo "You do not have astyle installed so your code style is not being checked!" 122 fi 123 return $rc 124} 125 126function check_comment_style() { 127 local rc=0 128 129 echo -n "Checking comment style..." 130 131 git grep --line-number -e '\/[*][^ *-]' -- '*.[ch]' > comment.log || true 132 git grep --line-number -e '[^ ][*]\/' -- '*.[ch]' ':!lib/rte_vhost*/*' >> comment.log || true 133 git grep --line-number -e '^[*]' -- '*.[ch]' >> comment.log || true 134 git grep --line-number -e '\s\/\/' -- '*.[ch]' >> comment.log || true 135 git grep --line-number -e '^\/\/' -- '*.[ch]' >> comment.log || true 136 137 if [ -s comment.log ]; then 138 echo " Incorrect comment formatting detected" 139 cat comment.log 140 rc=1 141 else 142 echo " OK" 143 fi 144 rm -f comment.log 145 146 return $rc 147} 148 149function check_spaces_before_tabs() { 150 local rc=0 151 152 echo -n "Checking for spaces before tabs..." 153 git grep --line-number $' \t' -- './*' ':!*.patch' > whitespace.log || true 154 if [ -s whitespace.log ]; then 155 echo " Spaces before tabs detected" 156 cat whitespace.log 157 rc=1 158 else 159 echo " OK" 160 fi 161 rm -f whitespace.log 162 163 return $rc 164} 165 166function check_trailing_whitespace() { 167 local rc=0 168 169 echo -n "Checking trailing whitespace in output strings..." 170 171 git grep --line-number -e ' \\n"' -- '*.[ch]' > whitespace.log || true 172 173 if [ -s whitespace.log ]; then 174 echo " Incorrect trailing whitespace detected" 175 cat whitespace.log 176 rc=1 177 else 178 echo " OK" 179 fi 180 rm -f whitespace.log 181 182 return $rc 183} 184 185function check_forbidden_functions() { 186 local rc=0 187 188 echo -n "Checking for use of forbidden library functions..." 189 190 git grep --line-number -w '\(atoi\|atol\|atoll\|strncpy\|strcpy\|strcat\|sprintf\|vsprintf\)' -- './*.c' ':!lib/rte_vhost*/**' > badfunc.log || true 191 if [ -s badfunc.log ]; then 192 echo " Forbidden library functions detected" 193 cat badfunc.log 194 rc=1 195 else 196 echo " OK" 197 fi 198 rm -f badfunc.log 199 200 return $rc 201} 202 203function check_cunit_style() { 204 local rc=0 205 206 echo -n "Checking for use of forbidden CUnit macros..." 207 208 git grep --line-number -w 'CU_ASSERT_FATAL' -- 'test/*' ':!test/spdk_cunit.h' > badcunit.log || true 209 if [ -s badcunit.log ]; then 210 echo " Forbidden CU_ASSERT_FATAL usage detected - use SPDK_CU_ASSERT_FATAL instead" 211 cat badcunit.log 212 rc=1 213 else 214 echo " OK" 215 fi 216 rm -f badcunit.log 217 218 return $rc 219} 220 221function check_eof() { 222 local rc=0 223 224 echo -n "Checking blank lines at end of file..." 225 226 if ! git grep -I -l -e . -z './*' ':!*.patch' \ 227 | xargs -0 -P$(nproc) -n1 scripts/eofnl > eofnl.log; then 228 echo " Incorrect end-of-file formatting detected" 229 cat eofnl.log 230 rc=1 231 else 232 echo " OK" 233 fi 234 rm -f eofnl.log 235 236 return $rc 237} 238 239function check_posix_includes() { 240 local rc=0 241 242 echo -n "Checking for POSIX includes..." 243 git grep -I -i -f scripts/posix.txt -- './*' ':!include/spdk/stdinc.h' ':!include/linux/**' ':!lib/rte_vhost*/**' ':!scripts/posix.txt' ':!*.patch' > scripts/posix.log || true 244 if [ -s scripts/posix.log ]; then 245 echo "POSIX includes detected. Please include spdk/stdinc.h instead." 246 cat scripts/posix.log 247 rc=1 248 else 249 echo " OK" 250 fi 251 rm -f scripts/posix.log 252 253 return $rc 254} 255 256function check_naming_conventions() { 257 local rc=0 258 259 echo -n "Checking for proper function naming conventions..." 260 # commit_to_compare = HEAD - 1. 261 commit_to_compare="$(git log --pretty=oneline --skip=1 -n 1 | awk '{print $1}')" 262 failed_naming_conventions=false 263 changed_c_libs=() 264 declared_symbols=() 265 266 # Build an array of all the modified C files. 267 mapfile -t changed_c_libs < <(git diff --name-only HEAD $commit_to_compare -- lib/**/*.c module/**/*.c) 268 # Matching groups are 1. qualifiers / return type. 2. function name 3. argument list / comments and stuff after that. 269 # Capture just the names of newly added (or modified) function definitions. 270 mapfile -t declared_symbols < <(git diff -U0 $commit_to_compare HEAD -- include/spdk*/*.h | sed -En 's/(^[+].*)(spdk[a-z,A-Z,0-9,_]*)(\(.*)/\2/p') 271 272 for c_file in "${changed_c_libs[@]}"; do 273 lib_map_file="mk/spdk_blank.map" 274 defined_symbols=() 275 removed_symbols=() 276 exported_symbols=() 277 if ls "$(dirname $c_file)"/*.map &> /dev/null; then 278 lib_map_file="$(ls "$(dirname $c_file)"/*.map)" 279 fi 280 # Matching groups are 1. leading +sign. 2, function name 3. argument list / anything after that. 281 # Capture just the names of newly added (or modified) functions that start with "spdk_" 282 mapfile -t defined_symbols < <(git diff -U0 $commit_to_compare HEAD -- $c_file | sed -En 's/(^[+])(spdk[a-z,A-Z,0-9,_]*)(\(.*)/\2/p') 283 # Capture the names of removed symbols to catch edge cases where we just move definitions around. 284 mapfile -t removed_symbols < <(git diff -U0 $commit_to_compare HEAD -- $c_file | sed -En 's/(^[-])(spdk[a-z,A-Z,0-9,_]*)(\(.*)/\2/p') 285 for symbol in "${removed_symbols[@]}"; do 286 for i in "${!defined_symbols[@]}"; do 287 if [[ ${defined_symbols[i]} = "$symbol" ]]; then 288 unset -v 'defined_symbols[i]' 289 fi 290 done 291 done 292 # It's possible that we just modified a functions arguments so unfortunately we can't just look at changed lines in this function. 293 # matching groups are 1. All leading whitespace 2. function name. Capture just the symbol name. 294 mapfile -t exported_symbols < <(sed -En 's/(^[[:space:]]*)(spdk[a-z,A-Z,0-9,_]*);/\2/p' < $lib_map_file) 295 for defined_symbol in "${defined_symbols[@]}"; do 296 # if the list of defined symbols is equal to the list of removed symbols, then we are left with a single empty element. skip it. 297 if [ "$defined_symbol" = '' ]; then 298 continue 299 fi 300 not_exported=true 301 not_declared=true 302 if array_contains_string exported_symbols $defined_symbol; then 303 not_exported=false 304 fi 305 306 if array_contains_string declared_symbols $defined_symbol; then 307 not_declared=false 308 fi 309 310 if $not_exported || $not_declared; then 311 if ! $failed_naming_conventions; then 312 echo " found naming convention errors." 313 fi 314 echo "function $defined_symbol starts with spdk_ which is reserved for public API functions." 315 echo "Please add this function to its corresponding map file and a public header or remove the spdk_ prefix." 316 failed_naming_conventions=true 317 rc=1 318 fi 319 done 320 done 321 322 if ! $failed_naming_conventions; then 323 echo " OK" 324 fi 325 326 return $rc 327} 328 329function check_include_style() { 330 local rc=0 331 332 echo -n "Checking #include style..." 333 git grep -I -i --line-number "#include <spdk/" -- '*.[ch]' > scripts/includes.log || true 334 if [ -s scripts/includes.log ]; then 335 echo "Incorrect #include syntax. #includes of spdk/ files should use quotes." 336 cat scripts/includes.log 337 rc=1 338 else 339 echo " OK" 340 fi 341 rm -f scripts/includes.log 342 343 return $rc 344} 345 346function check_python_style() { 347 local rc=0 348 349 if hash pycodestyle 2> /dev/null; then 350 PEP8=pycodestyle 351 elif hash pep8 2> /dev/null; then 352 PEP8=pep8 353 fi 354 355 if [ -n "${PEP8}" ]; then 356 echo -n "Checking Python style..." 357 358 PEP8_ARGS+=" --max-line-length=140" 359 360 error=0 361 git ls-files '*.py' | xargs -P$(nproc) -n1 $PEP8 $PEP8_ARGS > pep8.log || error=1 362 if [ $error -ne 0 ]; then 363 echo " Python formatting errors detected" 364 cat pep8.log 365 rc=1 366 else 367 echo " OK" 368 fi 369 rm -f pep8.log 370 else 371 echo "You do not have pycodestyle or pep8 installed so your Python style is not being checked!" 372 fi 373 374 return $rc 375} 376 377function get_bash_files() { 378 local sh shebang 379 380 mapfile -t sh < <(git ls-files '*.sh') 381 mapfile -t shebang < <(git grep -l '^#!.*bash') 382 383 printf '%s\n' "${sh[@]}" "${shebang[@]}" | sort -u 384} 385 386function check_bash_style() { 387 local rc=0 388 389 # find compatible shfmt binary 390 shfmt_bins=$(compgen -c | grep '^shfmt' || true) 391 for bin in $shfmt_bins; do 392 if version_lt "$("$bin" --version)" "3.1.0"; then 393 shfmt=$bin 394 break 395 fi 396 done 397 398 if [ -n "$shfmt" ]; then 399 shfmt_cmdline=() silly_plural=() 400 401 silly_plural[1]="s" 402 403 commits=() sh_files=() sh_files_repo=() sh_files_staged=() 404 mapfile -t sh_files_repo < <(get_bash_files) 405 # Fetch .sh files only from the commits that are targeted for merge 406 while read -r _ commit; do 407 commits+=("$commit") 408 done < <(git cherry -v origin/master) 409 410 mapfile -t sh_files < <(git diff --name-only HEAD origin/master "${sh_files_repo[@]}") 411 # In case of a call from a pre-commit git hook 412 mapfile -t sh_files_staged < <( 413 IFS="|" 414 git diff --cached --name-only "${sh_files_repo[@]}" | grep -v "${sh_files[*]}" 415 ) 416 417 if ((${#sh_files[@]})); then 418 printf 'Checking .sh formatting style...' 419 420 if ((${#sh_files_staged[@]})); then 421 sh_files+=("${sh_files_staged[@]}") 422 fi 423 424 shfmt_cmdline+=(-i 0) # indent_style = tab|indent_size = 0 425 shfmt_cmdline+=(-bn) # binary_next_line = true 426 shfmt_cmdline+=(-ci) # switch_case_indent = true 427 shfmt_cmdline+=(-ln bash) # shell_variant = bash (default) 428 shfmt_cmdline+=(-d) # diffOut - print diff of the changes and exit with != 0 429 shfmt_cmdline+=(-sr) # redirect operators will be followed by a space 430 431 diff=${output_dir:-$PWD}/$shfmt.patch 432 433 # Explicitly tell shfmt to not look for .editorconfig. .editorconfig is also not looked up 434 # in case any formatting arguments has been passed on its cmdline. 435 if ! SHFMT_NO_EDITORCONFIG=true "$shfmt" "${shfmt_cmdline[@]}" "${sh_files[@]}" > "$diff"; then 436 # In case shfmt detects an actual syntax error it will write out a proper message on 437 # its stderr, hence the diff file should remain empty. 438 if [[ -s $diff ]]; then 439 diff_out=$(< "$diff") 440 fi 441 442 cat <<- ERROR_SHFMT 443 444 * Errors in style formatting have been detected. 445 ${diff_out:+* Please, review the generated patch at $diff 446 447 # _START_OF_THE_DIFF 448 449 ${diff_out:-ERROR} 450 451 # _END_OF_THE_DIFF 452 } 453 454 ERROR_SHFMT 455 rc=1 456 else 457 rm -f "$diff" 458 printf ' OK\n' 459 fi 460 fi 461 else 462 echo "shfmt not detected, Bash style formatting check is skipped" 463 fi 464 465 return $rc 466} 467 468function check_bash_static_analysis() { 469 local rc=0 470 471 if hash shellcheck 2> /dev/null; then 472 echo -n "Checking Bash style..." 473 474 shellcheck_v=$(shellcheck --version | grep -P "version: [0-9\.]+" | cut -d " " -f2) 475 476 # SHCK_EXCLUDE contains a list of all of the spellcheck errors found in SPDK scripts 477 # currently. New errors should only be added to this list if the cost of fixing them 478 # is deemed too high. For more information about the errors, go to: 479 # https://github.com/koalaman/shellcheck/wiki/Checks 480 # Error descriptions can also be found at: https://github.com/koalaman/shellcheck/wiki 481 # SPDK fails some error checks which have been deprecated in later versions of shellcheck. 482 # We will not try to fix these error checks, but instead just leave the error types here 483 # so that we can still run with older versions of shellcheck. 484 SHCK_EXCLUDE="SC1117" 485 # SPDK has decided to not fix violations of these errors. 486 # We are aware about below exclude list and we want this errors to be excluded. 487 # SC1083: This {/} is literal. Check expression (missing ;/\n?) or quote it. 488 # SC1090: Can't follow non-constant source. Use a directive to specify location. 489 # SC1091: Not following: (error message here) 490 # SC2001: See if you can use ${variable//search/replace} instead. 491 # SC2010: Don't use ls | grep. Use a glob or a for loop with a condition to allow non-alphanumeric filenames. 492 # SC2015: Note that A && B || C is not if-then-else. C may run when A is true. 493 # SC2016: Expressions don't expand in single quotes, use double quotes for that. 494 # SC2034: foo appears unused. Verify it or export it. 495 # SC2046: Quote this to prevent word splitting. 496 # SC2086: Double quote to prevent globbing and word splitting. 497 # SC2119: Use foo "$@" if function's $1 should mean script's $1. 498 # SC2120: foo references arguments, but none are ever passed. 499 # SC2148: Add shebang to the top of your script. 500 # SC2153: Possible Misspelling: MYVARIABLE may not be assigned, but MY_VARIABLE is. 501 # SC2154: var is referenced but not assigned. 502 # SC2164: Use cd ... || exit in case cd fails. 503 # SC2174: When used with -p, -m only applies to the deepest directory. 504 # SC2206: Quote to prevent word splitting/globbing, 505 # or split robustly with mapfile or read -a. 506 # SC2207: Prefer mapfile or read -a to split command output (or quote to avoid splitting). 507 # SC2223: This default assignment may cause DoS due to globbing. Quote it. 508 SHCK_EXCLUDE="$SHCK_EXCLUDE,SC1083,SC1090,SC1091,SC2010,SC2015,SC2016,SC2034,SC2046,SC2086,\ 509SC2119,SC2120,SC2148,SC2153,SC2154,SC2164,SC2174,SC2001,SC2206,SC2207,SC2223" 510 511 SHCK_FORMAT="tty" 512 SHCK_APPLY=false 513 SHCH_ARGS=" -x -e $SHCK_EXCLUDE -f $SHCK_FORMAT" 514 515 get_bash_files | xargs -P$(nproc) -n1 shellcheck $SHCH_ARGS &> shellcheck.log 516 if [[ -s shellcheck.log ]]; then 517 echo " Bash formatting errors detected!" 518 519 cat shellcheck.log 520 if $SHCK_APPLY; then 521 git apply shellcheck.log 522 echo "Bash errors were automatically corrected." 523 echo "Please remember to add the changes to your commit." 524 fi 525 rc=1 526 else 527 echo " OK" 528 fi 529 rm -f shellcheck.log 530 else 531 echo "You do not have shellcheck installed so your Bash style is not being checked!" 532 fi 533 534 return $rc 535} 536 537function check_changelog() { 538 local rc=0 539 540 # Check if any of the public interfaces were modified by this patch. 541 # Warn the user to consider updating the changelog any changes 542 # are detected. 543 echo -n "Checking whether CHANGELOG.md should be updated..." 544 staged=$(git diff --name-only --cached .) 545 working=$(git status -s --porcelain --ignore-submodules | grep -iv "??" | awk '{print $2}') 546 files="$staged $working" 547 if [[ "$files" = " " ]]; then 548 files=$(git diff-tree --no-commit-id --name-only -r HEAD) 549 fi 550 551 has_changelog=0 552 for f in $files; do 553 if [[ $f == CHANGELOG.md ]]; then 554 # The user has a changelog entry, so exit. 555 has_changelog=1 556 break 557 fi 558 done 559 560 needs_changelog=0 561 if [ $has_changelog -eq 0 ]; then 562 for f in $files; do 563 if [[ $f == include/spdk/* ]] || [[ $f == scripts/rpc.py ]] || [[ $f == etc/* ]]; then 564 echo "" 565 echo -n "$f was modified. Consider updating CHANGELOG.md." 566 needs_changelog=1 567 fi 568 done 569 fi 570 571 if [ $needs_changelog -eq 0 ]; then 572 echo " OK" 573 else 574 echo "" 575 fi 576 577 return $rc 578} 579 580rc=0 581 582check_permissions || rc=1 583check_c_style || rc=1 584 585GIT_VERSION=$(git --version | cut -d' ' -f3) 586 587if version_lt "1.9.5" "${GIT_VERSION}"; then 588 # git <1.9.5 doesn't support pathspec magic exclude 589 echo " Your git version is too old to perform all tests. Please update git to at least 1.9.5 version..." 590 exit $rc 591fi 592 593check_comment_style || rc=1 594check_spaces_before_tabs || rc=1 595check_trailing_whitespace || rc=1 596check_forbidden_functions || rc=1 597check_cunit_style || rc=1 598check_eof || rc=1 599check_posix_includes || rc=1 600check_naming_conventions || rc=1 601check_include_style || rc=1 602check_python_style || rc=1 603check_bash_style || rc=1 604check_bash_static_analysis || rc=1 605check_changelog || rc=1 606 607exit $rc 608