1#!/usr/bin/env bash 2 3readonly BASEDIR=$(readlink -f $(dirname $0))/.. 4cd $BASEDIR 5 6# exit on errors 7set -e 8 9if ! hash nproc 2> /dev/null; then 10 11 function nproc() { 12 echo 8 13 } 14 15fi 16 17function version_lt() { 18 [ $(echo -e "$1\n$2" | sort -V | head -1) != "$1" ] 19} 20 21function array_contains_string() { 22 name="$1[@]" 23 array=("${!name}") 24 25 for element in "${array[@]}"; do 26 if [ "$element" = "$2" ]; then 27 return $(true) 28 fi 29 done 30 31 return $(false) 32} 33 34rc=0 35 36function check_permissions() { 37 echo -n "Checking file permissions..." 38 39 local rc=0 40 41 while read -r perm _res0 _res1 path; do 42 if [ ! -f "$path" ]; then 43 continue 44 fi 45 46 # Skip symlinks 47 if [[ -L $path ]]; then 48 continue 49 fi 50 fname=$(basename -- "$path") 51 52 case ${fname##*.} in 53 c | h | cpp | cc | cxx | hh | hpp | md | html | js | json | svg | Doxyfile | yml | LICENSE | README | conf | in | Makefile | mk | gitignore | go | txt) 54 # These file types should never be executable 55 if [ "$perm" -eq 100755 ]; then 56 echo "ERROR: $path is marked executable but is a code file." 57 rc=1 58 fi 59 ;; 60 *) 61 shebang=$(head -n 1 $path | cut -c1-3) 62 63 # git only tracks the execute bit, so will only ever return 755 or 644 as the permission. 64 if [ "$perm" -eq 100755 ]; then 65 # If the file has execute permission, it should start with a shebang. 66 if [ "$shebang" != "#!/" ]; then 67 echo "ERROR: $path is marked executable but does not start with a shebang." 68 rc=1 69 fi 70 else 71 # If the file doesnot have execute permissions, it should not start with a shebang. 72 if [ "$shebang" = "#!/" ]; then 73 echo "ERROR: $path is not marked executable but starts with a shebang." 74 rc=1 75 fi 76 fi 77 ;; 78 esac 79 80 done <<< "$(git grep -I --name-only --untracked -e . | git ls-files -s)" 81 82 if [ $rc -eq 0 ]; then 83 echo " OK" 84 fi 85 86 return $rc 87} 88 89function check_c_style() { 90 local rc=0 91 92 if hash astyle; then 93 echo -n "Checking coding style..." 94 if [ "$(astyle -V)" \< "Artistic Style Version 3" ]; then 95 echo -n " Your astyle version is too old so skipping coding style checks. Please update astyle to at least 3.0.1 version..." 96 else 97 rm -f astyle.log 98 touch astyle.log 99 # Exclude rte_vhost code imported from DPDK - we want to keep the original code 100 # as-is to enable ongoing work to synch with a generic upstream DPDK vhost library, 101 # rather than making diffs more complicated by a lot of changes to follow SPDK 102 # coding standards. 103 git ls-files '*.[ch]' '*.cpp' '*.cc' '*.cxx' '*.hh' '*.hpp' \ 104 | grep -v rte_vhost | grep -v cpp_headers \ 105 | xargs -P$(nproc) -n10 astyle --options=.astylerc >> astyle.log 106 if grep -q "^Formatted" astyle.log; then 107 echo " errors detected" 108 git diff --ignore-submodules=all 109 sed -i -e 's/ / /g' astyle.log 110 grep --color=auto "^Formatted.*" astyle.log 111 echo "Incorrect code style detected in one or more files." 112 echo "The files have been automatically formatted." 113 echo "Remember to add the files to your commit." 114 rc=1 115 else 116 echo " OK" 117 fi 118 rm -f astyle.log 119 fi 120 else 121 echo "You do not have astyle installed so your code style is not being checked!" 122 fi 123 return $rc 124} 125 126function check_comment_style() { 127 local rc=0 128 129 echo -n "Checking comment style..." 130 131 git grep --line-number -e '\/[*][^ *-]' -- '*.[ch]' > comment.log || true 132 git grep --line-number -e '[^ ][*]\/' -- '*.[ch]' ':!lib/rte_vhost*/*' >> comment.log || true 133 git grep --line-number -e '^[*]' -- '*.[ch]' >> comment.log || true 134 git grep --line-number -e '\s\/\/' -- '*.[ch]' >> comment.log || true 135 git grep --line-number -e '^\/\/' -- '*.[ch]' >> comment.log || true 136 137 if [ -s comment.log ]; then 138 echo " Incorrect comment formatting detected" 139 cat comment.log 140 rc=1 141 else 142 echo " OK" 143 fi 144 rm -f comment.log 145 146 return $rc 147} 148 149function check_spaces_before_tabs() { 150 local rc=0 151 152 echo -n "Checking for spaces before tabs..." 153 git grep --line-number $' \t' -- './*' ':!*.patch' > whitespace.log || true 154 if [ -s whitespace.log ]; then 155 echo " Spaces before tabs detected" 156 cat whitespace.log 157 rc=1 158 else 159 echo " OK" 160 fi 161 rm -f whitespace.log 162 163 return $rc 164} 165 166function check_trailing_whitespace() { 167 local rc=0 168 169 echo -n "Checking trailing whitespace in output strings..." 170 171 git grep --line-number -e ' \\n"' -- '*.[ch]' > whitespace.log || true 172 173 if [ -s whitespace.log ]; then 174 echo " Incorrect trailing whitespace detected" 175 cat whitespace.log 176 rc=1 177 else 178 echo " OK" 179 fi 180 rm -f whitespace.log 181 182 return $rc 183} 184 185function check_forbidden_functions() { 186 local rc=0 187 188 echo -n "Checking for use of forbidden library functions..." 189 190 git grep --line-number -w '\(atoi\|atol\|atoll\|strncpy\|strcpy\|strcat\|sprintf\|vsprintf\)' -- './*.c' ':!lib/rte_vhost*/**' > badfunc.log || true 191 if [ -s badfunc.log ]; then 192 echo " Forbidden library functions detected" 193 cat badfunc.log 194 rc=1 195 else 196 echo " OK" 197 fi 198 rm -f badfunc.log 199 200 return $rc 201} 202 203function check_cunit_style() { 204 local rc=0 205 206 echo -n "Checking for use of forbidden CUnit macros..." 207 208 git grep --line-number -w 'CU_ASSERT_FATAL' -- 'test/*' ':!test/spdk_cunit.h' > badcunit.log || true 209 if [ -s badcunit.log ]; then 210 echo " Forbidden CU_ASSERT_FATAL usage detected - use SPDK_CU_ASSERT_FATAL instead" 211 cat badcunit.log 212 rc=1 213 else 214 echo " OK" 215 fi 216 rm -f badcunit.log 217 218 return $rc 219} 220 221function check_eof() { 222 local rc=0 223 224 echo -n "Checking blank lines at end of file..." 225 226 if ! git grep -I -l -e . -z './*' ':!*.patch' \ 227 | xargs -0 -P$(nproc) -n1 scripts/eofnl > eofnl.log; then 228 echo " Incorrect end-of-file formatting detected" 229 cat eofnl.log 230 rc=1 231 else 232 echo " OK" 233 fi 234 rm -f eofnl.log 235 236 return $rc 237} 238 239function check_posix_includes() { 240 local rc=0 241 242 echo -n "Checking for POSIX includes..." 243 git grep -I -i -f scripts/posix.txt -- './*' ':!include/spdk/stdinc.h' ':!include/linux/**' ':!lib/rte_vhost*/**' ':!scripts/posix.txt' ':!*.patch' > scripts/posix.log || true 244 if [ -s scripts/posix.log ]; then 245 echo "POSIX includes detected. Please include spdk/stdinc.h instead." 246 cat scripts/posix.log 247 rc=1 248 else 249 echo " OK" 250 fi 251 rm -f scripts/posix.log 252 253 return $rc 254} 255 256function check_naming_conventions() { 257 local rc=0 258 259 echo -n "Checking for proper function naming conventions..." 260 # commit_to_compare = HEAD - 1. 261 commit_to_compare="$(git log --pretty=oneline --skip=1 -n 1 | awk '{print $1}')" 262 failed_naming_conventions=false 263 changed_c_libs=() 264 declared_symbols=() 265 266 # Build an array of all the modified C files. 267 mapfile -t changed_c_libs < <(git diff --name-only HEAD $commit_to_compare -- lib/**/*.c module/**/*.c) 268 # Matching groups are 1. qualifiers / return type. 2. function name 3. argument list / comments and stuff after that. 269 # Capture just the names of newly added (or modified) function definitions. 270 mapfile -t declared_symbols < <(git diff -U0 $commit_to_compare HEAD -- include/spdk*/*.h | sed -En 's/(^[+].*)(spdk[a-z,A-Z,0-9,_]*)(\(.*)/\2/p') 271 272 for c_file in "${changed_c_libs[@]}"; do 273 lib_map_file="mk/spdk_blank.map" 274 defined_symbols=() 275 removed_symbols=() 276 exported_symbols=() 277 if ls "$(dirname $c_file)"/*.map &> /dev/null; then 278 lib_map_file="$(ls "$(dirname $c_file)"/*.map)" 279 fi 280 # Matching groups are 1. leading +sign. 2, function name 3. argument list / anything after that. 281 # Capture just the names of newly added (or modified) functions that start with "spdk_" 282 mapfile -t defined_symbols < <(git diff -U0 $commit_to_compare HEAD -- $c_file | sed -En 's/(^[+])(spdk[a-z,A-Z,0-9,_]*)(\(.*)/\2/p') 283 # Capture the names of removed symbols to catch edge cases where we just move definitions around. 284 mapfile -t removed_symbols < <(git diff -U0 $commit_to_compare HEAD -- $c_file | sed -En 's/(^[-])(spdk[a-z,A-Z,0-9,_]*)(\(.*)/\2/p') 285 for symbol in "${removed_symbols[@]}"; do 286 for i in "${!defined_symbols[@]}"; do 287 if [[ ${defined_symbols[i]} = "$symbol" ]]; then 288 unset -v 'defined_symbols[i]' 289 fi 290 done 291 done 292 # It's possible that we just modified a functions arguments so unfortunately we can't just look at changed lines in this function. 293 # matching groups are 1. All leading whitespace 2. function name. Capture just the symbol name. 294 mapfile -t exported_symbols < <(sed -En 's/(^[[:space:]]*)(spdk[a-z,A-Z,0-9,_]*);/\2/p' < $lib_map_file) 295 for defined_symbol in "${defined_symbols[@]}"; do 296 # if the list of defined symbols is equal to the list of removed symbols, then we are left with a single empty element. skip it. 297 if [ "$defined_symbol" = '' ]; then 298 continue 299 fi 300 not_exported=true 301 not_declared=true 302 if array_contains_string exported_symbols $defined_symbol; then 303 not_exported=false 304 fi 305 306 if array_contains_string declared_symbols $defined_symbol; then 307 not_declared=false 308 fi 309 310 if $not_exported || $not_declared; then 311 if ! $failed_naming_conventions; then 312 echo " found naming convention errors." 313 fi 314 echo "function $defined_symbol starts with spdk_ which is reserved for public API functions." 315 echo "Please add this function to its corresponding map file and a public header or remove the spdk_ prefix." 316 failed_naming_conventions=true 317 rc=1 318 fi 319 done 320 done 321 322 if ! $failed_naming_conventions; then 323 echo " OK" 324 fi 325 326 return $rc 327} 328 329function check_include_style() { 330 local rc=0 331 332 echo -n "Checking #include style..." 333 git grep -I -i --line-number "#include <spdk/" -- '*.[ch]' > scripts/includes.log || true 334 if [ -s scripts/includes.log ]; then 335 echo "Incorrect #include syntax. #includes of spdk/ files should use quotes." 336 cat scripts/includes.log 337 rc=1 338 else 339 echo " OK" 340 fi 341 rm -f scripts/includes.log 342 343 return $rc 344} 345 346function check_python_style() { 347 local rc=0 348 349 if hash pycodestyle 2> /dev/null; then 350 PEP8=pycodestyle 351 elif hash pep8 2> /dev/null; then 352 PEP8=pep8 353 fi 354 355 if [ -n "${PEP8}" ]; then 356 echo -n "Checking Python style..." 357 358 PEP8_ARGS+=" --max-line-length=140" 359 360 error=0 361 git ls-files '*.py' | xargs -P$(nproc) -n1 $PEP8 $PEP8_ARGS > pep8.log || error=1 362 if [ $error -ne 0 ]; then 363 echo " Python formatting errors detected" 364 cat pep8.log 365 rc=1 366 else 367 echo " OK" 368 fi 369 rm -f pep8.log 370 else 371 echo "You do not have pycodestyle or pep8 installed so your Python style is not being checked!" 372 fi 373 374 return $rc 375} 376 377function check_bash_style() { 378 local rc=0 379 380 # find compatible shfmt binary 381 shfmt_bins=$(compgen -c | grep '^shfmt' || true) 382 for bin in $shfmt_bins; do 383 if version_lt "$("$bin" --version)" "3.1.0"; then 384 shfmt=$bin 385 break 386 fi 387 done 388 389 if [ -n "$shfmt" ]; then 390 shfmt_cmdline=() silly_plural=() 391 392 silly_plural[1]="s" 393 394 commits=() sh_files=() sh_files_repo=() sh_files_staged=() 395 396 mapfile -t sh_files_repo < <(git ls-files '*.sh') 397 # Fetch .sh files only from the commits that are targeted for merge 398 while read -r _ commit; do 399 commits+=("$commit") 400 done < <(git cherry -v origin/master) 401 402 mapfile -t sh_files < <(git diff --name-only HEAD origin/master "${sh_files_repo[@]}") 403 # In case of a call from a pre-commit git hook 404 mapfile -t sh_files_staged < <( 405 IFS="|" 406 git diff --cached --name-only "${sh_files_repo[@]}" | grep -v "${sh_files[*]}" 407 ) 408 409 if ((${#sh_files[@]})); then 410 printf 'Checking .sh formatting style...' 411 412 if ((${#sh_files_staged[@]})); then 413 sh_files+=("${sh_files_staged[@]}") 414 fi 415 416 shfmt_cmdline+=(-i 0) # indent_style = tab|indent_size = 0 417 shfmt_cmdline+=(-bn) # binary_next_line = true 418 shfmt_cmdline+=(-ci) # switch_case_indent = true 419 shfmt_cmdline+=(-ln bash) # shell_variant = bash (default) 420 shfmt_cmdline+=(-d) # diffOut - print diff of the changes and exit with != 0 421 shfmt_cmdline+=(-sr) # redirect operators will be followed by a space 422 423 diff=${output_dir:-$PWD}/$shfmt.patch 424 425 # Explicitly tell shfmt to not look for .editorconfig. .editorconfig is also not looked up 426 # in case any formatting arguments has been passed on its cmdline. 427 if ! SHFMT_NO_EDITORCONFIG=true "$shfmt" "${shfmt_cmdline[@]}" "${sh_files[@]}" > "$diff"; then 428 # In case shfmt detects an actual syntax error it will write out a proper message on 429 # its stderr, hence the diff file should remain empty. 430 if [[ -s $diff ]]; then 431 diff_out=$(< "$diff") 432 fi 433 434 cat <<- ERROR_SHFMT 435 436 * Errors in style formatting have been detected. 437 ${diff_out:+* Please, review the generated patch at $diff 438 439 # _START_OF_THE_DIFF 440 441 ${diff_out:-ERROR} 442 443 # _END_OF_THE_DIFF 444 } 445 446 ERROR_SHFMT 447 rc=1 448 else 449 rm -f "$diff" 450 printf ' OK\n' 451 fi 452 fi 453 else 454 echo "shfmt not detected, Bash style formatting check is skipped" 455 fi 456 457 return $rc 458} 459 460function check_bash_static_analysis() { 461 local rc=0 462 463 if hash shellcheck 2> /dev/null; then 464 echo -n "Checking Bash style..." 465 466 shellcheck_v=$(shellcheck --version | grep -P "version: [0-9\.]+" | cut -d " " -f2) 467 468 # SHCK_EXCLUDE contains a list of all of the spellcheck errors found in SPDK scripts 469 # currently. New errors should only be added to this list if the cost of fixing them 470 # is deemed too high. For more information about the errors, go to: 471 # https://github.com/koalaman/shellcheck/wiki/Checks 472 # Error descriptions can also be found at: https://github.com/koalaman/shellcheck/wiki 473 # SPDK fails some error checks which have been deprecated in later versions of shellcheck. 474 # We will not try to fix these error checks, but instead just leave the error types here 475 # so that we can still run with older versions of shellcheck. 476 SHCK_EXCLUDE="SC1117" 477 # SPDK has decided to not fix violations of these errors. 478 # We are aware about below exclude list and we want this errors to be excluded. 479 # SC1083: This {/} is literal. Check expression (missing ;/\n?) or quote it. 480 # SC1090: Can't follow non-constant source. Use a directive to specify location. 481 # SC1091: Not following: (error message here) 482 # SC2001: See if you can use ${variable//search/replace} instead. 483 # SC2010: Don't use ls | grep. Use a glob or a for loop with a condition to allow non-alphanumeric filenames. 484 # SC2015: Note that A && B || C is not if-then-else. C may run when A is true. 485 # SC2016: Expressions don't expand in single quotes, use double quotes for that. 486 # SC2034: foo appears unused. Verify it or export it. 487 # SC2046: Quote this to prevent word splitting. 488 # SC2086: Double quote to prevent globbing and word splitting. 489 # SC2119: Use foo "$@" if function's $1 should mean script's $1. 490 # SC2120: foo references arguments, but none are ever passed. 491 # SC2148: Add shebang to the top of your script. 492 # SC2153: Possible Misspelling: MYVARIABLE may not be assigned, but MY_VARIABLE is. 493 # SC2154: var is referenced but not assigned. 494 # SC2164: Use cd ... || exit in case cd fails. 495 # SC2174: When used with -p, -m only applies to the deepest directory. 496 # SC2206: Quote to prevent word splitting/globbing, 497 # or split robustly with mapfile or read -a. 498 # SC2207: Prefer mapfile or read -a to split command output (or quote to avoid splitting). 499 # SC2223: This default assignment may cause DoS due to globbing. Quote it. 500 SHCK_EXCLUDE="$SHCK_EXCLUDE,SC1083,SC1090,SC1091,SC2010,SC2015,SC2016,SC2034,SC2046,SC2086,\ 501SC2119,SC2120,SC2148,SC2153,SC2154,SC2164,SC2174,SC2001,SC2206,SC2207,SC2223" 502 503 SHCK_FORMAT="tty" 504 SHCK_APPLY=false 505 SHCH_ARGS=" -x -e $SHCK_EXCLUDE -f $SHCK_FORMAT" 506 507 git ls-files '*.sh' | xargs -P$(nproc) -n1 shellcheck $SHCH_ARGS &> shellcheck.log 508 if [[ -s shellcheck.log ]]; then 509 echo " Bash formatting errors detected!" 510 511 cat shellcheck.log 512 if $SHCK_APPLY; then 513 git apply shellcheck.log 514 echo "Bash errors were automatically corrected." 515 echo "Please remember to add the changes to your commit." 516 fi 517 rc=1 518 else 519 echo " OK" 520 fi 521 rm -f shellcheck.log 522 else 523 echo "You do not have shellcheck installed so your Bash style is not being checked!" 524 fi 525 526 return $rc 527} 528 529function check_changelog() { 530 local rc=0 531 532 # Check if any of the public interfaces were modified by this patch. 533 # Warn the user to consider updating the changelog any changes 534 # are detected. 535 echo -n "Checking whether CHANGELOG.md should be updated..." 536 staged=$(git diff --name-only --cached .) 537 working=$(git status -s --porcelain --ignore-submodules | grep -iv "??" | awk '{print $2}') 538 files="$staged $working" 539 if [[ "$files" = " " ]]; then 540 files=$(git diff-tree --no-commit-id --name-only -r HEAD) 541 fi 542 543 has_changelog=0 544 for f in $files; do 545 if [[ $f == CHANGELOG.md ]]; then 546 # The user has a changelog entry, so exit. 547 has_changelog=1 548 break 549 fi 550 done 551 552 needs_changelog=0 553 if [ $has_changelog -eq 0 ]; then 554 for f in $files; do 555 if [[ $f == include/spdk/* ]] || [[ $f == scripts/rpc.py ]] || [[ $f == etc/* ]]; then 556 echo "" 557 echo -n "$f was modified. Consider updating CHANGELOG.md." 558 needs_changelog=1 559 fi 560 done 561 fi 562 563 if [ $needs_changelog -eq 0 ]; then 564 echo " OK" 565 else 566 echo "" 567 fi 568 569 return $rc 570} 571 572rc=0 573 574check_permissions || rc=1 575check_c_style || rc=1 576 577GIT_VERSION=$(git --version | cut -d' ' -f3) 578 579if version_lt "1.9.5" "${GIT_VERSION}"; then 580 # git <1.9.5 doesn't support pathspec magic exclude 581 echo " Your git version is too old to perform all tests. Please update git to at least 1.9.5 version..." 582 exit $rc 583fi 584 585check_comment_style || rc=1 586check_spaces_before_tabs || rc=1 587check_trailing_whitespace || rc=1 588check_forbidden_functions || rc=1 589check_cunit_style || rc=1 590check_eof || rc=1 591check_posix_includes || rc=1 592check_naming_conventions || rc=1 593check_include_style || rc=1 594check_python_style || rc=1 595check_bash_style || rc=1 596check_bash_static_analysis || rc=1 597check_changelog || rc=1 598 599exit $rc 600