1#!/usr/bin/env bash 2 3if [[ $(uname -s) == Darwin ]]; then 4 # SPDK is not supported on MacOS, but as a developer 5 # convenience we support running the check_format.sh 6 # script on MacOS. 7 # Running "brew install bash greadlink ggrep" should be 8 # sufficient to get the correct versions of these utilities. 9 if [[ $(type -t mapfile) != builtin ]]; then 10 # We need bash version >= 4.0 for mapfile builtin 11 echo "Please install bash version >= 4.0" 12 exit 1 13 fi 14 if ! hash greadlink 2> /dev/null; then 15 # We need GNU readlink for -f option 16 echo "Please install GNU readlink" 17 exit 1 18 fi 19 if ! hash ggrep 2> /dev/null; then 20 # We need GNU grep for -P option 21 echo "Please install GNU grep" 22 exit 1 23 fi 24 GNU_READLINK="greadlink" 25 GNU_GREP="ggrep" 26else 27 GNU_READLINK="readlink" 28 GNU_GREP="grep" 29fi 30 31rootdir=$($GNU_READLINK -f "$(dirname "$0")")/.. 32source "$rootdir/scripts/common.sh" 33 34cd "$rootdir" 35 36# exit on errors 37set -e 38 39if ! hash nproc 2> /dev/null; then 40 41 function nproc() { 42 echo 8 43 } 44 45fi 46 47function version_lt() { 48 [ $(echo -e "$1\n$2" | sort -V | head -1) != "$1" ] 49} 50 51function array_contains_string() { 52 name="$1[@]" 53 array=("${!name}") 54 55 for element in "${array[@]}"; do 56 if [ "$element" = "$2" ]; then 57 return $(true) 58 fi 59 done 60 61 return $(false) 62} 63 64rc=0 65 66function check_permissions() { 67 echo -n "Checking file permissions..." 68 69 local rc=0 70 71 while read -r perm _res0 _res1 path; do 72 if [ ! -f "$path" ]; then 73 continue 74 fi 75 76 # Skip symlinks 77 if [[ -L $path ]]; then 78 continue 79 fi 80 fname=$(basename -- "$path") 81 82 case ${fname##*.} in 83 c | h | cpp | cc | cxx | hh | hpp | md | html | js | json | svg | Doxyfile | yml | LICENSE | README | conf | in | Makefile | mk | gitignore | go | txt) 84 # These file types should never be executable 85 if [ "$perm" -eq 100755 ]; then 86 echo "ERROR: $path is marked executable but is a code file." 87 rc=1 88 fi 89 ;; 90 *) 91 shebang=$(head -n 1 $path | cut -c1-3) 92 93 # git only tracks the execute bit, so will only ever return 755 or 644 as the permission. 94 if [ "$perm" -eq 100755 ]; then 95 # If the file has execute permission, it should start with a shebang. 96 if [ "$shebang" != "#!/" ]; then 97 echo "ERROR: $path is marked executable but does not start with a shebang." 98 rc=1 99 fi 100 else 101 # If the file does not have execute permissions, it should not start with a shebang. 102 if [ "$shebang" = "#!/" ]; then 103 echo "ERROR: $path is not marked executable but starts with a shebang." 104 rc=1 105 fi 106 fi 107 ;; 108 esac 109 110 done <<< "$(git grep -I --name-only --untracked -e . | git ls-files -s)" 111 112 if [ $rc -eq 0 ]; then 113 echo " OK" 114 fi 115 116 return $rc 117} 118 119function check_c_style() { 120 local rc=0 121 122 if hash astyle; then 123 echo -n "Checking coding style..." 124 if [ "$(astyle -V)" \< "Artistic Style Version 3" ]; then 125 echo -n " Your astyle version is too old so skipping coding style checks. Please update astyle to at least 3.0.1 version..." 126 else 127 rm -f astyle.log 128 touch astyle.log 129 # Exclude rte_vhost code imported from DPDK - we want to keep the original code 130 # as-is to enable ongoing work to synch with a generic upstream DPDK vhost library, 131 # rather than making diffs more complicated by a lot of changes to follow SPDK 132 # coding standards. 133 git ls-files '*.[ch]' \ 134 | grep -v rte_vhost | grep -v cpp_headers \ 135 | xargs -P$(nproc) -n10 astyle --break-return-type --attach-return-type-decl \ 136 --options=.astylerc >> astyle.log 137 git ls-files '*.cpp' '*.cc' '*.cxx' '*.hh' '*.hpp' \ 138 | grep -v rte_vhost | grep -v cpp_headers \ 139 | xargs -P$(nproc) -n10 astyle --options=.astylerc >> astyle.log 140 if grep -q "^Formatted" astyle.log; then 141 echo " errors detected" 142 git diff --ignore-submodules=all 143 sed -i -e 's/ / /g' astyle.log 144 grep --color=auto "^Formatted.*" astyle.log 145 echo "Incorrect code style detected in one or more files." 146 echo "The files have been automatically formatted." 147 echo "Remember to add the files to your commit." 148 rc=1 149 else 150 echo " OK" 151 fi 152 rm -f astyle.log 153 fi 154 else 155 echo "You do not have astyle installed so your code style is not being checked!" 156 fi 157 return $rc 158} 159 160function check_comment_style() { 161 local rc=0 162 163 echo -n "Checking comment style..." 164 165 git grep --line-number -e '\/[*][^ *-]' -- '*.[ch]' > comment.log || true 166 git grep --line-number -e '[^ ][*]\/' -- '*.[ch]' ':!lib/rte_vhost*/*' >> comment.log || true 167 git grep --line-number -e '^[*]' -- '*.[ch]' >> comment.log || true 168 git grep --line-number -e '\s\/\/' -- '*.[ch]' >> comment.log || true 169 git grep --line-number -e '^\/\/' -- '*.[ch]' >> comment.log || true 170 171 if [ -s comment.log ]; then 172 echo " Incorrect comment formatting detected" 173 cat comment.log 174 rc=1 175 else 176 echo " OK" 177 fi 178 rm -f comment.log 179 180 return $rc 181} 182 183function check_spaces_before_tabs() { 184 local rc=0 185 186 echo -n "Checking for spaces before tabs..." 187 git grep --line-number $' \t' -- './*' ':!*.patch' > whitespace.log || true 188 if [ -s whitespace.log ]; then 189 echo " Spaces before tabs detected" 190 cat whitespace.log 191 rc=1 192 else 193 echo " OK" 194 fi 195 rm -f whitespace.log 196 197 return $rc 198} 199 200function check_trailing_whitespace() { 201 local rc=0 202 203 echo -n "Checking trailing whitespace in output strings..." 204 205 git grep --line-number -e ' \\n"' -- '*.[ch]' > whitespace.log || true 206 207 if [ -s whitespace.log ]; then 208 echo " Incorrect trailing whitespace detected" 209 cat whitespace.log 210 rc=1 211 else 212 echo " OK" 213 fi 214 rm -f whitespace.log 215 216 return $rc 217} 218 219function check_forbidden_functions() { 220 local rc=0 221 222 echo -n "Checking for use of forbidden library functions..." 223 224 git grep --line-number -w '\(atoi\|atol\|atoll\|strncpy\|strcpy\|strcat\|sprintf\|vsprintf\)' -- './*.c' ':!lib/rte_vhost*/**' > badfunc.log || true 225 if [ -s badfunc.log ]; then 226 echo " Forbidden library functions detected" 227 cat badfunc.log 228 rc=1 229 else 230 echo " OK" 231 fi 232 rm -f badfunc.log 233 234 return $rc 235} 236 237function check_cunit_style() { 238 local rc=0 239 240 echo -n "Checking for use of forbidden CUnit macros..." 241 242 git grep --line-number -w 'CU_ASSERT_FATAL' -- 'test/*' ':!test/spdk_cunit.h' > badcunit.log || true 243 if [ -s badcunit.log ]; then 244 echo " Forbidden CU_ASSERT_FATAL usage detected - use SPDK_CU_ASSERT_FATAL instead" 245 cat badcunit.log 246 rc=1 247 else 248 echo " OK" 249 fi 250 rm -f badcunit.log 251 252 return $rc 253} 254 255function check_eof() { 256 local rc=0 257 258 echo -n "Checking blank lines at end of file..." 259 260 if ! git grep -I -l -e . -z './*' ':!*.patch' \ 261 | xargs -0 -P$(nproc) -n1 scripts/eofnl > eofnl.log; then 262 echo " Incorrect end-of-file formatting detected" 263 cat eofnl.log 264 rc=1 265 else 266 echo " OK" 267 fi 268 rm -f eofnl.log 269 270 return $rc 271} 272 273function check_posix_includes() { 274 local rc=0 275 276 echo -n "Checking for POSIX includes..." 277 git grep -I -i -f scripts/posix.txt -- './*' ':!include/spdk/stdinc.h' ':!include/linux/**' ':!lib/rte_vhost*/**' ':!scripts/posix.txt' ':!*.patch' > scripts/posix.log || true 278 if [ -s scripts/posix.log ]; then 279 echo "POSIX includes detected. Please include spdk/stdinc.h instead." 280 cat scripts/posix.log 281 rc=1 282 else 283 echo " OK" 284 fi 285 rm -f scripts/posix.log 286 287 return $rc 288} 289 290function check_naming_conventions() { 291 local rc=0 292 293 echo -n "Checking for proper function naming conventions..." 294 # commit_to_compare = HEAD - 1. 295 commit_to_compare="$(git log --pretty=oneline --skip=1 -n 1 | awk '{print $1}')" 296 failed_naming_conventions=false 297 changed_c_libs=() 298 declared_symbols=() 299 300 # Build an array of all the modified C libraries. 301 mapfile -t changed_c_libs < <(git diff --name-only HEAD $commit_to_compare -- lib/**/*.c module/**/*.c | xargs -r dirname | sort | uniq) 302 # Matching groups are 1. qualifiers / return type. 2. function name 3. argument list / comments and stuff after that. 303 # Capture just the names of newly added (or modified) function definitions. 304 mapfile -t declared_symbols < <(git diff -U0 $commit_to_compare HEAD -- include/spdk*/*.h | sed -En 's/(^[+].*)(spdk[a-z,A-Z,0-9,_]*)(\(.*)/\2/p') 305 306 for c_lib in "${changed_c_libs[@]}"; do 307 lib_map_file="mk/spdk_blank.map" 308 defined_symbols=() 309 removed_symbols=() 310 exported_symbols=() 311 if ls "$c_lib"/*.map &> /dev/null; then 312 lib_map_file="$(ls "$c_lib"/*.map)" 313 fi 314 # Matching groups are 1. leading +sign. 2, function name 3. argument list / anything after that. 315 # Capture just the names of newly added (or modified) functions that start with "spdk_" 316 mapfile -t defined_symbols < <(git diff -U0 $commit_to_compare HEAD -- $c_lib | sed -En 's/(^[+])(spdk[a-z,A-Z,0-9,_]*)(\(.*)/\2/p') 317 # Capture the names of removed symbols to catch edge cases where we just move definitions around. 318 mapfile -t removed_symbols < <(git diff -U0 $commit_to_compare HEAD -- $c_lib | sed -En 's/(^[-])(spdk[a-z,A-Z,0-9,_]*)(\(.*)/\2/p') 319 for symbol in "${removed_symbols[@]}"; do 320 for i in "${!defined_symbols[@]}"; do 321 if [[ ${defined_symbols[i]} = "$symbol" ]]; then 322 unset -v 'defined_symbols[i]' 323 fi 324 done 325 done 326 # It's possible that we just modified a functions arguments so unfortunately we can't just look at changed lines in this function. 327 # matching groups are 1. All leading whitespace 2. function name. Capture just the symbol name. 328 mapfile -t exported_symbols < <(sed -En 's/(^[[:space:]]*)(spdk[a-z,A-Z,0-9,_]*);/\2/p' < $lib_map_file) 329 for defined_symbol in "${defined_symbols[@]}"; do 330 # if the list of defined symbols is equal to the list of removed symbols, then we are left with a single empty element. skip it. 331 if [ "$defined_symbol" = '' ]; then 332 continue 333 fi 334 not_exported=true 335 not_declared=true 336 if array_contains_string exported_symbols $defined_symbol; then 337 not_exported=false 338 fi 339 340 if array_contains_string declared_symbols $defined_symbol; then 341 not_declared=false 342 fi 343 344 if $not_exported || $not_declared; then 345 if ! $failed_naming_conventions; then 346 echo " found naming convention errors." 347 fi 348 echo "function $defined_symbol starts with spdk_ which is reserved for public API functions." 349 echo "Please add this function to its corresponding map file and a public header or remove the spdk_ prefix." 350 failed_naming_conventions=true 351 rc=1 352 fi 353 done 354 done 355 356 if ! $failed_naming_conventions; then 357 echo " OK" 358 fi 359 360 return $rc 361} 362 363function check_include_style() { 364 local rc=0 365 366 echo -n "Checking #include style..." 367 git grep -I -i --line-number "#include <spdk/" -- '*.[ch]' > scripts/includes.log || true 368 if [ -s scripts/includes.log ]; then 369 echo "Incorrect #include syntax. #includes of spdk/ files should use quotes." 370 cat scripts/includes.log 371 rc=1 372 else 373 echo " OK" 374 fi 375 rm -f scripts/includes.log 376 377 return $rc 378} 379 380function check_python_style() { 381 local rc=0 382 383 if hash pycodestyle 2> /dev/null; then 384 PEP8=pycodestyle 385 elif hash pep8 2> /dev/null; then 386 PEP8=pep8 387 fi 388 389 if [ -n "${PEP8}" ]; then 390 echo -n "Checking Python style..." 391 392 PEP8_ARGS+=" --max-line-length=140" 393 394 error=0 395 git ls-files '*.py' | xargs -P$(nproc) -n1 $PEP8 $PEP8_ARGS > pep8.log || error=1 396 if [ $error -ne 0 ]; then 397 echo " Python formatting errors detected" 398 cat pep8.log 399 rc=1 400 else 401 echo " OK" 402 fi 403 rm -f pep8.log 404 else 405 echo "You do not have pycodestyle or pep8 installed so your Python style is not being checked!" 406 fi 407 408 return $rc 409} 410 411function get_bash_files() { 412 local sh shebang 413 414 mapfile -t sh < <(git ls-files '*.sh') 415 mapfile -t shebang < <(git grep -l '^#!.*bash') 416 417 printf '%s\n' "${sh[@]}" "${shebang[@]}" | sort -u 418} 419 420function check_bash_style() { 421 local rc=0 422 423 # find compatible shfmt binary 424 shfmt_bins=$(compgen -c | grep '^shfmt' | uniq || true) 425 for bin in $shfmt_bins; do 426 shfmt_version=$("$bin" --version) 427 if [ $shfmt_version != "v3.1.0" ]; then 428 echo "$bin version $shfmt_version not used (only v3.1.0 is supported)" 429 echo "v3.1.0 can be installed using 'scripts/pkgdep.sh -d'" 430 else 431 shfmt=$bin 432 break 433 fi 434 done 435 436 if [ -n "$shfmt" ]; then 437 shfmt_cmdline=() sh_files=() 438 439 mapfile -t sh_files < <(get_bash_files) 440 441 if ((${#sh_files[@]})); then 442 printf 'Checking .sh formatting style...' 443 444 shfmt_cmdline+=(-i 0) # indent_style = tab|indent_size = 0 445 shfmt_cmdline+=(-bn) # binary_next_line = true 446 shfmt_cmdline+=(-ci) # switch_case_indent = true 447 shfmt_cmdline+=(-ln bash) # shell_variant = bash (default) 448 shfmt_cmdline+=(-d) # diffOut - print diff of the changes and exit with != 0 449 shfmt_cmdline+=(-sr) # redirect operators will be followed by a space 450 451 diff=${output_dir:-$PWD}/$shfmt.patch 452 453 # Explicitly tell shfmt to not look for .editorconfig. .editorconfig is also not looked up 454 # in case any formatting arguments has been passed on its cmdline. 455 if ! SHFMT_NO_EDITORCONFIG=true "$shfmt" "${shfmt_cmdline[@]}" "${sh_files[@]}" > "$diff"; then 456 # In case shfmt detects an actual syntax error it will write out a proper message on 457 # its stderr, hence the diff file should remain empty. 458 if [[ -s $diff ]]; then 459 diff_out=$(< "$diff") 460 fi 461 462 cat <<- ERROR_SHFMT 463 464 * Errors in style formatting have been detected. 465 ${diff_out:+* Please, review the generated patch at $diff 466 467 # _START_OF_THE_DIFF 468 469 ${diff_out:-ERROR} 470 471 # _END_OF_THE_DIFF 472 } 473 474 ERROR_SHFMT 475 rc=1 476 else 477 rm -f "$diff" 478 printf ' OK\n' 479 fi 480 fi 481 else 482 echo "Supported version of shfmt not detected, Bash style formatting check is skipped" 483 fi 484 485 return $rc 486} 487 488function check_bash_static_analysis() { 489 local rc=0 490 491 if hash shellcheck 2> /dev/null; then 492 echo -n "Checking Bash static analysis with shellcheck..." 493 494 shellcheck_v=$(shellcheck --version | grep -P "version: [0-9\.]+" | cut -d " " -f2) 495 496 # SHCK_EXCLUDE contains a list of all of the spellcheck errors found in SPDK scripts 497 # currently. New errors should only be added to this list if the cost of fixing them 498 # is deemed too high. For more information about the errors, go to: 499 # https://github.com/koalaman/shellcheck/wiki/Checks 500 # Error descriptions can also be found at: https://github.com/koalaman/shellcheck/wiki 501 # SPDK fails some error checks which have been deprecated in later versions of shellcheck. 502 # We will not try to fix these error checks, but instead just leave the error types here 503 # so that we can still run with older versions of shellcheck. 504 SHCK_EXCLUDE="SC1117" 505 # SPDK has decided to not fix violations of these errors. 506 # We are aware about below exclude list and we want this errors to be excluded. 507 # SC1083: This {/} is literal. Check expression (missing ;/\n?) or quote it. 508 # SC1090: Can't follow non-constant source. Use a directive to specify location. 509 # SC1091: Not following: (error message here) 510 # SC2001: See if you can use ${variable//search/replace} instead. 511 # SC2010: Don't use ls | grep. Use a glob or a for loop with a condition to allow non-alphanumeric filenames. 512 # SC2015: Note that A && B || C is not if-then-else. C may run when A is true. 513 # SC2016: Expressions don't expand in single quotes, use double quotes for that. 514 # SC2034: foo appears unused. Verify it or export it. 515 # SC2046: Quote this to prevent word splitting. 516 # SC2086: Double quote to prevent globbing and word splitting. 517 # SC2119: Use foo "$@" if function's $1 should mean script's $1. 518 # SC2120: foo references arguments, but none are ever passed. 519 # SC2128: Expanding an array without an index only gives the first element. 520 # SC2148: Add shebang to the top of your script. 521 # SC2153: Possible Misspelling: MYVARIABLE may not be assigned, but MY_VARIABLE is. 522 # SC2154: var is referenced but not assigned. 523 # SC2164: Use cd ... || exit in case cd fails. 524 # SC2174: When used with -p, -m only applies to the deepest directory. 525 # SC2178: Variable was used as an array but is now assigned a string. 526 # SC2206: Quote to prevent word splitting/globbing, 527 # or split robustly with mapfile or read -a. 528 # SC2207: Prefer mapfile or read -a to split command output (or quote to avoid splitting). 529 # SC2223: This default assignment may cause DoS due to globbing. Quote it. 530 SHCK_EXCLUDE="$SHCK_EXCLUDE,SC1083,SC1090,SC1091,SC2010,SC2015,SC2016,SC2034,SC2046,SC2086,\ 531SC2119,SC2120,SC2128,SC2148,SC2153,SC2154,SC2164,SC2174,SC2178,SC2001,SC2206,SC2207,SC2223" 532 533 SHCK_FORMAT="tty" 534 SHCK_APPLY=false 535 SHCH_ARGS="-e $SHCK_EXCLUDE -f $SHCK_FORMAT" 536 537 if ge "$shellcheck_v" 0.4.0; then 538 SHCH_ARGS+=" -x" 539 else 540 echo "shellcheck $shellcheck_v detected, recommended >= 0.4.0." 541 fi 542 543 get_bash_files | xargs -P$(nproc) -n1 shellcheck $SHCH_ARGS &> shellcheck.log 544 if [[ -s shellcheck.log ]]; then 545 echo " Bash shellcheck errors detected!" 546 547 cat shellcheck.log 548 if $SHCK_APPLY; then 549 git apply shellcheck.log 550 echo "Bash errors were automatically corrected." 551 echo "Please remember to add the changes to your commit." 552 fi 553 rc=1 554 else 555 echo " OK" 556 fi 557 rm -f shellcheck.log 558 else 559 echo "You do not have shellcheck installed so your Bash static analysis is not being performed!" 560 fi 561 562 return $rc 563} 564 565function check_changelog() { 566 local rc=0 567 568 # Check if any of the public interfaces were modified by this patch. 569 # Warn the user to consider updating the changelog any changes 570 # are detected. 571 echo -n "Checking whether CHANGELOG.md should be updated..." 572 staged=$(git diff --name-only --cached .) 573 working=$(git status -s --porcelain --ignore-submodules | grep -iv "??" | awk '{print $2}') 574 files="$staged $working" 575 if [[ "$files" = " " ]]; then 576 files=$(git diff-tree --no-commit-id --name-only -r HEAD) 577 fi 578 579 has_changelog=0 580 for f in $files; do 581 if [[ $f == CHANGELOG.md ]]; then 582 # The user has a changelog entry, so exit. 583 has_changelog=1 584 break 585 fi 586 done 587 588 needs_changelog=0 589 if [ $has_changelog -eq 0 ]; then 590 for f in $files; do 591 if [[ $f == include/spdk/* ]] || [[ $f == scripts/rpc.py ]] || [[ $f == etc/* ]]; then 592 echo "" 593 echo -n "$f was modified. Consider updating CHANGELOG.md." 594 needs_changelog=1 595 fi 596 done 597 fi 598 599 if [ $needs_changelog -eq 0 ]; then 600 echo " OK" 601 else 602 echo "" 603 fi 604 605 return $rc 606} 607 608function check_json_rpc() { 609 local rc=0 610 611 echo -n "Checking that all RPCs are documented..." 612 while IFS='"' read -r _ rpc _; do 613 if ! grep -q "^### $rpc" doc/jsonrpc.md; then 614 echo "Missing JSON-RPC documentation for ${rpc}" 615 rc=1 616 fi 617 done < <(git grep -h -E "^SPDK_RPC_REGISTER\(" ':!test/*') 618 619 if [ $rc -eq 0 ]; then 620 echo " OK" 621 fi 622 return $rc 623} 624 625function check_markdown_format() { 626 local rc=0 627 628 if hash mdl 2> /dev/null; then 629 echo -n "Checking markdown files format..." 630 mdl -g -s $rootdir/mdl_rules.rb . > mdl.log || true 631 if [ -s mdl.log ]; then 632 echo " Errors in .md files detected:" 633 cat mdl.log 634 rc=1 635 else 636 echo " OK" 637 fi 638 rm -f mdl.log 639 else 640 echo "You do not have markdownlint installed so .md files not being checked!" 641 fi 642 643 return $rc 644} 645 646function check_rpc_args() { 647 local rc=0 648 649 echo -n "Checking rpc.py argument option names..." 650 grep add_argument scripts/rpc.py | $GNU_GREP -oP "(?<=--)[a-z0-9\-\_]*(?=\')" | grep "_" > badargs.log 651 652 if [[ -s badargs.log ]]; then 653 echo "rpc.py arguments with underscores detected!" 654 cat badargs.log 655 echo "Please convert the underscores to dashes." 656 rc=1 657 else 658 echo " OK" 659 fi 660 rm -f badargs.log 661 return $rc 662} 663 664rc=0 665 666check_permissions || rc=1 667check_c_style || rc=1 668 669GIT_VERSION=$(git --version | cut -d' ' -f3) 670 671if version_lt "1.9.5" "${GIT_VERSION}"; then 672 # git <1.9.5 doesn't support pathspec magic exclude 673 echo " Your git version is too old to perform all tests. Please update git to at least 1.9.5 version..." 674 exit $rc 675fi 676 677check_comment_style || rc=1 678check_markdown_format || rc=1 679check_spaces_before_tabs || rc=1 680check_trailing_whitespace || rc=1 681check_forbidden_functions || rc=1 682check_cunit_style || rc=1 683check_eof || rc=1 684check_posix_includes || rc=1 685check_naming_conventions || rc=1 686check_include_style || rc=1 687check_python_style || rc=1 688check_bash_style || rc=1 689check_bash_static_analysis || rc=1 690check_changelog || rc=1 691check_json_rpc || rc=1 692check_rpc_args || rc=1 693 694exit $rc 695