1#!/usr/bin/env bash 2 3if [[ $(uname -s) == Darwin ]]; then 4 # SPDK is not supported on MacOS, but as a developer 5 # convenience we support running the check_format.sh 6 # script on MacOS. 7 # Running "brew install bash coreutils grep" should be 8 # sufficient to get the correct versions of these utilities. 9 if [[ $(type -t mapfile) != builtin ]]; then 10 # We need bash version >= 4.0 for mapfile builtin 11 echo "Please install bash version >= 4.0" 12 exit 1 13 fi 14 if ! hash greadlink 2> /dev/null; then 15 # We need GNU readlink for -f option 16 echo "Please install GNU readlink" 17 exit 1 18 fi 19 if ! hash ggrep 2> /dev/null; then 20 # We need GNU grep for -P option 21 echo "Please install GNU grep" 22 exit 1 23 fi 24 GNU_READLINK="greadlink" 25 GNU_GREP="ggrep" 26else 27 GNU_READLINK="readlink" 28 GNU_GREP="grep" 29fi 30 31rootdir=$($GNU_READLINK -f "$(dirname "$0")")/.. 32source "$rootdir/scripts/common.sh" 33 34cd "$rootdir" 35 36# exit on errors 37set -e 38 39if ! hash nproc 2> /dev/null; then 40 41 function nproc() { 42 echo 8 43 } 44 45fi 46 47function version_lt() { 48 [ $(echo -e "$1\n$2" | sort -V | head -1) != "$1" ] 49} 50 51function array_contains_string() { 52 name="$1[@]" 53 array=("${!name}") 54 55 for element in "${array[@]}"; do 56 if [ "$element" = "$2" ]; then 57 return $(true) 58 fi 59 done 60 61 return $(false) 62} 63 64rc=0 65 66function check_permissions() { 67 echo -n "Checking file permissions..." 68 69 local rc=0 70 71 while read -r perm _res0 _res1 path; do 72 if [ ! -f "$path" ]; then 73 continue 74 fi 75 76 # Skip symlinks 77 if [[ -L $path ]]; then 78 continue 79 fi 80 fname=$(basename -- "$path") 81 82 case ${fname##*.} in 83 c | h | cpp | cc | cxx | hh | hpp | md | html | js | json | svg | Doxyfile | yml | LICENSE | README | conf | in | Makefile | mk | gitignore | go | txt) 84 # These file types should never be executable 85 if [ "$perm" -eq 100755 ]; then 86 echo "ERROR: $path is marked executable but is a code file." 87 rc=1 88 fi 89 ;; 90 *) 91 shebang=$(head -n 1 $path | cut -c1-3) 92 93 # git only tracks the execute bit, so will only ever return 755 or 644 as the permission. 94 if [ "$perm" -eq 100755 ]; then 95 # If the file has execute permission, it should start with a shebang. 96 if [ "$shebang" != "#!/" ]; then 97 echo "ERROR: $path is marked executable but does not start with a shebang." 98 rc=1 99 fi 100 else 101 # If the file does not have execute permissions, it should not start with a shebang. 102 if [ "$shebang" = "#!/" ]; then 103 echo "ERROR: $path is not marked executable but starts with a shebang." 104 rc=1 105 fi 106 fi 107 ;; 108 esac 109 110 done <<< "$(git grep -I --name-only --untracked -e . | git ls-files -s)" 111 112 if [ $rc -eq 0 ]; then 113 echo " OK" 114 fi 115 116 return $rc 117} 118 119function check_c_style() { 120 local rc=0 121 122 if hash astyle; then 123 echo -n "Checking coding style..." 124 if [ "$(astyle -V)" \< "Artistic Style Version 3" ]; then 125 echo -n " Your astyle version is too old so skipping coding style checks. Please update astyle to at least 3.0.1 version..." 126 else 127 rm -f astyle.log 128 touch astyle.log 129 # Exclude DPDK header files copied into our tree 130 git ls-files '*.[ch]' ':!:*/env_dpdk/*/*.h' \ 131 | xargs -P$(nproc) -n10 astyle --break-return-type --attach-return-type-decl \ 132 --options=.astylerc >> astyle.log 133 git ls-files '*.cpp' '*.cc' '*.cxx' '*.hh' '*.hpp' \ 134 | xargs -P$(nproc) -n10 astyle --options=.astylerc >> astyle.log 135 if grep -q "^Formatted" astyle.log; then 136 echo " errors detected" 137 git diff --ignore-submodules=all 138 sed -i -e 's/ / /g' astyle.log 139 grep --color=auto "^Formatted.*" astyle.log 140 echo "Incorrect code style detected in one or more files." 141 echo "The files have been automatically formatted." 142 echo "Remember to add the files to your commit." 143 rc=1 144 else 145 echo " OK" 146 fi 147 rm -f astyle.log 148 fi 149 else 150 echo "You do not have astyle installed so your code style is not being checked!" 151 fi 152 return $rc 153} 154 155function check_comment_style() { 156 local rc=0 157 158 echo -n "Checking comment style..." 159 160 git grep --line-number -e '\/[*][^ *-]' -- '*.[ch]' > comment.log || true 161 git grep --line-number -e '[^ ][*]\/' -- '*.[ch]' ':!lib/rte_vhost*/*' >> comment.log || true 162 git grep --line-number -e '^[*]' -- '*.[ch]' >> comment.log || true 163 git grep --line-number -e '\s\/\/' -- '*.[ch]' >> comment.log || true 164 git grep --line-number -e '^\/\/' -- '*.[ch]' >> comment.log || true 165 166 if [ -s comment.log ]; then 167 echo " Incorrect comment formatting detected" 168 cat comment.log 169 rc=1 170 else 171 echo " OK" 172 fi 173 rm -f comment.log 174 175 return $rc 176} 177 178function check_spaces_before_tabs() { 179 local rc=0 180 181 echo -n "Checking for spaces before tabs..." 182 git grep --line-number $' \t' -- './*' ':!*.patch' > whitespace.log || true 183 if [ -s whitespace.log ]; then 184 echo " Spaces before tabs detected" 185 cat whitespace.log 186 rc=1 187 else 188 echo " OK" 189 fi 190 rm -f whitespace.log 191 192 return $rc 193} 194 195function check_trailing_whitespace() { 196 local rc=0 197 198 echo -n "Checking trailing whitespace in output strings..." 199 200 git grep --line-number -e ' \\n"' -- '*.[ch]' > whitespace.log || true 201 202 if [ -s whitespace.log ]; then 203 echo " Incorrect trailing whitespace detected" 204 cat whitespace.log 205 rc=1 206 else 207 echo " OK" 208 fi 209 rm -f whitespace.log 210 211 return $rc 212} 213 214function check_forbidden_functions() { 215 local rc=0 216 217 echo -n "Checking for use of forbidden library functions..." 218 219 git grep --line-number -w '\(atoi\|atol\|atoll\|strncpy\|strcpy\|strcat\|sprintf\|vsprintf\)' -- './*.c' ':!lib/rte_vhost*/**' > badfunc.log || true 220 if [ -s badfunc.log ]; then 221 echo " Forbidden library functions detected" 222 cat badfunc.log 223 rc=1 224 else 225 echo " OK" 226 fi 227 rm -f badfunc.log 228 229 return $rc 230} 231 232function check_cunit_style() { 233 local rc=0 234 235 echo -n "Checking for use of forbidden CUnit macros..." 236 237 git grep --line-number -w 'CU_ASSERT_FATAL' -- 'test/*' ':!test/spdk_cunit.h' > badcunit.log || true 238 if [ -s badcunit.log ]; then 239 echo " Forbidden CU_ASSERT_FATAL usage detected - use SPDK_CU_ASSERT_FATAL instead" 240 cat badcunit.log 241 rc=1 242 else 243 echo " OK" 244 fi 245 rm -f badcunit.log 246 247 return $rc 248} 249 250function check_eof() { 251 local rc=0 252 253 echo -n "Checking blank lines at end of file..." 254 255 if ! git grep -I -l -e . -z './*' ':!*.patch' \ 256 | xargs -0 -P$(nproc) -n1 scripts/eofnl > eofnl.log; then 257 echo " Incorrect end-of-file formatting detected" 258 cat eofnl.log 259 rc=1 260 else 261 echo " OK" 262 fi 263 rm -f eofnl.log 264 265 return $rc 266} 267 268function check_posix_includes() { 269 local rc=0 270 271 echo -n "Checking for POSIX includes..." 272 git grep -I -i -f scripts/posix.txt -- './*' ':!include/spdk/stdinc.h' \ 273 ':!include/linux/**' ':!scripts/posix.txt' ':!lib/env_dpdk/*/*.h' \ 274 ':!*.patch' ':!configure' > scripts/posix.log || true 275 if [ -s scripts/posix.log ]; then 276 echo "POSIX includes detected. Please include spdk/stdinc.h instead." 277 cat scripts/posix.log 278 rc=1 279 else 280 echo " OK" 281 fi 282 rm -f scripts/posix.log 283 284 return $rc 285} 286 287function check_naming_conventions() { 288 local rc=0 289 290 echo -n "Checking for proper function naming conventions..." 291 # commit_to_compare = HEAD - 1. 292 commit_to_compare="$(git log --pretty=oneline --skip=1 -n 1 | awk '{print $1}')" 293 failed_naming_conventions=false 294 changed_c_libs=() 295 declared_symbols=() 296 297 # Build an array of all the modified C libraries. 298 mapfile -t changed_c_libs < <(git diff --name-only HEAD $commit_to_compare -- lib/**/*.c module/**/*.c | xargs -r dirname | sort | uniq) 299 # Matching groups are 1. qualifiers / return type. 2. function name 3. argument list / comments and stuff after that. 300 # Capture just the names of newly added (or modified) function definitions. 301 mapfile -t declared_symbols < <(git diff -U0 $commit_to_compare HEAD -- include/spdk*/*.h | sed -En 's/(^[+].*)(spdk[a-z,A-Z,0-9,_]*)(\(.*)/\2/p') 302 303 for c_lib in "${changed_c_libs[@]}"; do 304 lib_map_file="mk/spdk_blank.map" 305 defined_symbols=() 306 removed_symbols=() 307 exported_symbols=() 308 if ls "$c_lib"/*.map &> /dev/null; then 309 lib_map_file="$(ls "$c_lib"/*.map)" 310 fi 311 # Matching groups are 1. leading +sign. 2, function name 3. argument list / anything after that. 312 # Capture just the names of newly added (or modified) functions that start with "spdk_" 313 mapfile -t defined_symbols < <(git diff -U0 $commit_to_compare HEAD -- $c_lib | sed -En 's/(^[+])(spdk[a-z,A-Z,0-9,_]*)(\(.*)/\2/p') 314 # Capture the names of removed symbols to catch edge cases where we just move definitions around. 315 mapfile -t removed_symbols < <(git diff -U0 $commit_to_compare HEAD -- $c_lib | sed -En 's/(^[-])(spdk[a-z,A-Z,0-9,_]*)(\(.*)/\2/p') 316 for symbol in "${removed_symbols[@]}"; do 317 for i in "${!defined_symbols[@]}"; do 318 if [[ ${defined_symbols[i]} = "$symbol" ]]; then 319 unset -v 'defined_symbols[i]' 320 fi 321 done 322 done 323 # It's possible that we just modified a functions arguments so unfortunately we can't just look at changed lines in this function. 324 # matching groups are 1. All leading whitespace 2. function name. Capture just the symbol name. 325 mapfile -t exported_symbols < <(sed -En 's/(^[[:space:]]*)(spdk[a-z,A-Z,0-9,_]*);/\2/p' < $lib_map_file) 326 for defined_symbol in "${defined_symbols[@]}"; do 327 # if the list of defined symbols is equal to the list of removed symbols, then we are left with a single empty element. skip it. 328 if [ "$defined_symbol" = '' ]; then 329 continue 330 fi 331 not_exported=true 332 not_declared=true 333 if array_contains_string exported_symbols $defined_symbol; then 334 not_exported=false 335 fi 336 337 if array_contains_string declared_symbols $defined_symbol; then 338 not_declared=false 339 fi 340 341 if $not_exported || $not_declared; then 342 if ! $failed_naming_conventions; then 343 echo " found naming convention errors." 344 fi 345 echo "function $defined_symbol starts with spdk_ which is reserved for public API functions." 346 echo "Please add this function to its corresponding map file and a public header or remove the spdk_ prefix." 347 failed_naming_conventions=true 348 rc=1 349 fi 350 done 351 done 352 353 if ! $failed_naming_conventions; then 354 echo " OK" 355 fi 356 357 return $rc 358} 359 360function check_include_style() { 361 local rc=0 362 363 echo -n "Checking #include style..." 364 git grep -I -i --line-number "#include <spdk/" -- '*.[ch]' > scripts/includes.log || true 365 if [ -s scripts/includes.log ]; then 366 echo "Incorrect #include syntax. #includes of spdk/ files should use quotes." 367 cat scripts/includes.log 368 rc=1 369 else 370 echo " OK" 371 fi 372 rm -f scripts/includes.log 373 374 return $rc 375} 376 377function check_python_style() { 378 local rc=0 379 380 if hash pycodestyle 2> /dev/null; then 381 PEP8=pycodestyle 382 elif hash pep8 2> /dev/null; then 383 PEP8=pep8 384 fi 385 386 if [ -n "${PEP8}" ]; then 387 echo -n "Checking Python style..." 388 389 PEP8_ARGS+=" --max-line-length=140" 390 391 error=0 392 git ls-files '*.py' | xargs -P$(nproc) -n1 $PEP8 $PEP8_ARGS > pep8.log || error=1 393 if [ $error -ne 0 ]; then 394 echo " Python formatting errors detected" 395 cat pep8.log 396 rc=1 397 else 398 echo " OK" 399 fi 400 rm -f pep8.log 401 else 402 echo "You do not have pycodestyle or pep8 installed so your Python style is not being checked!" 403 fi 404 405 return $rc 406} 407 408function get_bash_files() { 409 local sh shebang 410 411 mapfile -t sh < <(git ls-files '*.sh') 412 mapfile -t shebang < <(git grep -l '^#!.*bash') 413 414 printf '%s\n' "${sh[@]}" "${shebang[@]}" | sort -u 415} 416 417function check_bash_style() { 418 local rc=0 419 420 # find compatible shfmt binary 421 shfmt_bins=$(compgen -c | grep '^shfmt' | uniq || true) 422 for bin in $shfmt_bins; do 423 shfmt_version=$("$bin" --version) 424 if [ $shfmt_version != "v3.1.0" ]; then 425 echo "$bin version $shfmt_version not used (only v3.1.0 is supported)" 426 echo "v3.1.0 can be installed using 'scripts/pkgdep.sh -d'" 427 else 428 shfmt=$bin 429 break 430 fi 431 done 432 433 if [ -n "$shfmt" ]; then 434 shfmt_cmdline=() sh_files=() 435 436 mapfile -t sh_files < <(get_bash_files) 437 438 if ((${#sh_files[@]})); then 439 printf 'Checking .sh formatting style...' 440 441 shfmt_cmdline+=(-i 0) # indent_style = tab|indent_size = 0 442 shfmt_cmdline+=(-bn) # binary_next_line = true 443 shfmt_cmdline+=(-ci) # switch_case_indent = true 444 shfmt_cmdline+=(-ln bash) # shell_variant = bash (default) 445 shfmt_cmdline+=(-d) # diffOut - print diff of the changes and exit with != 0 446 shfmt_cmdline+=(-sr) # redirect operators will be followed by a space 447 448 diff=${output_dir:-$PWD}/$shfmt.patch 449 450 # Explicitly tell shfmt to not look for .editorconfig. .editorconfig is also not looked up 451 # in case any formatting arguments has been passed on its cmdline. 452 if ! SHFMT_NO_EDITORCONFIG=true "$shfmt" "${shfmt_cmdline[@]}" "${sh_files[@]}" > "$diff"; then 453 # In case shfmt detects an actual syntax error it will write out a proper message on 454 # its stderr, hence the diff file should remain empty. 455 if [[ -s $diff ]]; then 456 diff_out=$(< "$diff") 457 fi 458 459 cat <<- ERROR_SHFMT 460 461 * Errors in style formatting have been detected. 462 ${diff_out:+* Please, review the generated patch at $diff 463 464 # _START_OF_THE_DIFF 465 466 ${diff_out:-ERROR} 467 468 # _END_OF_THE_DIFF 469 } 470 471 ERROR_SHFMT 472 rc=1 473 else 474 rm -f "$diff" 475 printf ' OK\n' 476 fi 477 fi 478 else 479 echo "Supported version of shfmt not detected, Bash style formatting check is skipped" 480 fi 481 482 return $rc 483} 484 485function check_bash_static_analysis() { 486 local rc=0 487 488 if hash shellcheck 2> /dev/null; then 489 echo -n "Checking Bash static analysis with shellcheck..." 490 491 shellcheck_v=$(shellcheck --version | grep -P "version: [0-9\.]+" | cut -d " " -f2) 492 493 # SHCK_EXCLUDE contains a list of all of the spellcheck errors found in SPDK scripts 494 # currently. New errors should only be added to this list if the cost of fixing them 495 # is deemed too high. For more information about the errors, go to: 496 # https://github.com/koalaman/shellcheck/wiki/Checks 497 # Error descriptions can also be found at: https://github.com/koalaman/shellcheck/wiki 498 # SPDK fails some error checks which have been deprecated in later versions of shellcheck. 499 # We will not try to fix these error checks, but instead just leave the error types here 500 # so that we can still run with older versions of shellcheck. 501 SHCK_EXCLUDE="SC1117" 502 # SPDK has decided to not fix violations of these errors. 503 # We are aware about below exclude list and we want this errors to be excluded. 504 # SC1083: This {/} is literal. Check expression (missing ;/\n?) or quote it. 505 # SC1090: Can't follow non-constant source. Use a directive to specify location. 506 # SC1091: Not following: (error message here) 507 # SC2001: See if you can use ${variable//search/replace} instead. 508 # SC2010: Don't use ls | grep. Use a glob or a for loop with a condition to allow non-alphanumeric filenames. 509 # SC2015: Note that A && B || C is not if-then-else. C may run when A is true. 510 # SC2016: Expressions don't expand in single quotes, use double quotes for that. 511 # SC2034: foo appears unused. Verify it or export it. 512 # SC2046: Quote this to prevent word splitting. 513 # SC2086: Double quote to prevent globbing and word splitting. 514 # SC2119: Use foo "$@" if function's $1 should mean script's $1. 515 # SC2120: foo references arguments, but none are ever passed. 516 # SC2128: Expanding an array without an index only gives the first element. 517 # SC2148: Add shebang to the top of your script. 518 # SC2153: Possible Misspelling: MYVARIABLE may not be assigned, but MY_VARIABLE is. 519 # SC2154: var is referenced but not assigned. 520 # SC2164: Use cd ... || exit in case cd fails. 521 # SC2174: When used with -p, -m only applies to the deepest directory. 522 # SC2178: Variable was used as an array but is now assigned a string. 523 # SC2206: Quote to prevent word splitting/globbing, 524 # or split robustly with mapfile or read -a. 525 # SC2207: Prefer mapfile or read -a to split command output (or quote to avoid splitting). 526 # SC2223: This default assignment may cause DoS due to globbing. Quote it. 527 SHCK_EXCLUDE="$SHCK_EXCLUDE,SC1083,SC1090,SC1091,SC2010,SC2015,SC2016,SC2034,SC2046,SC2086,\ 528SC2119,SC2120,SC2128,SC2148,SC2153,SC2154,SC2164,SC2174,SC2178,SC2001,SC2206,SC2207,SC2223" 529 530 SHCK_FORMAT="tty" 531 SHCK_APPLY=false 532 SHCH_ARGS="-e $SHCK_EXCLUDE -f $SHCK_FORMAT" 533 534 if ge "$shellcheck_v" 0.4.0; then 535 SHCH_ARGS+=" -x" 536 else 537 echo "shellcheck $shellcheck_v detected, recommended >= 0.4.0." 538 fi 539 540 get_bash_files | xargs -P$(nproc) -n1 shellcheck $SHCH_ARGS &> shellcheck.log 541 if [[ -s shellcheck.log ]]; then 542 echo " Bash shellcheck errors detected!" 543 544 cat shellcheck.log 545 if $SHCK_APPLY; then 546 git apply shellcheck.log 547 echo "Bash errors were automatically corrected." 548 echo "Please remember to add the changes to your commit." 549 fi 550 rc=1 551 else 552 echo " OK" 553 fi 554 rm -f shellcheck.log 555 else 556 echo "You do not have shellcheck installed so your Bash static analysis is not being performed!" 557 fi 558 559 return $rc 560} 561 562function check_changelog() { 563 local rc=0 564 565 # Check if any of the public interfaces were modified by this patch. 566 # Warn the user to consider updating the changelog any changes 567 # are detected. 568 echo -n "Checking whether CHANGELOG.md should be updated..." 569 staged=$(git diff --name-only --cached .) 570 working=$(git status -s --porcelain --ignore-submodules | grep -iv "??" | awk '{print $2}') 571 files="$staged $working" 572 if [[ "$files" = " " ]]; then 573 files=$(git diff-tree --no-commit-id --name-only -r HEAD) 574 fi 575 576 has_changelog=0 577 for f in $files; do 578 if [[ $f == CHANGELOG.md ]]; then 579 # The user has a changelog entry, so exit. 580 has_changelog=1 581 break 582 fi 583 done 584 585 needs_changelog=0 586 if [ $has_changelog -eq 0 ]; then 587 for f in $files; do 588 if [[ $f == include/spdk/* ]] || [[ $f == scripts/rpc.py ]] || [[ $f == etc/* ]]; then 589 echo "" 590 echo -n "$f was modified. Consider updating CHANGELOG.md." 591 needs_changelog=1 592 fi 593 done 594 fi 595 596 if [ $needs_changelog -eq 0 ]; then 597 echo " OK" 598 else 599 echo "" 600 fi 601 602 return $rc 603} 604 605function check_json_rpc() { 606 local rc=0 607 608 echo -n "Checking that all RPCs are documented..." 609 while IFS='"' read -r _ rpc _; do 610 if ! grep -q "^### $rpc" doc/jsonrpc.md; then 611 echo "Missing JSON-RPC documentation for ${rpc}" 612 rc=1 613 fi 614 done < <(git grep -h -E "^SPDK_RPC_REGISTER\(" ':!test/*' ':!examples/nvme/hotplug/*') 615 616 if [ $rc -eq 0 ]; then 617 echo " OK" 618 fi 619 return $rc 620} 621 622function check_markdown_format() { 623 local rc=0 624 625 if hash mdl 2> /dev/null; then 626 echo -n "Checking markdown files format..." 627 mdl -g -s $rootdir/mdl_rules.rb . > mdl.log || true 628 if [ -s mdl.log ]; then 629 echo " Errors in .md files detected:" 630 cat mdl.log 631 rc=1 632 else 633 echo " OK" 634 fi 635 rm -f mdl.log 636 else 637 echo "You do not have markdownlint installed so .md files not being checked!" 638 fi 639 640 return $rc 641} 642 643function check_rpc_args() { 644 local rc=0 645 646 echo -n "Checking rpc.py argument option names..." 647 grep add_argument scripts/rpc.py | $GNU_GREP -oP "(?<=--)[a-z0-9\-\_]*(?=\')" | grep "_" > badargs.log 648 649 if [[ -s badargs.log ]]; then 650 echo "rpc.py arguments with underscores detected!" 651 cat badargs.log 652 echo "Please convert the underscores to dashes." 653 rc=1 654 else 655 echo " OK" 656 fi 657 rm -f badargs.log 658 return $rc 659} 660 661rc=0 662 663check_permissions || rc=1 664check_c_style || rc=1 665 666GIT_VERSION=$(git --version | cut -d' ' -f3) 667 668if version_lt "1.9.5" "${GIT_VERSION}"; then 669 # git <1.9.5 doesn't support pathspec magic exclude 670 echo " Your git version is too old to perform all tests. Please update git to at least 1.9.5 version..." 671 exit $rc 672fi 673 674check_comment_style || rc=1 675check_markdown_format || rc=1 676check_spaces_before_tabs || rc=1 677check_trailing_whitespace || rc=1 678check_forbidden_functions || rc=1 679check_cunit_style || rc=1 680check_eof || rc=1 681check_posix_includes || rc=1 682check_naming_conventions || rc=1 683check_include_style || rc=1 684check_python_style || rc=1 685check_bash_style || rc=1 686check_bash_static_analysis || rc=1 687check_changelog || rc=1 688check_json_rpc || rc=1 689check_rpc_args || rc=1 690 691exit $rc 692