1#!/usr/bin/env bash 2 3readonly BASEDIR=$(readlink -f $(dirname $0))/.. 4cd $BASEDIR 5 6# exit on errors 7set -e 8 9if ! hash nproc 2> /dev/null; then 10 11 function nproc() { 12 echo 8 13 } 14 15fi 16 17function version_lt() { 18 [ $(echo -e "$1\n$2" | sort -V | head -1) != "$1" ] 19} 20 21function array_contains_string() { 22 name="$1[@]" 23 array=("${!name}") 24 25 for element in "${array[@]}"; do 26 if [ "$element" = "$2" ]; then 27 return $(true) 28 fi 29 done 30 31 return $(false) 32} 33 34rc=0 35 36function check_permissions() { 37 echo -n "Checking file permissions..." 38 39 local rc=0 40 41 while read -r perm _res0 _res1 path; do 42 if [ ! -f "$path" ]; then 43 continue 44 fi 45 46 # Skip symlinks 47 if [[ -L $path ]]; then 48 continue 49 fi 50 fname=$(basename -- "$path") 51 52 case ${fname##*.} in 53 c | h | cpp | cc | cxx | hh | hpp | md | html | js | json | svg | Doxyfile | yml | LICENSE | README | conf | in | Makefile | mk | gitignore | go | txt) 54 # These file types should never be executable 55 if [ "$perm" -eq 100755 ]; then 56 echo "ERROR: $path is marked executable but is a code file." 57 rc=1 58 fi 59 ;; 60 *) 61 shebang=$(head -n 1 $path | cut -c1-3) 62 63 # git only tracks the execute bit, so will only ever return 755 or 644 as the permission. 64 if [ "$perm" -eq 100755 ]; then 65 # If the file has execute permission, it should start with a shebang. 66 if [ "$shebang" != "#!/" ]; then 67 echo "ERROR: $path is marked executable but does not start with a shebang." 68 rc=1 69 fi 70 else 71 # If the file doesnot have execute permissions, it should not start with a shebang. 72 if [ "$shebang" = "#!/" ]; then 73 echo "ERROR: $path is not marked executable but starts with a shebang." 74 rc=1 75 fi 76 fi 77 ;; 78 esac 79 80 done <<< "$(git grep -I --name-only --untracked -e . | git ls-files -s)" 81 82 if [ $rc -eq 0 ]; then 83 echo " OK" 84 fi 85 86 return $rc 87} 88 89function check_c_style() { 90 local rc=0 91 92 if hash astyle; then 93 echo -n "Checking coding style..." 94 if [ "$(astyle -V)" \< "Artistic Style Version 3" ]; then 95 echo -n " Your astyle version is too old so skipping coding style checks. Please update astyle to at least 3.0.1 version..." 96 else 97 rm -f astyle.log 98 touch astyle.log 99 # Exclude rte_vhost code imported from DPDK - we want to keep the original code 100 # as-is to enable ongoing work to synch with a generic upstream DPDK vhost library, 101 # rather than making diffs more complicated by a lot of changes to follow SPDK 102 # coding standards. 103 git ls-files '*.[ch]' '*.cpp' '*.cc' '*.cxx' '*.hh' '*.hpp' \ 104 | grep -v rte_vhost | grep -v cpp_headers \ 105 | xargs -P$(nproc) -n10 astyle --options=.astylerc >> astyle.log 106 if grep -q "^Formatted" astyle.log; then 107 echo " errors detected" 108 git diff --ignore-submodules=all 109 sed -i -e 's/ / /g' astyle.log 110 grep --color=auto "^Formatted.*" astyle.log 111 echo "Incorrect code style detected in one or more files." 112 echo "The files have been automatically formatted." 113 echo "Remember to add the files to your commit." 114 rc=1 115 else 116 echo " OK" 117 fi 118 rm -f astyle.log 119 fi 120 else 121 echo "You do not have astyle installed so your code style is not being checked!" 122 fi 123 return $rc 124} 125 126function check_comment_style() { 127 local rc=0 128 129 echo -n "Checking comment style..." 130 131 git grep --line-number -e '\/[*][^ *-]' -- '*.[ch]' > comment.log || true 132 git grep --line-number -e '[^ ][*]\/' -- '*.[ch]' ':!lib/rte_vhost*/*' >> comment.log || true 133 git grep --line-number -e '^[*]' -- '*.[ch]' >> comment.log || true 134 git grep --line-number -e '\s\/\/' -- '*.[ch]' >> comment.log || true 135 git grep --line-number -e '^\/\/' -- '*.[ch]' >> comment.log || true 136 137 if [ -s comment.log ]; then 138 echo " Incorrect comment formatting detected" 139 cat comment.log 140 rc=1 141 else 142 echo " OK" 143 fi 144 rm -f comment.log 145 146 return $rc 147} 148 149function check_spaces_before_tabs() { 150 local rc=0 151 152 echo -n "Checking for spaces before tabs..." 153 git grep --line-number $' \t' -- './*' ':!*.patch' > whitespace.log || true 154 if [ -s whitespace.log ]; then 155 echo " Spaces before tabs detected" 156 cat whitespace.log 157 rc=1 158 else 159 echo " OK" 160 fi 161 rm -f whitespace.log 162 163 return $rc 164} 165 166function check_trailing_whitespace() { 167 local rc=0 168 169 echo -n "Checking trailing whitespace in output strings..." 170 171 git grep --line-number -e ' \\n"' -- '*.[ch]' > whitespace.log || true 172 173 if [ -s whitespace.log ]; then 174 echo " Incorrect trailing whitespace detected" 175 cat whitespace.log 176 rc=1 177 else 178 echo " OK" 179 fi 180 rm -f whitespace.log 181 182 return $rc 183} 184 185function check_forbidden_functions() { 186 local rc=0 187 188 echo -n "Checking for use of forbidden library functions..." 189 190 git grep --line-number -w '\(atoi\|atol\|atoll\|strncpy\|strcpy\|strcat\|sprintf\|vsprintf\)' -- './*.c' ':!lib/rte_vhost*/**' > badfunc.log || true 191 if [ -s badfunc.log ]; then 192 echo " Forbidden library functions detected" 193 cat badfunc.log 194 rc=1 195 else 196 echo " OK" 197 fi 198 rm -f badfunc.log 199 200 return $rc 201} 202 203function check_cunit_style() { 204 local rc=0 205 206 echo -n "Checking for use of forbidden CUnit macros..." 207 208 git grep --line-number -w 'CU_ASSERT_FATAL' -- 'test/*' ':!test/spdk_cunit.h' > badcunit.log || true 209 if [ -s badcunit.log ]; then 210 echo " Forbidden CU_ASSERT_FATAL usage detected - use SPDK_CU_ASSERT_FATAL instead" 211 cat badcunit.log 212 rc=1 213 else 214 echo " OK" 215 fi 216 rm -f badcunit.log 217 218 return $rc 219} 220 221function check_eof() { 222 local rc=0 223 224 echo -n "Checking blank lines at end of file..." 225 226 if ! git grep -I -l -e . -z './*' ':!*.patch' \ 227 | xargs -0 -P$(nproc) -n1 scripts/eofnl > eofnl.log; then 228 echo " Incorrect end-of-file formatting detected" 229 cat eofnl.log 230 rc=1 231 else 232 echo " OK" 233 fi 234 rm -f eofnl.log 235 236 return $rc 237} 238 239function check_posix_includes() { 240 local rc=0 241 242 echo -n "Checking for POSIX includes..." 243 git grep -I -i -f scripts/posix.txt -- './*' ':!include/spdk/stdinc.h' ':!include/linux/**' ':!lib/rte_vhost*/**' ':!scripts/posix.txt' ':!*.patch' > scripts/posix.log || true 244 if [ -s scripts/posix.log ]; then 245 echo "POSIX includes detected. Please include spdk/stdinc.h instead." 246 cat scripts/posix.log 247 rc=1 248 else 249 echo " OK" 250 fi 251 rm -f scripts/posix.log 252 253 return $rc 254} 255 256function check_naming_conventions() { 257 local rc=0 258 259 echo -n "Checking for proper function naming conventions..." 260 # commit_to_compare = HEAD - 1. 261 commit_to_compare="$(git log --pretty=oneline --skip=1 -n 1 | awk '{print $1}')" 262 failed_naming_conventions=false 263 changed_c_libs=() 264 declared_symbols=() 265 266 # Build an array of all the modified C files. 267 mapfile -t changed_c_libs < <(git diff --name-only HEAD $commit_to_compare -- lib/**/*.c module/**/*.c) 268 # Matching groups are 1. qualifiers / return type. 2. function name 3. argument list / comments and stuff after that. 269 # Capture just the names of newly added (or modified) function definitions. 270 mapfile -t declared_symbols < <(git diff -U0 $commit_to_compare HEAD -- include/spdk*/*.h | sed -En 's/(^[+].*)(spdk[a-z,A-Z,0-9,_]*)(\(.*)/\2/p') 271 272 for c_file in "${changed_c_libs[@]}"; do 273 lib_map_file="mk/spdk_blank.map" 274 defined_symbols=() 275 removed_symbols=() 276 exported_symbols=() 277 if ls "$(dirname $c_file)"/*.map &> /dev/null; then 278 lib_map_file="$(ls "$(dirname $c_file)"/*.map)" 279 fi 280 # Matching groups are 1. leading +sign. 2, function name 3. argument list / anything after that. 281 # Capture just the names of newly added (or modified) functions that start with "spdk_" 282 mapfile -t defined_symbols < <(git diff -U0 $commit_to_compare HEAD -- $c_file | sed -En 's/(^[+])(spdk[a-z,A-Z,0-9,_]*)(\(.*)/\2/p') 283 # Capture the names of removed symbols to catch edge cases where we just move definitions around. 284 mapfile -t removed_symbols < <(git diff -U0 $commit_to_compare HEAD -- $c_file | sed -En 's/(^[-])(spdk[a-z,A-Z,0-9,_]*)(\(.*)/\2/p') 285 for symbol in "${removed_symbols[@]}"; do 286 for i in "${!defined_symbols[@]}"; do 287 if [[ ${defined_symbols[i]} = "$symbol" ]]; then 288 unset -v 'defined_symbols[i]' 289 fi 290 done 291 done 292 # It's possible that we just modified a functions arguments so unfortunately we can't just look at changed lines in this function. 293 # matching groups are 1. All leading whitespace 2. function name. Capture just the symbol name. 294 mapfile -t exported_symbols < <(sed -En 's/(^[[:space:]]*)(spdk[a-z,A-Z,0-9,_]*);/\2/p' < $lib_map_file) 295 for defined_symbol in "${defined_symbols[@]}"; do 296 # if the list of defined symbols is equal to the list of removed symbols, then we are left with a single empty element. skip it. 297 if [ "$defined_symbol" = '' ]; then 298 continue 299 fi 300 not_exported=true 301 not_declared=true 302 if array_contains_string exported_symbols $defined_symbol; then 303 not_exported=false 304 fi 305 306 if array_contains_string declared_symbols $defined_symbol; then 307 not_declared=false 308 fi 309 310 if $not_exported || $not_declared; then 311 if ! $failed_naming_conventions; then 312 echo " found naming convention errors." 313 fi 314 echo "function $defined_symbol starts with spdk_ which is reserved for public API functions." 315 echo "Please add this function to its corresponding map file and a public header or remove the spdk_ prefix." 316 failed_naming_conventions=true 317 rc=1 318 fi 319 done 320 done 321 322 if ! $failed_naming_conventions; then 323 echo " OK" 324 fi 325 326 return $rc 327} 328 329function check_include_style() { 330 local rc=0 331 332 echo -n "Checking #include style..." 333 git grep -I -i --line-number "#include <spdk/" -- '*.[ch]' > scripts/includes.log || true 334 if [ -s scripts/includes.log ]; then 335 echo "Incorrect #include syntax. #includes of spdk/ files should use quotes." 336 cat scripts/includes.log 337 rc=1 338 else 339 echo " OK" 340 fi 341 rm -f scripts/includes.log 342 343 return $rc 344} 345 346function check_python_style() { 347 local rc=0 348 349 if hash pycodestyle 2> /dev/null; then 350 PEP8=pycodestyle 351 elif hash pep8 2> /dev/null; then 352 PEP8=pep8 353 fi 354 355 if [ -n "${PEP8}" ]; then 356 echo -n "Checking Python style..." 357 358 PEP8_ARGS+=" --max-line-length=140" 359 360 error=0 361 git ls-files '*.py' | xargs -P$(nproc) -n1 $PEP8 $PEP8_ARGS > pep8.log || error=1 362 if [ $error -ne 0 ]; then 363 echo " Python formatting errors detected" 364 cat pep8.log 365 rc=1 366 else 367 echo " OK" 368 fi 369 rm -f pep8.log 370 else 371 echo "You do not have pycodestyle or pep8 installed so your Python style is not being checked!" 372 fi 373 374 return $rc 375} 376 377function get_bash_files() { 378 local sh shebang 379 380 mapfile -t sh < <(git ls-files '*.sh') 381 mapfile -t shebang < <(git grep -l '^#!.*bash') 382 383 printf '%s\n' "${sh[@]}" "${shebang[@]}" | sort -u 384} 385 386function check_bash_style() { 387 local rc=0 388 389 # find compatible shfmt binary 390 shfmt_bins=$(compgen -c | grep '^shfmt' || true) 391 for bin in $shfmt_bins; do 392 if version_lt "$("$bin" --version)" "3.1.0"; then 393 shfmt=$bin 394 break 395 fi 396 done 397 398 if [ -n "$shfmt" ]; then 399 shfmt_cmdline=() sh_files=() 400 401 mapfile -t sh_files < <(get_bash_files) 402 403 if ((${#sh_files[@]})); then 404 printf 'Checking .sh formatting style...' 405 406 shfmt_cmdline+=(-i 0) # indent_style = tab|indent_size = 0 407 shfmt_cmdline+=(-bn) # binary_next_line = true 408 shfmt_cmdline+=(-ci) # switch_case_indent = true 409 shfmt_cmdline+=(-ln bash) # shell_variant = bash (default) 410 shfmt_cmdline+=(-d) # diffOut - print diff of the changes and exit with != 0 411 shfmt_cmdline+=(-sr) # redirect operators will be followed by a space 412 413 diff=${output_dir:-$PWD}/$shfmt.patch 414 415 # Explicitly tell shfmt to not look for .editorconfig. .editorconfig is also not looked up 416 # in case any formatting arguments has been passed on its cmdline. 417 if ! SHFMT_NO_EDITORCONFIG=true "$shfmt" "${shfmt_cmdline[@]}" "${sh_files[@]}" > "$diff"; then 418 # In case shfmt detects an actual syntax error it will write out a proper message on 419 # its stderr, hence the diff file should remain empty. 420 if [[ -s $diff ]]; then 421 diff_out=$(< "$diff") 422 fi 423 424 cat <<- ERROR_SHFMT 425 426 * Errors in style formatting have been detected. 427 ${diff_out:+* Please, review the generated patch at $diff 428 429 # _START_OF_THE_DIFF 430 431 ${diff_out:-ERROR} 432 433 # _END_OF_THE_DIFF 434 } 435 436 ERROR_SHFMT 437 rc=1 438 else 439 rm -f "$diff" 440 printf ' OK\n' 441 fi 442 fi 443 else 444 echo "shfmt not detected, Bash style formatting check is skipped" 445 fi 446 447 return $rc 448} 449 450function check_bash_static_analysis() { 451 local rc=0 452 453 if hash shellcheck 2> /dev/null; then 454 echo -n "Checking Bash style..." 455 456 shellcheck_v=$(shellcheck --version | grep -P "version: [0-9\.]+" | cut -d " " -f2) 457 458 # SHCK_EXCLUDE contains a list of all of the spellcheck errors found in SPDK scripts 459 # currently. New errors should only be added to this list if the cost of fixing them 460 # is deemed too high. For more information about the errors, go to: 461 # https://github.com/koalaman/shellcheck/wiki/Checks 462 # Error descriptions can also be found at: https://github.com/koalaman/shellcheck/wiki 463 # SPDK fails some error checks which have been deprecated in later versions of shellcheck. 464 # We will not try to fix these error checks, but instead just leave the error types here 465 # so that we can still run with older versions of shellcheck. 466 SHCK_EXCLUDE="SC1117" 467 # SPDK has decided to not fix violations of these errors. 468 # We are aware about below exclude list and we want this errors to be excluded. 469 # SC1083: This {/} is literal. Check expression (missing ;/\n?) or quote it. 470 # SC1090: Can't follow non-constant source. Use a directive to specify location. 471 # SC1091: Not following: (error message here) 472 # SC2001: See if you can use ${variable//search/replace} instead. 473 # SC2010: Don't use ls | grep. Use a glob or a for loop with a condition to allow non-alphanumeric filenames. 474 # SC2015: Note that A && B || C is not if-then-else. C may run when A is true. 475 # SC2016: Expressions don't expand in single quotes, use double quotes for that. 476 # SC2034: foo appears unused. Verify it or export it. 477 # SC2046: Quote this to prevent word splitting. 478 # SC2086: Double quote to prevent globbing and word splitting. 479 # SC2119: Use foo "$@" if function's $1 should mean script's $1. 480 # SC2120: foo references arguments, but none are ever passed. 481 # SC2148: Add shebang to the top of your script. 482 # SC2153: Possible Misspelling: MYVARIABLE may not be assigned, but MY_VARIABLE is. 483 # SC2154: var is referenced but not assigned. 484 # SC2164: Use cd ... || exit in case cd fails. 485 # SC2174: When used with -p, -m only applies to the deepest directory. 486 # SC2206: Quote to prevent word splitting/globbing, 487 # or split robustly with mapfile or read -a. 488 # SC2207: Prefer mapfile or read -a to split command output (or quote to avoid splitting). 489 # SC2223: This default assignment may cause DoS due to globbing. Quote it. 490 SHCK_EXCLUDE="$SHCK_EXCLUDE,SC1083,SC1090,SC1091,SC2010,SC2015,SC2016,SC2034,SC2046,SC2086,\ 491SC2119,SC2120,SC2148,SC2153,SC2154,SC2164,SC2174,SC2001,SC2206,SC2207,SC2223" 492 493 SHCK_FORMAT="tty" 494 SHCK_APPLY=false 495 SHCH_ARGS=" -x -e $SHCK_EXCLUDE -f $SHCK_FORMAT" 496 497 get_bash_files | xargs -P$(nproc) -n1 shellcheck $SHCH_ARGS &> shellcheck.log 498 if [[ -s shellcheck.log ]]; then 499 echo " Bash formatting errors detected!" 500 501 cat shellcheck.log 502 if $SHCK_APPLY; then 503 git apply shellcheck.log 504 echo "Bash errors were automatically corrected." 505 echo "Please remember to add the changes to your commit." 506 fi 507 rc=1 508 else 509 echo " OK" 510 fi 511 rm -f shellcheck.log 512 else 513 echo "You do not have shellcheck installed so your Bash style is not being checked!" 514 fi 515 516 return $rc 517} 518 519function check_changelog() { 520 local rc=0 521 522 # Check if any of the public interfaces were modified by this patch. 523 # Warn the user to consider updating the changelog any changes 524 # are detected. 525 echo -n "Checking whether CHANGELOG.md should be updated..." 526 staged=$(git diff --name-only --cached .) 527 working=$(git status -s --porcelain --ignore-submodules | grep -iv "??" | awk '{print $2}') 528 files="$staged $working" 529 if [[ "$files" = " " ]]; then 530 files=$(git diff-tree --no-commit-id --name-only -r HEAD) 531 fi 532 533 has_changelog=0 534 for f in $files; do 535 if [[ $f == CHANGELOG.md ]]; then 536 # The user has a changelog entry, so exit. 537 has_changelog=1 538 break 539 fi 540 done 541 542 needs_changelog=0 543 if [ $has_changelog -eq 0 ]; then 544 for f in $files; do 545 if [[ $f == include/spdk/* ]] || [[ $f == scripts/rpc.py ]] || [[ $f == etc/* ]]; then 546 echo "" 547 echo -n "$f was modified. Consider updating CHANGELOG.md." 548 needs_changelog=1 549 fi 550 done 551 fi 552 553 if [ $needs_changelog -eq 0 ]; then 554 echo " OK" 555 else 556 echo "" 557 fi 558 559 return $rc 560} 561 562rc=0 563 564check_permissions || rc=1 565check_c_style || rc=1 566 567GIT_VERSION=$(git --version | cut -d' ' -f3) 568 569if version_lt "1.9.5" "${GIT_VERSION}"; then 570 # git <1.9.5 doesn't support pathspec magic exclude 571 echo " Your git version is too old to perform all tests. Please update git to at least 1.9.5 version..." 572 exit $rc 573fi 574 575check_comment_style || rc=1 576check_spaces_before_tabs || rc=1 577check_trailing_whitespace || rc=1 578check_forbidden_functions || rc=1 579check_cunit_style || rc=1 580check_eof || rc=1 581check_posix_includes || rc=1 582check_naming_conventions || rc=1 583check_include_style || rc=1 584check_python_style || rc=1 585check_bash_style || rc=1 586check_bash_static_analysis || rc=1 587check_changelog || rc=1 588 589exit $rc 590