1#!/usr/bin/env bash 2 3set -e 4 5# If the configuration of tests is not provided, no tests will be carried out. 6if [[ ! -f $1 ]]; then 7 echo "ERROR: SPDK test configuration not specified" 8 exit 1 9fi 10 11rootdir=$(readlink -f $(dirname $0)) 12 13source "$1" 14source "$rootdir/test/common/autotest_common.sh" 15source "$rootdir/scripts/common.sh" 16 17out=$output_dir 18if [ -n "$SPDK_TEST_NATIVE_DPDK" ]; then 19 scanbuild_exclude=" --exclude $(dirname $SPDK_RUN_EXTERNAL_DPDK)" 20else 21 scanbuild_exclude="--exclude $rootdir/dpdk/" 22fi 23scanbuild="scan-build -o $output_dir/scan-build-tmp $scanbuild_exclude --status-bugs" 24config_params=$(get_config_params) 25 26trap '[[ -d $SPDK_WORKSPACE ]] && rm -rf "$SPDK_WORKSPACE"' 0 27 28SPDK_WORKSPACE=$(mktemp -dt "spdk_$(date +%s).XXXXXX") 29export SPDK_WORKSPACE 30 31umask 022 32cd $rootdir 33 34# Print some test system info out for the log 35date -u 36git describe --tags 37 38function ocf_precompile() { 39 # We compile OCF sources ourselves 40 # They don't need to be checked with scanbuild and code coverage is not applicable 41 # So we precompile OCF now for further use as standalone static library 42 ./configure $(echo $config_params | sed 's/--enable-coverage//g') 43 $MAKE $MAKEFLAGS include/spdk/config.h 44 CC=gcc CCAR=ar $MAKE $MAKEFLAGS -C lib/env_ocf exportlib O=$rootdir/build/ocf.a 45 # Set config to use precompiled library 46 config_params="$config_params --with-ocf=/$rootdir/build/ocf.a" 47 # need to reconfigure to avoid clearing ocf related files on future make clean. 48 ./configure $config_params 49} 50 51function build_native_dpdk() { 52 local external_dpdk_dir 53 local external_dpdk_base_dir 54 local gcc_version 55 56 gcc_version=$(gcc -dumpversion) 57 gcc_version=${gcc_version//./} 58 external_dpdk_dir="$SPDK_RUN_EXTERNAL_DPDK" 59 external_dpdk_base_dir="$(dirname $external_dpdk_dir)" 60 61 if [[ ! -d "$external_dpdk_base_dir" ]]; then 62 sudo mkdir -p "$external_dpdk_base_dir" 63 sudo chown -R $(whoami) "$external_dpdk_base_dir"/.. 64 fi 65 orgdir=$PWD 66 67 rm -rf "$external_dpdk_base_dir" 68 git clone --branch $SPDK_TEST_NATIVE_DPDK --depth 1 http://dpdk.org/git/dpdk "$external_dpdk_base_dir" 69 git -C "$external_dpdk_base_dir" log --oneline -n 5 70 71 dpdk_cflags="-fPIC -g -fcommon" 72 dpdk_ldflags="" 73 74 if [[ $gcc_version -ge 5 ]]; then 75 dpdk_cflags+=" -Werror" 76 fi 77 78 if [[ $gcc_version -ge 10 ]]; then 79 dpdk_cflags+=" -Wno-stringop-overflow" 80 fi 81 82 # the drivers we use 83 # net/i40e driver is not really needed by us, but it's built as a workaround 84 # for DPDK issue: https://bugs.dpdk.org/show_bug.cgi?id=576 85 DPDK_DRIVERS=("bus" "bus/pci" "bus/vdev" "mempool/ring" "net/i40e" "net/i40e/base") 86 # all possible DPDK drivers 87 DPDK_ALL_DRIVERS=($(find "$external_dpdk_base_dir/drivers" -mindepth 1 -type d | sed -n "s#^$external_dpdk_base_dir/drivers/##p")) 88 89 if [[ "$SPDK_TEST_CRYPTO" -eq 1 ]]; then 90 git clone --branch v0.54 --depth 1 https://github.com/intel/intel-ipsec-mb.git "$external_dpdk_base_dir/intel-ipsec-mb" 91 cd "$external_dpdk_base_dir/intel-ipsec-mb" 92 $MAKE $MAKEFLAGS all SHARED=y EXTRA_CFLAGS=-fPIC 93 DPDK_DRIVERS+=("crypto") 94 DPDK_DRIVERS+=("crypto/aesni_mb") 95 DPDK_DRIVERS+=("crypto/qat") 96 DPDK_DRIVERS+=("compress/qat") 97 DPDK_DRIVERS+=("common/qat") 98 dpdk_cflags+=" -I$external_dpdk_base_dir/intel-ipsec-mb" 99 dpdk_ldflags+=" -L$external_dpdk_base_dir/intel-ipsec-mb" 100 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$external_dpdk_base_dir/intel-ipsec-mb 101 fi 102 103 if [[ "$SPDK_TEST_REDUCE" -eq 1 ]]; then 104 isal_dir="$external_dpdk_base_dir/isa-l" 105 git clone --branch v2.29.0 --depth 1 https://github.com/intel/isa-l.git "$isal_dir" 106 107 cd $isal_dir 108 ./autogen.sh 109 ./configure CFLAGS="-fPIC -g -O2" --enable-shared=yes --prefix="$isal_dir/build" 110 ln -s $PWD/include $PWD/isa-l 111 $MAKE $MAKEFLAGS all 112 $MAKE install 113 DPDK_DRIVERS+=("compress") 114 DPDK_DRIVERS+=("compress/isal") 115 DPDK_DRIVERS+=("compress/qat") 116 DPDK_DRIVERS+=("common/qat") 117 export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:$isal_dir/build/lib/pkgconfig" 118 export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$isal_dir/build/lib" 119 fi 120 121 # Use difference between DPDK_ALL_DRIVERS and DPDK_DRIVERS as a set of DPDK drivers we don't want or 122 # don't need to build. 123 DPDK_DISABLED_DRIVERS=($(sort <(printf "%s\n" "${DPDK_DRIVERS[@]}") <(printf "%s\n" "${DPDK_ALL_DRIVERS[@]}") | uniq -u)) 124 125 cd $external_dpdk_base_dir 126 if [ "$(uname -s)" = "Linux" ]; then 127 # Fix for freeing device if not kernel driver configured. 128 # TODO: Remove once this is merged in upstream DPDK 129 if grep "20.08.0" $external_dpdk_base_dir/VERSION; then 130 wget https://github.com/spdk/dpdk/commit/64f1ced13f974e8b3d46b87c361a09eca68126f9.patch -O dpdk-pci.patch 131 wget https://github.com/spdk/dpdk/commit/c2c273d5c8fbf673623b427f8f4ab5af5ddf0e08.patch -O dpdk-qat.patch 132 elif grep "20.11\|21.02" $external_dpdk_base_dir/VERSION; then 133 wget https://github.com/karlatec/dpdk/commit/3219c0cfc38803aec10c809dde16e013b370bda9.patch -O dpdk-pci.patch 134 wget https://github.com/karlatec/dpdk/commit/adf8f7638de29bc4bf9ba3faf12bbdae73acda0c.patch -O dpdk-qat.patch 135 else 136 wget https://github.com/karlatec/dpdk/commit/eac05db0580091ef8e4d338aa5d2210695521894.patch -O dpdk-pci.patch 137 wget https://github.com/karlatec/dpdk/commit/d649d5efb7bb404ce59dea81768adeb994b284f7.patch -O dpdk-qat.patch 138 fi 139 git config --local user.name "spdk" 140 git config --local user.email "nomail@all.com" 141 git am dpdk-pci.patch 142 git am dpdk-qat.patch 143 fi 144 145 meson build-tmp --prefix="$external_dpdk_dir" --libdir lib \ 146 -Denable_docs=false -Denable_kmods=false -Dtests=false \ 147 -Dc_link_args="$dpdk_ldflags" -Dc_args="$dpdk_cflags" \ 148 -Dmachine=native -Ddisable_drivers=$(printf "%s," "${DPDK_DISABLED_DRIVERS[@]}") 149 ninja -C "$external_dpdk_base_dir/build-tmp" $MAKEFLAGS 150 ninja -C "$external_dpdk_base_dir/build-tmp" $MAKEFLAGS install 151 152 # Save this path. In tests are run using autorun.sh then autotest.sh 153 # script will be unaware of LD_LIBRARY_PATH and will fail tests. 154 echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH" > /tmp/spdk-ld-path 155 156 cd "$orgdir" 157} 158 159function make_fail_cleanup() { 160 if [ -d $out/scan-build-tmp ]; then 161 scanoutput=$(ls -1 $out/scan-build-tmp/) 162 mv $out/scan-build-tmp/$scanoutput $out/scan-build 163 rm -rf $out/scan-build-tmp 164 chmod -R a+rX $out/scan-build 165 fi 166 false 167} 168 169function scanbuild_make() { 170 pass=true 171 $scanbuild $MAKE $MAKEFLAGS > $out/build_output.txt && rm -rf $out/scan-build-tmp || make_fail_cleanup 172 xtrace_disable 173 174 rm -f $out/*files.txt 175 for ent in $(find app examples lib module test -type f | grep -vF ".h"); do 176 if [[ $ent == lib/env_ocf* ]]; then continue; fi 177 if file -bi $ent | grep -q 'text/x-c'; then 178 echo $ent | sed 's/\.cp\{0,2\}$//g' >> $out/all_c_files.txt 179 fi 180 done 181 xtrace_restore 182 183 grep -E "CC|CXX" $out/build_output.txt | sed 's/\s\s\(CC\|CXX\)\s//g' | sed 's/\.o//g' > $out/built_c_files.txt 184 cat $rootdir/test/common/skipped_build_files.txt >> $out/built_c_files.txt 185 186 sort -o $out/all_c_files.txt $out/all_c_files.txt 187 sort -o $out/built_c_files.txt $out/built_c_files.txt 188 # from comm manual: 189 # -2 suppress column 2 (lines unique to FILE2) 190 # -3 suppress column 3 (lines that appear in both files) 191 # comm may exit 1 if no lines were printed (undocumented, unreliable) 192 comm -2 -3 $out/all_c_files.txt $out/built_c_files.txt > $out/unbuilt_c_files.txt || true 193 194 if [ $(wc -l < $out/unbuilt_c_files.txt) -ge 1 ]; then 195 echo "missing files" 196 cat $out/unbuilt_c_files.txt 197 pass=false 198 fi 199 200 $pass 201} 202 203function porcelain_check() { 204 if [ $(git status --porcelain --ignore-submodules | wc -l) -ne 0 ]; then 205 echo "Generated files missing from .gitignore:" 206 git status --porcelain --ignore-submodules 207 exit 1 208 fi 209} 210 211# Check that header file dependencies are working correctly by 212# capturing a binary's stat data before and after touching a 213# header file and re-making. 214function header_dependency_check() { 215 STAT1=$(stat $SPDK_BIN_DIR/spdk_tgt) 216 sleep 1 217 touch lib/nvme/nvme_internal.h 218 $MAKE $MAKEFLAGS 219 STAT2=$(stat $SPDK_BIN_DIR/spdk_tgt) 220 221 if [ "$STAT1" == "$STAT2" ]; then 222 echo "Header dependency check failed" 223 false 224 fi 225} 226 227function test_make_uninstall() { 228 # Create empty file to check if it is not deleted by target uninstall 229 touch "$SPDK_WORKSPACE/usr/lib/sample_xyz.a" 230 $MAKE $MAKEFLAGS uninstall DESTDIR="$SPDK_WORKSPACE" prefix=/usr 231 if [[ $(find "$SPDK_WORKSPACE/usr" -maxdepth 1 -mindepth 1 | wc -l) -ne 2 ]] || [[ $(find "$SPDK_WORKSPACE/usr/lib/" -maxdepth 1 -mindepth 1 | wc -l) -ne 1 ]]; then 232 ls -lR "$SPDK_WORKSPACE" 233 echo "Make uninstall failed" 234 exit 1 235 fi 236} 237 238function build_doc() { 239 local doxygenv 240 doxygenv=$(doxygen --version) 241 242 $MAKE -C "$rootdir"/doc --no-print-directory $MAKEFLAGS &> "$out"/doxygen.log 243 if [ -s "$out"/doxygen.log ]; then 244 cat "$out"/doxygen.log 245 echo "Doxygen errors found!" 246 eq "$doxygenv" 1.8.20 || exit 1 247 echo "Doxygen $doxygenv detected, all warnings are potentially false positives, continuing the test" 248 fi 249 if hash pdflatex 2> /dev/null; then 250 $MAKE -C "$rootdir"/doc/output/latex --no-print-directory $MAKEFLAGS &>> "$out"/doxygen.log 251 fi 252 mkdir -p "$out"/doc 253 mv "$rootdir"/doc/output/html "$out"/doc 254 if [ -f "$rootdir"/doc/output/latex/refman.pdf ]; then 255 mv "$rootdir"/doc/output/latex/refman.pdf "$out"/doc/spdk.pdf 256 fi 257 $MAKE -C "$rootdir"/doc --no-print-directory $MAKEFLAGS clean &>> "$out"/doxygen.log 258 if [ -s "$out"/doxygen.log ]; then 259 # Save the log as an artifact in case we are working with potentially broken version 260 eq "$doxygenv" 1.8.20 || rm "$out"/doxygen.log 261 fi 262 rm -rf "$rootdir"/doc/output 263} 264 265function autobuild_test_suite() { 266 run_test "autobuild_check_format" ./scripts/check_format.sh 267 run_test "autobuild_external_code" sudo -E --preserve-env=PATH LD_LIBRARY_PATH=$LD_LIBRARY_PATH $rootdir/test/external_code/test_make.sh $rootdir 268 if [ "$SPDK_TEST_OCF" -eq 1 ]; then 269 run_test "autobuild_ocf_precompile" ocf_precompile 270 fi 271 run_test "autobuild_check_so_deps" $rootdir/test/make/check_so_deps.sh $1 272 ./configure $config_params --without-shared 273 run_test "scanbuild_make" scanbuild_make 274 run_test "autobuild_generated_files_check" porcelain_check 275 run_test "autobuild_header_dependency_check" header_dependency_check 276 run_test "autobuild_make_install" $MAKE $MAKEFLAGS install DESTDIR="$SPDK_WORKSPACE" prefix=/usr 277 run_test "autobuild_make_uninstall" test_make_uninstall 278 run_test "autobuild_build_doc" build_doc 279} 280 281if [ $SPDK_RUN_VALGRIND -eq 1 ]; then 282 run_test "valgrind" echo "using valgrind" 283fi 284 285if [ $SPDK_RUN_ASAN -eq 1 ]; then 286 run_test "asan" echo "using asan" 287fi 288 289if [ $SPDK_RUN_UBSAN -eq 1 ]; then 290 run_test "ubsan" echo "using ubsan" 291fi 292 293if [ -n "$SPDK_TEST_NATIVE_DPDK" ]; then 294 run_test "build_native_dpdk" build_native_dpdk 295fi 296 297./configure $config_params 298echo "** START ** Info for Hostname: $HOSTNAME" 299uname -a 300$MAKE cc_version 301$MAKE cxx_version 302echo "** END ** Info for Hostname: $HOSTNAME" 303 304if [ "$SPDK_TEST_AUTOBUILD" -eq 1 ]; then 305 run_test "autobuild" autobuild_test_suite $1 306else 307 if [ "$SPDK_TEST_OCF" -eq 1 ]; then 308 run_test "autobuild_ocf_precompile" ocf_precompile 309 fi 310 # if we aren't testing the unittests, build with shared objects. 311 ./configure $config_params --with-shared 312 run_test "make" $MAKE $MAKEFLAGS 313fi 314