1#!/usr/bin/env bash 2 3set -e 4 5# If the configuration of tests is not provided, no tests will be carried out. 6if [[ ! -f $1 ]]; then 7 echo "ERROR: SPDK test configuration not specified" 8 exit 1 9fi 10 11rootdir=$(readlink -f $(dirname $0)) 12 13source "$1" 14source "$rootdir/test/common/autotest_common.sh" 15 16out=$output_dir 17if [ -n "$SPDK_TEST_NATIVE_DPDK" ]; then 18 scanbuild_exclude=" --exclude $(dirname $SPDK_RUN_EXTERNAL_DPDK)" 19else 20 scanbuild_exclude="--exclude $rootdir/dpdk/" 21fi 22scanbuild="scan-build -o $output_dir/scan-build-tmp $scanbuild_exclude --status-bugs" 23config_params=$(get_config_params) 24 25trap '[[ -d $SPDK_WORKSPACE ]] && rm -rf "$SPDK_WORKSPACE"' 0 26 27SPDK_WORKSPACE=$(mktemp -dt "spdk_$(date +%s).XXXXXX") 28export SPDK_WORKSPACE 29 30umask 022 31cd $rootdir 32 33# Print some test system info out for the log 34date -u 35git describe --tags 36 37function ocf_precompile() { 38 # We compile OCF sources ourselves 39 # They don't need to be checked with scanbuild and code coverage is not applicable 40 # So we precompile OCF now for further use as standalone static library 41 ./configure $(echo $config_params | sed 's/--enable-coverage//g') 42 $MAKE $MAKEFLAGS include/spdk/config.h 43 CC=gcc CCAR=ar $MAKE $MAKEFLAGS -C lib/env_ocf exportlib O=$rootdir/build/ocf.a 44 # Set config to use precompiled library 45 config_params="$config_params --with-ocf=/$rootdir/build/ocf.a" 46 # need to reconfigure to avoid clearing ocf related files on future make clean. 47 ./configure $config_params 48} 49 50function build_native_dpdk() { 51 local external_dpdk_dir 52 local external_dpdk_base_dir 53 54 external_dpdk_dir="$SPDK_RUN_EXTERNAL_DPDK" 55 external_dpdk_base_dir="$(dirname $external_dpdk_dir)" 56 57 if [[ ! -d "$external_dpdk_base_dir" ]]; then 58 sudo mkdir -p "$external_dpdk_base_dir" 59 sudo chown -R $(whoami) "$external_dpdk_base_dir"/.. 60 fi 61 orgdir=$PWD 62 63 rm -rf "$external_dpdk_base_dir" 64 git clone --branch $SPDK_TEST_NATIVE_DPDK --depth 1 http://dpdk.org/git/dpdk "$external_dpdk_base_dir" 65 git -C "$external_dpdk_base_dir" log --oneline -n 5 66 67 dpdk_cflags="-fPIC -g -Werror -fcommon" 68 dpdk_ldflags="" 69 70 # the drivers we use 71 DPDK_DRIVERS=("bus" "bus/pci" "bus/vdev" "mempool/ring") 72 # all possible DPDK drivers 73 DPDK_ALL_DRIVERS=($(find "$external_dpdk_base_dir/drivers" -mindepth 1 -type d | sed -n "s#^$external_dpdk_base_dir/drivers/##p")) 74 75 if [[ "$SPDK_TEST_CRYPTO" -eq 1 ]]; then 76 git clone --branch v0.54 --depth 1 https://github.com/intel/intel-ipsec-mb.git "$external_dpdk_base_dir/intel-ipsec-mb" 77 cd "$external_dpdk_base_dir/intel-ipsec-mb" 78 $MAKE $MAKEFLAGS all SHARED=y EXTRA_CFLAGS=-fPIC 79 DPDK_DRIVERS+=("crypto") 80 DPDK_DRIVERS+=("crypto/aesni_mb") 81 DPDK_DRIVERS+=("crypto/qat") 82 DPDK_DRIVERS+=("compress/qat") 83 DPDK_DRIVERS+=("common/qat") 84 dpdk_cflags+=" -I$external_dpdk_base_dir/intel-ipsec-mb" 85 dpdk_ldflags+=" -L$external_dpdk_base_dir/intel-ipsec-mb" 86 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$external_dpdk_base_dir/intel-ipsec-mb 87 fi 88 89 if [[ "$SPDK_TEST_REDUCE" -eq 1 ]]; then 90 isal_dir="$external_dpdk_base_dir/isa-l" 91 git clone --branch v2.29.0 --depth 1 https://github.com/intel/isa-l.git "$isal_dir" 92 93 cd $isal_dir 94 ./autogen.sh 95 ./configure CFLAGS="-fPIC -g -O2" --enable-shared=yes --prefix="$isal_dir/build" 96 ln -s $PWD/include $PWD/isa-l 97 $MAKE $MAKEFLAGS all 98 $MAKE install 99 DPDK_DRIVERS+=("compress") 100 DPDK_DRIVERS+=("compress/isal") 101 DPDK_DRIVERS+=("compress/qat") 102 DPDK_DRIVERS+=("common/qat") 103 export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:$isal_dir/build/lib/pkgconfig" 104 export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$isal_dir/build/lib" 105 fi 106 107 # Use difference between DPDK_ALL_DRIVERS and DPDK_DRIVERS as a set of DPDK drivers we don't want or 108 # don't need to build. 109 DPDK_DISABLED_DRIVERS=($(sort <(printf "%s\n" "${DPDK_DRIVERS[@]}") <(printf "%s\n" "${DPDK_ALL_DRIVERS[@]}") | uniq -u)) 110 111 cd $external_dpdk_base_dir 112 if [ "$(uname -s)" = "Linux" ]; then 113 dpdk_cflags+=" -Wno-stringop-overflow" 114 # Fix for freeing device if not kernel driver configured. 115 # TODO: Remove once this is merged in upstream DPDK 116 if grep "20.08.0" $external_dpdk_base_dir/VERSION; then 117 wget https://github.com/spdk/dpdk/commit/64f1ced13f974e8b3d46b87c361a09eca68126f9.patch -O dpdk-pci.patch 118 wget https://github.com/spdk/dpdk/commit/c2c273d5c8fbf673623b427f8f4ab5af5ddf0e08.patch -O dpdk-qat.patch 119 else 120 wget https://github.com/karlatec/dpdk/commit/3219c0cfc38803aec10c809dde16e013b370bda9.patch -O dpdk-pci.patch 121 wget https://github.com/karlatec/dpdk/commit/adf8f7638de29bc4bf9ba3faf12bbdae73acda0c.patch -O dpdk-qat.patch 122 fi 123 git config --local user.name "spdk" 124 git config --local user.email "nomail@all.com" 125 git am dpdk-pci.patch 126 git am dpdk-qat.patch 127 fi 128 129 meson build-tmp --prefix="$external_dpdk_dir" --libdir lib \ 130 -Denable_docs=false -Denable_kmods=false -Dtests=false \ 131 -Dc_link_args="$dpdk_ldflags" -Dc_args="$dpdk_cflags" \ 132 -Dmachine=native -Ddisable_drivers=$(printf "%s," "${DPDK_DISABLED_DRIVERS[@]}") 133 ninja -C "$external_dpdk_base_dir/build-tmp" $MAKEFLAGS 134 ninja -C "$external_dpdk_base_dir/build-tmp" $MAKEFLAGS install 135 136 # Save this path. In tests are run using autorun.sh then autotest.sh 137 # script will be unaware of LD_LIBRARY_PATH and will fail tests. 138 echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH" > /tmp/spdk-ld-path 139 140 cd "$orgdir" 141} 142 143function make_fail_cleanup() { 144 if [ -d $out/scan-build-tmp ]; then 145 scanoutput=$(ls -1 $out/scan-build-tmp/) 146 mv $out/scan-build-tmp/$scanoutput $out/scan-build 147 rm -rf $out/scan-build-tmp 148 chmod -R a+rX $out/scan-build 149 fi 150 false 151} 152 153function scanbuild_make() { 154 pass=true 155 $scanbuild $MAKE $MAKEFLAGS > $out/build_output.txt && rm -rf $out/scan-build-tmp || make_fail_cleanup 156 xtrace_disable 157 158 rm -f $out/*files.txt 159 for ent in $(find app examples lib module test -type f | grep -vF ".h"); do 160 if [[ $ent == lib/env_ocf* ]]; then continue; fi 161 if file -bi $ent | grep -q 'text/x-c'; then 162 echo $ent | sed 's/\.cp\{0,2\}$//g' >> $out/all_c_files.txt 163 fi 164 done 165 xtrace_restore 166 167 grep -E "CC|CXX" $out/build_output.txt | sed 's/\s\s\(CC\|CXX\)\s//g' | sed 's/\.o//g' > $out/built_c_files.txt 168 cat $rootdir/test/common/skipped_build_files.txt >> $out/built_c_files.txt 169 170 sort -o $out/all_c_files.txt $out/all_c_files.txt 171 sort -o $out/built_c_files.txt $out/built_c_files.txt 172 # from comm manual: 173 # -2 suppress column 2 (lines unique to FILE2) 174 # -3 suppress column 3 (lines that appear in both files) 175 # comm may exit 1 if no lines were printed (undocumented, unreliable) 176 comm -2 -3 $out/all_c_files.txt $out/built_c_files.txt > $out/unbuilt_c_files.txt || true 177 178 if [ $(wc -l < $out/unbuilt_c_files.txt) -ge 1 ]; then 179 echo "missing files" 180 cat $out/unbuilt_c_files.txt 181 pass=false 182 fi 183 184 $pass 185} 186 187function porcelain_check() { 188 if [ $(git status --porcelain --ignore-submodules | wc -l) -ne 0 ]; then 189 echo "Generated files missing from .gitignore:" 190 git status --porcelain --ignore-submodules 191 exit 1 192 fi 193} 194 195# Check that header file dependencies are working correctly by 196# capturing a binary's stat data before and after touching a 197# header file and re-making. 198function header_dependency_check() { 199 STAT1=$(stat $SPDK_BIN_DIR/spdk_tgt) 200 sleep 1 201 touch lib/nvme/nvme_internal.h 202 $MAKE $MAKEFLAGS 203 STAT2=$(stat $SPDK_BIN_DIR/spdk_tgt) 204 205 if [ "$STAT1" == "$STAT2" ]; then 206 echo "Header dependency check failed" 207 false 208 fi 209} 210 211function test_make_uninstall() { 212 # Create empty file to check if it is not deleted by target uninstall 213 touch "$SPDK_WORKSPACE/usr/lib/sample_xyz.a" 214 $MAKE $MAKEFLAGS uninstall DESTDIR="$SPDK_WORKSPACE" prefix=/usr 215 if [[ $(find "$SPDK_WORKSPACE/usr" -maxdepth 1 -mindepth 1 | wc -l) -ne 2 ]] || [[ $(find "$SPDK_WORKSPACE/usr/lib/" -maxdepth 1 -mindepth 1 | wc -l) -ne 1 ]]; then 216 ls -lR "$SPDK_WORKSPACE" 217 echo "Make uninstall failed" 218 exit 1 219 fi 220} 221 222function build_doc() { 223 $MAKE -C "$rootdir"/doc --no-print-directory $MAKEFLAGS &> "$out"/doxygen.log 224 if [ -s "$out"/doxygen.log ]; then 225 cat "$out"/doxygen.log 226 echo "Doxygen errors found!" 227 exit 1 228 fi 229 if hash pdflatex 2> /dev/null; then 230 $MAKE -C "$rootdir"/doc/output/latex --no-print-directory $MAKEFLAGS &>> "$out"/doxygen.log 231 fi 232 mkdir -p "$out"/doc 233 mv "$rootdir"/doc/output/html "$out"/doc 234 if [ -f "$rootdir"/doc/output/latex/refman.pdf ]; then 235 mv "$rootdir"/doc/output/latex/refman.pdf "$out"/doc/spdk.pdf 236 fi 237 $MAKE -C "$rootdir"/doc --no-print-directory $MAKEFLAGS clean &>> "$out"/doxygen.log 238 if [ -s "$out"/doxygen.log ]; then 239 rm "$out"/doxygen.log 240 fi 241 rm -rf "$rootdir"/doc/output 242} 243 244function autobuild_test_suite() { 245 run_test "autobuild_check_format" ./scripts/check_format.sh 246 run_test "autobuild_external_code" sudo -E --preserve-env=PATH LD_LIBRARY_PATH=$LD_LIBRARY_PATH $rootdir/test/external_code/test_make.sh $rootdir 247 if [ "$SPDK_TEST_OCF" -eq 1 ]; then 248 run_test "autobuild_ocf_precompile" ocf_precompile 249 fi 250 run_test "autobuild_check_so_deps" $rootdir/test/make/check_so_deps.sh $1 251 ./configure $config_params --without-shared 252 run_test "scanbuild_make" scanbuild_make 253 run_test "autobuild_generated_files_check" porcelain_check 254 run_test "autobuild_header_dependency_check" header_dependency_check 255 run_test "autobuild_make_install" $MAKE $MAKEFLAGS install DESTDIR="$SPDK_WORKSPACE" prefix=/usr 256 run_test "autobuild_make_uninstall" test_make_uninstall 257 run_test "autobuild_build_doc" build_doc 258} 259 260if [ $SPDK_RUN_VALGRIND -eq 1 ]; then 261 run_test "valgrind" echo "using valgrind" 262fi 263 264if [ $SPDK_RUN_ASAN -eq 1 ]; then 265 run_test "asan" echo "using asan" 266fi 267 268if [ $SPDK_RUN_UBSAN -eq 1 ]; then 269 run_test "ubsan" echo "using ubsan" 270fi 271 272if [ -n "$SPDK_TEST_NATIVE_DPDK" ]; then 273 run_test "build_native_dpdk" build_native_dpdk 274fi 275 276./configure $config_params 277echo "** START ** Info for Hostname: $HOSTNAME" 278uname -a 279$MAKE cc_version 280$MAKE cxx_version 281echo "** END ** Info for Hostname: $HOSTNAME" 282 283if [ "$SPDK_TEST_AUTOBUILD" -eq 1 ]; then 284 run_test "autobuild" autobuild_test_suite $1 285else 286 if [ "$SPDK_TEST_OCF" -eq 1 ]; then 287 run_test "autobuild_ocf_precompile" ocf_precompile 288 fi 289 # if we aren't testing the unittests, build with shared objects. 290 ./configure $config_params --with-shared 291 run_test "make" $MAKE $MAKEFLAGS 292fi 293