1#!/usr/bin/env bash 2# SPDX-License-Identifier: BSD-3-Clause 3# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. 4# All rights reserved. 5# 6 7testdir=$(readlink -f $(dirname $0)) 8rootdir=$(readlink -f $testdir/../../..) 9source $rootdir/test/common/autotest_common.sh 10source $rootdir/test/nvmf/common.sh 11 12allowed_devices=${1:-"mlx5_0"} 13 14MALLOC_BDEV_SIZE=256 15MALLOC_BLOCK_SIZE=512 16app_sock=/var/tmp/bdev.sock 17 18function gen_accel_mlx5_driver_rdma_json() { 19 accel_qp_size=${1:-256} 20 accel_num_requests=${2:-2047} 21 22 jq . <<- JSON 23 { 24 "subsystems": [ 25 { 26 "subsystem": "accel", 27 "config": [ 28 { 29 "method": "mlx5_scan_accel_module", 30 "params": { 31 "allowed_devs": "${allowed_devices}", 32 "qp_size": ${accel_qp_size}, 33 "num_requests": ${accel_num_requests}, 34 "enable_driver": true 35 } 36 } 37 ] 38 }, 39 { 40 "subsystem": "bdev", 41 "config": [ 42 { 43 "method": "bdev_nvme_set_options", 44 "params": { 45 "allow_accel_sequence": true, 46 "rdma_umr_per_io": true 47 } 48 }, 49 { 50 "method": "bdev_nvme_attach_controller", 51 "params": { 52 "name": "Nvme0", 53 "trtype": "$TEST_TRANSPORT", 54 "adrfam": "IPv4", 55 "traddr": "$NVMF_FIRST_TARGET_IP", 56 "trsvcid": "$NVMF_PORT", 57 "subnqn": "nqn.2016-06.io.spdk:cnode0", 58 "ddgst": true 59 } 60 }, 61 { 62 "method": "bdev_wait_for_examine" 63 } 64 ] 65 } 66 ] 67 } 68 JSON 69} 70 71validate_mkey_stats() { 72 rpc_sock=$1 73 stats=$($rpc_py -s $rpc_sock accel_mlx5_dump_stats -l total) 74 75 val=$(echo $stats | jq -r '.total.umrs.umrs') 76 if [ "$val" == 0 ]; then 77 echo "Unexpected number of umrs: $val, expected > 0" 78 return 1 79 fi 80 val=$(echo $stats | jq -r '.total.umrs.crypto_umrs') 81 if [ "$val" != 0 ]; then 82 echo "Unexpected number of crypto_umrs: $val, expected 0" 83 return 1 84 fi 85 val=$(echo $stats | jq -r '.total.umrs.sig_umrs') 86 if [ "$val" != 0 ]; then 87 echo "Unexpected number of sig_umrs: $val, expected 0" 88 return 1 89 fi 90 val=$(echo $stats | jq -r '.total.rdma.total') 91 if [ "$val" != 0 ]; then 92 echo "Unexpected number of RDMA operations: $val, expected 0" 93 return 1 94 fi 95 val=$(echo $stats | jq -r '.total.tasks.mkey') 96 if [ $val != 0 ] && [ $val != $(echo $stats | jq -r '.total.tasks.total') ]; then 97 echo "Unexpected number of tasks operations: $val, expected > 0 and no other tasks" 98 return 1 99 fi 100} 101 102if [ "$TEST_TRANSPORT" != "rdma" ]; then 103 exit 0 104fi 105 106# Test mlx5 platform driver with crypto bdev and bdev_nvme rdma 107nvmftestinit 108nvmfappstart -m 0x3 109 110$rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS 111$rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE -b Malloc0 112$rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode0 -a -s SPDK00000000000001 113$rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode0 Malloc0 114$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode0 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT 115 116sleep 1 117 118# Test with bdevperf, without src memory domain 119bdevperf=$rootdir/build/examples/bdevperf 120$bdevperf --json <(gen_accel_mlx5_driver_rdma_json) -q 64 -o 4096 -t 10 -w verify -M 50 -m 0xc -r $app_sock 121$bdevperf --json <(gen_accel_mlx5_driver_rdma_json) -q 64 -o 131072 -t 10 -w verify -M 50 -m 0xc -r $app_sock 122 123## By killing the target, we trigger qpair disconnect with outstanding IOs and test that nvme_rdma<->accel_mlx5 124## interaction works well. No hang or crash expected. 125$bdevperf --json <(gen_accel_mlx5_driver_rdma_json) -q 64 -o 4096 -t 60 -w rw -M 50 -m 0xc -r $app_sock & 126bdev_perf_pid=$! 127waitforlisten $bdev_perf_pid $app_sock 128sleep 5 129validate_mkey_stats $app_sock 130sleep 1 131killprocess $nvmfpid 132wait $bdev_perf_pid || true 133 134nvmfappstart -m 0x3 135$rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS 136$rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE -b Malloc0 137$rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode0 -a -s SPDK00000000000001 138$rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode0 Malloc0 139$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode0 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT 140 141$bdevperf --json <(gen_accel_mlx5_driver_rdma_json) -q 64 -o 4096 -t 10 -w rw -M 50 -m 0xc -r $app_sock & 142bdev_perf_pid=$! 143waitforlisten $bdev_perf_pid $app_sock 144sleep 5 145validate_mkey_stats $app_sock 146sleep 1 147wait $bdev_perf_pid 148 149# Test with dma app which uses memory domains 150testdma="$rootdir/test/dma/test_dma/test_dma" 151$testdma --json <(gen_accel_mlx5_driver_rdma_json) -q 64 -o 4096 -t 10 -w verify -M 50 -m 0xc -r $app_sock -b "Nvme0n1" -f -x translate & 152testdma_pid=$! 153waitforlisten $testdma_pid $app_sock 154sleep 5 155validate_mkey_stats $app_sock 156sleep 1 157wait $testdma_pid 158 159# Test with fragmented payload 160testdma="$rootdir/test/dma/test_dma/test_dma" 161$testdma --json <(gen_accel_mlx5_driver_rdma_json) -q 64 -o 4096 -O 18 -t 10 -w verify -M 50 -m 0xc -r $app_sock -b "Nvme0n1" -f -x translate & 162testdma_pid=$! 163waitforlisten $testdma_pid $app_sock 164sleep 5 165validate_mkey_stats $app_sock 166sleep 1 167wait $testdma_pid 168 169# Test small qp size and number of MRs 170testdma="$rootdir/test/dma/test_dma/test_dma" 171$testdma --json <(gen_accel_mlx5_driver_rdma_json 16 32) -q 64 -o 32768 -t 10 -w verify -M 50 -m 0xc -r $app_sock -b "Nvme0n1" -f -x translate & 172testdma_pid=$! 173waitforlisten $testdma_pid $app_sock 174sleep 5 175validate_mkey_stats $app_sock 176sleep 1 177wait $testdma_pid 178 179# Test mkey corruption 180testdma="$rootdir/test/dma/test_dma/test_dma" 181$testdma --json <(gen_accel_mlx5_driver_rdma_json) -q 64 -o 4096 -t 10 -w randrw -M 50 -m 0xc -r $app_sock -b "Nvme0n1" -f -x translate -Y 500000 & 182testdma_pid=$! 183waitforlisten $testdma_pid $app_sock 184sleep 5 185validate_mkey_stats $app_sock 186sleep 1 187wait $testdma_pid || true 188 189nvmftestfini 190 191trap - SIGINT SIGTERM EXIT 192