1#!/usr/bin/env bash 2# SPDX-License-Identifier: BSD-3-Clause 3# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. 4# All rights reserved. 5# 6 7testdir=$(readlink -f $(dirname $0)) 8rootdir=$(readlink -f $testdir/../../..) 9source $rootdir/test/common/autotest_common.sh 10source $rootdir/test/nvmf/common.sh 11 12MALLOC_BDEV_SIZE=256 13MALLOC_BLOCK_SIZE=512 14app_sock=/var/tmp/bdev.sock 15 16function gen_accel_mlx5_driver_crypto_rdma_json() { 17 accel_qp_size=${1:-256} 18 accel_num_requests=${2:-2047} 19 20 jq . <<- JSON 21 { 22 "subsystems": [ 23 { 24 "subsystem": "accel", 25 "config": [ 26 { 27 "method": "mlx5_scan_accel_module", 28 "params": { 29 "qp_size": ${accel_qp_size}, 30 "num_requests": ${accel_num_requests}, 31 "enable_driver": true 32 } 33 }, 34 { 35 "method": "accel_crypto_key_create", 36 "params": { 37 "name": "test_dek", 38 "cipher": "AES_XTS", 39 "key": "00112233445566778899001122334455", 40 "key2": "11223344556677889900112233445500" 41 } 42 } 43 ] 44 }, 45 { 46 "subsystem": "bdev", 47 "config": [ 48 { 49 "method": "bdev_nvme_set_options", 50 "params": { 51 "allow_accel_sequence": true 52 } 53 }, 54 { 55 "method": "bdev_nvme_attach_controller", 56 "params": { 57 "name": "Nvme0", 58 "trtype": "$TEST_TRANSPORT", 59 "adrfam": "IPv4", 60 "traddr": "$NVMF_FIRST_TARGET_IP", 61 "trsvcid": "$NVMF_PORT", 62 "subnqn": "nqn.2016-06.io.spdk:cnode0", 63 "ddgst": true 64 } 65 }, 66 { 67 "method": "bdev_crypto_create", 68 "params": { 69 "base_bdev_name": "Nvme0n1", 70 "name": "Crypto0", 71 "key_name": "test_dek" 72 } 73 }, 74 { 75 "method": "bdev_wait_for_examine" 76 } 77 ] 78 } 79 ] 80 } 81 JSON 82} 83 84validate_crypto_umr_stats() { 85 rpc_sock=$1 86 stats=$($rpc_py -s $rpc_sock accel_mlx5_dump_stats -l total) 87 88 val=$(echo $stats | jq -r '.total.umrs.crypto_umrs') 89 if [ "$val" == 0 ]; then 90 echo "Unexpected number of crypto_umrs: $val, expected > 0" 91 return 1 92 fi 93 val=$(echo $stats | jq -r '.total.umrs.sig_umrs') 94 if [ "$val" != 0 ]; then 95 echo "Unexpected number of sig_umrs: $val, expected 0" 96 return 1 97 fi 98 val=$(echo $stats | jq -r '.total.rdma.total') 99 if [ "$val" != 0 ]; then 100 echo "Unexpected number of RDMA operations: $val, expected 0" 101 return 1 102 fi 103 val=$(echo $stats | jq -r '.total.tasks.crypto_mkey') 104 if [ $val != 0 ] && [ $val != $(echo $stats | jq -r '.total.tasks.total') ]; then 105 echo "Unexpected number of tasks operations: $val, expected > 0 and no other tasks" 106 return 1 107 fi 108} 109 110if [ "$TEST_TRANSPORT" != "rdma" ]; then 111 exit 0 112fi 113 114# Test mlx5 platform driver with crypto bdev and bdev_nvme rdma 115nvmftestinit 116nvmfappstart -m 0x3 117 118$rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS 119$rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE -b Malloc0 120$rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode0 -a -s SPDK00000000000001 121$rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode0 Malloc0 122$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode0 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT 123 124sleep 1 125 126# Test with bdevperf, without src memory domain 127bdevperf=$rootdir/build/examples/bdevperf 128$bdevperf --json <(gen_accel_mlx5_driver_crypto_rdma_json) -q 64 -o 4096 -t 10 -w verify -M 50 -m 0xc -r $app_sock 129$bdevperf --json <(gen_accel_mlx5_driver_crypto_rdma_json) -q 64 -o 131072 -t 10 -w verify -M 50 -m 0xc -r $app_sock 130 131## By killing the target, we trigger qpair disconnect with outstanding IOs and test that nvme_rdma<->accel_mlx5 132## interaction works well. No hang or crash expected. 133$bdevperf --json <(gen_accel_mlx5_driver_crypto_rdma_json) -q 64 -o 4096 -t 60 -w rw -M 50 -m 0xc -r $app_sock & 134bdev_perf_pid=$! 135waitforlisten $bdev_perf_pid $app_sock 136sleep 5 137validate_crypto_umr_stats $app_sock 138sleep 1 139killprocess $nvmfpid 140wait $bdev_perf_pid || true 141 142nvmfappstart -m 0x3 143$rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS 144$rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE -b Malloc0 145$rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode0 -a -s SPDK00000000000001 146$rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode0 Malloc0 147$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode0 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT 148 149$bdevperf --json <(gen_accel_mlx5_driver_crypto_rdma_json) -q 64 -o 4096 -t 10 -w rw -M 50 -m 0xc -r $app_sock & 150bdev_perf_pid=$! 151waitforlisten $bdev_perf_pid $app_sock 152sleep 5 153validate_crypto_umr_stats $app_sock 154sleep 1 155wait $bdev_perf_pid 156 157# Test with dma app which uses memory domains 158testdma="$rootdir/test/dma/test_dma/test_dma" 159$testdma --json <(gen_accel_mlx5_driver_crypto_rdma_json) -q 64 -o 4096 -t 10 -w verify -M 50 -m 0xc -r $app_sock -b "Crypto0" -f -x translate & 160testdma_pid=$! 161waitforlisten $testdma_pid $app_sock 162sleep 5 163validate_crypto_umr_stats $app_sock 164sleep 1 165wait $testdma_pid 166 167# Test small qp size and number of MRs 168testdma="$rootdir/test/dma/test_dma/test_dma" 169$testdma --json <(gen_accel_mlx5_driver_crypto_rdma_json 16 32) -q 64 -o 32768 -t 10 -w verify -M 50 -m 0xc -r $app_sock -b "Crypto0" -f -x translate & 170testdma_pid=$! 171waitforlisten $testdma_pid $app_sock 172sleep 5 173validate_crypto_umr_stats $app_sock 174sleep 1 175wait $testdma_pid 176 177# Test mkey corruption 178testdma="$rootdir/test/dma/test_dma/test_dma" 179$testdma --json <(gen_accel_mlx5_driver_crypto_rdma_json) -q 64 -o 4096 -t 10 -w randrw -M 50 -m 0xc -r $app_sock -b "Crypto0" -f -x translate -Y 500000 & 180testdma_pid=$! 181waitforlisten $testdma_pid $app_sock 182sleep 5 183validate_crypto_umr_stats $app_sock 184sleep 1 185wait $testdma_pid || true 186 187nvmftestfini 188 189trap - SIGINT SIGTERM EXIT 190