1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #ifndef SPDK_INTERNAL_NVME_TCP_H 35 #define SPDK_INTERNAL_NVME_TCP_H 36 37 #include "spdk/sock.h" 38 39 #define SPDK_CRC32C_XOR 0xffffffffUL 40 #define SPDK_NVME_TCP_DIGEST_LEN 4 41 #define SPDK_NVME_TCP_DIGEST_ALIGNMENT 4 42 #define SPDK_NVME_TCP_QPAIR_EXIT_TIMEOUT 30 43 44 #define MAKE_DIGEST_WORD(BUF, CRC32C) \ 45 ( ((*((uint8_t *)(BUF)+0)) = (uint8_t)((uint32_t)(CRC32C) >> 0)), \ 46 ((*((uint8_t *)(BUF)+1)) = (uint8_t)((uint32_t)(CRC32C) >> 8)), \ 47 ((*((uint8_t *)(BUF)+2)) = (uint8_t)((uint32_t)(CRC32C) >> 16)), \ 48 ((*((uint8_t *)(BUF)+3)) = (uint8_t)((uint32_t)(CRC32C) >> 24))) 49 50 #define MATCH_DIGEST_WORD(BUF, CRC32C) \ 51 ( ((((uint32_t) *((uint8_t *)(BUF)+0)) << 0) \ 52 | (((uint32_t) *((uint8_t *)(BUF)+1)) << 8) \ 53 | (((uint32_t) *((uint8_t *)(BUF)+2)) << 16) \ 54 | (((uint32_t) *((uint8_t *)(BUF)+3)) << 24)) \ 55 == (CRC32C)) 56 57 #define DGET32(B) \ 58 ((( (uint32_t) *((uint8_t *)(B)+0)) << 0) \ 59 | (((uint32_t) *((uint8_t *)(B)+1)) << 8) \ 60 | (((uint32_t) *((uint8_t *)(B)+2)) << 16) \ 61 | (((uint32_t) *((uint8_t *)(B)+3)) << 24)) 62 63 #define DSET32(B,D) \ 64 (((*((uint8_t *)(B)+0)) = (uint8_t)((uint32_t)(D) >> 0)), \ 65 ((*((uint8_t *)(B)+1)) = (uint8_t)((uint32_t)(D) >> 8)), \ 66 ((*((uint8_t *)(B)+2)) = (uint8_t)((uint32_t)(D) >> 16)), \ 67 ((*((uint8_t *)(B)+3)) = (uint8_t)((uint32_t)(D) >> 24))) 68 69 typedef void (*nvme_tcp_qpair_xfer_complete_cb)(void *cb_arg); 70 71 struct _iov_ctx { 72 struct iovec *iov; 73 int num_iovs; 74 uint32_t iov_offset; 75 int iovcnt; 76 uint32_t mapped_len; 77 }; 78 79 struct nvme_tcp_pdu { 80 union { 81 /* to hold error pdu data */ 82 uint8_t raw[SPDK_NVME_TCP_TERM_REQ_PDU_MAX_SIZE]; 83 struct spdk_nvme_tcp_common_pdu_hdr common; 84 struct spdk_nvme_tcp_ic_req ic_req; 85 struct spdk_nvme_tcp_term_req_hdr term_req; 86 struct spdk_nvme_tcp_cmd capsule_cmd; 87 struct spdk_nvme_tcp_h2c_data_hdr h2c_data; 88 struct spdk_nvme_tcp_ic_resp ic_resp; 89 struct spdk_nvme_tcp_rsp capsule_resp; 90 struct spdk_nvme_tcp_c2h_data_hdr c2h_data; 91 struct spdk_nvme_tcp_r2t_hdr r2t; 92 93 } hdr; 94 95 bool has_hdgst; 96 bool ddgst_enable; 97 uint8_t data_digest[SPDK_NVME_TCP_DIGEST_LEN]; 98 int32_t padding_valid_bytes; 99 100 uint32_t ch_valid_bytes; 101 uint32_t psh_valid_bytes; 102 103 nvme_tcp_qpair_xfer_complete_cb cb_fn; 104 void *cb_arg; 105 int ref; 106 void *data; 107 uint32_t data_len; 108 109 uint32_t readv_offset; 110 uint32_t writev_offset; 111 TAILQ_ENTRY(nvme_tcp_pdu) tailq; 112 uint32_t remaining; 113 uint32_t padding_len; 114 struct _iov_ctx iov_ctx; 115 116 void *ctx; /* data tied to a tcp request */ 117 }; 118 119 enum nvme_tcp_pdu_recv_state { 120 /* Ready to wait for PDU */ 121 NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY, 122 123 /* Active tqpair waiting for any PDU common header */ 124 NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_CH, 125 126 /* Active tqpair waiting for any PDU specific header */ 127 NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PSH, 128 129 /* Active tqpair waiting for payload */ 130 NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PAYLOAD, 131 132 /* Active tqpair does not wait for payload */ 133 NVME_TCP_PDU_RECV_STATE_ERROR, 134 }; 135 136 enum nvme_tcp_error_codes { 137 NVME_TCP_PDU_IN_PROGRESS = 0, 138 NVME_TCP_CONNECTION_FATAL = -1, 139 NVME_TCP_PDU_FATAL = -2, 140 }; 141 142 enum nvme_tcp_qpair_state { 143 NVME_TCP_QPAIR_STATE_INVALID = 0, 144 NVME_TCP_QPAIR_STATE_RUNNING = 1, 145 NVME_TCP_QPAIR_STATE_EXITING = 2, 146 NVME_TCP_QPAIR_STATE_EXITED = 3, 147 }; 148 149 static uint32_t 150 nvme_tcp_pdu_calc_header_digest(struct nvme_tcp_pdu *pdu) 151 { 152 uint32_t crc32c; 153 uint32_t hlen = pdu->hdr.common.hlen; 154 155 crc32c = spdk_crc32c_update(&pdu->hdr.raw, hlen, ~0); 156 crc32c = crc32c ^ SPDK_CRC32C_XOR; 157 return crc32c; 158 } 159 160 static uint32_t 161 nvme_tcp_pdu_calc_data_digest(struct nvme_tcp_pdu *pdu) 162 { 163 uint32_t crc32c; 164 uint32_t mod; 165 166 assert(pdu->data != NULL); 167 assert(pdu->data_len != 0); 168 169 crc32c = spdk_crc32c_update(pdu->data, pdu->data_len, ~0); 170 171 mod = pdu->data_len % SPDK_NVME_TCP_DIGEST_ALIGNMENT; 172 if (mod != 0) { 173 uint32_t pad_length = SPDK_NVME_TCP_DIGEST_ALIGNMENT - mod; 174 uint8_t pad[3] = {0, 0, 0}; 175 176 assert(pad_length > 0); 177 assert(pad_length <= sizeof(pad)); 178 crc32c = spdk_crc32c_update(pad, pad_length, crc32c); 179 } 180 181 crc32c = crc32c ^ SPDK_CRC32C_XOR; 182 return crc32c; 183 } 184 185 static inline void 186 _iov_ctx_init(struct _iov_ctx *ctx, struct iovec *iovs, int num_iovs, 187 uint32_t iov_offset) 188 { 189 ctx->iov = iovs; 190 ctx->num_iovs = num_iovs; 191 ctx->iov_offset = iov_offset; 192 ctx->iovcnt = 0; 193 ctx->mapped_len = 0; 194 } 195 196 static inline bool 197 _iov_ctx_set_iov(struct _iov_ctx *ctx, uint8_t *data, uint32_t data_len) 198 { 199 if (ctx->iov_offset >= data_len) { 200 ctx->iov_offset -= data_len; 201 } else { 202 ctx->iov->iov_base = data + ctx->iov_offset; 203 ctx->iov->iov_len = data_len - ctx->iov_offset; 204 ctx->mapped_len += data_len - ctx->iov_offset; 205 ctx->iov_offset = 0; 206 ctx->iov++; 207 ctx->iovcnt++; 208 if (ctx->iovcnt == ctx->num_iovs) { 209 return false; 210 } 211 } 212 213 return true; 214 } 215 216 static int 217 nvme_tcp_build_iovecs(struct iovec *iovec, int num_iovs, struct nvme_tcp_pdu *pdu, 218 bool hdgst_enable, bool ddgst_enable, uint32_t *_mapped_length) 219 { 220 int enable_digest; 221 uint32_t hlen, plen; 222 struct _iov_ctx *ctx; 223 224 if (num_iovs == 0) { 225 return 0; 226 } 227 228 ctx = &pdu->iov_ctx; 229 _iov_ctx_init(ctx, iovec, num_iovs, pdu->writev_offset); 230 hlen = pdu->hdr.common.hlen; 231 enable_digest = 1; 232 if (pdu->hdr.common.pdu_type == SPDK_NVME_TCP_PDU_TYPE_IC_REQ || 233 pdu->hdr.common.pdu_type == SPDK_NVME_TCP_PDU_TYPE_IC_RESP || 234 pdu->hdr.common.pdu_type == SPDK_NVME_TCP_PDU_TYPE_H2C_TERM_REQ || 235 pdu->hdr.common.pdu_type == SPDK_NVME_TCP_PDU_TYPE_C2H_TERM_REQ) { 236 /* this PDU should be sent without digest */ 237 enable_digest = 0; 238 } 239 240 /* Header Digest */ 241 if (enable_digest && hdgst_enable) { 242 hlen += SPDK_NVME_TCP_DIGEST_LEN; 243 } 244 245 plen = hlen; 246 if (!pdu->data_len || !pdu->data) { 247 /* PDU header + possible header digest */ 248 _iov_ctx_set_iov(ctx, (uint8_t *)&pdu->hdr.raw, hlen); 249 goto end; 250 } 251 252 /* Padding */ 253 if (pdu->padding_len > 0) { 254 hlen += pdu->padding_len; 255 plen = hlen; 256 } 257 258 if (!_iov_ctx_set_iov(ctx, (uint8_t *)&pdu->hdr.raw, hlen)) { 259 goto end; 260 } 261 262 /* Data Segment */ 263 plen += pdu->data_len; 264 if (!_iov_ctx_set_iov(ctx, pdu->data, pdu->data_len)) { 265 goto end; 266 } 267 268 /* Data Digest */ 269 if (enable_digest && ddgst_enable) { 270 plen += SPDK_NVME_TCP_DIGEST_LEN; 271 _iov_ctx_set_iov(ctx, pdu->data_digest, SPDK_NVME_TCP_DIGEST_LEN); 272 } 273 274 end: 275 if (_mapped_length != NULL) { 276 *_mapped_length = ctx->mapped_len; 277 } 278 279 /* check the plen for the first time constructing iov */ 280 if (!pdu->writev_offset) { 281 assert(plen == pdu->hdr.common.plen); 282 } 283 284 return ctx->iovcnt; 285 } 286 287 static int 288 nvme_tcp_build_payload_iovecs(struct iovec *iovec, int num_iovs, struct nvme_tcp_pdu *pdu, 289 bool ddgst_enable, uint32_t *_mapped_length) 290 { 291 struct _iov_ctx *ctx; 292 293 if (num_iovs == 0) { 294 return 0; 295 } 296 297 ctx = &pdu->iov_ctx; 298 _iov_ctx_init(ctx, iovec, num_iovs, pdu->readv_offset); 299 if (!_iov_ctx_set_iov(ctx, pdu->data, pdu->data_len)) { 300 goto end; 301 } 302 303 /* Data Digest */ 304 if (ddgst_enable) { 305 _iov_ctx_set_iov(ctx, pdu->data_digest, SPDK_NVME_TCP_DIGEST_LEN); 306 } 307 308 end: 309 if (_mapped_length != NULL) { 310 *_mapped_length = ctx->mapped_len; 311 } 312 return ctx->iovcnt; 313 } 314 315 static int 316 nvme_tcp_read_data(struct spdk_sock *sock, int bytes, 317 void *buf) 318 { 319 int ret; 320 321 ret = spdk_sock_recv(sock, buf, bytes); 322 323 if (ret > 0) { 324 return ret; 325 } 326 327 if (ret < 0) { 328 if (errno == EAGAIN || errno == EWOULDBLOCK) { 329 return 0; 330 } 331 332 /* For connect reset issue, do not output error log */ 333 if (errno == ECONNRESET) { 334 SPDK_DEBUGLOG(SPDK_LOG_NVME, "spdk_sock_recv() failed, errno %d: %s\n", 335 errno, spdk_strerror(errno)); 336 } else { 337 SPDK_ERRLOG("spdk_sock_recv() failed, errno %d: %s\n", 338 errno, spdk_strerror(errno)); 339 } 340 } 341 342 /* connection closed */ 343 return NVME_TCP_CONNECTION_FATAL; 344 } 345 346 static int 347 nvme_tcp_readv_data(struct spdk_sock *sock, struct iovec *iov, int iovcnt) 348 { 349 int ret; 350 351 assert(sock != NULL); 352 if (iov == NULL || iovcnt == 0) { 353 return 0; 354 } 355 356 if (iovcnt == 1) { 357 return nvme_tcp_read_data(sock, iov->iov_len, iov->iov_base); 358 } 359 360 ret = spdk_sock_readv(sock, iov, iovcnt); 361 362 if (ret > 0) { 363 return ret; 364 } 365 366 if (ret < 0) { 367 if (errno == EAGAIN || errno == EWOULDBLOCK) { 368 return 0; 369 } 370 371 /* For connect reset issue, do not output error log */ 372 if (errno == ECONNRESET) { 373 SPDK_DEBUGLOG(SPDK_LOG_NVME, "spdk_sock_readv() failed, errno %d: %s\n", 374 errno, spdk_strerror(errno)); 375 } else { 376 SPDK_ERRLOG("spdk_sock_readv() failed, errno %d: %s\n", 377 errno, spdk_strerror(errno)); 378 } 379 } 380 381 /* connection closed */ 382 return NVME_TCP_CONNECTION_FATAL; 383 } 384 385 386 static int 387 nvme_tcp_read_payload_data(struct spdk_sock *sock, struct nvme_tcp_pdu *pdu) 388 { 389 struct iovec iovec_array[2]; 390 struct iovec *iov = iovec_array; 391 int iovec_cnt; 392 393 iovec_cnt = nvme_tcp_build_payload_iovecs(iovec_array, 2, pdu, pdu->ddgst_enable, NULL); 394 assert(iovec_cnt >= 0); 395 396 return nvme_tcp_readv_data(sock, iov, iovec_cnt); 397 } 398 399 #endif /* SPDK_INTERNAL_NVME_TCP_H */ 400