1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. All rights reserved. 5 * Copyright (c) 2019, 2020 Mellanox Technologies LTD. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 #include "spdk/crc32.h" 36 #include "spdk/endian.h" 37 #include "spdk/assert.h" 38 #include "spdk/thread.h" 39 #include "spdk/nvmf_transport.h" 40 #include "spdk/sock.h" 41 #include "spdk/string.h" 42 #include "spdk/trace.h" 43 #include "spdk/util.h" 44 45 #include "spdk_internal/assert.h" 46 #include "spdk_internal/log.h" 47 #include "spdk_internal/nvme_tcp.h" 48 49 #define NVMF_TCP_MAX_ACCEPT_SOCK_ONE_TIME 16 50 #define SPDK_NVMF_TCP_DEFAULT_MAX_SOCK_PRIORITY 6 51 #define SPDK_NVMF_TCP_RECV_BUF_SIZE_FACTOR 4 52 53 const struct spdk_nvmf_transport_ops spdk_nvmf_transport_tcp; 54 55 /* spdk nvmf related structure */ 56 enum spdk_nvmf_tcp_req_state { 57 58 /* The request is not currently in use */ 59 TCP_REQUEST_STATE_FREE = 0, 60 61 /* Initial state when request first received */ 62 TCP_REQUEST_STATE_NEW, 63 64 /* The request is queued until a data buffer is available. */ 65 TCP_REQUEST_STATE_NEED_BUFFER, 66 67 /* The request is currently transferring data from the host to the controller. */ 68 TCP_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER, 69 70 /* The request is waiting for the R2T send acknowledgement. */ 71 TCP_REQUEST_STATE_AWAITING_R2T_ACK, 72 73 /* The request is ready to execute at the block device */ 74 TCP_REQUEST_STATE_READY_TO_EXECUTE, 75 76 /* The request is currently executing at the block device */ 77 TCP_REQUEST_STATE_EXECUTING, 78 79 /* The request finished executing at the block device */ 80 TCP_REQUEST_STATE_EXECUTED, 81 82 /* The request is ready to send a completion */ 83 TCP_REQUEST_STATE_READY_TO_COMPLETE, 84 85 /* The request is currently transferring final pdus from the controller to the host. */ 86 TCP_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST, 87 88 /* The request completed and can be marked free. */ 89 TCP_REQUEST_STATE_COMPLETED, 90 91 /* Terminator */ 92 TCP_REQUEST_NUM_STATES, 93 }; 94 95 static const char *spdk_nvmf_tcp_term_req_fes_str[] = { 96 "Invalid PDU Header Field", 97 "PDU Sequence Error", 98 "Header Digiest Error", 99 "Data Transfer Out of Range", 100 "R2T Limit Exceeded", 101 "Unsupported parameter", 102 }; 103 104 #define OBJECT_NVMF_TCP_IO 0x80 105 106 #define TRACE_GROUP_NVMF_TCP 0x5 107 #define TRACE_TCP_REQUEST_STATE_NEW SPDK_TPOINT_ID(TRACE_GROUP_NVMF_TCP, 0x0) 108 #define TRACE_TCP_REQUEST_STATE_NEED_BUFFER SPDK_TPOINT_ID(TRACE_GROUP_NVMF_TCP, 0x1) 109 #define TRACE_TCP_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER SPDK_TPOINT_ID(TRACE_GROUP_NVMF_TCP, 0x2) 110 #define TRACE_TCP_REQUEST_STATE_READY_TO_EXECUTE SPDK_TPOINT_ID(TRACE_GROUP_NVMF_TCP, 0x3) 111 #define TRACE_TCP_REQUEST_STATE_EXECUTING SPDK_TPOINT_ID(TRACE_GROUP_NVMF_TCP, 0x4) 112 #define TRACE_TCP_REQUEST_STATE_EXECUTED SPDK_TPOINT_ID(TRACE_GROUP_NVMF_TCP, 0x5) 113 #define TRACE_TCP_REQUEST_STATE_READY_TO_COMPLETE SPDK_TPOINT_ID(TRACE_GROUP_NVMF_TCP, 0x6) 114 #define TRACE_TCP_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST SPDK_TPOINT_ID(TRACE_GROUP_NVMF_TCP, 0x7) 115 #define TRACE_TCP_REQUEST_STATE_COMPLETED SPDK_TPOINT_ID(TRACE_GROUP_NVMF_TCP, 0x8) 116 #define TRACE_TCP_FLUSH_WRITEBUF_START SPDK_TPOINT_ID(TRACE_GROUP_NVMF_TCP, 0x9) 117 #define TRACE_TCP_FLUSH_WRITEBUF_DONE SPDK_TPOINT_ID(TRACE_GROUP_NVMF_TCP, 0xA) 118 #define TRACE_TCP_READ_FROM_SOCKET_DONE SPDK_TPOINT_ID(TRACE_GROUP_NVMF_TCP, 0xB) 119 #define TRACE_TCP_REQUEST_STATE_AWAIT_R2T_ACK SPDK_TPOINT_ID(TRACE_GROUP_NVMF_TCP, 0xC) 120 121 SPDK_TRACE_REGISTER_FN(nvmf_tcp_trace, "nvmf_tcp", TRACE_GROUP_NVMF_TCP) 122 { 123 spdk_trace_register_object(OBJECT_NVMF_TCP_IO, 'r'); 124 spdk_trace_register_description("TCP_REQ_NEW", 125 TRACE_TCP_REQUEST_STATE_NEW, 126 OWNER_NONE, OBJECT_NVMF_TCP_IO, 1, 1, ""); 127 spdk_trace_register_description("TCP_REQ_NEED_BUFFER", 128 TRACE_TCP_REQUEST_STATE_NEED_BUFFER, 129 OWNER_NONE, OBJECT_NVMF_TCP_IO, 0, 1, ""); 130 spdk_trace_register_description("TCP_REQ_TX_H_TO_C", 131 TRACE_TCP_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER, 132 OWNER_NONE, OBJECT_NVMF_TCP_IO, 0, 1, ""); 133 spdk_trace_register_description("TCP_REQ_RDY_TO_EXECUTE", 134 TRACE_TCP_REQUEST_STATE_READY_TO_EXECUTE, 135 OWNER_NONE, OBJECT_NVMF_TCP_IO, 0, 1, ""); 136 spdk_trace_register_description("TCP_REQ_EXECUTING", 137 TRACE_TCP_REQUEST_STATE_EXECUTING, 138 OWNER_NONE, OBJECT_NVMF_TCP_IO, 0, 1, ""); 139 spdk_trace_register_description("TCP_REQ_EXECUTED", 140 TRACE_TCP_REQUEST_STATE_EXECUTED, 141 OWNER_NONE, OBJECT_NVMF_TCP_IO, 0, 1, ""); 142 spdk_trace_register_description("TCP_REQ_RDY_TO_COMPLETE", 143 TRACE_TCP_REQUEST_STATE_READY_TO_COMPLETE, 144 OWNER_NONE, OBJECT_NVMF_TCP_IO, 0, 1, ""); 145 spdk_trace_register_description("TCP_REQ_TRANSFER_C2H", 146 TRACE_TCP_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST, 147 OWNER_NONE, OBJECT_NVMF_TCP_IO, 0, 1, ""); 148 spdk_trace_register_description("TCP_REQ_COMPLETED", 149 TRACE_TCP_REQUEST_STATE_COMPLETED, 150 OWNER_NONE, OBJECT_NVMF_TCP_IO, 0, 1, ""); 151 spdk_trace_register_description("TCP_WRITE_START", 152 TRACE_TCP_FLUSH_WRITEBUF_START, 153 OWNER_NONE, OBJECT_NONE, 0, 0, ""); 154 spdk_trace_register_description("TCP_WRITE_DONE", 155 TRACE_TCP_FLUSH_WRITEBUF_DONE, 156 OWNER_NONE, OBJECT_NONE, 0, 0, ""); 157 spdk_trace_register_description("TCP_READ_DONE", 158 TRACE_TCP_READ_FROM_SOCKET_DONE, 159 OWNER_NONE, OBJECT_NONE, 0, 0, ""); 160 spdk_trace_register_description("TCP_REQ_AWAIT_R2T_ACK", 161 TRACE_TCP_REQUEST_STATE_AWAIT_R2T_ACK, 162 OWNER_NONE, OBJECT_NVMF_TCP_IO, 0, 1, ""); 163 } 164 165 struct spdk_nvmf_tcp_req { 166 struct spdk_nvmf_request req; 167 struct spdk_nvme_cpl rsp; 168 struct spdk_nvme_cmd cmd; 169 170 /* A PDU that can be used for sending responses. This is 171 * not the incoming PDU! */ 172 struct nvme_tcp_pdu *pdu; 173 174 /* 175 * The PDU for a request may be used multiple times in serial over 176 * the request's lifetime. For example, first to send an R2T, then 177 * to send a completion. To catch mistakes where the PDU is used 178 * twice at the same time, add a debug flag here for init/fini. 179 */ 180 bool pdu_in_use; 181 182 /* In-capsule data buffer */ 183 uint8_t *buf; 184 185 bool has_incapsule_data; 186 187 /* transfer_tag */ 188 uint16_t ttag; 189 190 enum spdk_nvmf_tcp_req_state state; 191 192 /* 193 * h2c_offset is used when we receive the h2c_data PDU. 194 */ 195 uint32_t h2c_offset; 196 197 STAILQ_ENTRY(spdk_nvmf_tcp_req) link; 198 TAILQ_ENTRY(spdk_nvmf_tcp_req) state_link; 199 }; 200 201 struct spdk_nvmf_tcp_qpair { 202 struct spdk_nvmf_qpair qpair; 203 struct spdk_nvmf_tcp_poll_group *group; 204 struct spdk_nvmf_tcp_port *port; 205 struct spdk_sock *sock; 206 207 enum nvme_tcp_pdu_recv_state recv_state; 208 enum nvme_tcp_qpair_state state; 209 210 /* PDU being actively received */ 211 struct nvme_tcp_pdu pdu_in_progress; 212 213 /* This is a spare PDU used for sending special management 214 * operations. Primarily, this is used for the initial 215 * connection response and c2h termination request. */ 216 struct nvme_tcp_pdu mgmt_pdu; 217 218 TAILQ_HEAD(, nvme_tcp_pdu) send_queue; 219 220 /* Arrays of in-capsule buffers, requests, and pdus. 221 * Each array is 'resource_count' number of elements */ 222 void *bufs; 223 struct spdk_nvmf_tcp_req *reqs; 224 struct nvme_tcp_pdu *pdus; 225 uint32_t resource_count; 226 227 /* Queues to track the requests in all states */ 228 TAILQ_HEAD(, spdk_nvmf_tcp_req) state_queue[TCP_REQUEST_NUM_STATES]; 229 /* Number of requests in each state */ 230 uint32_t state_cntr[TCP_REQUEST_NUM_STATES]; 231 232 uint8_t cpda; 233 234 bool host_hdgst_enable; 235 bool host_ddgst_enable; 236 237 /* IP address */ 238 char initiator_addr[SPDK_NVMF_TRADDR_MAX_LEN]; 239 char target_addr[SPDK_NVMF_TRADDR_MAX_LEN]; 240 241 /* IP port */ 242 uint16_t initiator_port; 243 uint16_t target_port; 244 245 /* Timer used to destroy qpair after detecting transport error issue if initiator does 246 * not close the connection. 247 */ 248 struct spdk_poller *timeout_poller; 249 250 TAILQ_ENTRY(spdk_nvmf_tcp_qpair) link; 251 }; 252 253 struct spdk_nvmf_tcp_poll_group { 254 struct spdk_nvmf_transport_poll_group group; 255 struct spdk_sock_group *sock_group; 256 257 TAILQ_HEAD(, spdk_nvmf_tcp_qpair) qpairs; 258 TAILQ_HEAD(, spdk_nvmf_tcp_qpair) await_req; 259 }; 260 261 struct spdk_nvmf_tcp_port { 262 const struct spdk_nvme_transport_id *trid; 263 struct spdk_sock *listen_sock; 264 TAILQ_ENTRY(spdk_nvmf_tcp_port) link; 265 }; 266 267 struct spdk_nvmf_tcp_transport { 268 struct spdk_nvmf_transport transport; 269 270 pthread_mutex_t lock; 271 272 TAILQ_HEAD(, spdk_nvmf_tcp_port) ports; 273 }; 274 275 static bool spdk_nvmf_tcp_req_process(struct spdk_nvmf_tcp_transport *ttransport, 276 struct spdk_nvmf_tcp_req *tcp_req); 277 278 static void 279 spdk_nvmf_tcp_req_set_state(struct spdk_nvmf_tcp_req *tcp_req, 280 enum spdk_nvmf_tcp_req_state state) 281 { 282 struct spdk_nvmf_qpair *qpair; 283 struct spdk_nvmf_tcp_qpair *tqpair; 284 285 qpair = tcp_req->req.qpair; 286 tqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_tcp_qpair, qpair); 287 288 TAILQ_REMOVE(&tqpair->state_queue[tcp_req->state], tcp_req, state_link); 289 assert(tqpair->state_cntr[tcp_req->state] > 0); 290 tqpair->state_cntr[tcp_req->state]--; 291 292 TAILQ_INSERT_TAIL(&tqpair->state_queue[state], tcp_req, state_link); 293 tqpair->state_cntr[state]++; 294 295 tcp_req->state = state; 296 } 297 298 static inline struct nvme_tcp_pdu * 299 nvmf_tcp_req_pdu_init(struct spdk_nvmf_tcp_req *tcp_req) 300 { 301 assert(tcp_req->pdu_in_use == false); 302 tcp_req->pdu_in_use = true; 303 304 memset(tcp_req->pdu, 0, sizeof(*tcp_req->pdu)); 305 tcp_req->pdu->qpair = SPDK_CONTAINEROF(tcp_req->req.qpair, struct spdk_nvmf_tcp_qpair, qpair); 306 307 return tcp_req->pdu; 308 } 309 310 static inline void 311 nvmf_tcp_req_pdu_fini(struct spdk_nvmf_tcp_req *tcp_req) 312 { 313 tcp_req->pdu_in_use = false; 314 } 315 316 static struct spdk_nvmf_tcp_req * 317 spdk_nvmf_tcp_req_get(struct spdk_nvmf_tcp_qpair *tqpair) 318 { 319 struct spdk_nvmf_tcp_req *tcp_req; 320 321 tcp_req = TAILQ_FIRST(&tqpair->state_queue[TCP_REQUEST_STATE_FREE]); 322 if (!tcp_req) { 323 return NULL; 324 } 325 326 memset(&tcp_req->rsp, 0, sizeof(tcp_req->rsp)); 327 tcp_req->h2c_offset = 0; 328 tcp_req->has_incapsule_data = false; 329 tcp_req->req.dif.dif_insert_or_strip = false; 330 331 spdk_nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_NEW); 332 return tcp_req; 333 } 334 335 static void 336 nvmf_tcp_request_free(struct spdk_nvmf_tcp_req *tcp_req) 337 { 338 struct spdk_nvmf_tcp_transport *ttransport; 339 340 assert(tcp_req != NULL); 341 342 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "tcp_req=%p will be freed\n", tcp_req); 343 ttransport = SPDK_CONTAINEROF(tcp_req->req.qpair->transport, 344 struct spdk_nvmf_tcp_transport, transport); 345 spdk_nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_COMPLETED); 346 spdk_nvmf_tcp_req_process(ttransport, tcp_req); 347 } 348 349 static int 350 spdk_nvmf_tcp_req_free(struct spdk_nvmf_request *req) 351 { 352 struct spdk_nvmf_tcp_req *tcp_req = SPDK_CONTAINEROF(req, struct spdk_nvmf_tcp_req, req); 353 354 nvmf_tcp_request_free(tcp_req); 355 356 return 0; 357 } 358 359 static void 360 spdk_nvmf_tcp_drain_state_queue(struct spdk_nvmf_tcp_qpair *tqpair, 361 enum spdk_nvmf_tcp_req_state state) 362 { 363 struct spdk_nvmf_tcp_req *tcp_req, *req_tmp; 364 365 TAILQ_FOREACH_SAFE(tcp_req, &tqpair->state_queue[state], state_link, req_tmp) { 366 nvmf_tcp_request_free(tcp_req); 367 } 368 } 369 370 static void 371 spdk_nvmf_tcp_cleanup_all_states(struct spdk_nvmf_tcp_qpair *tqpair) 372 { 373 struct spdk_nvmf_tcp_req *tcp_req, *req_tmp; 374 375 assert(TAILQ_EMPTY(&tqpair->send_queue)); 376 377 spdk_nvmf_tcp_drain_state_queue(tqpair, TCP_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST); 378 spdk_nvmf_tcp_drain_state_queue(tqpair, TCP_REQUEST_STATE_NEW); 379 380 /* Wipe the requests waiting for buffer from the global list */ 381 TAILQ_FOREACH_SAFE(tcp_req, &tqpair->state_queue[TCP_REQUEST_STATE_NEED_BUFFER], state_link, 382 req_tmp) { 383 STAILQ_REMOVE(&tqpair->group->group.pending_buf_queue, &tcp_req->req, 384 spdk_nvmf_request, buf_link); 385 } 386 387 spdk_nvmf_tcp_drain_state_queue(tqpair, TCP_REQUEST_STATE_NEED_BUFFER); 388 spdk_nvmf_tcp_drain_state_queue(tqpair, TCP_REQUEST_STATE_EXECUTING); 389 spdk_nvmf_tcp_drain_state_queue(tqpair, TCP_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 390 spdk_nvmf_tcp_drain_state_queue(tqpair, TCP_REQUEST_STATE_AWAITING_R2T_ACK); 391 } 392 393 static void 394 nvmf_tcp_dump_qpair_req_contents(struct spdk_nvmf_tcp_qpair *tqpair) 395 { 396 int i; 397 struct spdk_nvmf_tcp_req *tcp_req; 398 399 SPDK_ERRLOG("Dumping contents of queue pair (QID %d)\n", tqpair->qpair.qid); 400 for (i = 1; i < TCP_REQUEST_NUM_STATES; i++) { 401 SPDK_ERRLOG("\tNum of requests in state[%d] = %u\n", i, tqpair->state_cntr[i]); 402 TAILQ_FOREACH(tcp_req, &tqpair->state_queue[i], state_link) { 403 SPDK_ERRLOG("\t\tRequest Data From Pool: %d\n", tcp_req->req.data_from_pool); 404 SPDK_ERRLOG("\t\tRequest opcode: %d\n", tcp_req->req.cmd->nvmf_cmd.opcode); 405 } 406 } 407 } 408 409 static void 410 spdk_nvmf_tcp_qpair_destroy(struct spdk_nvmf_tcp_qpair *tqpair) 411 { 412 int err = 0; 413 414 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "enter\n"); 415 416 err = spdk_sock_close(&tqpair->sock); 417 assert(err == 0); 418 spdk_nvmf_tcp_cleanup_all_states(tqpair); 419 420 if (tqpair->state_cntr[TCP_REQUEST_STATE_FREE] != tqpair->resource_count) { 421 SPDK_ERRLOG("tqpair(%p) free tcp request num is %u but should be %u\n", tqpair, 422 tqpair->state_cntr[TCP_REQUEST_STATE_FREE], 423 tqpair->resource_count); 424 err++; 425 } 426 427 if (err > 0) { 428 nvmf_tcp_dump_qpair_req_contents(tqpair); 429 } 430 431 spdk_dma_free(tqpair->pdus); 432 free(tqpair->reqs); 433 spdk_free(tqpair->bufs); 434 free(tqpair); 435 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "Leave\n"); 436 } 437 438 static int 439 spdk_nvmf_tcp_destroy(struct spdk_nvmf_transport *transport) 440 { 441 struct spdk_nvmf_tcp_transport *ttransport; 442 443 assert(transport != NULL); 444 ttransport = SPDK_CONTAINEROF(transport, struct spdk_nvmf_tcp_transport, transport); 445 446 pthread_mutex_destroy(&ttransport->lock); 447 free(ttransport); 448 return 0; 449 } 450 451 static struct spdk_nvmf_transport * 452 spdk_nvmf_tcp_create(struct spdk_nvmf_transport_opts *opts) 453 { 454 struct spdk_nvmf_tcp_transport *ttransport; 455 uint32_t sge_count; 456 uint32_t min_shared_buffers; 457 458 ttransport = calloc(1, sizeof(*ttransport)); 459 if (!ttransport) { 460 return NULL; 461 } 462 463 TAILQ_INIT(&ttransport->ports); 464 465 ttransport->transport.ops = &spdk_nvmf_transport_tcp; 466 467 SPDK_NOTICELOG("*** TCP Transport Init ***\n"); 468 469 SPDK_INFOLOG(SPDK_LOG_NVMF_TCP, "*** TCP Transport Init ***\n" 470 " Transport opts: max_ioq_depth=%d, max_io_size=%d,\n" 471 " max_qpairs_per_ctrlr=%d, io_unit_size=%d,\n" 472 " in_capsule_data_size=%d, max_aq_depth=%d\n" 473 " num_shared_buffers=%d, c2h_success=%d,\n" 474 " dif_insert_or_strip=%d, sock_priority=%d\n", 475 opts->max_queue_depth, 476 opts->max_io_size, 477 opts->max_qpairs_per_ctrlr, 478 opts->io_unit_size, 479 opts->in_capsule_data_size, 480 opts->max_aq_depth, 481 opts->num_shared_buffers, 482 opts->c2h_success, 483 opts->dif_insert_or_strip, 484 opts->sock_priority); 485 486 if (opts->sock_priority > SPDK_NVMF_TCP_DEFAULT_MAX_SOCK_PRIORITY) { 487 SPDK_ERRLOG("Unsupported socket_priority=%d, the current range is: 0 to %d\n" 488 "you can use man 7 socket to view the range of priority under SO_PRIORITY item\n", 489 opts->sock_priority, SPDK_NVMF_TCP_DEFAULT_MAX_SOCK_PRIORITY); 490 free(ttransport); 491 return NULL; 492 } 493 494 /* I/O unit size cannot be larger than max I/O size */ 495 if (opts->io_unit_size > opts->max_io_size) { 496 opts->io_unit_size = opts->max_io_size; 497 } 498 499 sge_count = opts->max_io_size / opts->io_unit_size; 500 if (sge_count > SPDK_NVMF_MAX_SGL_ENTRIES) { 501 SPDK_ERRLOG("Unsupported IO Unit size specified, %d bytes\n", opts->io_unit_size); 502 free(ttransport); 503 return NULL; 504 } 505 506 min_shared_buffers = spdk_thread_get_count() * opts->buf_cache_size; 507 if (min_shared_buffers > opts->num_shared_buffers) { 508 SPDK_ERRLOG("There are not enough buffers to satisfy" 509 "per-poll group caches for each thread. (%" PRIu32 ")" 510 "supplied. (%" PRIu32 ") required\n", opts->num_shared_buffers, min_shared_buffers); 511 SPDK_ERRLOG("Please specify a larger number of shared buffers\n"); 512 spdk_nvmf_tcp_destroy(&ttransport->transport); 513 return NULL; 514 } 515 516 pthread_mutex_init(&ttransport->lock, NULL); 517 518 return &ttransport->transport; 519 } 520 521 static int 522 _spdk_nvmf_tcp_trsvcid_to_int(const char *trsvcid) 523 { 524 unsigned long long ull; 525 char *end = NULL; 526 527 ull = strtoull(trsvcid, &end, 10); 528 if (end == NULL || end == trsvcid || *end != '\0') { 529 return -1; 530 } 531 532 /* Valid TCP/IP port numbers are in [0, 65535] */ 533 if (ull > 65535) { 534 return -1; 535 } 536 537 return (int)ull; 538 } 539 540 /** 541 * Canonicalize a listen address trid. 542 */ 543 static int 544 _spdk_nvmf_tcp_canon_listen_trid(struct spdk_nvme_transport_id *canon_trid, 545 const struct spdk_nvme_transport_id *trid) 546 { 547 int trsvcid_int; 548 549 trsvcid_int = _spdk_nvmf_tcp_trsvcid_to_int(trid->trsvcid); 550 if (trsvcid_int < 0) { 551 return -EINVAL; 552 } 553 554 memset(canon_trid, 0, sizeof(*canon_trid)); 555 spdk_nvme_trid_populate_transport(canon_trid, SPDK_NVME_TRANSPORT_TCP); 556 canon_trid->adrfam = trid->adrfam; 557 snprintf(canon_trid->traddr, sizeof(canon_trid->traddr), "%s", trid->traddr); 558 snprintf(canon_trid->trsvcid, sizeof(canon_trid->trsvcid), "%d", trsvcid_int); 559 560 return 0; 561 } 562 563 /** 564 * Find an existing listening port. 565 * 566 * Caller must hold ttransport->lock. 567 */ 568 static struct spdk_nvmf_tcp_port * 569 _spdk_nvmf_tcp_find_port(struct spdk_nvmf_tcp_transport *ttransport, 570 const struct spdk_nvme_transport_id *trid) 571 { 572 struct spdk_nvme_transport_id canon_trid; 573 struct spdk_nvmf_tcp_port *port; 574 575 if (_spdk_nvmf_tcp_canon_listen_trid(&canon_trid, trid) != 0) { 576 return NULL; 577 } 578 579 TAILQ_FOREACH(port, &ttransport->ports, link) { 580 if (spdk_nvme_transport_id_compare(&canon_trid, port->trid) == 0) { 581 return port; 582 } 583 } 584 585 return NULL; 586 } 587 588 static int 589 spdk_nvmf_tcp_listen(struct spdk_nvmf_transport *transport, 590 const struct spdk_nvme_transport_id *trid) 591 { 592 struct spdk_nvmf_tcp_transport *ttransport; 593 struct spdk_nvmf_tcp_port *port; 594 int trsvcid_int; 595 uint8_t adrfam; 596 597 ttransport = SPDK_CONTAINEROF(transport, struct spdk_nvmf_tcp_transport, transport); 598 599 trsvcid_int = _spdk_nvmf_tcp_trsvcid_to_int(trid->trsvcid); 600 if (trsvcid_int < 0) { 601 SPDK_ERRLOG("Invalid trsvcid '%s'\n", trid->trsvcid); 602 return -EINVAL; 603 } 604 605 pthread_mutex_lock(&ttransport->lock); 606 port = calloc(1, sizeof(*port)); 607 if (!port) { 608 SPDK_ERRLOG("Port allocation failed\n"); 609 pthread_mutex_unlock(&ttransport->lock); 610 return -ENOMEM; 611 } 612 613 port->trid = trid; 614 port->listen_sock = spdk_sock_listen(trid->traddr, trsvcid_int, NULL); 615 if (port->listen_sock == NULL) { 616 SPDK_ERRLOG("spdk_sock_listen(%s, %d) failed: %s (%d)\n", 617 trid->traddr, trsvcid_int, 618 spdk_strerror(errno), errno); 619 free(port); 620 pthread_mutex_unlock(&ttransport->lock); 621 return -errno; 622 } 623 624 if (spdk_sock_is_ipv4(port->listen_sock)) { 625 adrfam = SPDK_NVMF_ADRFAM_IPV4; 626 } else if (spdk_sock_is_ipv6(port->listen_sock)) { 627 adrfam = SPDK_NVMF_ADRFAM_IPV6; 628 } else { 629 SPDK_ERRLOG("Unhandled socket type\n"); 630 adrfam = 0; 631 } 632 633 if (adrfam != trid->adrfam) { 634 SPDK_ERRLOG("Socket address family mismatch\n"); 635 spdk_sock_close(&port->listen_sock); 636 free(port); 637 pthread_mutex_unlock(&ttransport->lock); 638 return -EINVAL; 639 } 640 641 SPDK_NOTICELOG("*** NVMe/TCP Target Listening on %s port %s ***\n", 642 trid->traddr, trid->trsvcid); 643 644 TAILQ_INSERT_TAIL(&ttransport->ports, port, link); 645 pthread_mutex_unlock(&ttransport->lock); 646 return 0; 647 } 648 649 static void 650 spdk_nvmf_tcp_stop_listen(struct spdk_nvmf_transport *transport, 651 const struct spdk_nvme_transport_id *trid) 652 { 653 struct spdk_nvmf_tcp_transport *ttransport; 654 struct spdk_nvmf_tcp_port *port; 655 656 ttransport = SPDK_CONTAINEROF(transport, struct spdk_nvmf_tcp_transport, transport); 657 658 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "Removing listen address %s port %s\n", 659 trid->traddr, trid->trsvcid); 660 661 pthread_mutex_lock(&ttransport->lock); 662 port = _spdk_nvmf_tcp_find_port(ttransport, trid); 663 if (port) { 664 TAILQ_REMOVE(&ttransport->ports, port, link); 665 spdk_sock_close(&port->listen_sock); 666 free(port); 667 } 668 669 pthread_mutex_unlock(&ttransport->lock); 670 } 671 672 static void spdk_nvmf_tcp_qpair_set_recv_state(struct spdk_nvmf_tcp_qpair *tqpair, 673 enum nvme_tcp_pdu_recv_state state); 674 675 static void 676 spdk_nvmf_tcp_qpair_disconnect(struct spdk_nvmf_tcp_qpair *tqpair) 677 { 678 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "Disconnecting qpair %p\n", tqpair); 679 680 if (tqpair->state <= NVME_TCP_QPAIR_STATE_RUNNING) { 681 tqpair->state = NVME_TCP_QPAIR_STATE_EXITING; 682 spdk_nvmf_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_ERROR); 683 spdk_poller_unregister(&tqpair->timeout_poller); 684 685 /* This will end up calling spdk_nvmf_tcp_close_qpair */ 686 spdk_nvmf_qpair_disconnect(&tqpair->qpair, NULL, NULL); 687 } 688 } 689 690 static void 691 _pdu_write_done(void *_pdu, int err) 692 { 693 struct nvme_tcp_pdu *pdu = _pdu; 694 struct spdk_nvmf_tcp_qpair *tqpair = pdu->qpair; 695 696 TAILQ_REMOVE(&tqpair->send_queue, pdu, tailq); 697 698 if (err != 0) { 699 spdk_nvmf_tcp_qpair_disconnect(tqpair); 700 return; 701 } 702 703 assert(pdu->cb_fn != NULL); 704 pdu->cb_fn(pdu->cb_arg); 705 } 706 707 static void 708 spdk_nvmf_tcp_qpair_write_pdu(struct spdk_nvmf_tcp_qpair *tqpair, 709 struct nvme_tcp_pdu *pdu, 710 nvme_tcp_qpair_xfer_complete_cb cb_fn, 711 void *cb_arg) 712 { 713 int hlen; 714 uint32_t crc32c; 715 uint32_t mapped_length = 0; 716 ssize_t rc; 717 718 assert(&tqpair->pdu_in_progress != pdu); 719 720 hlen = pdu->hdr.common.hlen; 721 722 /* Header Digest */ 723 if (g_nvme_tcp_hdgst[pdu->hdr.common.pdu_type] && tqpair->host_hdgst_enable) { 724 crc32c = nvme_tcp_pdu_calc_header_digest(pdu); 725 MAKE_DIGEST_WORD((uint8_t *)pdu->hdr.raw + hlen, crc32c); 726 } 727 728 /* Data Digest */ 729 if (pdu->data_len > 0 && g_nvme_tcp_ddgst[pdu->hdr.common.pdu_type] && tqpair->host_ddgst_enable) { 730 crc32c = nvme_tcp_pdu_calc_data_digest(pdu); 731 MAKE_DIGEST_WORD(pdu->data_digest, crc32c); 732 } 733 734 pdu->cb_fn = cb_fn; 735 pdu->cb_arg = cb_arg; 736 737 pdu->sock_req.iovcnt = nvme_tcp_build_iovs(pdu->iov, SPDK_COUNTOF(pdu->iov), pdu, 738 tqpair->host_hdgst_enable, tqpair->host_ddgst_enable, 739 &mapped_length); 740 pdu->sock_req.cb_fn = _pdu_write_done; 741 pdu->sock_req.cb_arg = pdu; 742 TAILQ_INSERT_TAIL(&tqpair->send_queue, pdu, tailq); 743 if (pdu->hdr.common.pdu_type == SPDK_NVME_TCP_PDU_TYPE_IC_RESP || 744 pdu->hdr.common.pdu_type == SPDK_NVME_TCP_PDU_TYPE_C2H_TERM_REQ) { 745 rc = spdk_sock_writev(tqpair->sock, pdu->iov, pdu->sock_req.iovcnt); 746 if (rc == mapped_length) { 747 _pdu_write_done(pdu, 0); 748 } else { 749 SPDK_ERRLOG("IC_RESP or TERM_REQ could not write to socket.\n"); 750 _pdu_write_done(pdu, -1); 751 } 752 } else { 753 spdk_sock_writev_async(tqpair->sock, &pdu->sock_req); 754 } 755 } 756 757 static int 758 spdk_nvmf_tcp_qpair_init_mem_resource(struct spdk_nvmf_tcp_qpair *tqpair) 759 { 760 uint32_t i; 761 struct spdk_nvmf_transport_opts *opts; 762 uint32_t in_capsule_data_size; 763 764 opts = &tqpair->qpair.transport->opts; 765 766 in_capsule_data_size = opts->in_capsule_data_size; 767 if (opts->dif_insert_or_strip) { 768 in_capsule_data_size = SPDK_BDEV_BUF_SIZE_WITH_MD(in_capsule_data_size); 769 } 770 771 tqpair->resource_count = opts->max_queue_depth; 772 773 tqpair->mgmt_pdu.qpair = tqpair; 774 775 tqpair->reqs = calloc(tqpair->resource_count, sizeof(*tqpair->reqs)); 776 if (!tqpair->reqs) { 777 SPDK_ERRLOG("Unable to allocate reqs on tqpair=%p\n", tqpair); 778 return -1; 779 } 780 781 if (in_capsule_data_size) { 782 tqpair->bufs = spdk_zmalloc(tqpair->resource_count * in_capsule_data_size, 0x1000, 783 NULL, SPDK_ENV_LCORE_ID_ANY, 784 SPDK_MALLOC_DMA); 785 if (!tqpair->bufs) { 786 SPDK_ERRLOG("Unable to allocate bufs on tqpair=%p.\n", tqpair); 787 return -1; 788 } 789 } 790 791 tqpair->pdus = spdk_dma_malloc(tqpair->resource_count * sizeof(*tqpair->pdus), 0x1000, NULL); 792 if (!tqpair->pdus) { 793 SPDK_ERRLOG("Unable to allocate pdu pool on tqpair =%p.\n", tqpair); 794 return -1; 795 } 796 797 for (i = 0; i < tqpair->resource_count; i++) { 798 struct spdk_nvmf_tcp_req *tcp_req = &tqpair->reqs[i]; 799 800 tcp_req->ttag = i + 1; 801 tcp_req->req.qpair = &tqpair->qpair; 802 803 tcp_req->pdu = &tqpair->pdus[i]; 804 tcp_req->pdu->qpair = tqpair; 805 806 /* Set up memory to receive commands */ 807 if (tqpair->bufs) { 808 tcp_req->buf = (void *)((uintptr_t)tqpair->bufs + (i * in_capsule_data_size)); 809 } 810 811 /* Set the cmdn and rsp */ 812 tcp_req->req.rsp = (union nvmf_c2h_msg *)&tcp_req->rsp; 813 tcp_req->req.cmd = (union nvmf_h2c_msg *)&tcp_req->cmd; 814 815 /* Initialize request state to FREE */ 816 tcp_req->state = TCP_REQUEST_STATE_FREE; 817 TAILQ_INSERT_TAIL(&tqpair->state_queue[tcp_req->state], tcp_req, state_link); 818 tqpair->state_cntr[TCP_REQUEST_STATE_FREE]++; 819 } 820 821 return 0; 822 } 823 824 static int 825 spdk_nvmf_tcp_qpair_init(struct spdk_nvmf_qpair *qpair) 826 { 827 struct spdk_nvmf_tcp_qpair *tqpair; 828 int i; 829 830 tqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_tcp_qpair, qpair); 831 832 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "New TCP Connection: %p\n", qpair); 833 834 TAILQ_INIT(&tqpair->send_queue); 835 836 /* Initialise request state queues of the qpair */ 837 for (i = TCP_REQUEST_STATE_FREE; i < TCP_REQUEST_NUM_STATES; i++) { 838 TAILQ_INIT(&tqpair->state_queue[i]); 839 } 840 841 tqpair->host_hdgst_enable = true; 842 tqpair->host_ddgst_enable = true; 843 844 return 0; 845 } 846 847 static int 848 spdk_nvmf_tcp_qpair_sock_init(struct spdk_nvmf_tcp_qpair *tqpair) 849 { 850 int rc; 851 852 /* set low water mark */ 853 rc = spdk_sock_set_recvlowat(tqpair->sock, sizeof(struct spdk_nvme_tcp_c2h_data_hdr)); 854 if (rc != 0) { 855 SPDK_ERRLOG("spdk_sock_set_recvlowat() failed\n"); 856 return rc; 857 } 858 859 return 0; 860 } 861 862 static void 863 _spdk_nvmf_tcp_handle_connect(struct spdk_nvmf_transport *transport, 864 struct spdk_nvmf_tcp_port *port, 865 struct spdk_sock *sock, 866 new_qpair_fn cb_fn, void *cb_arg) 867 { 868 struct spdk_nvmf_tcp_qpair *tqpair; 869 int rc; 870 871 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "New connection accepted on %s port %s\n", 872 port->trid->traddr, port->trid->trsvcid); 873 874 if (transport->opts.sock_priority) { 875 rc = spdk_sock_set_priority(sock, transport->opts.sock_priority); 876 if (rc) { 877 SPDK_ERRLOG("Failed to set the priority of the socket\n"); 878 spdk_sock_close(&sock); 879 return; 880 } 881 } 882 883 tqpair = calloc(1, sizeof(struct spdk_nvmf_tcp_qpair)); 884 if (tqpair == NULL) { 885 SPDK_ERRLOG("Could not allocate new connection.\n"); 886 spdk_sock_close(&sock); 887 return; 888 } 889 890 tqpair->sock = sock; 891 tqpair->state_cntr[TCP_REQUEST_STATE_FREE] = 0; 892 tqpair->port = port; 893 tqpair->qpair.transport = transport; 894 895 rc = spdk_sock_getaddr(tqpair->sock, tqpair->target_addr, 896 sizeof(tqpair->target_addr), &tqpair->target_port, 897 tqpair->initiator_addr, sizeof(tqpair->initiator_addr), 898 &tqpair->initiator_port); 899 if (rc < 0) { 900 SPDK_ERRLOG("spdk_sock_getaddr() failed of tqpair=%p\n", tqpair); 901 spdk_nvmf_tcp_qpair_destroy(tqpair); 902 return; 903 } 904 905 cb_fn(&tqpair->qpair, cb_arg); 906 } 907 908 static void 909 spdk_nvmf_tcp_port_accept(struct spdk_nvmf_transport *transport, struct spdk_nvmf_tcp_port *port, 910 new_qpair_fn cb_fn, void *cb_arg) 911 { 912 struct spdk_sock *sock; 913 int i; 914 915 for (i = 0; i < NVMF_TCP_MAX_ACCEPT_SOCK_ONE_TIME; i++) { 916 sock = spdk_sock_accept(port->listen_sock); 917 if (sock == NULL) { 918 break; 919 } 920 _spdk_nvmf_tcp_handle_connect(transport, port, sock, cb_fn, cb_arg); 921 } 922 } 923 924 static void 925 spdk_nvmf_tcp_accept(struct spdk_nvmf_transport *transport, new_qpair_fn cb_fn, void *cb_arg) 926 { 927 struct spdk_nvmf_tcp_transport *ttransport; 928 struct spdk_nvmf_tcp_port *port; 929 930 ttransport = SPDK_CONTAINEROF(transport, struct spdk_nvmf_tcp_transport, transport); 931 932 TAILQ_FOREACH(port, &ttransport->ports, link) { 933 spdk_nvmf_tcp_port_accept(transport, port, cb_fn, cb_arg); 934 } 935 } 936 937 static void 938 spdk_nvmf_tcp_discover(struct spdk_nvmf_transport *transport, 939 struct spdk_nvme_transport_id *trid, 940 struct spdk_nvmf_discovery_log_page_entry *entry) 941 { 942 entry->trtype = SPDK_NVMF_TRTYPE_TCP; 943 entry->adrfam = trid->adrfam; 944 entry->treq.secure_channel = SPDK_NVMF_TREQ_SECURE_CHANNEL_NOT_REQUIRED; 945 946 spdk_strcpy_pad(entry->trsvcid, trid->trsvcid, sizeof(entry->trsvcid), ' '); 947 spdk_strcpy_pad(entry->traddr, trid->traddr, sizeof(entry->traddr), ' '); 948 949 entry->tsas.tcp.sectype = SPDK_NVME_TCP_SECURITY_NONE; 950 } 951 952 static struct spdk_nvmf_transport_poll_group * 953 spdk_nvmf_tcp_poll_group_create(struct spdk_nvmf_transport *transport) 954 { 955 struct spdk_nvmf_tcp_poll_group *tgroup; 956 957 tgroup = calloc(1, sizeof(*tgroup)); 958 if (!tgroup) { 959 return NULL; 960 } 961 962 tgroup->sock_group = spdk_sock_group_create(&tgroup->group); 963 if (!tgroup->sock_group) { 964 goto cleanup; 965 } 966 967 TAILQ_INIT(&tgroup->qpairs); 968 TAILQ_INIT(&tgroup->await_req); 969 970 return &tgroup->group; 971 972 cleanup: 973 free(tgroup); 974 return NULL; 975 } 976 977 static struct spdk_nvmf_transport_poll_group * 978 spdk_nvmf_tcp_get_optimal_poll_group(struct spdk_nvmf_qpair *qpair) 979 { 980 struct spdk_nvmf_tcp_qpair *tqpair; 981 struct spdk_sock_group *group = NULL; 982 int rc; 983 984 tqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_tcp_qpair, qpair); 985 rc = spdk_sock_get_optimal_sock_group(tqpair->sock, &group); 986 if (!rc && group != NULL) { 987 return spdk_sock_group_get_ctx(group); 988 } 989 990 return NULL; 991 } 992 993 static void 994 spdk_nvmf_tcp_poll_group_destroy(struct spdk_nvmf_transport_poll_group *group) 995 { 996 struct spdk_nvmf_tcp_poll_group *tgroup; 997 998 tgroup = SPDK_CONTAINEROF(group, struct spdk_nvmf_tcp_poll_group, group); 999 spdk_sock_group_close(&tgroup->sock_group); 1000 1001 free(tgroup); 1002 } 1003 1004 static void 1005 spdk_nvmf_tcp_qpair_set_recv_state(struct spdk_nvmf_tcp_qpair *tqpair, 1006 enum nvme_tcp_pdu_recv_state state) 1007 { 1008 if (tqpair->recv_state == state) { 1009 SPDK_ERRLOG("The recv state of tqpair=%p is same with the state(%d) to be set\n", 1010 tqpair, state); 1011 return; 1012 } 1013 1014 if (tqpair->recv_state == NVME_TCP_PDU_RECV_STATE_AWAIT_REQ) { 1015 /* When leaving the await req state, move the qpair to the main list */ 1016 TAILQ_REMOVE(&tqpair->group->await_req, tqpair, link); 1017 TAILQ_INSERT_TAIL(&tqpair->group->qpairs, tqpair, link); 1018 } 1019 1020 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "tqpair(%p) recv state=%d\n", tqpair, state); 1021 tqpair->recv_state = state; 1022 1023 switch (state) { 1024 case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_CH: 1025 case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PSH: 1026 case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PAYLOAD: 1027 break; 1028 case NVME_TCP_PDU_RECV_STATE_AWAIT_REQ: 1029 TAILQ_REMOVE(&tqpair->group->qpairs, tqpair, link); 1030 TAILQ_INSERT_TAIL(&tqpair->group->await_req, tqpair, link); 1031 break; 1032 case NVME_TCP_PDU_RECV_STATE_ERROR: 1033 case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY: 1034 memset(&tqpair->pdu_in_progress, 0, sizeof(tqpair->pdu_in_progress)); 1035 break; 1036 default: 1037 SPDK_ERRLOG("The state(%d) is invalid\n", state); 1038 abort(); 1039 break; 1040 } 1041 } 1042 1043 static int 1044 spdk_nvmf_tcp_qpair_handle_timeout(void *ctx) 1045 { 1046 struct spdk_nvmf_tcp_qpair *tqpair = ctx; 1047 1048 assert(tqpair->recv_state == NVME_TCP_PDU_RECV_STATE_ERROR); 1049 1050 SPDK_ERRLOG("No pdu coming for tqpair=%p within %d seconds\n", tqpair, 1051 SPDK_NVME_TCP_QPAIR_EXIT_TIMEOUT); 1052 1053 spdk_nvmf_tcp_qpair_disconnect(tqpair); 1054 return 0; 1055 } 1056 1057 static void 1058 spdk_nvmf_tcp_send_c2h_term_req_complete(void *cb_arg) 1059 { 1060 struct spdk_nvmf_tcp_qpair *tqpair = (struct spdk_nvmf_tcp_qpair *)cb_arg; 1061 1062 if (!tqpair->timeout_poller) { 1063 tqpair->timeout_poller = spdk_poller_register(spdk_nvmf_tcp_qpair_handle_timeout, tqpair, 1064 SPDK_NVME_TCP_QPAIR_EXIT_TIMEOUT * 1000000); 1065 } 1066 } 1067 1068 static void 1069 spdk_nvmf_tcp_send_c2h_term_req(struct spdk_nvmf_tcp_qpair *tqpair, struct nvme_tcp_pdu *pdu, 1070 enum spdk_nvme_tcp_term_req_fes fes, uint32_t error_offset) 1071 { 1072 struct nvme_tcp_pdu *rsp_pdu; 1073 struct spdk_nvme_tcp_term_req_hdr *c2h_term_req; 1074 uint32_t c2h_term_req_hdr_len = sizeof(*c2h_term_req); 1075 uint32_t copy_len; 1076 1077 rsp_pdu = &tqpair->mgmt_pdu; 1078 1079 c2h_term_req = &rsp_pdu->hdr.term_req; 1080 c2h_term_req->common.pdu_type = SPDK_NVME_TCP_PDU_TYPE_C2H_TERM_REQ; 1081 c2h_term_req->common.hlen = c2h_term_req_hdr_len; 1082 1083 if ((fes == SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD) || 1084 (fes == SPDK_NVME_TCP_TERM_REQ_FES_INVALID_DATA_UNSUPPORTED_PARAMETER)) { 1085 DSET32(&c2h_term_req->fei, error_offset); 1086 } 1087 1088 copy_len = spdk_min(pdu->hdr.common.hlen, SPDK_NVME_TCP_TERM_REQ_ERROR_DATA_MAX_SIZE); 1089 1090 /* Copy the error info into the buffer */ 1091 memcpy((uint8_t *)rsp_pdu->hdr.raw + c2h_term_req_hdr_len, pdu->hdr.raw, copy_len); 1092 nvme_tcp_pdu_set_data(rsp_pdu, (uint8_t *)rsp_pdu->hdr.raw + c2h_term_req_hdr_len, copy_len); 1093 1094 /* Contain the header of the wrong received pdu */ 1095 c2h_term_req->common.plen = c2h_term_req->common.hlen + copy_len; 1096 spdk_nvmf_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_ERROR); 1097 spdk_nvmf_tcp_qpair_write_pdu(tqpair, rsp_pdu, spdk_nvmf_tcp_send_c2h_term_req_complete, tqpair); 1098 } 1099 1100 static void 1101 spdk_nvmf_tcp_capsule_cmd_hdr_handle(struct spdk_nvmf_tcp_transport *ttransport, 1102 struct spdk_nvmf_tcp_qpair *tqpair, 1103 struct nvme_tcp_pdu *pdu) 1104 { 1105 struct spdk_nvmf_tcp_req *tcp_req; 1106 1107 assert(pdu->psh_valid_bytes == pdu->psh_len); 1108 assert(pdu->hdr.common.pdu_type == SPDK_NVME_TCP_PDU_TYPE_CAPSULE_CMD); 1109 1110 tcp_req = spdk_nvmf_tcp_req_get(tqpair); 1111 if (!tcp_req) { 1112 /* Directly return and make the allocation retry again */ 1113 if (tqpair->state_cntr[TCP_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST] > 0) { 1114 return; 1115 } 1116 1117 /* The host sent more commands than the maximum queue depth. */ 1118 SPDK_ERRLOG("Cannot allocate tcp_req on tqpair=%p\n", tqpair); 1119 spdk_nvmf_tcp_qpair_disconnect(tqpair); 1120 return; 1121 } 1122 1123 pdu->req = tcp_req; 1124 assert(tcp_req->state == TCP_REQUEST_STATE_NEW); 1125 spdk_nvmf_tcp_req_process(ttransport, tcp_req); 1126 } 1127 1128 static void 1129 spdk_nvmf_tcp_capsule_cmd_payload_handle(struct spdk_nvmf_tcp_transport *ttransport, 1130 struct spdk_nvmf_tcp_qpair *tqpair, 1131 struct nvme_tcp_pdu *pdu) 1132 { 1133 struct spdk_nvmf_tcp_req *tcp_req; 1134 struct spdk_nvme_tcp_cmd *capsule_cmd; 1135 uint32_t error_offset = 0; 1136 enum spdk_nvme_tcp_term_req_fes fes; 1137 1138 capsule_cmd = &pdu->hdr.capsule_cmd; 1139 tcp_req = pdu->req; 1140 assert(tcp_req != NULL); 1141 if (capsule_cmd->common.pdo > SPDK_NVME_TCP_PDU_PDO_MAX_OFFSET) { 1142 SPDK_ERRLOG("Expected ICReq capsule_cmd pdu offset <= %d, got %c\n", 1143 SPDK_NVME_TCP_PDU_PDO_MAX_OFFSET, capsule_cmd->common.pdo); 1144 fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD; 1145 error_offset = offsetof(struct spdk_nvme_tcp_common_pdu_hdr, pdo); 1146 goto err; 1147 } 1148 1149 spdk_nvmf_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY); 1150 spdk_nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_READY_TO_EXECUTE); 1151 spdk_nvmf_tcp_req_process(ttransport, tcp_req); 1152 1153 return; 1154 err: 1155 spdk_nvmf_tcp_send_c2h_term_req(tqpair, pdu, fes, error_offset); 1156 } 1157 1158 static int 1159 nvmf_tcp_find_req_in_state(struct spdk_nvmf_tcp_qpair *tqpair, 1160 enum spdk_nvmf_tcp_req_state state, 1161 uint16_t cid, uint16_t tag, 1162 struct spdk_nvmf_tcp_req **req) 1163 { 1164 struct spdk_nvmf_tcp_req *tcp_req = NULL; 1165 1166 TAILQ_FOREACH(tcp_req, &tqpair->state_queue[state], state_link) { 1167 if (tcp_req->req.cmd->nvme_cmd.cid != cid) { 1168 continue; 1169 } 1170 1171 if (tcp_req->ttag == tag) { 1172 *req = tcp_req; 1173 return 0; 1174 } 1175 1176 *req = NULL; 1177 return -1; 1178 } 1179 1180 /* Didn't find it, but not an error */ 1181 *req = NULL; 1182 return 0; 1183 } 1184 1185 static void 1186 spdk_nvmf_tcp_h2c_data_hdr_handle(struct spdk_nvmf_tcp_transport *ttransport, 1187 struct spdk_nvmf_tcp_qpair *tqpair, 1188 struct nvme_tcp_pdu *pdu) 1189 { 1190 struct spdk_nvmf_tcp_req *tcp_req; 1191 uint32_t error_offset = 0; 1192 enum spdk_nvme_tcp_term_req_fes fes = 0; 1193 struct spdk_nvme_tcp_h2c_data_hdr *h2c_data; 1194 int rc; 1195 1196 h2c_data = &pdu->hdr.h2c_data; 1197 1198 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "tqpair=%p, r2t_info: datao=%u, datal=%u, cccid=%u, ttag=%u\n", 1199 tqpair, h2c_data->datao, h2c_data->datal, h2c_data->cccid, h2c_data->ttag); 1200 1201 rc = nvmf_tcp_find_req_in_state(tqpair, TCP_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER, 1202 h2c_data->cccid, h2c_data->ttag, &tcp_req); 1203 if (rc == 0 && tcp_req == NULL) { 1204 rc = nvmf_tcp_find_req_in_state(tqpair, TCP_REQUEST_STATE_AWAITING_R2T_ACK, h2c_data->cccid, 1205 h2c_data->ttag, &tcp_req); 1206 } 1207 1208 if (!tcp_req) { 1209 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "tcp_req is not found for tqpair=%p\n", tqpair); 1210 fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_DATA_UNSUPPORTED_PARAMETER; 1211 if (rc == 0) { 1212 error_offset = offsetof(struct spdk_nvme_tcp_h2c_data_hdr, cccid); 1213 } else { 1214 error_offset = offsetof(struct spdk_nvme_tcp_h2c_data_hdr, ttag); 1215 } 1216 goto err; 1217 } 1218 1219 if (tcp_req->h2c_offset != h2c_data->datao) { 1220 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, 1221 "tcp_req(%p), tqpair=%p, expected data offset %u, but data offset is %u\n", 1222 tcp_req, tqpair, tcp_req->h2c_offset, h2c_data->datao); 1223 fes = SPDK_NVME_TCP_TERM_REQ_FES_DATA_TRANSFER_OUT_OF_RANGE; 1224 goto err; 1225 } 1226 1227 if ((h2c_data->datao + h2c_data->datal) > tcp_req->req.length) { 1228 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, 1229 "tcp_req(%p), tqpair=%p, (datao=%u + datal=%u) execeeds requested length=%u\n", 1230 tcp_req, tqpair, h2c_data->datao, h2c_data->datal, tcp_req->req.length); 1231 fes = SPDK_NVME_TCP_TERM_REQ_FES_DATA_TRANSFER_OUT_OF_RANGE; 1232 goto err; 1233 } 1234 1235 pdu->req = tcp_req; 1236 1237 if (spdk_unlikely(tcp_req->req.dif.dif_insert_or_strip)) { 1238 pdu->dif_ctx = &tcp_req->req.dif.dif_ctx; 1239 } 1240 1241 nvme_tcp_pdu_set_data_buf(pdu, tcp_req->req.iov, tcp_req->req.iovcnt, 1242 h2c_data->datao, h2c_data->datal); 1243 spdk_nvmf_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PAYLOAD); 1244 return; 1245 1246 err: 1247 spdk_nvmf_tcp_send_c2h_term_req(tqpair, pdu, fes, error_offset); 1248 } 1249 1250 static void 1251 spdk_nvmf_tcp_pdu_cmd_complete(void *cb_arg) 1252 { 1253 struct spdk_nvmf_tcp_req *tcp_req = cb_arg; 1254 nvmf_tcp_request_free(tcp_req); 1255 } 1256 1257 static void 1258 spdk_nvmf_tcp_send_capsule_resp_pdu(struct spdk_nvmf_tcp_req *tcp_req, 1259 struct spdk_nvmf_tcp_qpair *tqpair) 1260 { 1261 struct nvme_tcp_pdu *rsp_pdu; 1262 struct spdk_nvme_tcp_rsp *capsule_resp; 1263 1264 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "enter, tqpair=%p\n", tqpair); 1265 1266 rsp_pdu = nvmf_tcp_req_pdu_init(tcp_req); 1267 assert(rsp_pdu != NULL); 1268 1269 capsule_resp = &rsp_pdu->hdr.capsule_resp; 1270 capsule_resp->common.pdu_type = SPDK_NVME_TCP_PDU_TYPE_CAPSULE_RESP; 1271 capsule_resp->common.plen = capsule_resp->common.hlen = sizeof(*capsule_resp); 1272 capsule_resp->rccqe = tcp_req->req.rsp->nvme_cpl; 1273 if (tqpair->host_hdgst_enable) { 1274 capsule_resp->common.flags |= SPDK_NVME_TCP_CH_FLAGS_HDGSTF; 1275 capsule_resp->common.plen += SPDK_NVME_TCP_DIGEST_LEN; 1276 } 1277 1278 spdk_nvmf_tcp_qpair_write_pdu(tqpair, rsp_pdu, spdk_nvmf_tcp_pdu_cmd_complete, tcp_req); 1279 } 1280 1281 static void 1282 spdk_nvmf_tcp_pdu_c2h_data_complete(void *cb_arg) 1283 { 1284 struct spdk_nvmf_tcp_req *tcp_req = cb_arg; 1285 struct spdk_nvmf_tcp_qpair *tqpair = SPDK_CONTAINEROF(tcp_req->req.qpair, 1286 struct spdk_nvmf_tcp_qpair, qpair); 1287 1288 assert(tqpair != NULL); 1289 if (tqpair->qpair.transport->opts.c2h_success) { 1290 nvmf_tcp_request_free(tcp_req); 1291 } else { 1292 nvmf_tcp_req_pdu_fini(tcp_req); 1293 spdk_nvmf_tcp_send_capsule_resp_pdu(tcp_req, tqpair); 1294 } 1295 } 1296 1297 static void 1298 spdk_nvmf_tcp_r2t_complete(void *cb_arg) 1299 { 1300 struct spdk_nvmf_tcp_req *tcp_req = cb_arg; 1301 struct spdk_nvmf_tcp_transport *ttransport; 1302 1303 nvmf_tcp_req_pdu_fini(tcp_req); 1304 1305 ttransport = SPDK_CONTAINEROF(tcp_req->req.qpair->transport, 1306 struct spdk_nvmf_tcp_transport, transport); 1307 1308 spdk_nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 1309 1310 if (tcp_req->h2c_offset == tcp_req->req.length) { 1311 spdk_nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_READY_TO_EXECUTE); 1312 spdk_nvmf_tcp_req_process(ttransport, tcp_req); 1313 } 1314 } 1315 1316 static void 1317 spdk_nvmf_tcp_send_r2t_pdu(struct spdk_nvmf_tcp_qpair *tqpair, 1318 struct spdk_nvmf_tcp_req *tcp_req) 1319 { 1320 struct nvme_tcp_pdu *rsp_pdu; 1321 struct spdk_nvme_tcp_r2t_hdr *r2t; 1322 1323 rsp_pdu = nvmf_tcp_req_pdu_init(tcp_req); 1324 assert(rsp_pdu != NULL); 1325 1326 r2t = &rsp_pdu->hdr.r2t; 1327 r2t->common.pdu_type = SPDK_NVME_TCP_PDU_TYPE_R2T; 1328 r2t->common.plen = r2t->common.hlen = sizeof(*r2t); 1329 1330 if (tqpair->host_hdgst_enable) { 1331 r2t->common.flags |= SPDK_NVME_TCP_CH_FLAGS_HDGSTF; 1332 r2t->common.plen += SPDK_NVME_TCP_DIGEST_LEN; 1333 } 1334 1335 r2t->cccid = tcp_req->req.cmd->nvme_cmd.cid; 1336 r2t->ttag = tcp_req->ttag; 1337 r2t->r2to = tcp_req->h2c_offset; 1338 r2t->r2tl = tcp_req->req.length; 1339 1340 spdk_nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_AWAITING_R2T_ACK); 1341 1342 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, 1343 "tcp_req(%p) on tqpair(%p), r2t_info: cccid=%u, ttag=%u, r2to=%u, r2tl=%u\n", 1344 tcp_req, tqpair, r2t->cccid, r2t->ttag, r2t->r2to, r2t->r2tl); 1345 spdk_nvmf_tcp_qpair_write_pdu(tqpair, rsp_pdu, spdk_nvmf_tcp_r2t_complete, tcp_req); 1346 } 1347 1348 static void 1349 spdk_nvmf_tcp_h2c_data_payload_handle(struct spdk_nvmf_tcp_transport *ttransport, 1350 struct spdk_nvmf_tcp_qpair *tqpair, 1351 struct nvme_tcp_pdu *pdu) 1352 { 1353 struct spdk_nvmf_tcp_req *tcp_req; 1354 1355 tcp_req = pdu->req; 1356 assert(tcp_req != NULL); 1357 1358 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "enter\n"); 1359 1360 tcp_req->h2c_offset += pdu->data_len; 1361 1362 spdk_nvmf_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY); 1363 1364 /* Wait for all of the data to arrive AND for the initial R2T PDU send to be 1365 * acknowledged before moving on. */ 1366 if (tcp_req->h2c_offset == tcp_req->req.length && 1367 tcp_req->state == TCP_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER) { 1368 spdk_nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_READY_TO_EXECUTE); 1369 spdk_nvmf_tcp_req_process(ttransport, tcp_req); 1370 } 1371 } 1372 1373 static void 1374 spdk_nvmf_tcp_h2c_term_req_dump(struct spdk_nvme_tcp_term_req_hdr *h2c_term_req) 1375 { 1376 SPDK_ERRLOG("Error info of pdu(%p): %s\n", h2c_term_req, 1377 spdk_nvmf_tcp_term_req_fes_str[h2c_term_req->fes]); 1378 if ((h2c_term_req->fes == SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD) || 1379 (h2c_term_req->fes == SPDK_NVME_TCP_TERM_REQ_FES_INVALID_DATA_UNSUPPORTED_PARAMETER)) { 1380 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "The offset from the start of the PDU header is %u\n", 1381 DGET32(h2c_term_req->fei)); 1382 } 1383 } 1384 1385 static void 1386 spdk_nvmf_tcp_h2c_term_req_hdr_handle(struct spdk_nvmf_tcp_qpair *tqpair, 1387 struct nvme_tcp_pdu *pdu) 1388 { 1389 struct spdk_nvme_tcp_term_req_hdr *h2c_term_req = &pdu->hdr.term_req; 1390 uint32_t error_offset = 0; 1391 enum spdk_nvme_tcp_term_req_fes fes; 1392 1393 1394 if (h2c_term_req->fes > SPDK_NVME_TCP_TERM_REQ_FES_INVALID_DATA_UNSUPPORTED_PARAMETER) { 1395 SPDK_ERRLOG("Fatal Error Stauts(FES) is unknown for h2c_term_req pdu=%p\n", pdu); 1396 fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD; 1397 error_offset = offsetof(struct spdk_nvme_tcp_term_req_hdr, fes); 1398 goto end; 1399 } 1400 1401 /* set the data buffer */ 1402 nvme_tcp_pdu_set_data(pdu, (uint8_t *)pdu->hdr.raw + h2c_term_req->common.hlen, 1403 h2c_term_req->common.plen - h2c_term_req->common.hlen); 1404 spdk_nvmf_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PAYLOAD); 1405 return; 1406 end: 1407 spdk_nvmf_tcp_send_c2h_term_req(tqpair, pdu, fes, error_offset); 1408 } 1409 1410 static void 1411 spdk_nvmf_tcp_h2c_term_req_payload_handle(struct spdk_nvmf_tcp_qpair *tqpair, 1412 struct nvme_tcp_pdu *pdu) 1413 { 1414 struct spdk_nvme_tcp_term_req_hdr *h2c_term_req = &pdu->hdr.term_req; 1415 1416 spdk_nvmf_tcp_h2c_term_req_dump(h2c_term_req); 1417 spdk_nvmf_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_ERROR); 1418 } 1419 1420 static void 1421 spdk_nvmf_tcp_pdu_payload_handle(struct spdk_nvmf_tcp_qpair *tqpair, 1422 struct spdk_nvmf_tcp_transport *ttransport) 1423 { 1424 int rc = 0; 1425 struct nvme_tcp_pdu *pdu; 1426 uint32_t crc32c, error_offset = 0; 1427 enum spdk_nvme_tcp_term_req_fes fes; 1428 1429 assert(tqpair->recv_state == NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PAYLOAD); 1430 pdu = &tqpair->pdu_in_progress; 1431 1432 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "enter\n"); 1433 /* check data digest if need */ 1434 if (pdu->ddgst_enable) { 1435 crc32c = nvme_tcp_pdu_calc_data_digest(pdu); 1436 rc = MATCH_DIGEST_WORD(pdu->data_digest, crc32c); 1437 if (rc == 0) { 1438 SPDK_ERRLOG("Data digest error on tqpair=(%p) with pdu=%p\n", tqpair, pdu); 1439 fes = SPDK_NVME_TCP_TERM_REQ_FES_HDGST_ERROR; 1440 spdk_nvmf_tcp_send_c2h_term_req(tqpair, pdu, fes, error_offset); 1441 return; 1442 1443 } 1444 } 1445 1446 switch (pdu->hdr.common.pdu_type) { 1447 case SPDK_NVME_TCP_PDU_TYPE_CAPSULE_CMD: 1448 spdk_nvmf_tcp_capsule_cmd_payload_handle(ttransport, tqpair, pdu); 1449 break; 1450 case SPDK_NVME_TCP_PDU_TYPE_H2C_DATA: 1451 spdk_nvmf_tcp_h2c_data_payload_handle(ttransport, tqpair, pdu); 1452 break; 1453 1454 case SPDK_NVME_TCP_PDU_TYPE_H2C_TERM_REQ: 1455 spdk_nvmf_tcp_h2c_term_req_payload_handle(tqpair, pdu); 1456 break; 1457 1458 default: 1459 /* The code should not go to here */ 1460 SPDK_ERRLOG("The code should not go to here\n"); 1461 break; 1462 } 1463 } 1464 1465 static void 1466 spdk_nvmf_tcp_send_icresp_complete(void *cb_arg) 1467 { 1468 struct spdk_nvmf_tcp_qpair *tqpair = cb_arg; 1469 1470 tqpair->state = NVME_TCP_QPAIR_STATE_RUNNING; 1471 } 1472 1473 static void 1474 spdk_nvmf_tcp_icreq_handle(struct spdk_nvmf_tcp_transport *ttransport, 1475 struct spdk_nvmf_tcp_qpair *tqpair, 1476 struct nvme_tcp_pdu *pdu) 1477 { 1478 struct spdk_nvme_tcp_ic_req *ic_req = &pdu->hdr.ic_req; 1479 struct nvme_tcp_pdu *rsp_pdu; 1480 struct spdk_nvme_tcp_ic_resp *ic_resp; 1481 uint32_t error_offset = 0; 1482 enum spdk_nvme_tcp_term_req_fes fes; 1483 1484 /* Only PFV 0 is defined currently */ 1485 if (ic_req->pfv != 0) { 1486 SPDK_ERRLOG("Expected ICReq PFV %u, got %u\n", 0u, ic_req->pfv); 1487 fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD; 1488 error_offset = offsetof(struct spdk_nvme_tcp_ic_req, pfv); 1489 goto end; 1490 } 1491 1492 /* MAXR2T is 0's based */ 1493 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "maxr2t =%u\n", (ic_req->maxr2t + 1u)); 1494 1495 tqpair->host_hdgst_enable = ic_req->dgst.bits.hdgst_enable ? true : false; 1496 tqpair->host_ddgst_enable = ic_req->dgst.bits.ddgst_enable ? true : false; 1497 1498 tqpair->cpda = spdk_min(ic_req->hpda, SPDK_NVME_TCP_CPDA_MAX); 1499 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "cpda of tqpair=(%p) is : %u\n", tqpair, tqpair->cpda); 1500 1501 rsp_pdu = &tqpair->mgmt_pdu; 1502 1503 ic_resp = &rsp_pdu->hdr.ic_resp; 1504 ic_resp->common.pdu_type = SPDK_NVME_TCP_PDU_TYPE_IC_RESP; 1505 ic_resp->common.hlen = ic_resp->common.plen = sizeof(*ic_resp); 1506 ic_resp->pfv = 0; 1507 ic_resp->cpda = tqpair->cpda; 1508 ic_resp->maxh2cdata = ttransport->transport.opts.max_io_size; 1509 ic_resp->dgst.bits.hdgst_enable = tqpair->host_hdgst_enable ? 1 : 0; 1510 ic_resp->dgst.bits.ddgst_enable = tqpair->host_ddgst_enable ? 1 : 0; 1511 1512 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "host_hdgst_enable: %u\n", tqpair->host_hdgst_enable); 1513 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "host_ddgst_enable: %u\n", tqpair->host_ddgst_enable); 1514 1515 tqpair->state = NVME_TCP_QPAIR_STATE_INITIALIZING; 1516 spdk_nvmf_tcp_qpair_write_pdu(tqpair, rsp_pdu, spdk_nvmf_tcp_send_icresp_complete, tqpair); 1517 spdk_nvmf_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY); 1518 return; 1519 end: 1520 spdk_nvmf_tcp_send_c2h_term_req(tqpair, pdu, fes, error_offset); 1521 } 1522 1523 static void 1524 spdk_nvmf_tcp_pdu_psh_handle(struct spdk_nvmf_tcp_qpair *tqpair, 1525 struct spdk_nvmf_tcp_transport *ttransport) 1526 { 1527 struct nvme_tcp_pdu *pdu; 1528 int rc; 1529 uint32_t crc32c, error_offset = 0; 1530 enum spdk_nvme_tcp_term_req_fes fes; 1531 1532 assert(tqpair->recv_state == NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PSH); 1533 pdu = &tqpair->pdu_in_progress; 1534 1535 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "pdu type of tqpair(%p) is %d\n", tqpair, 1536 pdu->hdr.common.pdu_type); 1537 /* check header digest if needed */ 1538 if (pdu->has_hdgst) { 1539 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "Compare the header of pdu=%p on tqpair=%p\n", pdu, tqpair); 1540 crc32c = nvme_tcp_pdu_calc_header_digest(pdu); 1541 rc = MATCH_DIGEST_WORD((uint8_t *)pdu->hdr.raw + pdu->hdr.common.hlen, crc32c); 1542 if (rc == 0) { 1543 SPDK_ERRLOG("Header digest error on tqpair=(%p) with pdu=%p\n", tqpair, pdu); 1544 fes = SPDK_NVME_TCP_TERM_REQ_FES_HDGST_ERROR; 1545 spdk_nvmf_tcp_send_c2h_term_req(tqpair, pdu, fes, error_offset); 1546 return; 1547 1548 } 1549 } 1550 1551 switch (pdu->hdr.common.pdu_type) { 1552 case SPDK_NVME_TCP_PDU_TYPE_IC_REQ: 1553 spdk_nvmf_tcp_icreq_handle(ttransport, tqpair, pdu); 1554 break; 1555 case SPDK_NVME_TCP_PDU_TYPE_CAPSULE_CMD: 1556 spdk_nvmf_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_REQ); 1557 break; 1558 case SPDK_NVME_TCP_PDU_TYPE_H2C_DATA: 1559 spdk_nvmf_tcp_h2c_data_hdr_handle(ttransport, tqpair, pdu); 1560 break; 1561 1562 case SPDK_NVME_TCP_PDU_TYPE_H2C_TERM_REQ: 1563 spdk_nvmf_tcp_h2c_term_req_hdr_handle(tqpair, pdu); 1564 break; 1565 1566 default: 1567 SPDK_ERRLOG("Unexpected PDU type 0x%02x\n", tqpair->pdu_in_progress.hdr.common.pdu_type); 1568 fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD; 1569 error_offset = 1; 1570 spdk_nvmf_tcp_send_c2h_term_req(tqpair, pdu, fes, error_offset); 1571 break; 1572 } 1573 } 1574 1575 static void 1576 spdk_nvmf_tcp_pdu_ch_handle(struct spdk_nvmf_tcp_qpair *tqpair) 1577 { 1578 struct nvme_tcp_pdu *pdu; 1579 uint32_t error_offset = 0; 1580 enum spdk_nvme_tcp_term_req_fes fes; 1581 uint8_t expected_hlen, pdo; 1582 bool plen_error = false, pdo_error = false; 1583 1584 assert(tqpair->recv_state == NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_CH); 1585 pdu = &tqpair->pdu_in_progress; 1586 1587 if (pdu->hdr.common.pdu_type == SPDK_NVME_TCP_PDU_TYPE_IC_REQ) { 1588 if (tqpair->state != NVME_TCP_QPAIR_STATE_INVALID) { 1589 SPDK_ERRLOG("Already received ICreq PDU, and reject this pdu=%p\n", pdu); 1590 fes = SPDK_NVME_TCP_TERM_REQ_FES_PDU_SEQUENCE_ERROR; 1591 goto err; 1592 } 1593 expected_hlen = sizeof(struct spdk_nvme_tcp_ic_req); 1594 if (pdu->hdr.common.plen != expected_hlen) { 1595 plen_error = true; 1596 } 1597 } else { 1598 if (tqpair->state != NVME_TCP_QPAIR_STATE_RUNNING) { 1599 SPDK_ERRLOG("The TCP/IP connection is not negotitated\n"); 1600 fes = SPDK_NVME_TCP_TERM_REQ_FES_PDU_SEQUENCE_ERROR; 1601 goto err; 1602 } 1603 1604 switch (pdu->hdr.common.pdu_type) { 1605 case SPDK_NVME_TCP_PDU_TYPE_CAPSULE_CMD: 1606 expected_hlen = sizeof(struct spdk_nvme_tcp_cmd); 1607 pdo = pdu->hdr.common.pdo; 1608 if ((tqpair->cpda != 0) && (pdo != ((tqpair->cpda + 1) << 2))) { 1609 pdo_error = true; 1610 break; 1611 } 1612 1613 if (pdu->hdr.common.plen < expected_hlen) { 1614 plen_error = true; 1615 } 1616 break; 1617 case SPDK_NVME_TCP_PDU_TYPE_H2C_DATA: 1618 expected_hlen = sizeof(struct spdk_nvme_tcp_h2c_data_hdr); 1619 pdo = pdu->hdr.common.pdo; 1620 if ((tqpair->cpda != 0) && (pdo != ((tqpair->cpda + 1) << 2))) { 1621 pdo_error = true; 1622 break; 1623 } 1624 if (pdu->hdr.common.plen < expected_hlen) { 1625 plen_error = true; 1626 } 1627 break; 1628 1629 case SPDK_NVME_TCP_PDU_TYPE_H2C_TERM_REQ: 1630 expected_hlen = sizeof(struct spdk_nvme_tcp_term_req_hdr); 1631 if ((pdu->hdr.common.plen <= expected_hlen) || 1632 (pdu->hdr.common.plen > SPDK_NVME_TCP_TERM_REQ_PDU_MAX_SIZE)) { 1633 plen_error = true; 1634 } 1635 break; 1636 1637 default: 1638 SPDK_ERRLOG("Unexpected PDU type 0x%02x\n", pdu->hdr.common.pdu_type); 1639 fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD; 1640 error_offset = offsetof(struct spdk_nvme_tcp_common_pdu_hdr, pdu_type); 1641 goto err; 1642 } 1643 } 1644 1645 if (pdu->hdr.common.hlen != expected_hlen) { 1646 SPDK_ERRLOG("PDU type=0x%02x, Expected ICReq header length %u, got %u on tqpair=%p\n", 1647 pdu->hdr.common.pdu_type, 1648 expected_hlen, pdu->hdr.common.hlen, tqpair); 1649 fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD; 1650 error_offset = offsetof(struct spdk_nvme_tcp_common_pdu_hdr, hlen); 1651 goto err; 1652 } else if (pdo_error) { 1653 fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD; 1654 error_offset = offsetof(struct spdk_nvme_tcp_common_pdu_hdr, pdo); 1655 } else if (plen_error) { 1656 fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD; 1657 error_offset = offsetof(struct spdk_nvme_tcp_common_pdu_hdr, plen); 1658 goto err; 1659 } else { 1660 spdk_nvmf_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PSH); 1661 nvme_tcp_pdu_calc_psh_len(&tqpair->pdu_in_progress, tqpair->host_hdgst_enable); 1662 return; 1663 } 1664 err: 1665 spdk_nvmf_tcp_send_c2h_term_req(tqpair, pdu, fes, error_offset); 1666 } 1667 1668 static int 1669 nvmf_tcp_pdu_payload_insert_dif(struct nvme_tcp_pdu *pdu, uint32_t read_offset, 1670 int read_len) 1671 { 1672 int rc; 1673 1674 rc = spdk_dif_generate_stream(pdu->data_iov, pdu->data_iovcnt, 1675 read_offset, read_len, pdu->dif_ctx); 1676 if (rc != 0) { 1677 SPDK_ERRLOG("DIF generate failed\n"); 1678 } 1679 1680 return rc; 1681 } 1682 1683 #define MAX_NVME_TCP_PDU_LOOP_COUNT 32 1684 1685 static int 1686 spdk_nvmf_tcp_sock_process(struct spdk_nvmf_tcp_qpair *tqpair) 1687 { 1688 int rc = 0; 1689 struct nvme_tcp_pdu *pdu; 1690 enum nvme_tcp_pdu_recv_state prev_state; 1691 uint32_t data_len, current_pdu_num = 0; 1692 struct spdk_nvmf_tcp_transport *ttransport = SPDK_CONTAINEROF(tqpair->qpair.transport, 1693 struct spdk_nvmf_tcp_transport, transport); 1694 1695 /* The loop here is to allow for several back-to-back state changes. */ 1696 do { 1697 prev_state = tqpair->recv_state; 1698 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "tqpair(%p) recv pdu entering state %d\n", tqpair, prev_state); 1699 1700 pdu = &tqpair->pdu_in_progress; 1701 switch (tqpair->recv_state) { 1702 /* Wait for the common header */ 1703 case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY: 1704 case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_CH: 1705 if (spdk_unlikely(tqpair->state == NVME_TCP_QPAIR_STATE_INITIALIZING)) { 1706 return rc; 1707 } 1708 1709 rc = nvme_tcp_read_data(tqpair->sock, 1710 sizeof(struct spdk_nvme_tcp_common_pdu_hdr) - pdu->ch_valid_bytes, 1711 (void *)&pdu->hdr.common + pdu->ch_valid_bytes); 1712 if (rc < 0) { 1713 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "will disconnect tqpair=%p\n", tqpair); 1714 return NVME_TCP_PDU_FATAL; 1715 } else if (rc > 0) { 1716 pdu->ch_valid_bytes += rc; 1717 spdk_trace_record(TRACE_TCP_READ_FROM_SOCKET_DONE, 0, rc, 0, 0); 1718 if (spdk_likely(tqpair->recv_state == NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY)) { 1719 spdk_nvmf_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_CH); 1720 } 1721 } 1722 1723 if (pdu->ch_valid_bytes < sizeof(struct spdk_nvme_tcp_common_pdu_hdr)) { 1724 return NVME_TCP_PDU_IN_PROGRESS; 1725 } 1726 1727 /* The command header of this PDU has now been read from the socket. */ 1728 spdk_nvmf_tcp_pdu_ch_handle(tqpair); 1729 break; 1730 /* Wait for the pdu specific header */ 1731 case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PSH: 1732 rc = nvme_tcp_read_data(tqpair->sock, 1733 pdu->psh_len - pdu->psh_valid_bytes, 1734 (void *)&pdu->hdr.raw + sizeof(struct spdk_nvme_tcp_common_pdu_hdr) + pdu->psh_valid_bytes); 1735 if (rc < 0) { 1736 return NVME_TCP_PDU_FATAL; 1737 } else if (rc > 0) { 1738 spdk_trace_record(TRACE_TCP_READ_FROM_SOCKET_DONE, 1739 0, rc, 0, 0); 1740 pdu->psh_valid_bytes += rc; 1741 } 1742 if (pdu->psh_valid_bytes < pdu->psh_len) { 1743 return NVME_TCP_PDU_IN_PROGRESS; 1744 } 1745 1746 /* All header(ch, psh, head digist) of this PDU has now been read from the socket. */ 1747 spdk_nvmf_tcp_pdu_psh_handle(tqpair, ttransport); 1748 if (tqpair->recv_state == NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY) { 1749 current_pdu_num++; 1750 } 1751 break; 1752 /* Wait for the req slot */ 1753 case NVME_TCP_PDU_RECV_STATE_AWAIT_REQ: 1754 spdk_nvmf_tcp_capsule_cmd_hdr_handle(ttransport, tqpair, pdu); 1755 break; 1756 case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PAYLOAD: 1757 /* check whether the data is valid, if not we just return */ 1758 if (!pdu->data_len) { 1759 return NVME_TCP_PDU_IN_PROGRESS; 1760 } 1761 1762 data_len = pdu->data_len; 1763 /* data digest */ 1764 if (spdk_unlikely((pdu->hdr.common.pdu_type != SPDK_NVME_TCP_PDU_TYPE_H2C_TERM_REQ) && 1765 tqpair->host_ddgst_enable)) { 1766 data_len += SPDK_NVME_TCP_DIGEST_LEN; 1767 pdu->ddgst_enable = true; 1768 } 1769 1770 rc = nvme_tcp_read_payload_data(tqpair->sock, pdu); 1771 if (rc < 0) { 1772 return NVME_TCP_PDU_IN_PROGRESS; 1773 } 1774 pdu->readv_offset += rc; 1775 1776 if (spdk_unlikely(pdu->dif_ctx != NULL)) { 1777 rc = nvmf_tcp_pdu_payload_insert_dif(pdu, pdu->readv_offset - rc, rc); 1778 if (rc != 0) { 1779 return NVME_TCP_PDU_FATAL; 1780 } 1781 } 1782 1783 if (pdu->readv_offset < data_len) { 1784 return NVME_TCP_PDU_IN_PROGRESS; 1785 } 1786 1787 /* All of this PDU has now been read from the socket. */ 1788 spdk_nvmf_tcp_pdu_payload_handle(tqpair, ttransport); 1789 current_pdu_num++; 1790 break; 1791 case NVME_TCP_PDU_RECV_STATE_ERROR: 1792 if (!spdk_sock_is_connected(tqpair->sock)) { 1793 return NVME_TCP_PDU_FATAL; 1794 } 1795 break; 1796 default: 1797 assert(0); 1798 SPDK_ERRLOG("code should not come to here"); 1799 break; 1800 } 1801 } while ((tqpair->recv_state != prev_state) && (current_pdu_num < MAX_NVME_TCP_PDU_LOOP_COUNT)); 1802 1803 return rc; 1804 } 1805 1806 static int 1807 spdk_nvmf_tcp_req_parse_sgl(struct spdk_nvmf_tcp_req *tcp_req, 1808 struct spdk_nvmf_transport *transport, 1809 struct spdk_nvmf_transport_poll_group *group) 1810 { 1811 struct spdk_nvmf_request *req = &tcp_req->req; 1812 struct spdk_nvme_cmd *cmd; 1813 struct spdk_nvme_cpl *rsp; 1814 struct spdk_nvme_sgl_descriptor *sgl; 1815 uint32_t length; 1816 1817 cmd = &req->cmd->nvme_cmd; 1818 rsp = &req->rsp->nvme_cpl; 1819 sgl = &cmd->dptr.sgl1; 1820 1821 length = sgl->unkeyed.length; 1822 1823 if (sgl->generic.type == SPDK_NVME_SGL_TYPE_TRANSPORT_DATA_BLOCK && 1824 sgl->unkeyed.subtype == SPDK_NVME_SGL_SUBTYPE_TRANSPORT) { 1825 if (length > transport->opts.max_io_size) { 1826 SPDK_ERRLOG("SGL length 0x%x exceeds max io size 0x%x\n", 1827 length, transport->opts.max_io_size); 1828 rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID; 1829 return -1; 1830 } 1831 1832 /* fill request length and populate iovs */ 1833 req->length = length; 1834 1835 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "Data requested length= 0x%x\n", length); 1836 1837 if (spdk_unlikely(req->dif.dif_insert_or_strip)) { 1838 req->dif.orig_length = length; 1839 length = spdk_dif_get_length_with_md(length, &req->dif.dif_ctx); 1840 req->dif.elba_length = length; 1841 } 1842 1843 if (spdk_nvmf_request_get_buffers(req, group, transport, length)) { 1844 /* No available buffers. Queue this request up. */ 1845 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "No available large data buffers. Queueing request %p\n", 1846 tcp_req); 1847 return 0; 1848 } 1849 1850 /* backward compatible */ 1851 req->data = req->iov[0].iov_base; 1852 1853 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "Request %p took %d buffer/s from central pool, and data=%p\n", 1854 tcp_req, req->iovcnt, req->data); 1855 1856 return 0; 1857 } else if (sgl->generic.type == SPDK_NVME_SGL_TYPE_DATA_BLOCK && 1858 sgl->unkeyed.subtype == SPDK_NVME_SGL_SUBTYPE_OFFSET) { 1859 uint64_t offset = sgl->address; 1860 uint32_t max_len = transport->opts.in_capsule_data_size; 1861 1862 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "In-capsule data: offset 0x%" PRIx64 ", length 0x%x\n", 1863 offset, length); 1864 1865 if (offset > max_len) { 1866 SPDK_ERRLOG("In-capsule offset 0x%" PRIx64 " exceeds capsule length 0x%x\n", 1867 offset, max_len); 1868 rsp->status.sc = SPDK_NVME_SC_INVALID_SGL_OFFSET; 1869 return -1; 1870 } 1871 max_len -= (uint32_t)offset; 1872 1873 if (length > max_len) { 1874 SPDK_ERRLOG("In-capsule data length 0x%x exceeds capsule length 0x%x\n", 1875 length, max_len); 1876 rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID; 1877 return -1; 1878 } 1879 1880 req->data = tcp_req->buf + offset; 1881 req->data_from_pool = false; 1882 req->length = length; 1883 1884 if (spdk_unlikely(req->dif.dif_insert_or_strip)) { 1885 length = spdk_dif_get_length_with_md(length, &req->dif.dif_ctx); 1886 req->dif.elba_length = length; 1887 } 1888 1889 req->iov[0].iov_base = req->data; 1890 req->iov[0].iov_len = length; 1891 req->iovcnt = 1; 1892 1893 return 0; 1894 } 1895 1896 SPDK_ERRLOG("Invalid NVMf I/O Command SGL: Type 0x%x, Subtype 0x%x\n", 1897 sgl->generic.type, sgl->generic.subtype); 1898 rsp->status.sc = SPDK_NVME_SC_SGL_DESCRIPTOR_TYPE_INVALID; 1899 return -1; 1900 } 1901 1902 static inline enum spdk_nvme_media_error_status_code 1903 nvmf_tcp_dif_error_to_compl_status(uint8_t err_type) { 1904 enum spdk_nvme_media_error_status_code result; 1905 1906 switch (err_type) 1907 { 1908 case SPDK_DIF_REFTAG_ERROR: 1909 result = SPDK_NVME_SC_REFERENCE_TAG_CHECK_ERROR; 1910 break; 1911 case SPDK_DIF_APPTAG_ERROR: 1912 result = SPDK_NVME_SC_APPLICATION_TAG_CHECK_ERROR; 1913 break; 1914 case SPDK_DIF_GUARD_ERROR: 1915 result = SPDK_NVME_SC_GUARD_CHECK_ERROR; 1916 break; 1917 default: 1918 SPDK_UNREACHABLE(); 1919 break; 1920 } 1921 1922 return result; 1923 } 1924 1925 static void 1926 spdk_nvmf_tcp_send_c2h_data(struct spdk_nvmf_tcp_qpair *tqpair, 1927 struct spdk_nvmf_tcp_req *tcp_req) 1928 { 1929 struct nvme_tcp_pdu *rsp_pdu; 1930 struct spdk_nvme_tcp_c2h_data_hdr *c2h_data; 1931 uint32_t plen, pdo, alignment; 1932 int rc; 1933 1934 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "enter\n"); 1935 1936 rsp_pdu = nvmf_tcp_req_pdu_init(tcp_req); 1937 assert(rsp_pdu != NULL); 1938 1939 c2h_data = &rsp_pdu->hdr.c2h_data; 1940 c2h_data->common.pdu_type = SPDK_NVME_TCP_PDU_TYPE_C2H_DATA; 1941 plen = c2h_data->common.hlen = sizeof(*c2h_data); 1942 1943 if (tqpair->host_hdgst_enable) { 1944 plen += SPDK_NVME_TCP_DIGEST_LEN; 1945 c2h_data->common.flags |= SPDK_NVME_TCP_CH_FLAGS_HDGSTF; 1946 } 1947 1948 /* set the psh */ 1949 c2h_data->cccid = tcp_req->req.cmd->nvme_cmd.cid; 1950 c2h_data->datal = tcp_req->req.length; 1951 c2h_data->datao = 0; 1952 1953 /* set the padding */ 1954 rsp_pdu->padding_len = 0; 1955 pdo = plen; 1956 if (tqpair->cpda) { 1957 alignment = (tqpair->cpda + 1) << 2; 1958 if (alignment > plen) { 1959 rsp_pdu->padding_len = alignment - plen; 1960 pdo = plen = alignment; 1961 } 1962 } 1963 1964 c2h_data->common.pdo = pdo; 1965 plen += c2h_data->datal; 1966 if (tqpair->host_ddgst_enable) { 1967 c2h_data->common.flags |= SPDK_NVME_TCP_CH_FLAGS_DDGSTF; 1968 plen += SPDK_NVME_TCP_DIGEST_LEN; 1969 } 1970 1971 c2h_data->common.plen = plen; 1972 1973 if (spdk_unlikely(tcp_req->req.dif.dif_insert_or_strip)) { 1974 rsp_pdu->dif_ctx = &tcp_req->req.dif.dif_ctx; 1975 } 1976 1977 nvme_tcp_pdu_set_data_buf(rsp_pdu, tcp_req->req.iov, tcp_req->req.iovcnt, 1978 c2h_data->datao, c2h_data->datal); 1979 1980 if (spdk_unlikely(tcp_req->req.dif.dif_insert_or_strip)) { 1981 struct spdk_nvme_cpl *rsp = &tcp_req->req.rsp->nvme_cpl; 1982 struct spdk_dif_error err_blk = {}; 1983 1984 rc = spdk_dif_verify_stream(rsp_pdu->data_iov, rsp_pdu->data_iovcnt, 1985 0, rsp_pdu->data_len, rsp_pdu->dif_ctx, &err_blk); 1986 if (rc != 0) { 1987 SPDK_ERRLOG("DIF error detected. type=%d, offset=%" PRIu32 "\n", 1988 err_blk.err_type, err_blk.err_offset); 1989 rsp->status.sct = SPDK_NVME_SCT_MEDIA_ERROR; 1990 rsp->status.sc = nvmf_tcp_dif_error_to_compl_status(err_blk.err_type); 1991 nvmf_tcp_req_pdu_fini(tcp_req); 1992 spdk_nvmf_tcp_send_capsule_resp_pdu(tcp_req, tqpair); 1993 return; 1994 } 1995 } 1996 1997 c2h_data->common.flags |= SPDK_NVME_TCP_C2H_DATA_FLAGS_LAST_PDU; 1998 if (tqpair->qpair.transport->opts.c2h_success) { 1999 c2h_data->common.flags |= SPDK_NVME_TCP_C2H_DATA_FLAGS_SUCCESS; 2000 } 2001 2002 spdk_nvmf_tcp_qpair_write_pdu(tqpair, rsp_pdu, spdk_nvmf_tcp_pdu_c2h_data_complete, tcp_req); 2003 } 2004 2005 static int 2006 request_transfer_out(struct spdk_nvmf_request *req) 2007 { 2008 struct spdk_nvmf_tcp_req *tcp_req; 2009 struct spdk_nvmf_qpair *qpair; 2010 struct spdk_nvmf_tcp_qpair *tqpair; 2011 struct spdk_nvme_cpl *rsp; 2012 2013 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "enter\n"); 2014 2015 qpair = req->qpair; 2016 rsp = &req->rsp->nvme_cpl; 2017 tcp_req = SPDK_CONTAINEROF(req, struct spdk_nvmf_tcp_req, req); 2018 2019 /* Advance our sq_head pointer */ 2020 if (qpair->sq_head == qpair->sq_head_max) { 2021 qpair->sq_head = 0; 2022 } else { 2023 qpair->sq_head++; 2024 } 2025 rsp->sqhd = qpair->sq_head; 2026 2027 tqpair = SPDK_CONTAINEROF(tcp_req->req.qpair, struct spdk_nvmf_tcp_qpair, qpair); 2028 spdk_nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST); 2029 if (rsp->status.sc == SPDK_NVME_SC_SUCCESS && req->xfer == SPDK_NVME_DATA_CONTROLLER_TO_HOST) { 2030 spdk_nvmf_tcp_send_c2h_data(tqpair, tcp_req); 2031 } else { 2032 spdk_nvmf_tcp_send_capsule_resp_pdu(tcp_req, tqpair); 2033 } 2034 2035 return 0; 2036 } 2037 2038 static void 2039 spdk_nvmf_tcp_set_incapsule_data(struct spdk_nvmf_tcp_qpair *tqpair, 2040 struct spdk_nvmf_tcp_req *tcp_req) 2041 { 2042 struct nvme_tcp_pdu *pdu; 2043 uint32_t plen = 0; 2044 2045 pdu = &tqpair->pdu_in_progress; 2046 plen = pdu->hdr.common.hlen; 2047 2048 if (tqpair->host_hdgst_enable) { 2049 plen += SPDK_NVME_TCP_DIGEST_LEN; 2050 } 2051 2052 if (pdu->hdr.common.plen != plen) { 2053 tcp_req->has_incapsule_data = true; 2054 } 2055 } 2056 2057 static bool 2058 spdk_nvmf_tcp_req_process(struct spdk_nvmf_tcp_transport *ttransport, 2059 struct spdk_nvmf_tcp_req *tcp_req) 2060 { 2061 struct spdk_nvmf_tcp_qpair *tqpair; 2062 int rc; 2063 enum spdk_nvmf_tcp_req_state prev_state; 2064 bool progress = false; 2065 struct spdk_nvmf_transport *transport = &ttransport->transport; 2066 struct spdk_nvmf_transport_poll_group *group; 2067 2068 tqpair = SPDK_CONTAINEROF(tcp_req->req.qpair, struct spdk_nvmf_tcp_qpair, qpair); 2069 group = &tqpair->group->group; 2070 assert(tcp_req->state != TCP_REQUEST_STATE_FREE); 2071 2072 /* The loop here is to allow for several back-to-back state changes. */ 2073 do { 2074 prev_state = tcp_req->state; 2075 2076 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "Request %p entering state %d on tqpair=%p\n", tcp_req, prev_state, 2077 tqpair); 2078 2079 switch (tcp_req->state) { 2080 case TCP_REQUEST_STATE_FREE: 2081 /* Some external code must kick a request into TCP_REQUEST_STATE_NEW 2082 * to escape this state. */ 2083 break; 2084 case TCP_REQUEST_STATE_NEW: 2085 spdk_trace_record(TRACE_TCP_REQUEST_STATE_NEW, 0, 0, (uintptr_t)tcp_req, 0); 2086 2087 /* copy the cmd from the receive pdu */ 2088 tcp_req->cmd = tqpair->pdu_in_progress.hdr.capsule_cmd.ccsqe; 2089 2090 if (spdk_unlikely(spdk_nvmf_request_get_dif_ctx(&tcp_req->req, &tcp_req->req.dif.dif_ctx))) { 2091 tcp_req->req.dif.dif_insert_or_strip = true; 2092 tqpair->pdu_in_progress.dif_ctx = &tcp_req->req.dif.dif_ctx; 2093 } 2094 2095 /* The next state transition depends on the data transfer needs of this request. */ 2096 tcp_req->req.xfer = spdk_nvmf_req_get_xfer(&tcp_req->req); 2097 2098 /* If no data to transfer, ready to execute. */ 2099 if (tcp_req->req.xfer == SPDK_NVME_DATA_NONE) { 2100 /* Reset the tqpair receving pdu state */ 2101 spdk_nvmf_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY); 2102 spdk_nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_READY_TO_EXECUTE); 2103 break; 2104 } 2105 2106 spdk_nvmf_tcp_set_incapsule_data(tqpair, tcp_req); 2107 2108 if (!tcp_req->has_incapsule_data) { 2109 spdk_nvmf_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY); 2110 } 2111 2112 spdk_nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_NEED_BUFFER); 2113 STAILQ_INSERT_TAIL(&group->pending_buf_queue, &tcp_req->req, buf_link); 2114 break; 2115 case TCP_REQUEST_STATE_NEED_BUFFER: 2116 spdk_trace_record(TRACE_TCP_REQUEST_STATE_NEED_BUFFER, 0, 0, (uintptr_t)tcp_req, 0); 2117 2118 assert(tcp_req->req.xfer != SPDK_NVME_DATA_NONE); 2119 2120 if (!tcp_req->has_incapsule_data && (&tcp_req->req != STAILQ_FIRST(&group->pending_buf_queue))) { 2121 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, 2122 "Not the first element to wait for the buf for tcp_req(%p) on tqpair=%p\n", 2123 tcp_req, tqpair); 2124 /* This request needs to wait in line to obtain a buffer */ 2125 break; 2126 } 2127 2128 /* Try to get a data buffer */ 2129 rc = spdk_nvmf_tcp_req_parse_sgl(tcp_req, transport, group); 2130 if (rc < 0) { 2131 STAILQ_REMOVE_HEAD(&group->pending_buf_queue, buf_link); 2132 /* Reset the tqpair receving pdu state */ 2133 spdk_nvmf_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_ERROR); 2134 spdk_nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_READY_TO_COMPLETE); 2135 break; 2136 } 2137 2138 if (!tcp_req->req.data) { 2139 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "No buffer allocated for tcp_req(%p) on tqpair(%p\n)", 2140 tcp_req, tqpair); 2141 /* No buffers available. */ 2142 break; 2143 } 2144 2145 STAILQ_REMOVE(&group->pending_buf_queue, &tcp_req->req, spdk_nvmf_request, buf_link); 2146 2147 /* If data is transferring from host to controller, we need to do a transfer from the host. */ 2148 if (tcp_req->req.xfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER) { 2149 if (tcp_req->req.data_from_pool) { 2150 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "Sending R2T for tcp_req(%p) on tqpair=%p\n", tcp_req, tqpair); 2151 spdk_nvmf_tcp_send_r2t_pdu(tqpair, tcp_req); 2152 } else { 2153 struct nvme_tcp_pdu *pdu; 2154 2155 spdk_nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 2156 2157 pdu = &tqpair->pdu_in_progress; 2158 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "Not need to send r2t for tcp_req(%p) on tqpair=%p\n", tcp_req, 2159 tqpair); 2160 /* No need to send r2t, contained in the capsuled data */ 2161 nvme_tcp_pdu_set_data_buf(pdu, tcp_req->req.iov, tcp_req->req.iovcnt, 2162 0, tcp_req->req.length); 2163 spdk_nvmf_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PAYLOAD); 2164 } 2165 break; 2166 } 2167 2168 spdk_nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_READY_TO_EXECUTE); 2169 break; 2170 case TCP_REQUEST_STATE_AWAITING_R2T_ACK: 2171 spdk_trace_record(TRACE_TCP_REQUEST_STATE_AWAIT_R2T_ACK, 0, 0, (uintptr_t)tcp_req, 0); 2172 /* The R2T completion or the h2c data incoming will kick it out of this state. */ 2173 break; 2174 case TCP_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER: 2175 2176 spdk_trace_record(TRACE_TCP_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER, 0, 0, 2177 (uintptr_t)tcp_req, 0); 2178 /* Some external code must kick a request into TCP_REQUEST_STATE_READY_TO_EXECUTE 2179 * to escape this state. */ 2180 break; 2181 case TCP_REQUEST_STATE_READY_TO_EXECUTE: 2182 spdk_trace_record(TRACE_TCP_REQUEST_STATE_READY_TO_EXECUTE, 0, 0, (uintptr_t)tcp_req, 0); 2183 2184 if (spdk_unlikely(tcp_req->req.dif.dif_insert_or_strip)) { 2185 assert(tcp_req->req.dif.elba_length >= tcp_req->req.length); 2186 tcp_req->req.length = tcp_req->req.dif.elba_length; 2187 } 2188 2189 spdk_nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_EXECUTING); 2190 spdk_nvmf_request_exec(&tcp_req->req); 2191 break; 2192 case TCP_REQUEST_STATE_EXECUTING: 2193 spdk_trace_record(TRACE_TCP_REQUEST_STATE_EXECUTING, 0, 0, (uintptr_t)tcp_req, 0); 2194 /* Some external code must kick a request into TCP_REQUEST_STATE_EXECUTED 2195 * to escape this state. */ 2196 break; 2197 case TCP_REQUEST_STATE_EXECUTED: 2198 spdk_trace_record(TRACE_TCP_REQUEST_STATE_EXECUTED, 0, 0, (uintptr_t)tcp_req, 0); 2199 2200 if (spdk_unlikely(tcp_req->req.dif.dif_insert_or_strip)) { 2201 tcp_req->req.length = tcp_req->req.dif.orig_length; 2202 } 2203 2204 spdk_nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_READY_TO_COMPLETE); 2205 break; 2206 case TCP_REQUEST_STATE_READY_TO_COMPLETE: 2207 spdk_trace_record(TRACE_TCP_REQUEST_STATE_READY_TO_COMPLETE, 0, 0, (uintptr_t)tcp_req, 0); 2208 rc = request_transfer_out(&tcp_req->req); 2209 assert(rc == 0); /* No good way to handle this currently */ 2210 break; 2211 case TCP_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST: 2212 spdk_trace_record(TRACE_TCP_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST, 0, 0, 2213 (uintptr_t)tcp_req, 2214 0); 2215 /* Some external code must kick a request into TCP_REQUEST_STATE_COMPLETED 2216 * to escape this state. */ 2217 break; 2218 case TCP_REQUEST_STATE_COMPLETED: 2219 spdk_trace_record(TRACE_TCP_REQUEST_STATE_COMPLETED, 0, 0, (uintptr_t)tcp_req, 0); 2220 if (tcp_req->req.data_from_pool) { 2221 spdk_nvmf_request_free_buffers(&tcp_req->req, group, transport); 2222 } 2223 tcp_req->req.length = 0; 2224 tcp_req->req.iovcnt = 0; 2225 tcp_req->req.data = NULL; 2226 2227 nvmf_tcp_req_pdu_fini(tcp_req); 2228 2229 spdk_nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_FREE); 2230 break; 2231 case TCP_REQUEST_NUM_STATES: 2232 default: 2233 assert(0); 2234 break; 2235 } 2236 2237 if (tcp_req->state != prev_state) { 2238 progress = true; 2239 } 2240 } while (tcp_req->state != prev_state); 2241 2242 return progress; 2243 } 2244 2245 static void 2246 spdk_nvmf_tcp_sock_cb(void *arg, struct spdk_sock_group *group, struct spdk_sock *sock) 2247 { 2248 struct spdk_nvmf_tcp_qpair *tqpair = arg; 2249 int rc; 2250 2251 assert(tqpair != NULL); 2252 rc = spdk_nvmf_tcp_sock_process(tqpair); 2253 2254 /* If there was a new socket error, disconnect */ 2255 if (rc < 0) { 2256 spdk_nvmf_tcp_qpair_disconnect(tqpair); 2257 } 2258 } 2259 2260 static int 2261 spdk_nvmf_tcp_poll_group_add(struct spdk_nvmf_transport_poll_group *group, 2262 struct spdk_nvmf_qpair *qpair) 2263 { 2264 struct spdk_nvmf_tcp_poll_group *tgroup; 2265 struct spdk_nvmf_tcp_qpair *tqpair; 2266 int rc; 2267 2268 tgroup = SPDK_CONTAINEROF(group, struct spdk_nvmf_tcp_poll_group, group); 2269 tqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_tcp_qpair, qpair); 2270 2271 rc = spdk_sock_group_add_sock(tgroup->sock_group, tqpair->sock, 2272 spdk_nvmf_tcp_sock_cb, tqpair); 2273 if (rc != 0) { 2274 SPDK_ERRLOG("Could not add sock to sock_group: %s (%d)\n", 2275 spdk_strerror(errno), errno); 2276 return -1; 2277 } 2278 2279 rc = spdk_nvmf_tcp_qpair_sock_init(tqpair); 2280 if (rc != 0) { 2281 SPDK_ERRLOG("Cannot set sock opt for tqpair=%p\n", tqpair); 2282 return -1; 2283 } 2284 2285 rc = spdk_nvmf_tcp_qpair_init(&tqpair->qpair); 2286 if (rc < 0) { 2287 SPDK_ERRLOG("Cannot init tqpair=%p\n", tqpair); 2288 return -1; 2289 } 2290 2291 rc = spdk_nvmf_tcp_qpair_init_mem_resource(tqpair); 2292 if (rc < 0) { 2293 SPDK_ERRLOG("Cannot init memory resource info for tqpair=%p\n", tqpair); 2294 return -1; 2295 } 2296 2297 tqpair->group = tgroup; 2298 tqpair->state = NVME_TCP_QPAIR_STATE_INVALID; 2299 TAILQ_INSERT_TAIL(&tgroup->qpairs, tqpair, link); 2300 2301 return 0; 2302 } 2303 2304 static int 2305 spdk_nvmf_tcp_poll_group_remove(struct spdk_nvmf_transport_poll_group *group, 2306 struct spdk_nvmf_qpair *qpair) 2307 { 2308 struct spdk_nvmf_tcp_poll_group *tgroup; 2309 struct spdk_nvmf_tcp_qpair *tqpair; 2310 int rc; 2311 2312 tgroup = SPDK_CONTAINEROF(group, struct spdk_nvmf_tcp_poll_group, group); 2313 tqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_tcp_qpair, qpair); 2314 2315 assert(tqpair->group == tgroup); 2316 2317 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "remove tqpair=%p from the tgroup=%p\n", tqpair, tgroup); 2318 if (tqpair->recv_state == NVME_TCP_PDU_RECV_STATE_AWAIT_REQ) { 2319 TAILQ_REMOVE(&tgroup->await_req, tqpair, link); 2320 } else { 2321 TAILQ_REMOVE(&tgroup->qpairs, tqpair, link); 2322 } 2323 2324 rc = spdk_sock_group_remove_sock(tgroup->sock_group, tqpair->sock); 2325 if (rc != 0) { 2326 SPDK_ERRLOG("Could not remove sock from sock_group: %s (%d)\n", 2327 spdk_strerror(errno), errno); 2328 } 2329 2330 return rc; 2331 } 2332 2333 static int 2334 spdk_nvmf_tcp_req_complete(struct spdk_nvmf_request *req) 2335 { 2336 struct spdk_nvmf_tcp_transport *ttransport; 2337 struct spdk_nvmf_tcp_req *tcp_req; 2338 2339 ttransport = SPDK_CONTAINEROF(req->qpair->transport, struct spdk_nvmf_tcp_transport, transport); 2340 tcp_req = SPDK_CONTAINEROF(req, struct spdk_nvmf_tcp_req, req); 2341 2342 spdk_nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_EXECUTED); 2343 spdk_nvmf_tcp_req_process(ttransport, tcp_req); 2344 2345 return 0; 2346 } 2347 2348 static void 2349 spdk_nvmf_tcp_close_qpair(struct spdk_nvmf_qpair *qpair) 2350 { 2351 struct spdk_nvmf_tcp_qpair *tqpair; 2352 2353 SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "Qpair: %p\n", qpair); 2354 2355 tqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_tcp_qpair, qpair); 2356 tqpair->state = NVME_TCP_QPAIR_STATE_EXITED; 2357 spdk_nvmf_tcp_qpair_destroy(tqpair); 2358 } 2359 2360 static int 2361 spdk_nvmf_tcp_poll_group_poll(struct spdk_nvmf_transport_poll_group *group) 2362 { 2363 struct spdk_nvmf_tcp_poll_group *tgroup; 2364 int rc; 2365 struct spdk_nvmf_request *req, *req_tmp; 2366 struct spdk_nvmf_tcp_req *tcp_req; 2367 struct spdk_nvmf_tcp_qpair *tqpair, *tqpair_tmp; 2368 struct spdk_nvmf_tcp_transport *ttransport = SPDK_CONTAINEROF(group->transport, 2369 struct spdk_nvmf_tcp_transport, transport); 2370 2371 tgroup = SPDK_CONTAINEROF(group, struct spdk_nvmf_tcp_poll_group, group); 2372 2373 if (spdk_unlikely(TAILQ_EMPTY(&tgroup->qpairs) && TAILQ_EMPTY(&tgroup->await_req))) { 2374 return 0; 2375 } 2376 2377 STAILQ_FOREACH_SAFE(req, &group->pending_buf_queue, buf_link, req_tmp) { 2378 tcp_req = SPDK_CONTAINEROF(req, struct spdk_nvmf_tcp_req, req); 2379 if (spdk_nvmf_tcp_req_process(ttransport, tcp_req) == false) { 2380 break; 2381 } 2382 } 2383 2384 rc = spdk_sock_group_poll(tgroup->sock_group); 2385 if (rc < 0) { 2386 SPDK_ERRLOG("Failed to poll sock_group=%p\n", tgroup->sock_group); 2387 } 2388 2389 TAILQ_FOREACH_SAFE(tqpair, &tgroup->await_req, link, tqpair_tmp) { 2390 spdk_nvmf_tcp_sock_process(tqpair); 2391 } 2392 2393 return rc; 2394 } 2395 2396 static int 2397 spdk_nvmf_tcp_qpair_get_trid(struct spdk_nvmf_qpair *qpair, 2398 struct spdk_nvme_transport_id *trid, bool peer) 2399 { 2400 struct spdk_nvmf_tcp_qpair *tqpair; 2401 uint16_t port; 2402 2403 tqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_tcp_qpair, qpair); 2404 spdk_nvme_trid_populate_transport(trid, SPDK_NVME_TRANSPORT_TCP); 2405 2406 if (peer) { 2407 snprintf(trid->traddr, sizeof(trid->traddr), "%s", tqpair->initiator_addr); 2408 port = tqpair->initiator_port; 2409 } else { 2410 snprintf(trid->traddr, sizeof(trid->traddr), "%s", tqpair->target_addr); 2411 port = tqpair->target_port; 2412 } 2413 2414 if (spdk_sock_is_ipv4(tqpair->sock)) { 2415 trid->adrfam = SPDK_NVMF_ADRFAM_IPV4; 2416 } else if (spdk_sock_is_ipv6(tqpair->sock)) { 2417 trid->adrfam = SPDK_NVMF_ADRFAM_IPV6; 2418 } else { 2419 return -1; 2420 } 2421 2422 snprintf(trid->trsvcid, sizeof(trid->trsvcid), "%d", port); 2423 return 0; 2424 } 2425 2426 static int 2427 spdk_nvmf_tcp_qpair_get_local_trid(struct spdk_nvmf_qpair *qpair, 2428 struct spdk_nvme_transport_id *trid) 2429 { 2430 return spdk_nvmf_tcp_qpair_get_trid(qpair, trid, 0); 2431 } 2432 2433 static int 2434 spdk_nvmf_tcp_qpair_get_peer_trid(struct spdk_nvmf_qpair *qpair, 2435 struct spdk_nvme_transport_id *trid) 2436 { 2437 return spdk_nvmf_tcp_qpair_get_trid(qpair, trid, 1); 2438 } 2439 2440 static int 2441 spdk_nvmf_tcp_qpair_get_listen_trid(struct spdk_nvmf_qpair *qpair, 2442 struct spdk_nvme_transport_id *trid) 2443 { 2444 return spdk_nvmf_tcp_qpair_get_trid(qpair, trid, 0); 2445 } 2446 2447 #define SPDK_NVMF_TCP_DEFAULT_MAX_QUEUE_DEPTH 128 2448 #define SPDK_NVMF_TCP_DEFAULT_AQ_DEPTH 128 2449 #define SPDK_NVMF_TCP_DEFAULT_MAX_QPAIRS_PER_CTRLR 128 2450 #define SPDK_NVMF_TCP_DEFAULT_IN_CAPSULE_DATA_SIZE 4096 2451 #define SPDK_NVMF_TCP_DEFAULT_MAX_IO_SIZE 131072 2452 #define SPDK_NVMF_TCP_DEFAULT_IO_UNIT_SIZE 131072 2453 #define SPDK_NVMF_TCP_DEFAULT_NUM_SHARED_BUFFERS 511 2454 #define SPDK_NVMF_TCP_DEFAULT_BUFFER_CACHE_SIZE 32 2455 #define SPDK_NVMF_TCP_DEFAULT_SUCCESS_OPTIMIZATION true 2456 #define SPDK_NVMF_TCP_DEFAULT_DIF_INSERT_OR_STRIP false 2457 #define SPDK_NVMF_TCP_DEFAULT_SOCK_PRIORITY 0 2458 2459 static void 2460 spdk_nvmf_tcp_opts_init(struct spdk_nvmf_transport_opts *opts) 2461 { 2462 opts->max_queue_depth = SPDK_NVMF_TCP_DEFAULT_MAX_QUEUE_DEPTH; 2463 opts->max_qpairs_per_ctrlr = SPDK_NVMF_TCP_DEFAULT_MAX_QPAIRS_PER_CTRLR; 2464 opts->in_capsule_data_size = SPDK_NVMF_TCP_DEFAULT_IN_CAPSULE_DATA_SIZE; 2465 opts->max_io_size = SPDK_NVMF_TCP_DEFAULT_MAX_IO_SIZE; 2466 opts->io_unit_size = SPDK_NVMF_TCP_DEFAULT_IO_UNIT_SIZE; 2467 opts->max_aq_depth = SPDK_NVMF_TCP_DEFAULT_AQ_DEPTH; 2468 opts->num_shared_buffers = SPDK_NVMF_TCP_DEFAULT_NUM_SHARED_BUFFERS; 2469 opts->buf_cache_size = SPDK_NVMF_TCP_DEFAULT_BUFFER_CACHE_SIZE; 2470 opts->c2h_success = SPDK_NVMF_TCP_DEFAULT_SUCCESS_OPTIMIZATION; 2471 opts->dif_insert_or_strip = SPDK_NVMF_TCP_DEFAULT_DIF_INSERT_OR_STRIP; 2472 opts->sock_priority = SPDK_NVMF_TCP_DEFAULT_SOCK_PRIORITY; 2473 } 2474 2475 const struct spdk_nvmf_transport_ops spdk_nvmf_transport_tcp = { 2476 .name = "TCP", 2477 .type = SPDK_NVME_TRANSPORT_TCP, 2478 .opts_init = spdk_nvmf_tcp_opts_init, 2479 .create = spdk_nvmf_tcp_create, 2480 .destroy = spdk_nvmf_tcp_destroy, 2481 2482 .listen = spdk_nvmf_tcp_listen, 2483 .stop_listen = spdk_nvmf_tcp_stop_listen, 2484 .accept = spdk_nvmf_tcp_accept, 2485 2486 .listener_discover = spdk_nvmf_tcp_discover, 2487 2488 .poll_group_create = spdk_nvmf_tcp_poll_group_create, 2489 .get_optimal_poll_group = spdk_nvmf_tcp_get_optimal_poll_group, 2490 .poll_group_destroy = spdk_nvmf_tcp_poll_group_destroy, 2491 .poll_group_add = spdk_nvmf_tcp_poll_group_add, 2492 .poll_group_remove = spdk_nvmf_tcp_poll_group_remove, 2493 .poll_group_poll = spdk_nvmf_tcp_poll_group_poll, 2494 2495 .req_free = spdk_nvmf_tcp_req_free, 2496 .req_complete = spdk_nvmf_tcp_req_complete, 2497 2498 .qpair_fini = spdk_nvmf_tcp_close_qpair, 2499 .qpair_get_local_trid = spdk_nvmf_tcp_qpair_get_local_trid, 2500 .qpair_get_peer_trid = spdk_nvmf_tcp_qpair_get_peer_trid, 2501 .qpair_get_listen_trid = spdk_nvmf_tcp_qpair_get_listen_trid, 2502 }; 2503 2504 SPDK_NVMF_TRANSPORT_REGISTER(tcp, &spdk_nvmf_transport_tcp); 2505 SPDK_LOG_REGISTER_COMPONENT("nvmf_tcp", SPDK_LOG_NVMF_TCP) 2506