1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. All rights reserved. 5 * Copyright (c) 2019, 2020 Mellanox Technologies LTD. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 #include "spdk/crc32.h" 36 #include "spdk/endian.h" 37 #include "spdk/assert.h" 38 #include "spdk/thread.h" 39 #include "spdk/nvmf_transport.h" 40 #include "spdk/sock.h" 41 #include "spdk/string.h" 42 #include "spdk/trace.h" 43 #include "spdk/util.h" 44 45 #include "spdk_internal/assert.h" 46 #include "spdk/log.h" 47 #include "spdk_internal/nvme_tcp.h" 48 49 #include "nvmf_internal.h" 50 51 #define NVMF_TCP_MAX_ACCEPT_SOCK_ONE_TIME 16 52 #define SPDK_NVMF_TCP_DEFAULT_MAX_SOCK_PRIORITY 16 53 #define SPDK_NVMF_TCP_DEFAULT_SOCK_PRIORITY 0 54 #define SPDK_NVMF_TCP_DEFAULT_SUCCESS_OPTIMIZATION true 55 56 const struct spdk_nvmf_transport_ops spdk_nvmf_transport_tcp; 57 58 /* spdk nvmf related structure */ 59 enum spdk_nvmf_tcp_req_state { 60 61 /* The request is not currently in use */ 62 TCP_REQUEST_STATE_FREE = 0, 63 64 /* Initial state when request first received */ 65 TCP_REQUEST_STATE_NEW, 66 67 /* The request is queued until a data buffer is available. */ 68 TCP_REQUEST_STATE_NEED_BUFFER, 69 70 /* The request is currently transferring data from the host to the controller. */ 71 TCP_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER, 72 73 /* The request is waiting for the R2T send acknowledgement. */ 74 TCP_REQUEST_STATE_AWAITING_R2T_ACK, 75 76 /* The request is ready to execute at the block device */ 77 TCP_REQUEST_STATE_READY_TO_EXECUTE, 78 79 /* The request is currently executing at the block device */ 80 TCP_REQUEST_STATE_EXECUTING, 81 82 /* The request finished executing at the block device */ 83 TCP_REQUEST_STATE_EXECUTED, 84 85 /* The request is ready to send a completion */ 86 TCP_REQUEST_STATE_READY_TO_COMPLETE, 87 88 /* The request is currently transferring final pdus from the controller to the host. */ 89 TCP_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST, 90 91 /* The request completed and can be marked free. */ 92 TCP_REQUEST_STATE_COMPLETED, 93 94 /* Terminator */ 95 TCP_REQUEST_NUM_STATES, 96 }; 97 98 static const char *spdk_nvmf_tcp_term_req_fes_str[] = { 99 "Invalid PDU Header Field", 100 "PDU Sequence Error", 101 "Header Digiest Error", 102 "Data Transfer Out of Range", 103 "R2T Limit Exceeded", 104 "Unsupported parameter", 105 }; 106 107 #define OBJECT_NVMF_TCP_IO 0x80 108 109 #define TRACE_GROUP_NVMF_TCP 0x5 110 #define TRACE_TCP_REQUEST_STATE_NEW SPDK_TPOINT_ID(TRACE_GROUP_NVMF_TCP, 0x0) 111 #define TRACE_TCP_REQUEST_STATE_NEED_BUFFER SPDK_TPOINT_ID(TRACE_GROUP_NVMF_TCP, 0x1) 112 #define TRACE_TCP_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER SPDK_TPOINT_ID(TRACE_GROUP_NVMF_TCP, 0x2) 113 #define TRACE_TCP_REQUEST_STATE_READY_TO_EXECUTE SPDK_TPOINT_ID(TRACE_GROUP_NVMF_TCP, 0x3) 114 #define TRACE_TCP_REQUEST_STATE_EXECUTING SPDK_TPOINT_ID(TRACE_GROUP_NVMF_TCP, 0x4) 115 #define TRACE_TCP_REQUEST_STATE_EXECUTED SPDK_TPOINT_ID(TRACE_GROUP_NVMF_TCP, 0x5) 116 #define TRACE_TCP_REQUEST_STATE_READY_TO_COMPLETE SPDK_TPOINT_ID(TRACE_GROUP_NVMF_TCP, 0x6) 117 #define TRACE_TCP_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST SPDK_TPOINT_ID(TRACE_GROUP_NVMF_TCP, 0x7) 118 #define TRACE_TCP_REQUEST_STATE_COMPLETED SPDK_TPOINT_ID(TRACE_GROUP_NVMF_TCP, 0x8) 119 #define TRACE_TCP_FLUSH_WRITEBUF_START SPDK_TPOINT_ID(TRACE_GROUP_NVMF_TCP, 0x9) 120 #define TRACE_TCP_FLUSH_WRITEBUF_DONE SPDK_TPOINT_ID(TRACE_GROUP_NVMF_TCP, 0xA) 121 #define TRACE_TCP_READ_FROM_SOCKET_DONE SPDK_TPOINT_ID(TRACE_GROUP_NVMF_TCP, 0xB) 122 #define TRACE_TCP_REQUEST_STATE_AWAIT_R2T_ACK SPDK_TPOINT_ID(TRACE_GROUP_NVMF_TCP, 0xC) 123 124 SPDK_TRACE_REGISTER_FN(nvmf_tcp_trace, "nvmf_tcp", TRACE_GROUP_NVMF_TCP) 125 { 126 spdk_trace_register_object(OBJECT_NVMF_TCP_IO, 'r'); 127 spdk_trace_register_description("TCP_REQ_NEW", 128 TRACE_TCP_REQUEST_STATE_NEW, 129 OWNER_NONE, OBJECT_NVMF_TCP_IO, 1, 1, ""); 130 spdk_trace_register_description("TCP_REQ_NEED_BUFFER", 131 TRACE_TCP_REQUEST_STATE_NEED_BUFFER, 132 OWNER_NONE, OBJECT_NVMF_TCP_IO, 0, 1, ""); 133 spdk_trace_register_description("TCP_REQ_TX_H_TO_C", 134 TRACE_TCP_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER, 135 OWNER_NONE, OBJECT_NVMF_TCP_IO, 0, 1, ""); 136 spdk_trace_register_description("TCP_REQ_RDY_TO_EXECUTE", 137 TRACE_TCP_REQUEST_STATE_READY_TO_EXECUTE, 138 OWNER_NONE, OBJECT_NVMF_TCP_IO, 0, 1, ""); 139 spdk_trace_register_description("TCP_REQ_EXECUTING", 140 TRACE_TCP_REQUEST_STATE_EXECUTING, 141 OWNER_NONE, OBJECT_NVMF_TCP_IO, 0, 1, ""); 142 spdk_trace_register_description("TCP_REQ_EXECUTED", 143 TRACE_TCP_REQUEST_STATE_EXECUTED, 144 OWNER_NONE, OBJECT_NVMF_TCP_IO, 0, 1, ""); 145 spdk_trace_register_description("TCP_REQ_RDY_TO_COMPLETE", 146 TRACE_TCP_REQUEST_STATE_READY_TO_COMPLETE, 147 OWNER_NONE, OBJECT_NVMF_TCP_IO, 0, 1, ""); 148 spdk_trace_register_description("TCP_REQ_TRANSFER_C2H", 149 TRACE_TCP_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST, 150 OWNER_NONE, OBJECT_NVMF_TCP_IO, 0, 1, ""); 151 spdk_trace_register_description("TCP_REQ_COMPLETED", 152 TRACE_TCP_REQUEST_STATE_COMPLETED, 153 OWNER_NONE, OBJECT_NVMF_TCP_IO, 0, 1, ""); 154 spdk_trace_register_description("TCP_WRITE_START", 155 TRACE_TCP_FLUSH_WRITEBUF_START, 156 OWNER_NONE, OBJECT_NONE, 0, 0, ""); 157 spdk_trace_register_description("TCP_WRITE_DONE", 158 TRACE_TCP_FLUSH_WRITEBUF_DONE, 159 OWNER_NONE, OBJECT_NONE, 0, 0, ""); 160 spdk_trace_register_description("TCP_READ_DONE", 161 TRACE_TCP_READ_FROM_SOCKET_DONE, 162 OWNER_NONE, OBJECT_NONE, 0, 0, ""); 163 spdk_trace_register_description("TCP_REQ_AWAIT_R2T_ACK", 164 TRACE_TCP_REQUEST_STATE_AWAIT_R2T_ACK, 165 OWNER_NONE, OBJECT_NVMF_TCP_IO, 0, 1, ""); 166 } 167 168 struct spdk_nvmf_tcp_req { 169 struct spdk_nvmf_request req; 170 struct spdk_nvme_cpl rsp; 171 struct spdk_nvme_cmd cmd; 172 173 /* A PDU that can be used for sending responses. This is 174 * not the incoming PDU! */ 175 struct nvme_tcp_pdu *pdu; 176 177 /* 178 * The PDU for a request may be used multiple times in serial over 179 * the request's lifetime. For example, first to send an R2T, then 180 * to send a completion. To catch mistakes where the PDU is used 181 * twice at the same time, add a debug flag here for init/fini. 182 */ 183 bool pdu_in_use; 184 185 /* In-capsule data buffer */ 186 uint8_t *buf; 187 188 bool has_incapsule_data; 189 190 /* transfer_tag */ 191 uint16_t ttag; 192 193 enum spdk_nvmf_tcp_req_state state; 194 195 /* 196 * h2c_offset is used when we receive the h2c_data PDU. 197 */ 198 uint32_t h2c_offset; 199 200 STAILQ_ENTRY(spdk_nvmf_tcp_req) link; 201 TAILQ_ENTRY(spdk_nvmf_tcp_req) state_link; 202 }; 203 204 struct spdk_nvmf_tcp_qpair { 205 struct spdk_nvmf_qpair qpair; 206 struct spdk_nvmf_tcp_poll_group *group; 207 struct spdk_nvmf_tcp_port *port; 208 struct spdk_sock *sock; 209 210 enum nvme_tcp_pdu_recv_state recv_state; 211 enum nvme_tcp_qpair_state state; 212 213 /* PDU being actively received */ 214 struct nvme_tcp_pdu pdu_in_progress; 215 uint32_t recv_buf_size; 216 217 /* This is a spare PDU used for sending special management 218 * operations. Primarily, this is used for the initial 219 * connection response and c2h termination request. */ 220 struct nvme_tcp_pdu *mgmt_pdu; 221 222 TAILQ_HEAD(, nvme_tcp_pdu) send_queue; 223 224 /* Arrays of in-capsule buffers, requests, and pdus. 225 * Each array is 'resource_count' number of elements */ 226 void *bufs; 227 struct spdk_nvmf_tcp_req *reqs; 228 struct nvme_tcp_pdu *pdus; 229 uint32_t resource_count; 230 231 /* Queues to track the requests in all states */ 232 TAILQ_HEAD(, spdk_nvmf_tcp_req) state_queue[TCP_REQUEST_NUM_STATES]; 233 /* Number of requests in each state */ 234 uint32_t state_cntr[TCP_REQUEST_NUM_STATES]; 235 236 uint8_t cpda; 237 238 bool host_hdgst_enable; 239 bool host_ddgst_enable; 240 241 /* IP address */ 242 char initiator_addr[SPDK_NVMF_TRADDR_MAX_LEN]; 243 char target_addr[SPDK_NVMF_TRADDR_MAX_LEN]; 244 245 /* IP port */ 246 uint16_t initiator_port; 247 uint16_t target_port; 248 249 /* Timer used to destroy qpair after detecting transport error issue if initiator does 250 * not close the connection. 251 */ 252 struct spdk_poller *timeout_poller; 253 254 TAILQ_ENTRY(spdk_nvmf_tcp_qpair) link; 255 }; 256 257 struct spdk_nvmf_tcp_poll_group { 258 struct spdk_nvmf_transport_poll_group group; 259 struct spdk_sock_group *sock_group; 260 261 TAILQ_HEAD(, spdk_nvmf_tcp_qpair) qpairs; 262 TAILQ_HEAD(, spdk_nvmf_tcp_qpair) await_req; 263 }; 264 265 struct spdk_nvmf_tcp_port { 266 const struct spdk_nvme_transport_id *trid; 267 struct spdk_sock *listen_sock; 268 TAILQ_ENTRY(spdk_nvmf_tcp_port) link; 269 }; 270 271 struct tcp_transport_opts { 272 bool c2h_success; 273 uint32_t sock_priority; 274 }; 275 276 struct spdk_nvmf_tcp_transport { 277 struct spdk_nvmf_transport transport; 278 struct tcp_transport_opts tcp_opts; 279 280 pthread_mutex_t lock; 281 282 TAILQ_HEAD(, spdk_nvmf_tcp_port) ports; 283 }; 284 285 static const struct spdk_json_object_decoder tcp_transport_opts_decoder[] = { 286 { 287 "c2h_success", offsetof(struct tcp_transport_opts, c2h_success), 288 spdk_json_decode_bool, true 289 }, 290 { 291 "sock_priority", offsetof(struct tcp_transport_opts, sock_priority), 292 spdk_json_decode_uint32, true 293 }, 294 }; 295 296 static bool nvmf_tcp_req_process(struct spdk_nvmf_tcp_transport *ttransport, 297 struct spdk_nvmf_tcp_req *tcp_req); 298 299 static void 300 nvmf_tcp_req_set_state(struct spdk_nvmf_tcp_req *tcp_req, 301 enum spdk_nvmf_tcp_req_state state) 302 { 303 struct spdk_nvmf_qpair *qpair; 304 struct spdk_nvmf_tcp_qpair *tqpair; 305 306 qpair = tcp_req->req.qpair; 307 tqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_tcp_qpair, qpair); 308 309 TAILQ_REMOVE(&tqpair->state_queue[tcp_req->state], tcp_req, state_link); 310 assert(tqpair->state_cntr[tcp_req->state] > 0); 311 tqpair->state_cntr[tcp_req->state]--; 312 313 TAILQ_INSERT_TAIL(&tqpair->state_queue[state], tcp_req, state_link); 314 tqpair->state_cntr[state]++; 315 316 tcp_req->state = state; 317 } 318 319 static inline struct nvme_tcp_pdu * 320 nvmf_tcp_req_pdu_init(struct spdk_nvmf_tcp_req *tcp_req) 321 { 322 assert(tcp_req->pdu_in_use == false); 323 tcp_req->pdu_in_use = true; 324 325 memset(tcp_req->pdu, 0, sizeof(*tcp_req->pdu)); 326 tcp_req->pdu->qpair = SPDK_CONTAINEROF(tcp_req->req.qpair, struct spdk_nvmf_tcp_qpair, qpair); 327 328 return tcp_req->pdu; 329 } 330 331 static inline void 332 nvmf_tcp_req_pdu_fini(struct spdk_nvmf_tcp_req *tcp_req) 333 { 334 tcp_req->pdu_in_use = false; 335 } 336 337 static struct spdk_nvmf_tcp_req * 338 nvmf_tcp_req_get(struct spdk_nvmf_tcp_qpair *tqpair) 339 { 340 struct spdk_nvmf_tcp_req *tcp_req; 341 342 tcp_req = TAILQ_FIRST(&tqpair->state_queue[TCP_REQUEST_STATE_FREE]); 343 if (!tcp_req) { 344 return NULL; 345 } 346 347 memset(&tcp_req->rsp, 0, sizeof(tcp_req->rsp)); 348 tcp_req->h2c_offset = 0; 349 tcp_req->has_incapsule_data = false; 350 tcp_req->req.dif.dif_insert_or_strip = false; 351 352 nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_NEW); 353 return tcp_req; 354 } 355 356 static void 357 nvmf_tcp_request_free(struct spdk_nvmf_tcp_req *tcp_req) 358 { 359 struct spdk_nvmf_tcp_transport *ttransport; 360 361 assert(tcp_req != NULL); 362 363 SPDK_DEBUGLOG(nvmf_tcp, "tcp_req=%p will be freed\n", tcp_req); 364 ttransport = SPDK_CONTAINEROF(tcp_req->req.qpair->transport, 365 struct spdk_nvmf_tcp_transport, transport); 366 nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_COMPLETED); 367 nvmf_tcp_req_process(ttransport, tcp_req); 368 } 369 370 static int 371 nvmf_tcp_req_free(struct spdk_nvmf_request *req) 372 { 373 struct spdk_nvmf_tcp_req *tcp_req = SPDK_CONTAINEROF(req, struct spdk_nvmf_tcp_req, req); 374 375 nvmf_tcp_request_free(tcp_req); 376 377 return 0; 378 } 379 380 static void 381 nvmf_tcp_drain_state_queue(struct spdk_nvmf_tcp_qpair *tqpair, 382 enum spdk_nvmf_tcp_req_state state) 383 { 384 struct spdk_nvmf_tcp_req *tcp_req, *req_tmp; 385 386 TAILQ_FOREACH_SAFE(tcp_req, &tqpair->state_queue[state], state_link, req_tmp) { 387 nvmf_tcp_request_free(tcp_req); 388 } 389 } 390 391 static void 392 nvmf_tcp_cleanup_all_states(struct spdk_nvmf_tcp_qpair *tqpair) 393 { 394 struct spdk_nvmf_tcp_req *tcp_req, *req_tmp; 395 396 assert(TAILQ_EMPTY(&tqpair->send_queue)); 397 398 nvmf_tcp_drain_state_queue(tqpair, TCP_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST); 399 nvmf_tcp_drain_state_queue(tqpair, TCP_REQUEST_STATE_NEW); 400 401 /* Wipe the requests waiting for buffer from the global list */ 402 TAILQ_FOREACH_SAFE(tcp_req, &tqpair->state_queue[TCP_REQUEST_STATE_NEED_BUFFER], state_link, 403 req_tmp) { 404 STAILQ_REMOVE(&tqpair->group->group.pending_buf_queue, &tcp_req->req, 405 spdk_nvmf_request, buf_link); 406 } 407 408 nvmf_tcp_drain_state_queue(tqpair, TCP_REQUEST_STATE_NEED_BUFFER); 409 nvmf_tcp_drain_state_queue(tqpair, TCP_REQUEST_STATE_EXECUTING); 410 nvmf_tcp_drain_state_queue(tqpair, TCP_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 411 nvmf_tcp_drain_state_queue(tqpair, TCP_REQUEST_STATE_AWAITING_R2T_ACK); 412 } 413 414 static void 415 nvmf_tcp_dump_qpair_req_contents(struct spdk_nvmf_tcp_qpair *tqpair) 416 { 417 int i; 418 struct spdk_nvmf_tcp_req *tcp_req; 419 420 SPDK_ERRLOG("Dumping contents of queue pair (QID %d)\n", tqpair->qpair.qid); 421 for (i = 1; i < TCP_REQUEST_NUM_STATES; i++) { 422 SPDK_ERRLOG("\tNum of requests in state[%d] = %u\n", i, tqpair->state_cntr[i]); 423 TAILQ_FOREACH(tcp_req, &tqpair->state_queue[i], state_link) { 424 SPDK_ERRLOG("\t\tRequest Data From Pool: %d\n", tcp_req->req.data_from_pool); 425 SPDK_ERRLOG("\t\tRequest opcode: %d\n", tcp_req->req.cmd->nvmf_cmd.opcode); 426 } 427 } 428 } 429 430 static void 431 nvmf_tcp_qpair_destroy(struct spdk_nvmf_tcp_qpair *tqpair) 432 { 433 int err = 0; 434 435 SPDK_DEBUGLOG(nvmf_tcp, "enter\n"); 436 437 err = spdk_sock_close(&tqpair->sock); 438 assert(err == 0); 439 nvmf_tcp_cleanup_all_states(tqpair); 440 441 if (tqpair->state_cntr[TCP_REQUEST_STATE_FREE] != tqpair->resource_count) { 442 SPDK_ERRLOG("tqpair(%p) free tcp request num is %u but should be %u\n", tqpair, 443 tqpair->state_cntr[TCP_REQUEST_STATE_FREE], 444 tqpair->resource_count); 445 err++; 446 } 447 448 if (err > 0) { 449 nvmf_tcp_dump_qpair_req_contents(tqpair); 450 } 451 452 spdk_dma_free(tqpair->pdus); 453 free(tqpair->reqs); 454 spdk_free(tqpair->bufs); 455 free(tqpair); 456 SPDK_DEBUGLOG(nvmf_tcp, "Leave\n"); 457 } 458 459 static void 460 nvmf_tcp_dump_opts(struct spdk_nvmf_transport *transport, struct spdk_json_write_ctx *w) 461 { 462 struct spdk_nvmf_tcp_transport *ttransport; 463 assert(w != NULL); 464 465 ttransport = SPDK_CONTAINEROF(transport, struct spdk_nvmf_tcp_transport, transport); 466 spdk_json_write_named_bool(w, "c2h_success", ttransport->tcp_opts.c2h_success); 467 spdk_json_write_named_uint32(w, "sock_priority", ttransport->tcp_opts.sock_priority); 468 } 469 470 static int 471 nvmf_tcp_destroy(struct spdk_nvmf_transport *transport) 472 { 473 struct spdk_nvmf_tcp_transport *ttransport; 474 475 assert(transport != NULL); 476 ttransport = SPDK_CONTAINEROF(transport, struct spdk_nvmf_tcp_transport, transport); 477 478 pthread_mutex_destroy(&ttransport->lock); 479 free(ttransport); 480 return 0; 481 } 482 483 static struct spdk_nvmf_transport * 484 nvmf_tcp_create(struct spdk_nvmf_transport_opts *opts) 485 { 486 struct spdk_nvmf_tcp_transport *ttransport; 487 uint32_t sge_count; 488 uint32_t min_shared_buffers; 489 490 ttransport = calloc(1, sizeof(*ttransport)); 491 if (!ttransport) { 492 return NULL; 493 } 494 495 TAILQ_INIT(&ttransport->ports); 496 497 ttransport->transport.ops = &spdk_nvmf_transport_tcp; 498 499 ttransport->tcp_opts.c2h_success = SPDK_NVMF_TCP_DEFAULT_SUCCESS_OPTIMIZATION; 500 ttransport->tcp_opts.sock_priority = SPDK_NVMF_TCP_DEFAULT_SOCK_PRIORITY; 501 if (opts->transport_specific != NULL && 502 spdk_json_decode_object_relaxed(opts->transport_specific, tcp_transport_opts_decoder, 503 SPDK_COUNTOF(tcp_transport_opts_decoder), 504 &ttransport->tcp_opts)) { 505 SPDK_ERRLOG("spdk_json_decode_object_relaxed failed\n"); 506 free(ttransport); 507 return NULL; 508 } 509 510 SPDK_NOTICELOG("*** TCP Transport Init ***\n"); 511 512 SPDK_INFOLOG(nvmf_tcp, "*** TCP Transport Init ***\n" 513 " Transport opts: max_ioq_depth=%d, max_io_size=%d,\n" 514 " max_io_qpairs_per_ctrlr=%d, io_unit_size=%d,\n" 515 " in_capsule_data_size=%d, max_aq_depth=%d\n" 516 " num_shared_buffers=%d, c2h_success=%d,\n" 517 " dif_insert_or_strip=%d, sock_priority=%d\n" 518 " abort_timeout_sec=%d\n", 519 opts->max_queue_depth, 520 opts->max_io_size, 521 opts->max_qpairs_per_ctrlr - 1, 522 opts->io_unit_size, 523 opts->in_capsule_data_size, 524 opts->max_aq_depth, 525 opts->num_shared_buffers, 526 ttransport->tcp_opts.c2h_success, 527 opts->dif_insert_or_strip, 528 ttransport->tcp_opts.sock_priority, 529 opts->abort_timeout_sec); 530 531 if (ttransport->tcp_opts.sock_priority > SPDK_NVMF_TCP_DEFAULT_MAX_SOCK_PRIORITY) { 532 SPDK_ERRLOG("Unsupported socket_priority=%d, the current range is: 0 to %d\n" 533 "you can use man 7 socket to view the range of priority under SO_PRIORITY item\n", 534 ttransport->tcp_opts.sock_priority, SPDK_NVMF_TCP_DEFAULT_MAX_SOCK_PRIORITY); 535 free(ttransport); 536 return NULL; 537 } 538 539 /* I/O unit size cannot be larger than max I/O size */ 540 if (opts->io_unit_size > opts->max_io_size) { 541 opts->io_unit_size = opts->max_io_size; 542 } 543 544 sge_count = opts->max_io_size / opts->io_unit_size; 545 if (sge_count > SPDK_NVMF_MAX_SGL_ENTRIES) { 546 SPDK_ERRLOG("Unsupported IO Unit size specified, %d bytes\n", opts->io_unit_size); 547 free(ttransport); 548 return NULL; 549 } 550 551 min_shared_buffers = spdk_env_get_core_count() * opts->buf_cache_size; 552 if (min_shared_buffers > opts->num_shared_buffers) { 553 SPDK_ERRLOG("There are not enough buffers to satisfy" 554 "per-poll group caches for each thread. (%" PRIu32 ")" 555 "supplied. (%" PRIu32 ") required\n", opts->num_shared_buffers, min_shared_buffers); 556 SPDK_ERRLOG("Please specify a larger number of shared buffers\n"); 557 nvmf_tcp_destroy(&ttransport->transport); 558 return NULL; 559 } 560 561 pthread_mutex_init(&ttransport->lock, NULL); 562 563 return &ttransport->transport; 564 } 565 566 static int 567 nvmf_tcp_trsvcid_to_int(const char *trsvcid) 568 { 569 unsigned long long ull; 570 char *end = NULL; 571 572 ull = strtoull(trsvcid, &end, 10); 573 if (end == NULL || end == trsvcid || *end != '\0') { 574 return -1; 575 } 576 577 /* Valid TCP/IP port numbers are in [0, 65535] */ 578 if (ull > 65535) { 579 return -1; 580 } 581 582 return (int)ull; 583 } 584 585 /** 586 * Canonicalize a listen address trid. 587 */ 588 static int 589 nvmf_tcp_canon_listen_trid(struct spdk_nvme_transport_id *canon_trid, 590 const struct spdk_nvme_transport_id *trid) 591 { 592 int trsvcid_int; 593 594 trsvcid_int = nvmf_tcp_trsvcid_to_int(trid->trsvcid); 595 if (trsvcid_int < 0) { 596 return -EINVAL; 597 } 598 599 memset(canon_trid, 0, sizeof(*canon_trid)); 600 spdk_nvme_trid_populate_transport(canon_trid, SPDK_NVME_TRANSPORT_TCP); 601 canon_trid->adrfam = trid->adrfam; 602 snprintf(canon_trid->traddr, sizeof(canon_trid->traddr), "%s", trid->traddr); 603 snprintf(canon_trid->trsvcid, sizeof(canon_trid->trsvcid), "%d", trsvcid_int); 604 605 return 0; 606 } 607 608 /** 609 * Find an existing listening port. 610 * 611 * Caller must hold ttransport->lock. 612 */ 613 static struct spdk_nvmf_tcp_port * 614 nvmf_tcp_find_port(struct spdk_nvmf_tcp_transport *ttransport, 615 const struct spdk_nvme_transport_id *trid) 616 { 617 struct spdk_nvme_transport_id canon_trid; 618 struct spdk_nvmf_tcp_port *port; 619 620 if (nvmf_tcp_canon_listen_trid(&canon_trid, trid) != 0) { 621 return NULL; 622 } 623 624 TAILQ_FOREACH(port, &ttransport->ports, link) { 625 if (spdk_nvme_transport_id_compare(&canon_trid, port->trid) == 0) { 626 return port; 627 } 628 } 629 630 return NULL; 631 } 632 633 static int 634 nvmf_tcp_listen(struct spdk_nvmf_transport *transport, 635 const struct spdk_nvme_transport_id *trid) 636 { 637 struct spdk_nvmf_tcp_transport *ttransport; 638 struct spdk_nvmf_tcp_port *port; 639 int trsvcid_int; 640 uint8_t adrfam; 641 struct spdk_sock_opts opts; 642 643 ttransport = SPDK_CONTAINEROF(transport, struct spdk_nvmf_tcp_transport, transport); 644 645 trsvcid_int = nvmf_tcp_trsvcid_to_int(trid->trsvcid); 646 if (trsvcid_int < 0) { 647 SPDK_ERRLOG("Invalid trsvcid '%s'\n", trid->trsvcid); 648 return -EINVAL; 649 } 650 651 pthread_mutex_lock(&ttransport->lock); 652 port = calloc(1, sizeof(*port)); 653 if (!port) { 654 SPDK_ERRLOG("Port allocation failed\n"); 655 pthread_mutex_unlock(&ttransport->lock); 656 return -ENOMEM; 657 } 658 659 port->trid = trid; 660 opts.opts_size = sizeof(opts); 661 spdk_sock_get_default_opts(&opts); 662 opts.priority = ttransport->tcp_opts.sock_priority; 663 port->listen_sock = spdk_sock_listen_ext(trid->traddr, trsvcid_int, 664 NULL, &opts); 665 if (port->listen_sock == NULL) { 666 SPDK_ERRLOG("spdk_sock_listen(%s, %d) failed: %s (%d)\n", 667 trid->traddr, trsvcid_int, 668 spdk_strerror(errno), errno); 669 free(port); 670 pthread_mutex_unlock(&ttransport->lock); 671 return -errno; 672 } 673 674 if (spdk_sock_is_ipv4(port->listen_sock)) { 675 adrfam = SPDK_NVMF_ADRFAM_IPV4; 676 } else if (spdk_sock_is_ipv6(port->listen_sock)) { 677 adrfam = SPDK_NVMF_ADRFAM_IPV6; 678 } else { 679 SPDK_ERRLOG("Unhandled socket type\n"); 680 adrfam = 0; 681 } 682 683 if (adrfam != trid->adrfam) { 684 SPDK_ERRLOG("Socket address family mismatch\n"); 685 spdk_sock_close(&port->listen_sock); 686 free(port); 687 pthread_mutex_unlock(&ttransport->lock); 688 return -EINVAL; 689 } 690 691 SPDK_NOTICELOG("*** NVMe/TCP Target Listening on %s port %s ***\n", 692 trid->traddr, trid->trsvcid); 693 694 TAILQ_INSERT_TAIL(&ttransport->ports, port, link); 695 pthread_mutex_unlock(&ttransport->lock); 696 return 0; 697 } 698 699 static void 700 nvmf_tcp_stop_listen(struct spdk_nvmf_transport *transport, 701 const struct spdk_nvme_transport_id *trid) 702 { 703 struct spdk_nvmf_tcp_transport *ttransport; 704 struct spdk_nvmf_tcp_port *port; 705 706 ttransport = SPDK_CONTAINEROF(transport, struct spdk_nvmf_tcp_transport, transport); 707 708 SPDK_DEBUGLOG(nvmf_tcp, "Removing listen address %s port %s\n", 709 trid->traddr, trid->trsvcid); 710 711 pthread_mutex_lock(&ttransport->lock); 712 port = nvmf_tcp_find_port(ttransport, trid); 713 if (port) { 714 TAILQ_REMOVE(&ttransport->ports, port, link); 715 spdk_sock_close(&port->listen_sock); 716 free(port); 717 } 718 719 pthread_mutex_unlock(&ttransport->lock); 720 } 721 722 static void nvmf_tcp_qpair_set_recv_state(struct spdk_nvmf_tcp_qpair *tqpair, 723 enum nvme_tcp_pdu_recv_state state); 724 725 static void 726 nvmf_tcp_qpair_disconnect(struct spdk_nvmf_tcp_qpair *tqpair) 727 { 728 SPDK_DEBUGLOG(nvmf_tcp, "Disconnecting qpair %p\n", tqpair); 729 730 if (tqpair->state <= NVME_TCP_QPAIR_STATE_RUNNING) { 731 tqpair->state = NVME_TCP_QPAIR_STATE_EXITING; 732 nvmf_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_ERROR); 733 spdk_poller_unregister(&tqpair->timeout_poller); 734 735 /* This will end up calling nvmf_tcp_close_qpair */ 736 spdk_nvmf_qpair_disconnect(&tqpair->qpair, NULL, NULL); 737 } 738 } 739 740 static void 741 _pdu_write_done(void *_pdu, int err) 742 { 743 struct nvme_tcp_pdu *pdu = _pdu; 744 struct spdk_nvmf_tcp_qpair *tqpair = pdu->qpair; 745 746 TAILQ_REMOVE(&tqpair->send_queue, pdu, tailq); 747 748 if (err != 0) { 749 nvmf_tcp_qpair_disconnect(tqpair); 750 return; 751 } 752 753 assert(pdu->cb_fn != NULL); 754 pdu->cb_fn(pdu->cb_arg); 755 } 756 757 static void 758 nvmf_tcp_qpair_write_pdu(struct spdk_nvmf_tcp_qpair *tqpair, 759 struct nvme_tcp_pdu *pdu, 760 nvme_tcp_qpair_xfer_complete_cb cb_fn, 761 void *cb_arg) 762 { 763 int hlen; 764 uint32_t crc32c; 765 uint32_t mapped_length = 0; 766 ssize_t rc; 767 768 assert(&tqpair->pdu_in_progress != pdu); 769 770 hlen = pdu->hdr.common.hlen; 771 772 /* Header Digest */ 773 if (g_nvme_tcp_hdgst[pdu->hdr.common.pdu_type] && tqpair->host_hdgst_enable) { 774 crc32c = nvme_tcp_pdu_calc_header_digest(pdu); 775 MAKE_DIGEST_WORD((uint8_t *)pdu->hdr.raw + hlen, crc32c); 776 } 777 778 /* Data Digest */ 779 if (pdu->data_len > 0 && g_nvme_tcp_ddgst[pdu->hdr.common.pdu_type] && tqpair->host_ddgst_enable) { 780 crc32c = nvme_tcp_pdu_calc_data_digest(pdu); 781 MAKE_DIGEST_WORD(pdu->data_digest, crc32c); 782 } 783 784 pdu->cb_fn = cb_fn; 785 pdu->cb_arg = cb_arg; 786 787 pdu->sock_req.iovcnt = nvme_tcp_build_iovs(pdu->iov, SPDK_COUNTOF(pdu->iov), pdu, 788 tqpair->host_hdgst_enable, tqpair->host_ddgst_enable, 789 &mapped_length); 790 pdu->sock_req.cb_fn = _pdu_write_done; 791 pdu->sock_req.cb_arg = pdu; 792 TAILQ_INSERT_TAIL(&tqpair->send_queue, pdu, tailq); 793 if (pdu->hdr.common.pdu_type == SPDK_NVME_TCP_PDU_TYPE_IC_RESP || 794 pdu->hdr.common.pdu_type == SPDK_NVME_TCP_PDU_TYPE_C2H_TERM_REQ) { 795 rc = spdk_sock_writev(tqpair->sock, pdu->iov, pdu->sock_req.iovcnt); 796 if (rc == mapped_length) { 797 _pdu_write_done(pdu, 0); 798 } else { 799 SPDK_ERRLOG("IC_RESP or TERM_REQ could not write to socket.\n"); 800 _pdu_write_done(pdu, -1); 801 } 802 } else { 803 spdk_sock_writev_async(tqpair->sock, &pdu->sock_req); 804 } 805 } 806 807 static int 808 nvmf_tcp_qpair_init_mem_resource(struct spdk_nvmf_tcp_qpair *tqpair) 809 { 810 uint32_t i; 811 struct spdk_nvmf_transport_opts *opts; 812 uint32_t in_capsule_data_size; 813 814 opts = &tqpair->qpair.transport->opts; 815 816 in_capsule_data_size = opts->in_capsule_data_size; 817 if (opts->dif_insert_or_strip) { 818 in_capsule_data_size = SPDK_BDEV_BUF_SIZE_WITH_MD(in_capsule_data_size); 819 } 820 821 tqpair->resource_count = opts->max_queue_depth; 822 823 tqpair->reqs = calloc(tqpair->resource_count, sizeof(*tqpair->reqs)); 824 if (!tqpair->reqs) { 825 SPDK_ERRLOG("Unable to allocate reqs on tqpair=%p\n", tqpair); 826 return -1; 827 } 828 829 if (in_capsule_data_size) { 830 tqpair->bufs = spdk_zmalloc(tqpair->resource_count * in_capsule_data_size, 0x1000, 831 NULL, SPDK_ENV_LCORE_ID_ANY, 832 SPDK_MALLOC_DMA); 833 if (!tqpair->bufs) { 834 SPDK_ERRLOG("Unable to allocate bufs on tqpair=%p.\n", tqpair); 835 return -1; 836 } 837 } 838 839 /* Add addtional one member, which will be used for mgmt_pdu owned by the tqpair */ 840 tqpair->pdus = spdk_dma_malloc((tqpair->resource_count + 1) * sizeof(*tqpair->pdus), 0x1000, NULL); 841 if (!tqpair->pdus) { 842 SPDK_ERRLOG("Unable to allocate pdu pool on tqpair =%p.\n", tqpair); 843 return -1; 844 } 845 846 for (i = 0; i < tqpair->resource_count; i++) { 847 struct spdk_nvmf_tcp_req *tcp_req = &tqpair->reqs[i]; 848 849 tcp_req->ttag = i + 1; 850 tcp_req->req.qpair = &tqpair->qpair; 851 852 tcp_req->pdu = &tqpair->pdus[i]; 853 tcp_req->pdu->qpair = tqpair; 854 855 /* Set up memory to receive commands */ 856 if (tqpair->bufs) { 857 tcp_req->buf = (void *)((uintptr_t)tqpair->bufs + (i * in_capsule_data_size)); 858 } 859 860 /* Set the cmdn and rsp */ 861 tcp_req->req.rsp = (union nvmf_c2h_msg *)&tcp_req->rsp; 862 tcp_req->req.cmd = (union nvmf_h2c_msg *)&tcp_req->cmd; 863 864 /* Initialize request state to FREE */ 865 tcp_req->state = TCP_REQUEST_STATE_FREE; 866 TAILQ_INSERT_TAIL(&tqpair->state_queue[tcp_req->state], tcp_req, state_link); 867 tqpair->state_cntr[TCP_REQUEST_STATE_FREE]++; 868 } 869 870 tqpair->mgmt_pdu = &tqpair->pdus[i]; 871 tqpair->mgmt_pdu->qpair = tqpair; 872 873 tqpair->recv_buf_size = (in_capsule_data_size + sizeof(struct spdk_nvme_tcp_cmd) + 2 * 874 SPDK_NVME_TCP_DIGEST_LEN) * SPDK_NVMF_TCP_RECV_BUF_SIZE_FACTOR; 875 876 return 0; 877 } 878 879 static int 880 nvmf_tcp_qpair_init(struct spdk_nvmf_qpair *qpair) 881 { 882 struct spdk_nvmf_tcp_qpair *tqpair; 883 int i; 884 885 tqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_tcp_qpair, qpair); 886 887 SPDK_DEBUGLOG(nvmf_tcp, "New TCP Connection: %p\n", qpair); 888 889 TAILQ_INIT(&tqpair->send_queue); 890 891 /* Initialise request state queues of the qpair */ 892 for (i = TCP_REQUEST_STATE_FREE; i < TCP_REQUEST_NUM_STATES; i++) { 893 TAILQ_INIT(&tqpair->state_queue[i]); 894 } 895 896 tqpair->host_hdgst_enable = true; 897 tqpair->host_ddgst_enable = true; 898 899 return 0; 900 } 901 902 static int 903 nvmf_tcp_qpair_sock_init(struct spdk_nvmf_tcp_qpair *tqpair) 904 { 905 int rc; 906 907 /* set low water mark */ 908 rc = spdk_sock_set_recvlowat(tqpair->sock, sizeof(struct spdk_nvme_tcp_common_pdu_hdr)); 909 if (rc != 0) { 910 SPDK_ERRLOG("spdk_sock_set_recvlowat() failed\n"); 911 return rc; 912 } 913 914 return 0; 915 } 916 917 static void 918 nvmf_tcp_handle_connect(struct spdk_nvmf_transport *transport, 919 struct spdk_nvmf_tcp_port *port, 920 struct spdk_sock *sock) 921 { 922 struct spdk_nvmf_tcp_qpair *tqpair; 923 int rc; 924 925 SPDK_DEBUGLOG(nvmf_tcp, "New connection accepted on %s port %s\n", 926 port->trid->traddr, port->trid->trsvcid); 927 928 tqpair = calloc(1, sizeof(struct spdk_nvmf_tcp_qpair)); 929 if (tqpair == NULL) { 930 SPDK_ERRLOG("Could not allocate new connection.\n"); 931 spdk_sock_close(&sock); 932 return; 933 } 934 935 tqpair->sock = sock; 936 tqpair->state_cntr[TCP_REQUEST_STATE_FREE] = 0; 937 tqpair->port = port; 938 tqpair->qpair.transport = transport; 939 tqpair->qpair.trid = port->trid; 940 941 rc = spdk_sock_getaddr(tqpair->sock, tqpair->target_addr, 942 sizeof(tqpair->target_addr), &tqpair->target_port, 943 tqpair->initiator_addr, sizeof(tqpair->initiator_addr), 944 &tqpair->initiator_port); 945 if (rc < 0) { 946 SPDK_ERRLOG("spdk_sock_getaddr() failed of tqpair=%p\n", tqpair); 947 nvmf_tcp_qpair_destroy(tqpair); 948 return; 949 } 950 951 spdk_nvmf_tgt_new_qpair(transport->tgt, &tqpair->qpair); 952 } 953 954 static uint32_t 955 nvmf_tcp_port_accept(struct spdk_nvmf_transport *transport, struct spdk_nvmf_tcp_port *port) 956 { 957 struct spdk_sock *sock; 958 uint32_t count = 0; 959 int i; 960 961 for (i = 0; i < NVMF_TCP_MAX_ACCEPT_SOCK_ONE_TIME; i++) { 962 sock = spdk_sock_accept(port->listen_sock); 963 if (sock == NULL) { 964 break; 965 } 966 count++; 967 nvmf_tcp_handle_connect(transport, port, sock); 968 } 969 970 return count; 971 } 972 973 static uint32_t 974 nvmf_tcp_accept(struct spdk_nvmf_transport *transport) 975 { 976 struct spdk_nvmf_tcp_transport *ttransport; 977 struct spdk_nvmf_tcp_port *port; 978 uint32_t count = 0; 979 980 ttransport = SPDK_CONTAINEROF(transport, struct spdk_nvmf_tcp_transport, transport); 981 982 TAILQ_FOREACH(port, &ttransport->ports, link) { 983 count += nvmf_tcp_port_accept(transport, port); 984 } 985 986 return count; 987 } 988 989 static void 990 nvmf_tcp_discover(struct spdk_nvmf_transport *transport, 991 struct spdk_nvme_transport_id *trid, 992 struct spdk_nvmf_discovery_log_page_entry *entry) 993 { 994 entry->trtype = SPDK_NVMF_TRTYPE_TCP; 995 entry->adrfam = trid->adrfam; 996 entry->treq.secure_channel = SPDK_NVMF_TREQ_SECURE_CHANNEL_NOT_REQUIRED; 997 998 spdk_strcpy_pad(entry->trsvcid, trid->trsvcid, sizeof(entry->trsvcid), ' '); 999 spdk_strcpy_pad(entry->traddr, trid->traddr, sizeof(entry->traddr), ' '); 1000 1001 entry->tsas.tcp.sectype = SPDK_NVME_TCP_SECURITY_NONE; 1002 } 1003 1004 static struct spdk_nvmf_transport_poll_group * 1005 nvmf_tcp_poll_group_create(struct spdk_nvmf_transport *transport) 1006 { 1007 struct spdk_nvmf_tcp_poll_group *tgroup; 1008 1009 tgroup = calloc(1, sizeof(*tgroup)); 1010 if (!tgroup) { 1011 return NULL; 1012 } 1013 1014 tgroup->sock_group = spdk_sock_group_create(&tgroup->group); 1015 if (!tgroup->sock_group) { 1016 goto cleanup; 1017 } 1018 1019 TAILQ_INIT(&tgroup->qpairs); 1020 TAILQ_INIT(&tgroup->await_req); 1021 1022 return &tgroup->group; 1023 1024 cleanup: 1025 free(tgroup); 1026 return NULL; 1027 } 1028 1029 static struct spdk_nvmf_transport_poll_group * 1030 nvmf_tcp_get_optimal_poll_group(struct spdk_nvmf_qpair *qpair) 1031 { 1032 struct spdk_nvmf_tcp_qpair *tqpair; 1033 struct spdk_sock_group *group = NULL; 1034 int rc; 1035 1036 tqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_tcp_qpair, qpair); 1037 rc = spdk_sock_get_optimal_sock_group(tqpair->sock, &group); 1038 if (!rc && group != NULL) { 1039 return spdk_sock_group_get_ctx(group); 1040 } 1041 1042 return NULL; 1043 } 1044 1045 static void 1046 nvmf_tcp_poll_group_destroy(struct spdk_nvmf_transport_poll_group *group) 1047 { 1048 struct spdk_nvmf_tcp_poll_group *tgroup; 1049 1050 tgroup = SPDK_CONTAINEROF(group, struct spdk_nvmf_tcp_poll_group, group); 1051 spdk_sock_group_close(&tgroup->sock_group); 1052 1053 free(tgroup); 1054 } 1055 1056 static void 1057 nvmf_tcp_qpair_set_recv_state(struct spdk_nvmf_tcp_qpair *tqpair, 1058 enum nvme_tcp_pdu_recv_state state) 1059 { 1060 if (tqpair->recv_state == state) { 1061 SPDK_ERRLOG("The recv state of tqpair=%p is same with the state(%d) to be set\n", 1062 tqpair, state); 1063 return; 1064 } 1065 1066 if (tqpair->recv_state == NVME_TCP_PDU_RECV_STATE_AWAIT_REQ) { 1067 /* When leaving the await req state, move the qpair to the main list */ 1068 TAILQ_REMOVE(&tqpair->group->await_req, tqpair, link); 1069 TAILQ_INSERT_TAIL(&tqpair->group->qpairs, tqpair, link); 1070 } 1071 1072 SPDK_DEBUGLOG(nvmf_tcp, "tqpair(%p) recv state=%d\n", tqpair, state); 1073 tqpair->recv_state = state; 1074 1075 switch (state) { 1076 case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_CH: 1077 case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PSH: 1078 case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PAYLOAD: 1079 break; 1080 case NVME_TCP_PDU_RECV_STATE_AWAIT_REQ: 1081 TAILQ_REMOVE(&tqpair->group->qpairs, tqpair, link); 1082 TAILQ_INSERT_TAIL(&tqpair->group->await_req, tqpair, link); 1083 break; 1084 case NVME_TCP_PDU_RECV_STATE_ERROR: 1085 case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY: 1086 memset(&tqpair->pdu_in_progress, 0, sizeof(tqpair->pdu_in_progress)); 1087 break; 1088 default: 1089 SPDK_ERRLOG("The state(%d) is invalid\n", state); 1090 abort(); 1091 break; 1092 } 1093 } 1094 1095 static int 1096 nvmf_tcp_qpair_handle_timeout(void *ctx) 1097 { 1098 struct spdk_nvmf_tcp_qpair *tqpair = ctx; 1099 1100 assert(tqpair->recv_state == NVME_TCP_PDU_RECV_STATE_ERROR); 1101 1102 SPDK_ERRLOG("No pdu coming for tqpair=%p within %d seconds\n", tqpair, 1103 SPDK_NVME_TCP_QPAIR_EXIT_TIMEOUT); 1104 1105 nvmf_tcp_qpair_disconnect(tqpair); 1106 return SPDK_POLLER_BUSY; 1107 } 1108 1109 static void 1110 nvmf_tcp_send_c2h_term_req_complete(void *cb_arg) 1111 { 1112 struct spdk_nvmf_tcp_qpair *tqpair = (struct spdk_nvmf_tcp_qpair *)cb_arg; 1113 1114 if (!tqpair->timeout_poller) { 1115 tqpair->timeout_poller = SPDK_POLLER_REGISTER(nvmf_tcp_qpair_handle_timeout, tqpair, 1116 SPDK_NVME_TCP_QPAIR_EXIT_TIMEOUT * 1000000); 1117 } 1118 } 1119 1120 static void 1121 nvmf_tcp_send_c2h_term_req(struct spdk_nvmf_tcp_qpair *tqpair, struct nvme_tcp_pdu *pdu, 1122 enum spdk_nvme_tcp_term_req_fes fes, uint32_t error_offset) 1123 { 1124 struct nvme_tcp_pdu *rsp_pdu; 1125 struct spdk_nvme_tcp_term_req_hdr *c2h_term_req; 1126 uint32_t c2h_term_req_hdr_len = sizeof(*c2h_term_req); 1127 uint32_t copy_len; 1128 1129 rsp_pdu = tqpair->mgmt_pdu; 1130 1131 c2h_term_req = &rsp_pdu->hdr.term_req; 1132 c2h_term_req->common.pdu_type = SPDK_NVME_TCP_PDU_TYPE_C2H_TERM_REQ; 1133 c2h_term_req->common.hlen = c2h_term_req_hdr_len; 1134 1135 if ((fes == SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD) || 1136 (fes == SPDK_NVME_TCP_TERM_REQ_FES_INVALID_DATA_UNSUPPORTED_PARAMETER)) { 1137 DSET32(&c2h_term_req->fei, error_offset); 1138 } 1139 1140 copy_len = spdk_min(pdu->hdr.common.hlen, SPDK_NVME_TCP_TERM_REQ_ERROR_DATA_MAX_SIZE); 1141 1142 /* Copy the error info into the buffer */ 1143 memcpy((uint8_t *)rsp_pdu->hdr.raw + c2h_term_req_hdr_len, pdu->hdr.raw, copy_len); 1144 nvme_tcp_pdu_set_data(rsp_pdu, (uint8_t *)rsp_pdu->hdr.raw + c2h_term_req_hdr_len, copy_len); 1145 1146 /* Contain the header of the wrong received pdu */ 1147 c2h_term_req->common.plen = c2h_term_req->common.hlen + copy_len; 1148 nvmf_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_ERROR); 1149 nvmf_tcp_qpair_write_pdu(tqpair, rsp_pdu, nvmf_tcp_send_c2h_term_req_complete, tqpair); 1150 } 1151 1152 static void 1153 nvmf_tcp_capsule_cmd_hdr_handle(struct spdk_nvmf_tcp_transport *ttransport, 1154 struct spdk_nvmf_tcp_qpair *tqpair, 1155 struct nvme_tcp_pdu *pdu) 1156 { 1157 struct spdk_nvmf_tcp_req *tcp_req; 1158 1159 assert(pdu->psh_valid_bytes == pdu->psh_len); 1160 assert(pdu->hdr.common.pdu_type == SPDK_NVME_TCP_PDU_TYPE_CAPSULE_CMD); 1161 1162 tcp_req = nvmf_tcp_req_get(tqpair); 1163 if (!tcp_req) { 1164 /* Directly return and make the allocation retry again */ 1165 if (tqpair->state_cntr[TCP_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST] > 0) { 1166 return; 1167 } 1168 1169 /* The host sent more commands than the maximum queue depth. */ 1170 SPDK_ERRLOG("Cannot allocate tcp_req on tqpair=%p\n", tqpair); 1171 nvmf_tcp_qpair_disconnect(tqpair); 1172 return; 1173 } 1174 1175 pdu->req = tcp_req; 1176 assert(tcp_req->state == TCP_REQUEST_STATE_NEW); 1177 nvmf_tcp_req_process(ttransport, tcp_req); 1178 } 1179 1180 static void 1181 nvmf_tcp_capsule_cmd_payload_handle(struct spdk_nvmf_tcp_transport *ttransport, 1182 struct spdk_nvmf_tcp_qpair *tqpair, 1183 struct nvme_tcp_pdu *pdu) 1184 { 1185 struct spdk_nvmf_tcp_req *tcp_req; 1186 struct spdk_nvme_tcp_cmd *capsule_cmd; 1187 uint32_t error_offset = 0; 1188 enum spdk_nvme_tcp_term_req_fes fes; 1189 1190 capsule_cmd = &pdu->hdr.capsule_cmd; 1191 tcp_req = pdu->req; 1192 assert(tcp_req != NULL); 1193 if (capsule_cmd->common.pdo > SPDK_NVME_TCP_PDU_PDO_MAX_OFFSET) { 1194 SPDK_ERRLOG("Expected ICReq capsule_cmd pdu offset <= %d, got %c\n", 1195 SPDK_NVME_TCP_PDU_PDO_MAX_OFFSET, capsule_cmd->common.pdo); 1196 fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD; 1197 error_offset = offsetof(struct spdk_nvme_tcp_common_pdu_hdr, pdo); 1198 goto err; 1199 } 1200 1201 nvmf_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY); 1202 nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_READY_TO_EXECUTE); 1203 nvmf_tcp_req_process(ttransport, tcp_req); 1204 1205 return; 1206 err: 1207 nvmf_tcp_send_c2h_term_req(tqpair, pdu, fes, error_offset); 1208 } 1209 1210 static int 1211 nvmf_tcp_find_req_in_state(struct spdk_nvmf_tcp_qpair *tqpair, 1212 enum spdk_nvmf_tcp_req_state state, 1213 uint16_t cid, uint16_t tag, 1214 struct spdk_nvmf_tcp_req **req) 1215 { 1216 struct spdk_nvmf_tcp_req *tcp_req = NULL; 1217 1218 TAILQ_FOREACH(tcp_req, &tqpair->state_queue[state], state_link) { 1219 if (tcp_req->req.cmd->nvme_cmd.cid != cid) { 1220 continue; 1221 } 1222 1223 if (tcp_req->ttag == tag) { 1224 *req = tcp_req; 1225 return 0; 1226 } 1227 1228 *req = NULL; 1229 return -1; 1230 } 1231 1232 /* Didn't find it, but not an error */ 1233 *req = NULL; 1234 return 0; 1235 } 1236 1237 static void 1238 nvmf_tcp_h2c_data_hdr_handle(struct spdk_nvmf_tcp_transport *ttransport, 1239 struct spdk_nvmf_tcp_qpair *tqpair, 1240 struct nvme_tcp_pdu *pdu) 1241 { 1242 struct spdk_nvmf_tcp_req *tcp_req; 1243 uint32_t error_offset = 0; 1244 enum spdk_nvme_tcp_term_req_fes fes = 0; 1245 struct spdk_nvme_tcp_h2c_data_hdr *h2c_data; 1246 int rc; 1247 1248 h2c_data = &pdu->hdr.h2c_data; 1249 1250 SPDK_DEBUGLOG(nvmf_tcp, "tqpair=%p, r2t_info: datao=%u, datal=%u, cccid=%u, ttag=%u\n", 1251 tqpair, h2c_data->datao, h2c_data->datal, h2c_data->cccid, h2c_data->ttag); 1252 1253 rc = nvmf_tcp_find_req_in_state(tqpair, TCP_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER, 1254 h2c_data->cccid, h2c_data->ttag, &tcp_req); 1255 if (rc == 0 && tcp_req == NULL) { 1256 rc = nvmf_tcp_find_req_in_state(tqpair, TCP_REQUEST_STATE_AWAITING_R2T_ACK, h2c_data->cccid, 1257 h2c_data->ttag, &tcp_req); 1258 } 1259 1260 if (!tcp_req) { 1261 SPDK_DEBUGLOG(nvmf_tcp, "tcp_req is not found for tqpair=%p\n", tqpair); 1262 fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_DATA_UNSUPPORTED_PARAMETER; 1263 if (rc == 0) { 1264 error_offset = offsetof(struct spdk_nvme_tcp_h2c_data_hdr, cccid); 1265 } else { 1266 error_offset = offsetof(struct spdk_nvme_tcp_h2c_data_hdr, ttag); 1267 } 1268 goto err; 1269 } 1270 1271 if (tcp_req->h2c_offset != h2c_data->datao) { 1272 SPDK_DEBUGLOG(nvmf_tcp, 1273 "tcp_req(%p), tqpair=%p, expected data offset %u, but data offset is %u\n", 1274 tcp_req, tqpair, tcp_req->h2c_offset, h2c_data->datao); 1275 fes = SPDK_NVME_TCP_TERM_REQ_FES_DATA_TRANSFER_OUT_OF_RANGE; 1276 goto err; 1277 } 1278 1279 if ((h2c_data->datao + h2c_data->datal) > tcp_req->req.length) { 1280 SPDK_DEBUGLOG(nvmf_tcp, 1281 "tcp_req(%p), tqpair=%p, (datao=%u + datal=%u) execeeds requested length=%u\n", 1282 tcp_req, tqpair, h2c_data->datao, h2c_data->datal, tcp_req->req.length); 1283 fes = SPDK_NVME_TCP_TERM_REQ_FES_DATA_TRANSFER_OUT_OF_RANGE; 1284 goto err; 1285 } 1286 1287 pdu->req = tcp_req; 1288 1289 if (spdk_unlikely(tcp_req->req.dif.dif_insert_or_strip)) { 1290 pdu->dif_ctx = &tcp_req->req.dif.dif_ctx; 1291 } 1292 1293 nvme_tcp_pdu_set_data_buf(pdu, tcp_req->req.iov, tcp_req->req.iovcnt, 1294 h2c_data->datao, h2c_data->datal); 1295 nvmf_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PAYLOAD); 1296 return; 1297 1298 err: 1299 nvmf_tcp_send_c2h_term_req(tqpair, pdu, fes, error_offset); 1300 } 1301 1302 static void 1303 nvmf_tcp_pdu_cmd_complete(void *cb_arg) 1304 { 1305 struct spdk_nvmf_tcp_req *tcp_req = cb_arg; 1306 nvmf_tcp_request_free(tcp_req); 1307 } 1308 1309 static void 1310 nvmf_tcp_send_capsule_resp_pdu(struct spdk_nvmf_tcp_req *tcp_req, 1311 struct spdk_nvmf_tcp_qpair *tqpair) 1312 { 1313 struct nvme_tcp_pdu *rsp_pdu; 1314 struct spdk_nvme_tcp_rsp *capsule_resp; 1315 1316 SPDK_DEBUGLOG(nvmf_tcp, "enter, tqpair=%p\n", tqpair); 1317 1318 rsp_pdu = nvmf_tcp_req_pdu_init(tcp_req); 1319 assert(rsp_pdu != NULL); 1320 1321 capsule_resp = &rsp_pdu->hdr.capsule_resp; 1322 capsule_resp->common.pdu_type = SPDK_NVME_TCP_PDU_TYPE_CAPSULE_RESP; 1323 capsule_resp->common.plen = capsule_resp->common.hlen = sizeof(*capsule_resp); 1324 capsule_resp->rccqe = tcp_req->req.rsp->nvme_cpl; 1325 if (tqpair->host_hdgst_enable) { 1326 capsule_resp->common.flags |= SPDK_NVME_TCP_CH_FLAGS_HDGSTF; 1327 capsule_resp->common.plen += SPDK_NVME_TCP_DIGEST_LEN; 1328 } 1329 1330 nvmf_tcp_qpair_write_pdu(tqpair, rsp_pdu, nvmf_tcp_pdu_cmd_complete, tcp_req); 1331 } 1332 1333 static void 1334 nvmf_tcp_pdu_c2h_data_complete(void *cb_arg) 1335 { 1336 struct spdk_nvmf_tcp_req *tcp_req = cb_arg; 1337 struct spdk_nvmf_tcp_qpair *tqpair = SPDK_CONTAINEROF(tcp_req->req.qpair, 1338 struct spdk_nvmf_tcp_qpair, qpair); 1339 struct spdk_nvmf_tcp_transport *ttransport = SPDK_CONTAINEROF( 1340 tcp_req->req.qpair->transport, struct spdk_nvmf_tcp_transport, transport); 1341 1342 assert(tqpair != NULL); 1343 if (ttransport->tcp_opts.c2h_success) { 1344 nvmf_tcp_request_free(tcp_req); 1345 } else { 1346 nvmf_tcp_req_pdu_fini(tcp_req); 1347 nvmf_tcp_send_capsule_resp_pdu(tcp_req, tqpair); 1348 } 1349 } 1350 1351 static void 1352 nvmf_tcp_r2t_complete(void *cb_arg) 1353 { 1354 struct spdk_nvmf_tcp_req *tcp_req = cb_arg; 1355 struct spdk_nvmf_tcp_transport *ttransport; 1356 1357 nvmf_tcp_req_pdu_fini(tcp_req); 1358 1359 ttransport = SPDK_CONTAINEROF(tcp_req->req.qpair->transport, 1360 struct spdk_nvmf_tcp_transport, transport); 1361 1362 nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 1363 1364 if (tcp_req->h2c_offset == tcp_req->req.length) { 1365 nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_READY_TO_EXECUTE); 1366 nvmf_tcp_req_process(ttransport, tcp_req); 1367 } 1368 } 1369 1370 static void 1371 nvmf_tcp_send_r2t_pdu(struct spdk_nvmf_tcp_qpair *tqpair, 1372 struct spdk_nvmf_tcp_req *tcp_req) 1373 { 1374 struct nvme_tcp_pdu *rsp_pdu; 1375 struct spdk_nvme_tcp_r2t_hdr *r2t; 1376 1377 rsp_pdu = nvmf_tcp_req_pdu_init(tcp_req); 1378 assert(rsp_pdu != NULL); 1379 1380 r2t = &rsp_pdu->hdr.r2t; 1381 r2t->common.pdu_type = SPDK_NVME_TCP_PDU_TYPE_R2T; 1382 r2t->common.plen = r2t->common.hlen = sizeof(*r2t); 1383 1384 if (tqpair->host_hdgst_enable) { 1385 r2t->common.flags |= SPDK_NVME_TCP_CH_FLAGS_HDGSTF; 1386 r2t->common.plen += SPDK_NVME_TCP_DIGEST_LEN; 1387 } 1388 1389 r2t->cccid = tcp_req->req.cmd->nvme_cmd.cid; 1390 r2t->ttag = tcp_req->ttag; 1391 r2t->r2to = tcp_req->h2c_offset; 1392 r2t->r2tl = tcp_req->req.length; 1393 1394 nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_AWAITING_R2T_ACK); 1395 1396 SPDK_DEBUGLOG(nvmf_tcp, 1397 "tcp_req(%p) on tqpair(%p), r2t_info: cccid=%u, ttag=%u, r2to=%u, r2tl=%u\n", 1398 tcp_req, tqpair, r2t->cccid, r2t->ttag, r2t->r2to, r2t->r2tl); 1399 nvmf_tcp_qpair_write_pdu(tqpair, rsp_pdu, nvmf_tcp_r2t_complete, tcp_req); 1400 } 1401 1402 static void 1403 nvmf_tcp_h2c_data_payload_handle(struct spdk_nvmf_tcp_transport *ttransport, 1404 struct spdk_nvmf_tcp_qpair *tqpair, 1405 struct nvme_tcp_pdu *pdu) 1406 { 1407 struct spdk_nvmf_tcp_req *tcp_req; 1408 1409 tcp_req = pdu->req; 1410 assert(tcp_req != NULL); 1411 1412 SPDK_DEBUGLOG(nvmf_tcp, "enter\n"); 1413 1414 tcp_req->h2c_offset += pdu->data_len; 1415 1416 nvmf_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY); 1417 1418 /* Wait for all of the data to arrive AND for the initial R2T PDU send to be 1419 * acknowledged before moving on. */ 1420 if (tcp_req->h2c_offset == tcp_req->req.length && 1421 tcp_req->state == TCP_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER) { 1422 nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_READY_TO_EXECUTE); 1423 nvmf_tcp_req_process(ttransport, tcp_req); 1424 } 1425 } 1426 1427 static void 1428 nvmf_tcp_h2c_term_req_dump(struct spdk_nvme_tcp_term_req_hdr *h2c_term_req) 1429 { 1430 SPDK_ERRLOG("Error info of pdu(%p): %s\n", h2c_term_req, 1431 spdk_nvmf_tcp_term_req_fes_str[h2c_term_req->fes]); 1432 if ((h2c_term_req->fes == SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD) || 1433 (h2c_term_req->fes == SPDK_NVME_TCP_TERM_REQ_FES_INVALID_DATA_UNSUPPORTED_PARAMETER)) { 1434 SPDK_DEBUGLOG(nvmf_tcp, "The offset from the start of the PDU header is %u\n", 1435 DGET32(h2c_term_req->fei)); 1436 } 1437 } 1438 1439 static void 1440 nvmf_tcp_h2c_term_req_hdr_handle(struct spdk_nvmf_tcp_qpair *tqpair, 1441 struct nvme_tcp_pdu *pdu) 1442 { 1443 struct spdk_nvme_tcp_term_req_hdr *h2c_term_req = &pdu->hdr.term_req; 1444 uint32_t error_offset = 0; 1445 enum spdk_nvme_tcp_term_req_fes fes; 1446 1447 1448 if (h2c_term_req->fes > SPDK_NVME_TCP_TERM_REQ_FES_INVALID_DATA_UNSUPPORTED_PARAMETER) { 1449 SPDK_ERRLOG("Fatal Error Stauts(FES) is unknown for h2c_term_req pdu=%p\n", pdu); 1450 fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD; 1451 error_offset = offsetof(struct spdk_nvme_tcp_term_req_hdr, fes); 1452 goto end; 1453 } 1454 1455 /* set the data buffer */ 1456 nvme_tcp_pdu_set_data(pdu, (uint8_t *)pdu->hdr.raw + h2c_term_req->common.hlen, 1457 h2c_term_req->common.plen - h2c_term_req->common.hlen); 1458 nvmf_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PAYLOAD); 1459 return; 1460 end: 1461 nvmf_tcp_send_c2h_term_req(tqpair, pdu, fes, error_offset); 1462 } 1463 1464 static void 1465 nvmf_tcp_h2c_term_req_payload_handle(struct spdk_nvmf_tcp_qpair *tqpair, 1466 struct nvme_tcp_pdu *pdu) 1467 { 1468 struct spdk_nvme_tcp_term_req_hdr *h2c_term_req = &pdu->hdr.term_req; 1469 1470 nvmf_tcp_h2c_term_req_dump(h2c_term_req); 1471 nvmf_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_ERROR); 1472 } 1473 1474 static void 1475 nvmf_tcp_pdu_payload_handle(struct spdk_nvmf_tcp_qpair *tqpair, 1476 struct spdk_nvmf_tcp_transport *ttransport) 1477 { 1478 int rc = 0; 1479 struct nvme_tcp_pdu *pdu; 1480 uint32_t crc32c, error_offset = 0; 1481 enum spdk_nvme_tcp_term_req_fes fes; 1482 1483 assert(tqpair->recv_state == NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PAYLOAD); 1484 pdu = &tqpair->pdu_in_progress; 1485 1486 SPDK_DEBUGLOG(nvmf_tcp, "enter\n"); 1487 /* check data digest if need */ 1488 if (pdu->ddgst_enable) { 1489 crc32c = nvme_tcp_pdu_calc_data_digest(pdu); 1490 rc = MATCH_DIGEST_WORD(pdu->data_digest, crc32c); 1491 if (rc == 0) { 1492 SPDK_ERRLOG("Data digest error on tqpair=(%p) with pdu=%p\n", tqpair, pdu); 1493 fes = SPDK_NVME_TCP_TERM_REQ_FES_HDGST_ERROR; 1494 nvmf_tcp_send_c2h_term_req(tqpair, pdu, fes, error_offset); 1495 return; 1496 1497 } 1498 } 1499 1500 switch (pdu->hdr.common.pdu_type) { 1501 case SPDK_NVME_TCP_PDU_TYPE_CAPSULE_CMD: 1502 nvmf_tcp_capsule_cmd_payload_handle(ttransport, tqpair, pdu); 1503 break; 1504 case SPDK_NVME_TCP_PDU_TYPE_H2C_DATA: 1505 nvmf_tcp_h2c_data_payload_handle(ttransport, tqpair, pdu); 1506 break; 1507 1508 case SPDK_NVME_TCP_PDU_TYPE_H2C_TERM_REQ: 1509 nvmf_tcp_h2c_term_req_payload_handle(tqpair, pdu); 1510 break; 1511 1512 default: 1513 /* The code should not go to here */ 1514 SPDK_ERRLOG("The code should not go to here\n"); 1515 break; 1516 } 1517 } 1518 1519 static void 1520 nvmf_tcp_send_icresp_complete(void *cb_arg) 1521 { 1522 struct spdk_nvmf_tcp_qpair *tqpair = cb_arg; 1523 1524 tqpair->state = NVME_TCP_QPAIR_STATE_RUNNING; 1525 } 1526 1527 static void 1528 nvmf_tcp_icreq_handle(struct spdk_nvmf_tcp_transport *ttransport, 1529 struct spdk_nvmf_tcp_qpair *tqpair, 1530 struct nvme_tcp_pdu *pdu) 1531 { 1532 struct spdk_nvme_tcp_ic_req *ic_req = &pdu->hdr.ic_req; 1533 struct nvme_tcp_pdu *rsp_pdu; 1534 struct spdk_nvme_tcp_ic_resp *ic_resp; 1535 uint32_t error_offset = 0; 1536 enum spdk_nvme_tcp_term_req_fes fes; 1537 1538 /* Only PFV 0 is defined currently */ 1539 if (ic_req->pfv != 0) { 1540 SPDK_ERRLOG("Expected ICReq PFV %u, got %u\n", 0u, ic_req->pfv); 1541 fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD; 1542 error_offset = offsetof(struct spdk_nvme_tcp_ic_req, pfv); 1543 goto end; 1544 } 1545 1546 /* MAXR2T is 0's based */ 1547 SPDK_DEBUGLOG(nvmf_tcp, "maxr2t =%u\n", (ic_req->maxr2t + 1u)); 1548 1549 tqpair->host_hdgst_enable = ic_req->dgst.bits.hdgst_enable ? true : false; 1550 if (!tqpair->host_hdgst_enable) { 1551 tqpair->recv_buf_size -= SPDK_NVME_TCP_DIGEST_LEN * SPDK_NVMF_TCP_RECV_BUF_SIZE_FACTOR; 1552 } 1553 1554 tqpair->host_ddgst_enable = ic_req->dgst.bits.ddgst_enable ? true : false; 1555 if (!tqpair->host_ddgst_enable) { 1556 tqpair->recv_buf_size -= SPDK_NVME_TCP_DIGEST_LEN * SPDK_NVMF_TCP_RECV_BUF_SIZE_FACTOR; 1557 } 1558 1559 /* Now that we know whether digests are enabled, properly size the receive buffer */ 1560 if (spdk_sock_set_recvbuf(tqpair->sock, tqpair->recv_buf_size) < 0) { 1561 SPDK_WARNLOG("Unable to allocate enough memory for receive buffer on tqpair=%p with size=%d\n", 1562 tqpair, 1563 tqpair->recv_buf_size); 1564 /* Not fatal. */ 1565 } 1566 1567 tqpair->cpda = spdk_min(ic_req->hpda, SPDK_NVME_TCP_CPDA_MAX); 1568 SPDK_DEBUGLOG(nvmf_tcp, "cpda of tqpair=(%p) is : %u\n", tqpair, tqpair->cpda); 1569 1570 rsp_pdu = tqpair->mgmt_pdu; 1571 1572 ic_resp = &rsp_pdu->hdr.ic_resp; 1573 ic_resp->common.pdu_type = SPDK_NVME_TCP_PDU_TYPE_IC_RESP; 1574 ic_resp->common.hlen = ic_resp->common.plen = sizeof(*ic_resp); 1575 ic_resp->pfv = 0; 1576 ic_resp->cpda = tqpair->cpda; 1577 ic_resp->maxh2cdata = ttransport->transport.opts.max_io_size; 1578 ic_resp->dgst.bits.hdgst_enable = tqpair->host_hdgst_enable ? 1 : 0; 1579 ic_resp->dgst.bits.ddgst_enable = tqpair->host_ddgst_enable ? 1 : 0; 1580 1581 SPDK_DEBUGLOG(nvmf_tcp, "host_hdgst_enable: %u\n", tqpair->host_hdgst_enable); 1582 SPDK_DEBUGLOG(nvmf_tcp, "host_ddgst_enable: %u\n", tqpair->host_ddgst_enable); 1583 1584 tqpair->state = NVME_TCP_QPAIR_STATE_INITIALIZING; 1585 nvmf_tcp_qpair_write_pdu(tqpair, rsp_pdu, nvmf_tcp_send_icresp_complete, tqpair); 1586 nvmf_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY); 1587 return; 1588 end: 1589 nvmf_tcp_send_c2h_term_req(tqpair, pdu, fes, error_offset); 1590 } 1591 1592 static void 1593 nvmf_tcp_pdu_psh_handle(struct spdk_nvmf_tcp_qpair *tqpair, 1594 struct spdk_nvmf_tcp_transport *ttransport) 1595 { 1596 struct nvme_tcp_pdu *pdu; 1597 int rc; 1598 uint32_t crc32c, error_offset = 0; 1599 enum spdk_nvme_tcp_term_req_fes fes; 1600 1601 assert(tqpair->recv_state == NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PSH); 1602 pdu = &tqpair->pdu_in_progress; 1603 1604 SPDK_DEBUGLOG(nvmf_tcp, "pdu type of tqpair(%p) is %d\n", tqpair, 1605 pdu->hdr.common.pdu_type); 1606 /* check header digest if needed */ 1607 if (pdu->has_hdgst) { 1608 SPDK_DEBUGLOG(nvmf_tcp, "Compare the header of pdu=%p on tqpair=%p\n", pdu, tqpair); 1609 crc32c = nvme_tcp_pdu_calc_header_digest(pdu); 1610 rc = MATCH_DIGEST_WORD((uint8_t *)pdu->hdr.raw + pdu->hdr.common.hlen, crc32c); 1611 if (rc == 0) { 1612 SPDK_ERRLOG("Header digest error on tqpair=(%p) with pdu=%p\n", tqpair, pdu); 1613 fes = SPDK_NVME_TCP_TERM_REQ_FES_HDGST_ERROR; 1614 nvmf_tcp_send_c2h_term_req(tqpair, pdu, fes, error_offset); 1615 return; 1616 1617 } 1618 } 1619 1620 switch (pdu->hdr.common.pdu_type) { 1621 case SPDK_NVME_TCP_PDU_TYPE_IC_REQ: 1622 nvmf_tcp_icreq_handle(ttransport, tqpair, pdu); 1623 break; 1624 case SPDK_NVME_TCP_PDU_TYPE_CAPSULE_CMD: 1625 nvmf_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_REQ); 1626 break; 1627 case SPDK_NVME_TCP_PDU_TYPE_H2C_DATA: 1628 nvmf_tcp_h2c_data_hdr_handle(ttransport, tqpair, pdu); 1629 break; 1630 1631 case SPDK_NVME_TCP_PDU_TYPE_H2C_TERM_REQ: 1632 nvmf_tcp_h2c_term_req_hdr_handle(tqpair, pdu); 1633 break; 1634 1635 default: 1636 SPDK_ERRLOG("Unexpected PDU type 0x%02x\n", tqpair->pdu_in_progress.hdr.common.pdu_type); 1637 fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD; 1638 error_offset = 1; 1639 nvmf_tcp_send_c2h_term_req(tqpair, pdu, fes, error_offset); 1640 break; 1641 } 1642 } 1643 1644 static void 1645 nvmf_tcp_pdu_ch_handle(struct spdk_nvmf_tcp_qpair *tqpair) 1646 { 1647 struct nvme_tcp_pdu *pdu; 1648 uint32_t error_offset = 0; 1649 enum spdk_nvme_tcp_term_req_fes fes; 1650 uint8_t expected_hlen, pdo; 1651 bool plen_error = false, pdo_error = false; 1652 1653 assert(tqpair->recv_state == NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_CH); 1654 pdu = &tqpair->pdu_in_progress; 1655 1656 if (pdu->hdr.common.pdu_type == SPDK_NVME_TCP_PDU_TYPE_IC_REQ) { 1657 if (tqpair->state != NVME_TCP_QPAIR_STATE_INVALID) { 1658 SPDK_ERRLOG("Already received ICreq PDU, and reject this pdu=%p\n", pdu); 1659 fes = SPDK_NVME_TCP_TERM_REQ_FES_PDU_SEQUENCE_ERROR; 1660 goto err; 1661 } 1662 expected_hlen = sizeof(struct spdk_nvme_tcp_ic_req); 1663 if (pdu->hdr.common.plen != expected_hlen) { 1664 plen_error = true; 1665 } 1666 } else { 1667 if (tqpair->state != NVME_TCP_QPAIR_STATE_RUNNING) { 1668 SPDK_ERRLOG("The TCP/IP connection is not negotitated\n"); 1669 fes = SPDK_NVME_TCP_TERM_REQ_FES_PDU_SEQUENCE_ERROR; 1670 goto err; 1671 } 1672 1673 switch (pdu->hdr.common.pdu_type) { 1674 case SPDK_NVME_TCP_PDU_TYPE_CAPSULE_CMD: 1675 expected_hlen = sizeof(struct spdk_nvme_tcp_cmd); 1676 pdo = pdu->hdr.common.pdo; 1677 if ((tqpair->cpda != 0) && (pdo != ((tqpair->cpda + 1) << 2))) { 1678 pdo_error = true; 1679 break; 1680 } 1681 1682 if (pdu->hdr.common.plen < expected_hlen) { 1683 plen_error = true; 1684 } 1685 break; 1686 case SPDK_NVME_TCP_PDU_TYPE_H2C_DATA: 1687 expected_hlen = sizeof(struct spdk_nvme_tcp_h2c_data_hdr); 1688 pdo = pdu->hdr.common.pdo; 1689 if ((tqpair->cpda != 0) && (pdo != ((tqpair->cpda + 1) << 2))) { 1690 pdo_error = true; 1691 break; 1692 } 1693 if (pdu->hdr.common.plen < expected_hlen) { 1694 plen_error = true; 1695 } 1696 break; 1697 1698 case SPDK_NVME_TCP_PDU_TYPE_H2C_TERM_REQ: 1699 expected_hlen = sizeof(struct spdk_nvme_tcp_term_req_hdr); 1700 if ((pdu->hdr.common.plen <= expected_hlen) || 1701 (pdu->hdr.common.plen > SPDK_NVME_TCP_TERM_REQ_PDU_MAX_SIZE)) { 1702 plen_error = true; 1703 } 1704 break; 1705 1706 default: 1707 SPDK_ERRLOG("Unexpected PDU type 0x%02x\n", pdu->hdr.common.pdu_type); 1708 fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD; 1709 error_offset = offsetof(struct spdk_nvme_tcp_common_pdu_hdr, pdu_type); 1710 goto err; 1711 } 1712 } 1713 1714 if (pdu->hdr.common.hlen != expected_hlen) { 1715 SPDK_ERRLOG("PDU type=0x%02x, Expected ICReq header length %u, got %u on tqpair=%p\n", 1716 pdu->hdr.common.pdu_type, 1717 expected_hlen, pdu->hdr.common.hlen, tqpair); 1718 fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD; 1719 error_offset = offsetof(struct spdk_nvme_tcp_common_pdu_hdr, hlen); 1720 goto err; 1721 } else if (pdo_error) { 1722 fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD; 1723 error_offset = offsetof(struct spdk_nvme_tcp_common_pdu_hdr, pdo); 1724 } else if (plen_error) { 1725 fes = SPDK_NVME_TCP_TERM_REQ_FES_INVALID_HEADER_FIELD; 1726 error_offset = offsetof(struct spdk_nvme_tcp_common_pdu_hdr, plen); 1727 goto err; 1728 } else { 1729 nvmf_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PSH); 1730 nvme_tcp_pdu_calc_psh_len(&tqpair->pdu_in_progress, tqpair->host_hdgst_enable); 1731 return; 1732 } 1733 err: 1734 nvmf_tcp_send_c2h_term_req(tqpair, pdu, fes, error_offset); 1735 } 1736 1737 static int 1738 nvmf_tcp_pdu_payload_insert_dif(struct nvme_tcp_pdu *pdu, uint32_t read_offset, 1739 int read_len) 1740 { 1741 int rc; 1742 1743 rc = spdk_dif_generate_stream(pdu->data_iov, pdu->data_iovcnt, 1744 read_offset, read_len, pdu->dif_ctx); 1745 if (rc != 0) { 1746 SPDK_ERRLOG("DIF generate failed\n"); 1747 } 1748 1749 return rc; 1750 } 1751 1752 static int 1753 nvmf_tcp_sock_process(struct spdk_nvmf_tcp_qpair *tqpair) 1754 { 1755 int rc = 0; 1756 struct nvme_tcp_pdu *pdu; 1757 enum nvme_tcp_pdu_recv_state prev_state; 1758 uint32_t data_len; 1759 struct spdk_nvmf_tcp_transport *ttransport = SPDK_CONTAINEROF(tqpair->qpair.transport, 1760 struct spdk_nvmf_tcp_transport, transport); 1761 1762 /* The loop here is to allow for several back-to-back state changes. */ 1763 do { 1764 prev_state = tqpair->recv_state; 1765 SPDK_DEBUGLOG(nvmf_tcp, "tqpair(%p) recv pdu entering state %d\n", tqpair, prev_state); 1766 1767 pdu = &tqpair->pdu_in_progress; 1768 switch (tqpair->recv_state) { 1769 /* Wait for the common header */ 1770 case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY: 1771 case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_CH: 1772 if (spdk_unlikely(tqpair->state == NVME_TCP_QPAIR_STATE_INITIALIZING)) { 1773 return rc; 1774 } 1775 1776 rc = nvme_tcp_read_data(tqpair->sock, 1777 sizeof(struct spdk_nvme_tcp_common_pdu_hdr) - pdu->ch_valid_bytes, 1778 (void *)&pdu->hdr.common + pdu->ch_valid_bytes); 1779 if (rc < 0) { 1780 SPDK_DEBUGLOG(nvmf_tcp, "will disconnect tqpair=%p\n", tqpair); 1781 return NVME_TCP_PDU_FATAL; 1782 } else if (rc > 0) { 1783 pdu->ch_valid_bytes += rc; 1784 spdk_trace_record(TRACE_TCP_READ_FROM_SOCKET_DONE, 0, rc, 0, 0); 1785 if (spdk_likely(tqpair->recv_state == NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY)) { 1786 nvmf_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_CH); 1787 } 1788 } 1789 1790 if (pdu->ch_valid_bytes < sizeof(struct spdk_nvme_tcp_common_pdu_hdr)) { 1791 return NVME_TCP_PDU_IN_PROGRESS; 1792 } 1793 1794 /* The command header of this PDU has now been read from the socket. */ 1795 nvmf_tcp_pdu_ch_handle(tqpair); 1796 break; 1797 /* Wait for the pdu specific header */ 1798 case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PSH: 1799 rc = nvme_tcp_read_data(tqpair->sock, 1800 pdu->psh_len - pdu->psh_valid_bytes, 1801 (void *)&pdu->hdr.raw + sizeof(struct spdk_nvme_tcp_common_pdu_hdr) + pdu->psh_valid_bytes); 1802 if (rc < 0) { 1803 return NVME_TCP_PDU_FATAL; 1804 } else if (rc > 0) { 1805 spdk_trace_record(TRACE_TCP_READ_FROM_SOCKET_DONE, 1806 0, rc, 0, 0); 1807 pdu->psh_valid_bytes += rc; 1808 } 1809 1810 if (pdu->psh_valid_bytes < pdu->psh_len) { 1811 return NVME_TCP_PDU_IN_PROGRESS; 1812 } 1813 1814 /* All header(ch, psh, head digist) of this PDU has now been read from the socket. */ 1815 nvmf_tcp_pdu_psh_handle(tqpair, ttransport); 1816 break; 1817 /* Wait for the req slot */ 1818 case NVME_TCP_PDU_RECV_STATE_AWAIT_REQ: 1819 nvmf_tcp_capsule_cmd_hdr_handle(ttransport, tqpair, pdu); 1820 break; 1821 case NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PAYLOAD: 1822 /* check whether the data is valid, if not we just return */ 1823 if (!pdu->data_len) { 1824 return NVME_TCP_PDU_IN_PROGRESS; 1825 } 1826 1827 data_len = pdu->data_len; 1828 /* data digest */ 1829 if (spdk_unlikely((pdu->hdr.common.pdu_type != SPDK_NVME_TCP_PDU_TYPE_H2C_TERM_REQ) && 1830 tqpair->host_ddgst_enable)) { 1831 data_len += SPDK_NVME_TCP_DIGEST_LEN; 1832 pdu->ddgst_enable = true; 1833 } 1834 1835 rc = nvme_tcp_read_payload_data(tqpair->sock, pdu); 1836 if (rc < 0) { 1837 return NVME_TCP_PDU_FATAL; 1838 } 1839 pdu->readv_offset += rc; 1840 1841 if (spdk_unlikely(pdu->dif_ctx != NULL)) { 1842 rc = nvmf_tcp_pdu_payload_insert_dif(pdu, pdu->readv_offset - rc, rc); 1843 if (rc != 0) { 1844 return NVME_TCP_PDU_FATAL; 1845 } 1846 } 1847 1848 if (pdu->readv_offset < data_len) { 1849 return NVME_TCP_PDU_IN_PROGRESS; 1850 } 1851 1852 /* All of this PDU has now been read from the socket. */ 1853 nvmf_tcp_pdu_payload_handle(tqpair, ttransport); 1854 break; 1855 case NVME_TCP_PDU_RECV_STATE_ERROR: 1856 if (!spdk_sock_is_connected(tqpair->sock)) { 1857 return NVME_TCP_PDU_FATAL; 1858 } 1859 break; 1860 default: 1861 assert(0); 1862 SPDK_ERRLOG("code should not come to here"); 1863 break; 1864 } 1865 } while (tqpair->recv_state != prev_state); 1866 1867 return rc; 1868 } 1869 1870 static int 1871 nvmf_tcp_req_parse_sgl(struct spdk_nvmf_tcp_req *tcp_req, 1872 struct spdk_nvmf_transport *transport, 1873 struct spdk_nvmf_transport_poll_group *group) 1874 { 1875 struct spdk_nvmf_request *req = &tcp_req->req; 1876 struct spdk_nvme_cmd *cmd; 1877 struct spdk_nvme_cpl *rsp; 1878 struct spdk_nvme_sgl_descriptor *sgl; 1879 uint32_t length; 1880 1881 cmd = &req->cmd->nvme_cmd; 1882 rsp = &req->rsp->nvme_cpl; 1883 sgl = &cmd->dptr.sgl1; 1884 1885 length = sgl->unkeyed.length; 1886 1887 if (sgl->generic.type == SPDK_NVME_SGL_TYPE_TRANSPORT_DATA_BLOCK && 1888 sgl->unkeyed.subtype == SPDK_NVME_SGL_SUBTYPE_TRANSPORT) { 1889 if (length > transport->opts.max_io_size) { 1890 SPDK_ERRLOG("SGL length 0x%x exceeds max io size 0x%x\n", 1891 length, transport->opts.max_io_size); 1892 rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID; 1893 return -1; 1894 } 1895 1896 /* fill request length and populate iovs */ 1897 req->length = length; 1898 1899 SPDK_DEBUGLOG(nvmf_tcp, "Data requested length= 0x%x\n", length); 1900 1901 if (spdk_unlikely(req->dif.dif_insert_or_strip)) { 1902 req->dif.orig_length = length; 1903 length = spdk_dif_get_length_with_md(length, &req->dif.dif_ctx); 1904 req->dif.elba_length = length; 1905 } 1906 1907 if (spdk_nvmf_request_get_buffers(req, group, transport, length)) { 1908 /* No available buffers. Queue this request up. */ 1909 SPDK_DEBUGLOG(nvmf_tcp, "No available large data buffers. Queueing request %p\n", 1910 tcp_req); 1911 return 0; 1912 } 1913 1914 /* backward compatible */ 1915 req->data = req->iov[0].iov_base; 1916 1917 SPDK_DEBUGLOG(nvmf_tcp, "Request %p took %d buffer/s from central pool, and data=%p\n", 1918 tcp_req, req->iovcnt, req->data); 1919 1920 return 0; 1921 } else if (sgl->generic.type == SPDK_NVME_SGL_TYPE_DATA_BLOCK && 1922 sgl->unkeyed.subtype == SPDK_NVME_SGL_SUBTYPE_OFFSET) { 1923 uint64_t offset = sgl->address; 1924 uint32_t max_len = transport->opts.in_capsule_data_size; 1925 1926 SPDK_DEBUGLOG(nvmf_tcp, "In-capsule data: offset 0x%" PRIx64 ", length 0x%x\n", 1927 offset, length); 1928 1929 if (offset > max_len) { 1930 SPDK_ERRLOG("In-capsule offset 0x%" PRIx64 " exceeds capsule length 0x%x\n", 1931 offset, max_len); 1932 rsp->status.sc = SPDK_NVME_SC_INVALID_SGL_OFFSET; 1933 return -1; 1934 } 1935 max_len -= (uint32_t)offset; 1936 1937 if (length > max_len) { 1938 SPDK_ERRLOG("In-capsule data length 0x%x exceeds capsule length 0x%x\n", 1939 length, max_len); 1940 rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID; 1941 return -1; 1942 } 1943 1944 req->data = tcp_req->buf + offset; 1945 req->data_from_pool = false; 1946 req->length = length; 1947 1948 if (spdk_unlikely(req->dif.dif_insert_or_strip)) { 1949 length = spdk_dif_get_length_with_md(length, &req->dif.dif_ctx); 1950 req->dif.elba_length = length; 1951 } 1952 1953 req->iov[0].iov_base = req->data; 1954 req->iov[0].iov_len = length; 1955 req->iovcnt = 1; 1956 1957 return 0; 1958 } 1959 1960 SPDK_ERRLOG("Invalid NVMf I/O Command SGL: Type 0x%x, Subtype 0x%x\n", 1961 sgl->generic.type, sgl->generic.subtype); 1962 rsp->status.sc = SPDK_NVME_SC_SGL_DESCRIPTOR_TYPE_INVALID; 1963 return -1; 1964 } 1965 1966 static inline enum spdk_nvme_media_error_status_code 1967 nvmf_tcp_dif_error_to_compl_status(uint8_t err_type) { 1968 enum spdk_nvme_media_error_status_code result; 1969 1970 switch (err_type) 1971 { 1972 case SPDK_DIF_REFTAG_ERROR: 1973 result = SPDK_NVME_SC_REFERENCE_TAG_CHECK_ERROR; 1974 break; 1975 case SPDK_DIF_APPTAG_ERROR: 1976 result = SPDK_NVME_SC_APPLICATION_TAG_CHECK_ERROR; 1977 break; 1978 case SPDK_DIF_GUARD_ERROR: 1979 result = SPDK_NVME_SC_GUARD_CHECK_ERROR; 1980 break; 1981 default: 1982 SPDK_UNREACHABLE(); 1983 break; 1984 } 1985 1986 return result; 1987 } 1988 1989 static void 1990 nvmf_tcp_send_c2h_data(struct spdk_nvmf_tcp_qpair *tqpair, 1991 struct spdk_nvmf_tcp_req *tcp_req) 1992 { 1993 struct spdk_nvmf_tcp_transport *ttransport = SPDK_CONTAINEROF( 1994 tqpair->qpair.transport, struct spdk_nvmf_tcp_transport, transport); 1995 struct nvme_tcp_pdu *rsp_pdu; 1996 struct spdk_nvme_tcp_c2h_data_hdr *c2h_data; 1997 uint32_t plen, pdo, alignment; 1998 int rc; 1999 2000 SPDK_DEBUGLOG(nvmf_tcp, "enter\n"); 2001 2002 rsp_pdu = nvmf_tcp_req_pdu_init(tcp_req); 2003 assert(rsp_pdu != NULL); 2004 2005 c2h_data = &rsp_pdu->hdr.c2h_data; 2006 c2h_data->common.pdu_type = SPDK_NVME_TCP_PDU_TYPE_C2H_DATA; 2007 plen = c2h_data->common.hlen = sizeof(*c2h_data); 2008 2009 if (tqpair->host_hdgst_enable) { 2010 plen += SPDK_NVME_TCP_DIGEST_LEN; 2011 c2h_data->common.flags |= SPDK_NVME_TCP_CH_FLAGS_HDGSTF; 2012 } 2013 2014 /* set the psh */ 2015 c2h_data->cccid = tcp_req->req.cmd->nvme_cmd.cid; 2016 c2h_data->datal = tcp_req->req.length; 2017 c2h_data->datao = 0; 2018 2019 /* set the padding */ 2020 rsp_pdu->padding_len = 0; 2021 pdo = plen; 2022 if (tqpair->cpda) { 2023 alignment = (tqpair->cpda + 1) << 2; 2024 if (alignment > plen) { 2025 rsp_pdu->padding_len = alignment - plen; 2026 pdo = plen = alignment; 2027 } 2028 } 2029 2030 c2h_data->common.pdo = pdo; 2031 plen += c2h_data->datal; 2032 if (tqpair->host_ddgst_enable) { 2033 c2h_data->common.flags |= SPDK_NVME_TCP_CH_FLAGS_DDGSTF; 2034 plen += SPDK_NVME_TCP_DIGEST_LEN; 2035 } 2036 2037 c2h_data->common.plen = plen; 2038 2039 if (spdk_unlikely(tcp_req->req.dif.dif_insert_or_strip)) { 2040 rsp_pdu->dif_ctx = &tcp_req->req.dif.dif_ctx; 2041 } 2042 2043 nvme_tcp_pdu_set_data_buf(rsp_pdu, tcp_req->req.iov, tcp_req->req.iovcnt, 2044 c2h_data->datao, c2h_data->datal); 2045 2046 if (spdk_unlikely(tcp_req->req.dif.dif_insert_or_strip)) { 2047 struct spdk_nvme_cpl *rsp = &tcp_req->req.rsp->nvme_cpl; 2048 struct spdk_dif_error err_blk = {}; 2049 2050 rc = spdk_dif_verify_stream(rsp_pdu->data_iov, rsp_pdu->data_iovcnt, 2051 0, rsp_pdu->data_len, rsp_pdu->dif_ctx, &err_blk); 2052 if (rc != 0) { 2053 SPDK_ERRLOG("DIF error detected. type=%d, offset=%" PRIu32 "\n", 2054 err_blk.err_type, err_blk.err_offset); 2055 rsp->status.sct = SPDK_NVME_SCT_MEDIA_ERROR; 2056 rsp->status.sc = nvmf_tcp_dif_error_to_compl_status(err_blk.err_type); 2057 nvmf_tcp_req_pdu_fini(tcp_req); 2058 nvmf_tcp_send_capsule_resp_pdu(tcp_req, tqpair); 2059 return; 2060 } 2061 } 2062 2063 c2h_data->common.flags |= SPDK_NVME_TCP_C2H_DATA_FLAGS_LAST_PDU; 2064 if (ttransport->tcp_opts.c2h_success) { 2065 c2h_data->common.flags |= SPDK_NVME_TCP_C2H_DATA_FLAGS_SUCCESS; 2066 } 2067 2068 nvmf_tcp_qpair_write_pdu(tqpair, rsp_pdu, nvmf_tcp_pdu_c2h_data_complete, tcp_req); 2069 } 2070 2071 static int 2072 request_transfer_out(struct spdk_nvmf_request *req) 2073 { 2074 struct spdk_nvmf_tcp_req *tcp_req; 2075 struct spdk_nvmf_qpair *qpair; 2076 struct spdk_nvmf_tcp_qpair *tqpair; 2077 struct spdk_nvme_cpl *rsp; 2078 2079 SPDK_DEBUGLOG(nvmf_tcp, "enter\n"); 2080 2081 qpair = req->qpair; 2082 rsp = &req->rsp->nvme_cpl; 2083 tcp_req = SPDK_CONTAINEROF(req, struct spdk_nvmf_tcp_req, req); 2084 2085 /* Advance our sq_head pointer */ 2086 if (qpair->sq_head == qpair->sq_head_max) { 2087 qpair->sq_head = 0; 2088 } else { 2089 qpair->sq_head++; 2090 } 2091 rsp->sqhd = qpair->sq_head; 2092 2093 tqpair = SPDK_CONTAINEROF(tcp_req->req.qpair, struct spdk_nvmf_tcp_qpair, qpair); 2094 nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST); 2095 if (rsp->status.sc == SPDK_NVME_SC_SUCCESS && req->xfer == SPDK_NVME_DATA_CONTROLLER_TO_HOST) { 2096 nvmf_tcp_send_c2h_data(tqpair, tcp_req); 2097 } else { 2098 nvmf_tcp_send_capsule_resp_pdu(tcp_req, tqpair); 2099 } 2100 2101 return 0; 2102 } 2103 2104 static void 2105 nvmf_tcp_set_incapsule_data(struct spdk_nvmf_tcp_qpair *tqpair, 2106 struct spdk_nvmf_tcp_req *tcp_req) 2107 { 2108 struct nvme_tcp_pdu *pdu; 2109 uint32_t plen = 0; 2110 2111 pdu = &tqpair->pdu_in_progress; 2112 plen = pdu->hdr.common.hlen; 2113 2114 if (tqpair->host_hdgst_enable) { 2115 plen += SPDK_NVME_TCP_DIGEST_LEN; 2116 } 2117 2118 if (pdu->hdr.common.plen != plen) { 2119 tcp_req->has_incapsule_data = true; 2120 } 2121 } 2122 2123 static bool 2124 nvmf_tcp_req_process(struct spdk_nvmf_tcp_transport *ttransport, 2125 struct spdk_nvmf_tcp_req *tcp_req) 2126 { 2127 struct spdk_nvmf_tcp_qpair *tqpair; 2128 int rc; 2129 enum spdk_nvmf_tcp_req_state prev_state; 2130 bool progress = false; 2131 struct spdk_nvmf_transport *transport = &ttransport->transport; 2132 struct spdk_nvmf_transport_poll_group *group; 2133 2134 tqpair = SPDK_CONTAINEROF(tcp_req->req.qpair, struct spdk_nvmf_tcp_qpair, qpair); 2135 group = &tqpair->group->group; 2136 assert(tcp_req->state != TCP_REQUEST_STATE_FREE); 2137 2138 /* If the qpair is not active, we need to abort the outstanding requests. */ 2139 if (tqpair->qpair.state != SPDK_NVMF_QPAIR_ACTIVE) { 2140 if (tcp_req->state == TCP_REQUEST_STATE_NEED_BUFFER) { 2141 STAILQ_REMOVE(&group->pending_buf_queue, &tcp_req->req, spdk_nvmf_request, buf_link); 2142 } 2143 nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_COMPLETED); 2144 } 2145 2146 /* The loop here is to allow for several back-to-back state changes. */ 2147 do { 2148 prev_state = tcp_req->state; 2149 2150 SPDK_DEBUGLOG(nvmf_tcp, "Request %p entering state %d on tqpair=%p\n", tcp_req, prev_state, 2151 tqpair); 2152 2153 switch (tcp_req->state) { 2154 case TCP_REQUEST_STATE_FREE: 2155 /* Some external code must kick a request into TCP_REQUEST_STATE_NEW 2156 * to escape this state. */ 2157 break; 2158 case TCP_REQUEST_STATE_NEW: 2159 spdk_trace_record(TRACE_TCP_REQUEST_STATE_NEW, 0, 0, (uintptr_t)tcp_req, 0); 2160 2161 /* copy the cmd from the receive pdu */ 2162 tcp_req->cmd = tqpair->pdu_in_progress.hdr.capsule_cmd.ccsqe; 2163 2164 if (spdk_unlikely(spdk_nvmf_request_get_dif_ctx(&tcp_req->req, &tcp_req->req.dif.dif_ctx))) { 2165 tcp_req->req.dif.dif_insert_or_strip = true; 2166 tqpair->pdu_in_progress.dif_ctx = &tcp_req->req.dif.dif_ctx; 2167 } 2168 2169 /* The next state transition depends on the data transfer needs of this request. */ 2170 tcp_req->req.xfer = spdk_nvmf_req_get_xfer(&tcp_req->req); 2171 2172 if (spdk_unlikely(tcp_req->req.xfer == SPDK_NVME_DATA_BIDIRECTIONAL)) { 2173 tcp_req->req.rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 2174 tcp_req->req.rsp->nvme_cpl.status.sct = SPDK_NVME_SC_INVALID_OPCODE; 2175 nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_READY_TO_COMPLETE); 2176 SPDK_DEBUGLOG(nvmf_tcp, "Request %p: invalid xfer type (BIDIRECTIONAL)\n", tcp_req); 2177 break; 2178 } 2179 2180 /* If no data to transfer, ready to execute. */ 2181 if (tcp_req->req.xfer == SPDK_NVME_DATA_NONE) { 2182 /* Reset the tqpair receving pdu state */ 2183 nvmf_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY); 2184 nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_READY_TO_EXECUTE); 2185 break; 2186 } 2187 2188 nvmf_tcp_set_incapsule_data(tqpair, tcp_req); 2189 2190 if (!tcp_req->has_incapsule_data) { 2191 nvmf_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_READY); 2192 } 2193 2194 nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_NEED_BUFFER); 2195 STAILQ_INSERT_TAIL(&group->pending_buf_queue, &tcp_req->req, buf_link); 2196 break; 2197 case TCP_REQUEST_STATE_NEED_BUFFER: 2198 spdk_trace_record(TRACE_TCP_REQUEST_STATE_NEED_BUFFER, 0, 0, (uintptr_t)tcp_req, 0); 2199 2200 assert(tcp_req->req.xfer != SPDK_NVME_DATA_NONE); 2201 2202 if (!tcp_req->has_incapsule_data && (&tcp_req->req != STAILQ_FIRST(&group->pending_buf_queue))) { 2203 SPDK_DEBUGLOG(nvmf_tcp, 2204 "Not the first element to wait for the buf for tcp_req(%p) on tqpair=%p\n", 2205 tcp_req, tqpair); 2206 /* This request needs to wait in line to obtain a buffer */ 2207 break; 2208 } 2209 2210 /* Try to get a data buffer */ 2211 rc = nvmf_tcp_req_parse_sgl(tcp_req, transport, group); 2212 if (rc < 0) { 2213 STAILQ_REMOVE_HEAD(&group->pending_buf_queue, buf_link); 2214 /* Reset the tqpair receving pdu state */ 2215 nvmf_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_ERROR); 2216 nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_READY_TO_COMPLETE); 2217 break; 2218 } 2219 2220 if (!tcp_req->req.data) { 2221 SPDK_DEBUGLOG(nvmf_tcp, "No buffer allocated for tcp_req(%p) on tqpair(%p\n)", 2222 tcp_req, tqpair); 2223 /* No buffers available. */ 2224 break; 2225 } 2226 2227 STAILQ_REMOVE(&group->pending_buf_queue, &tcp_req->req, spdk_nvmf_request, buf_link); 2228 2229 /* If data is transferring from host to controller, we need to do a transfer from the host. */ 2230 if (tcp_req->req.xfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER) { 2231 if (tcp_req->req.data_from_pool) { 2232 SPDK_DEBUGLOG(nvmf_tcp, "Sending R2T for tcp_req(%p) on tqpair=%p\n", tcp_req, tqpair); 2233 nvmf_tcp_send_r2t_pdu(tqpair, tcp_req); 2234 } else { 2235 struct nvme_tcp_pdu *pdu; 2236 2237 nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER); 2238 2239 pdu = &tqpair->pdu_in_progress; 2240 SPDK_DEBUGLOG(nvmf_tcp, "Not need to send r2t for tcp_req(%p) on tqpair=%p\n", tcp_req, 2241 tqpair); 2242 /* No need to send r2t, contained in the capsuled data */ 2243 nvme_tcp_pdu_set_data_buf(pdu, tcp_req->req.iov, tcp_req->req.iovcnt, 2244 0, tcp_req->req.length); 2245 nvmf_tcp_qpair_set_recv_state(tqpair, NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PAYLOAD); 2246 } 2247 break; 2248 } 2249 2250 nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_READY_TO_EXECUTE); 2251 break; 2252 case TCP_REQUEST_STATE_AWAITING_R2T_ACK: 2253 spdk_trace_record(TRACE_TCP_REQUEST_STATE_AWAIT_R2T_ACK, 0, 0, (uintptr_t)tcp_req, 0); 2254 /* The R2T completion or the h2c data incoming will kick it out of this state. */ 2255 break; 2256 case TCP_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER: 2257 2258 spdk_trace_record(TRACE_TCP_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER, 0, 0, 2259 (uintptr_t)tcp_req, 0); 2260 /* Some external code must kick a request into TCP_REQUEST_STATE_READY_TO_EXECUTE 2261 * to escape this state. */ 2262 break; 2263 case TCP_REQUEST_STATE_READY_TO_EXECUTE: 2264 spdk_trace_record(TRACE_TCP_REQUEST_STATE_READY_TO_EXECUTE, 0, 0, (uintptr_t)tcp_req, 0); 2265 2266 if (spdk_unlikely(tcp_req->req.dif.dif_insert_or_strip)) { 2267 assert(tcp_req->req.dif.elba_length >= tcp_req->req.length); 2268 tcp_req->req.length = tcp_req->req.dif.elba_length; 2269 } 2270 2271 nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_EXECUTING); 2272 spdk_nvmf_request_exec(&tcp_req->req); 2273 break; 2274 case TCP_REQUEST_STATE_EXECUTING: 2275 spdk_trace_record(TRACE_TCP_REQUEST_STATE_EXECUTING, 0, 0, (uintptr_t)tcp_req, 0); 2276 /* Some external code must kick a request into TCP_REQUEST_STATE_EXECUTED 2277 * to escape this state. */ 2278 break; 2279 case TCP_REQUEST_STATE_EXECUTED: 2280 spdk_trace_record(TRACE_TCP_REQUEST_STATE_EXECUTED, 0, 0, (uintptr_t)tcp_req, 0); 2281 2282 if (spdk_unlikely(tcp_req->req.dif.dif_insert_or_strip)) { 2283 tcp_req->req.length = tcp_req->req.dif.orig_length; 2284 } 2285 2286 nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_READY_TO_COMPLETE); 2287 break; 2288 case TCP_REQUEST_STATE_READY_TO_COMPLETE: 2289 spdk_trace_record(TRACE_TCP_REQUEST_STATE_READY_TO_COMPLETE, 0, 0, (uintptr_t)tcp_req, 0); 2290 rc = request_transfer_out(&tcp_req->req); 2291 assert(rc == 0); /* No good way to handle this currently */ 2292 break; 2293 case TCP_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST: 2294 spdk_trace_record(TRACE_TCP_REQUEST_STATE_TRANSFERRING_CONTROLLER_TO_HOST, 0, 0, 2295 (uintptr_t)tcp_req, 2296 0); 2297 /* Some external code must kick a request into TCP_REQUEST_STATE_COMPLETED 2298 * to escape this state. */ 2299 break; 2300 case TCP_REQUEST_STATE_COMPLETED: 2301 spdk_trace_record(TRACE_TCP_REQUEST_STATE_COMPLETED, 0, 0, (uintptr_t)tcp_req, 0); 2302 if (tcp_req->req.data_from_pool) { 2303 spdk_nvmf_request_free_buffers(&tcp_req->req, group, transport); 2304 } 2305 tcp_req->req.length = 0; 2306 tcp_req->req.iovcnt = 0; 2307 tcp_req->req.data = NULL; 2308 2309 nvmf_tcp_req_pdu_fini(tcp_req); 2310 2311 nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_FREE); 2312 break; 2313 case TCP_REQUEST_NUM_STATES: 2314 default: 2315 assert(0); 2316 break; 2317 } 2318 2319 if (tcp_req->state != prev_state) { 2320 progress = true; 2321 } 2322 } while (tcp_req->state != prev_state); 2323 2324 return progress; 2325 } 2326 2327 static void 2328 nvmf_tcp_sock_cb(void *arg, struct spdk_sock_group *group, struct spdk_sock *sock) 2329 { 2330 struct spdk_nvmf_tcp_qpair *tqpair = arg; 2331 int rc; 2332 2333 assert(tqpair != NULL); 2334 rc = nvmf_tcp_sock_process(tqpair); 2335 2336 /* If there was a new socket error, disconnect */ 2337 if (rc < 0) { 2338 nvmf_tcp_qpair_disconnect(tqpair); 2339 } 2340 } 2341 2342 static int 2343 nvmf_tcp_poll_group_add(struct spdk_nvmf_transport_poll_group *group, 2344 struct spdk_nvmf_qpair *qpair) 2345 { 2346 struct spdk_nvmf_tcp_poll_group *tgroup; 2347 struct spdk_nvmf_tcp_qpair *tqpair; 2348 int rc; 2349 2350 tgroup = SPDK_CONTAINEROF(group, struct spdk_nvmf_tcp_poll_group, group); 2351 tqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_tcp_qpair, qpair); 2352 2353 rc = spdk_sock_group_add_sock(tgroup->sock_group, tqpair->sock, 2354 nvmf_tcp_sock_cb, tqpair); 2355 if (rc != 0) { 2356 SPDK_ERRLOG("Could not add sock to sock_group: %s (%d)\n", 2357 spdk_strerror(errno), errno); 2358 return -1; 2359 } 2360 2361 rc = nvmf_tcp_qpair_sock_init(tqpair); 2362 if (rc != 0) { 2363 SPDK_ERRLOG("Cannot set sock opt for tqpair=%p\n", tqpair); 2364 return -1; 2365 } 2366 2367 rc = nvmf_tcp_qpair_init(&tqpair->qpair); 2368 if (rc < 0) { 2369 SPDK_ERRLOG("Cannot init tqpair=%p\n", tqpair); 2370 return -1; 2371 } 2372 2373 rc = nvmf_tcp_qpair_init_mem_resource(tqpair); 2374 if (rc < 0) { 2375 SPDK_ERRLOG("Cannot init memory resource info for tqpair=%p\n", tqpair); 2376 return -1; 2377 } 2378 2379 tqpair->group = tgroup; 2380 tqpair->state = NVME_TCP_QPAIR_STATE_INVALID; 2381 TAILQ_INSERT_TAIL(&tgroup->qpairs, tqpair, link); 2382 2383 return 0; 2384 } 2385 2386 static int 2387 nvmf_tcp_poll_group_remove(struct spdk_nvmf_transport_poll_group *group, 2388 struct spdk_nvmf_qpair *qpair) 2389 { 2390 struct spdk_nvmf_tcp_poll_group *tgroup; 2391 struct spdk_nvmf_tcp_qpair *tqpair; 2392 int rc; 2393 2394 tgroup = SPDK_CONTAINEROF(group, struct spdk_nvmf_tcp_poll_group, group); 2395 tqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_tcp_qpair, qpair); 2396 2397 assert(tqpair->group == tgroup); 2398 2399 SPDK_DEBUGLOG(nvmf_tcp, "remove tqpair=%p from the tgroup=%p\n", tqpair, tgroup); 2400 if (tqpair->recv_state == NVME_TCP_PDU_RECV_STATE_AWAIT_REQ) { 2401 TAILQ_REMOVE(&tgroup->await_req, tqpair, link); 2402 } else { 2403 TAILQ_REMOVE(&tgroup->qpairs, tqpair, link); 2404 } 2405 2406 rc = spdk_sock_group_remove_sock(tgroup->sock_group, tqpair->sock); 2407 if (rc != 0) { 2408 SPDK_ERRLOG("Could not remove sock from sock_group: %s (%d)\n", 2409 spdk_strerror(errno), errno); 2410 } 2411 2412 return rc; 2413 } 2414 2415 static int 2416 nvmf_tcp_req_complete(struct spdk_nvmf_request *req) 2417 { 2418 struct spdk_nvmf_tcp_transport *ttransport; 2419 struct spdk_nvmf_tcp_req *tcp_req; 2420 2421 ttransport = SPDK_CONTAINEROF(req->qpair->transport, struct spdk_nvmf_tcp_transport, transport); 2422 tcp_req = SPDK_CONTAINEROF(req, struct spdk_nvmf_tcp_req, req); 2423 2424 nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_EXECUTED); 2425 nvmf_tcp_req_process(ttransport, tcp_req); 2426 2427 return 0; 2428 } 2429 2430 static void 2431 nvmf_tcp_close_qpair(struct spdk_nvmf_qpair *qpair) 2432 { 2433 struct spdk_nvmf_tcp_qpair *tqpair; 2434 2435 SPDK_DEBUGLOG(nvmf_tcp, "Qpair: %p\n", qpair); 2436 2437 tqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_tcp_qpair, qpair); 2438 tqpair->state = NVME_TCP_QPAIR_STATE_EXITED; 2439 nvmf_tcp_qpair_destroy(tqpair); 2440 } 2441 2442 static int 2443 nvmf_tcp_poll_group_poll(struct spdk_nvmf_transport_poll_group *group) 2444 { 2445 struct spdk_nvmf_tcp_poll_group *tgroup; 2446 int rc; 2447 struct spdk_nvmf_request *req, *req_tmp; 2448 struct spdk_nvmf_tcp_req *tcp_req; 2449 struct spdk_nvmf_tcp_qpair *tqpair, *tqpair_tmp; 2450 struct spdk_nvmf_tcp_transport *ttransport = SPDK_CONTAINEROF(group->transport, 2451 struct spdk_nvmf_tcp_transport, transport); 2452 2453 tgroup = SPDK_CONTAINEROF(group, struct spdk_nvmf_tcp_poll_group, group); 2454 2455 if (spdk_unlikely(TAILQ_EMPTY(&tgroup->qpairs) && TAILQ_EMPTY(&tgroup->await_req))) { 2456 return 0; 2457 } 2458 2459 STAILQ_FOREACH_SAFE(req, &group->pending_buf_queue, buf_link, req_tmp) { 2460 tcp_req = SPDK_CONTAINEROF(req, struct spdk_nvmf_tcp_req, req); 2461 if (nvmf_tcp_req_process(ttransport, tcp_req) == false) { 2462 break; 2463 } 2464 } 2465 2466 rc = spdk_sock_group_poll(tgroup->sock_group); 2467 if (rc < 0) { 2468 SPDK_ERRLOG("Failed to poll sock_group=%p\n", tgroup->sock_group); 2469 } 2470 2471 TAILQ_FOREACH_SAFE(tqpair, &tgroup->await_req, link, tqpair_tmp) { 2472 nvmf_tcp_sock_process(tqpair); 2473 } 2474 2475 return rc; 2476 } 2477 2478 static int 2479 nvmf_tcp_qpair_get_trid(struct spdk_nvmf_qpair *qpair, 2480 struct spdk_nvme_transport_id *trid, bool peer) 2481 { 2482 struct spdk_nvmf_tcp_qpair *tqpair; 2483 uint16_t port; 2484 2485 tqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_tcp_qpair, qpair); 2486 spdk_nvme_trid_populate_transport(trid, SPDK_NVME_TRANSPORT_TCP); 2487 2488 if (peer) { 2489 snprintf(trid->traddr, sizeof(trid->traddr), "%s", tqpair->initiator_addr); 2490 port = tqpair->initiator_port; 2491 } else { 2492 snprintf(trid->traddr, sizeof(trid->traddr), "%s", tqpair->target_addr); 2493 port = tqpair->target_port; 2494 } 2495 2496 if (spdk_sock_is_ipv4(tqpair->sock)) { 2497 trid->adrfam = SPDK_NVMF_ADRFAM_IPV4; 2498 } else if (spdk_sock_is_ipv6(tqpair->sock)) { 2499 trid->adrfam = SPDK_NVMF_ADRFAM_IPV6; 2500 } else { 2501 return -1; 2502 } 2503 2504 snprintf(trid->trsvcid, sizeof(trid->trsvcid), "%d", port); 2505 return 0; 2506 } 2507 2508 static int 2509 nvmf_tcp_qpair_get_local_trid(struct spdk_nvmf_qpair *qpair, 2510 struct spdk_nvme_transport_id *trid) 2511 { 2512 return nvmf_tcp_qpair_get_trid(qpair, trid, 0); 2513 } 2514 2515 static int 2516 nvmf_tcp_qpair_get_peer_trid(struct spdk_nvmf_qpair *qpair, 2517 struct spdk_nvme_transport_id *trid) 2518 { 2519 return nvmf_tcp_qpair_get_trid(qpair, trid, 1); 2520 } 2521 2522 static int 2523 nvmf_tcp_qpair_get_listen_trid(struct spdk_nvmf_qpair *qpair, 2524 struct spdk_nvme_transport_id *trid) 2525 { 2526 return nvmf_tcp_qpair_get_trid(qpair, trid, 0); 2527 } 2528 2529 static void 2530 nvmf_tcp_req_set_abort_status(struct spdk_nvmf_request *req, 2531 struct spdk_nvmf_tcp_req *tcp_req_to_abort) 2532 { 2533 tcp_req_to_abort->req.rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 2534 tcp_req_to_abort->req.rsp->nvme_cpl.status.sc = SPDK_NVME_SC_ABORTED_BY_REQUEST; 2535 2536 nvmf_tcp_req_set_state(tcp_req_to_abort, TCP_REQUEST_STATE_READY_TO_COMPLETE); 2537 2538 req->rsp->nvme_cpl.cdw0 &= ~1U; /* Command was successfully aborted. */ 2539 } 2540 2541 static int 2542 _nvmf_tcp_qpair_abort_request(void *ctx) 2543 { 2544 struct spdk_nvmf_request *req = ctx; 2545 struct spdk_nvmf_tcp_req *tcp_req_to_abort = SPDK_CONTAINEROF(req->req_to_abort, 2546 struct spdk_nvmf_tcp_req, req); 2547 struct spdk_nvmf_tcp_qpair *tqpair = SPDK_CONTAINEROF(req->req_to_abort->qpair, 2548 struct spdk_nvmf_tcp_qpair, qpair); 2549 int rc; 2550 2551 spdk_poller_unregister(&req->poller); 2552 2553 switch (tcp_req_to_abort->state) { 2554 case TCP_REQUEST_STATE_EXECUTING: 2555 rc = nvmf_ctrlr_abort_request(req); 2556 if (rc == SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS) { 2557 return SPDK_POLLER_BUSY; 2558 } 2559 break; 2560 2561 case TCP_REQUEST_STATE_NEED_BUFFER: 2562 STAILQ_REMOVE(&tqpair->group->group.pending_buf_queue, 2563 &tcp_req_to_abort->req, spdk_nvmf_request, buf_link); 2564 2565 nvmf_tcp_req_set_abort_status(req, tcp_req_to_abort); 2566 break; 2567 2568 case TCP_REQUEST_STATE_AWAITING_R2T_ACK: 2569 nvmf_tcp_req_set_abort_status(req, tcp_req_to_abort); 2570 break; 2571 2572 case TCP_REQUEST_STATE_TRANSFERRING_HOST_TO_CONTROLLER: 2573 if (spdk_get_ticks() < req->timeout_tsc) { 2574 req->poller = SPDK_POLLER_REGISTER(_nvmf_tcp_qpair_abort_request, req, 0); 2575 return SPDK_POLLER_BUSY; 2576 } 2577 break; 2578 2579 default: 2580 break; 2581 } 2582 2583 spdk_nvmf_request_complete(req); 2584 return SPDK_POLLER_BUSY; 2585 } 2586 2587 static void 2588 nvmf_tcp_qpair_abort_request(struct spdk_nvmf_qpair *qpair, 2589 struct spdk_nvmf_request *req) 2590 { 2591 struct spdk_nvmf_tcp_qpair *tqpair; 2592 struct spdk_nvmf_tcp_transport *ttransport; 2593 struct spdk_nvmf_transport *transport; 2594 uint16_t cid; 2595 uint32_t i; 2596 struct spdk_nvmf_tcp_req *tcp_req_to_abort = NULL; 2597 2598 tqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_tcp_qpair, qpair); 2599 ttransport = SPDK_CONTAINEROF(qpair->transport, struct spdk_nvmf_tcp_transport, transport); 2600 transport = &ttransport->transport; 2601 2602 cid = req->cmd->nvme_cmd.cdw10_bits.abort.cid; 2603 2604 for (i = 0; i < tqpair->resource_count; i++) { 2605 tcp_req_to_abort = &tqpair->reqs[i]; 2606 2607 if (tcp_req_to_abort->state != TCP_REQUEST_STATE_FREE && 2608 tcp_req_to_abort->req.cmd->nvme_cmd.cid == cid) { 2609 break; 2610 } 2611 } 2612 2613 if (tcp_req_to_abort == NULL) { 2614 spdk_nvmf_request_complete(req); 2615 return; 2616 } 2617 2618 req->req_to_abort = &tcp_req_to_abort->req; 2619 req->timeout_tsc = spdk_get_ticks() + 2620 transport->opts.abort_timeout_sec * spdk_get_ticks_hz(); 2621 req->poller = NULL; 2622 2623 _nvmf_tcp_qpair_abort_request(req); 2624 } 2625 2626 #define SPDK_NVMF_TCP_DEFAULT_MAX_QUEUE_DEPTH 128 2627 #define SPDK_NVMF_TCP_DEFAULT_AQ_DEPTH 128 2628 #define SPDK_NVMF_TCP_DEFAULT_MAX_QPAIRS_PER_CTRLR 128 2629 #define SPDK_NVMF_TCP_DEFAULT_IN_CAPSULE_DATA_SIZE 4096 2630 #define SPDK_NVMF_TCP_DEFAULT_MAX_IO_SIZE 131072 2631 #define SPDK_NVMF_TCP_DEFAULT_IO_UNIT_SIZE 131072 2632 #define SPDK_NVMF_TCP_DEFAULT_NUM_SHARED_BUFFERS 511 2633 #define SPDK_NVMF_TCP_DEFAULT_BUFFER_CACHE_SIZE 32 2634 #define SPDK_NVMF_TCP_DEFAULT_DIF_INSERT_OR_STRIP false 2635 #define SPDK_NVMF_TCP_DEFAULT_ABORT_TIMEOUT_SEC 1 2636 2637 static void 2638 nvmf_tcp_opts_init(struct spdk_nvmf_transport_opts *opts) 2639 { 2640 opts->max_queue_depth = SPDK_NVMF_TCP_DEFAULT_MAX_QUEUE_DEPTH; 2641 opts->max_qpairs_per_ctrlr = SPDK_NVMF_TCP_DEFAULT_MAX_QPAIRS_PER_CTRLR; 2642 opts->in_capsule_data_size = SPDK_NVMF_TCP_DEFAULT_IN_CAPSULE_DATA_SIZE; 2643 opts->max_io_size = SPDK_NVMF_TCP_DEFAULT_MAX_IO_SIZE; 2644 opts->io_unit_size = SPDK_NVMF_TCP_DEFAULT_IO_UNIT_SIZE; 2645 opts->max_aq_depth = SPDK_NVMF_TCP_DEFAULT_AQ_DEPTH; 2646 opts->num_shared_buffers = SPDK_NVMF_TCP_DEFAULT_NUM_SHARED_BUFFERS; 2647 opts->buf_cache_size = SPDK_NVMF_TCP_DEFAULT_BUFFER_CACHE_SIZE; 2648 opts->dif_insert_or_strip = SPDK_NVMF_TCP_DEFAULT_DIF_INSERT_OR_STRIP; 2649 opts->abort_timeout_sec = SPDK_NVMF_TCP_DEFAULT_ABORT_TIMEOUT_SEC; 2650 opts->transport_specific = NULL; 2651 } 2652 2653 const struct spdk_nvmf_transport_ops spdk_nvmf_transport_tcp = { 2654 .name = "TCP", 2655 .type = SPDK_NVME_TRANSPORT_TCP, 2656 .opts_init = nvmf_tcp_opts_init, 2657 .create = nvmf_tcp_create, 2658 .dump_opts = nvmf_tcp_dump_opts, 2659 .destroy = nvmf_tcp_destroy, 2660 2661 .listen = nvmf_tcp_listen, 2662 .stop_listen = nvmf_tcp_stop_listen, 2663 .accept = nvmf_tcp_accept, 2664 2665 .listener_discover = nvmf_tcp_discover, 2666 2667 .poll_group_create = nvmf_tcp_poll_group_create, 2668 .get_optimal_poll_group = nvmf_tcp_get_optimal_poll_group, 2669 .poll_group_destroy = nvmf_tcp_poll_group_destroy, 2670 .poll_group_add = nvmf_tcp_poll_group_add, 2671 .poll_group_remove = nvmf_tcp_poll_group_remove, 2672 .poll_group_poll = nvmf_tcp_poll_group_poll, 2673 2674 .req_free = nvmf_tcp_req_free, 2675 .req_complete = nvmf_tcp_req_complete, 2676 2677 .qpair_fini = nvmf_tcp_close_qpair, 2678 .qpair_get_local_trid = nvmf_tcp_qpair_get_local_trid, 2679 .qpair_get_peer_trid = nvmf_tcp_qpair_get_peer_trid, 2680 .qpair_get_listen_trid = nvmf_tcp_qpair_get_listen_trid, 2681 .qpair_abort_request = nvmf_tcp_qpair_abort_request, 2682 }; 2683 2684 SPDK_NVMF_TRANSPORT_REGISTER(tcp, &spdk_nvmf_transport_tcp); 2685 SPDK_LOG_REGISTER_COMPONENT(nvmf_tcp) 2686