1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. All rights reserved. 5 * Copyright (c) 2020, 2021 Mellanox Technologies LTD. All rights reserved. 6 * Copyright (c) 2021, 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 12 * * Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * * Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in 16 * the documentation and/or other materials provided with the 17 * distribution. 18 * * Neither the name of Intel Corporation nor the names of its 19 * contributors may be used to endorse or promote products derived 20 * from this software without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 23 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 24 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 25 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 26 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 27 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 28 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 32 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 */ 34 35 #ifndef __NVME_INTERNAL_H__ 36 #define __NVME_INTERNAL_H__ 37 38 #include "spdk/config.h" 39 #include "spdk/likely.h" 40 #include "spdk/stdinc.h" 41 42 #include "spdk/nvme.h" 43 44 #if defined(__i386__) || defined(__x86_64__) 45 #include <x86intrin.h> 46 #endif 47 48 #include "spdk/queue.h" 49 #include "spdk/barrier.h" 50 #include "spdk/bit_array.h" 51 #include "spdk/mmio.h" 52 #include "spdk/pci_ids.h" 53 #include "spdk/util.h" 54 #include "spdk/memory.h" 55 #include "spdk/nvme_intel.h" 56 #include "spdk/nvmf_spec.h" 57 #include "spdk/tree.h" 58 #include "spdk/uuid.h" 59 60 #include "spdk_internal/assert.h" 61 #include "spdk/log.h" 62 63 extern pid_t g_spdk_nvme_pid; 64 65 /* 66 * Some Intel devices support vendor-unique read latency log page even 67 * though the log page directory says otherwise. 68 */ 69 #define NVME_INTEL_QUIRK_READ_LATENCY 0x1 70 71 /* 72 * Some Intel devices support vendor-unique write latency log page even 73 * though the log page directory says otherwise. 74 */ 75 #define NVME_INTEL_QUIRK_WRITE_LATENCY 0x2 76 77 /* 78 * The controller needs a delay before starts checking the device 79 * readiness, which is done by reading the NVME_CSTS_RDY bit. 80 */ 81 #define NVME_QUIRK_DELAY_BEFORE_CHK_RDY 0x4 82 83 /* 84 * The controller performs best when I/O is split on particular 85 * LBA boundaries. 86 */ 87 #define NVME_INTEL_QUIRK_STRIPING 0x8 88 89 /* 90 * The controller needs a delay after allocating an I/O queue pair 91 * before it is ready to accept I/O commands. 92 */ 93 #define NVME_QUIRK_DELAY_AFTER_QUEUE_ALLOC 0x10 94 95 /* 96 * Earlier NVMe devices do not indicate whether unmapped blocks 97 * will read all zeroes or not. This define indicates that the 98 * device does in fact read all zeroes after an unmap event 99 */ 100 #define NVME_QUIRK_READ_ZERO_AFTER_DEALLOCATE 0x20 101 102 /* 103 * The controller doesn't handle Identify value others than 0 or 1 correctly. 104 */ 105 #define NVME_QUIRK_IDENTIFY_CNS 0x40 106 107 /* 108 * The controller supports Open Channel command set if matching additional 109 * condition, like the first byte (value 0x1) in the vendor specific 110 * bits of the namespace identify structure is set. 111 */ 112 #define NVME_QUIRK_OCSSD 0x80 113 114 /* 115 * The controller has an Intel vendor ID but does not support Intel vendor-specific 116 * log pages. This is primarily for QEMU emulated SSDs which report an Intel vendor 117 * ID but do not support these log pages. 118 */ 119 #define NVME_INTEL_QUIRK_NO_LOG_PAGES 0x100 120 121 /* 122 * The controller does not set SHST_COMPLETE in a reasonable amount of time. This 123 * is primarily seen in virtual VMWare NVMe SSDs. This quirk merely adds an additional 124 * error message that on VMWare NVMe SSDs, the shutdown timeout may be expected. 125 */ 126 #define NVME_QUIRK_SHST_COMPLETE 0x200 127 128 /* 129 * The controller requires an extra delay before starting the initialization process 130 * during attach. 131 */ 132 #define NVME_QUIRK_DELAY_BEFORE_INIT 0x400 133 134 /* 135 * Some SSDs exhibit poor performance with the default SPDK NVMe IO queue size. 136 * This quirk will increase the default to 1024 which matches other operating 137 * systems, at the cost of some extra memory usage. Users can still override 138 * the increased default by changing the spdk_nvme_io_qpair_opts when allocating 139 * a new queue pair. 140 */ 141 #define NVME_QUIRK_MINIMUM_IO_QUEUE_SIZE 0x800 142 143 /** 144 * The maximum access width to PCI memory space is 8 Bytes, don't use AVX2 or 145 * SSE instructions to optimize the memory access(memcpy or memset) larger than 146 * 8 Bytes. 147 */ 148 #define NVME_QUIRK_MAXIMUM_PCI_ACCESS_WIDTH 0x1000 149 150 /** 151 * The SSD does not support OPAL even through it sets the security bit in OACS. 152 */ 153 #define NVME_QUIRK_OACS_SECURITY 0x2000 154 155 /** 156 * Intel P55XX SSDs can't support Dataset Management command with SGL format, 157 * so use PRP with DSM command. 158 */ 159 #define NVME_QUIRK_NO_SGL_FOR_DSM 0x4000 160 161 /** 162 * Maximum Data Transfer Size(MDTS) excludes interleaved metadata. 163 */ 164 #define NVME_QUIRK_MDTS_EXCLUDE_MD 0x8000 165 166 /** 167 * Force not to use SGL even the controller report that it can 168 * support it. 169 */ 170 #define NVME_QUIRK_NOT_USE_SGL 0x10000 171 172 #define NVME_MAX_ASYNC_EVENTS (8) 173 174 #define NVME_MAX_ADMIN_TIMEOUT_IN_SECS (30) 175 176 /* Maximum log page size to fetch for AERs. */ 177 #define NVME_MAX_AER_LOG_SIZE (4096) 178 179 /* 180 * NVME_MAX_IO_QUEUES in nvme_spec.h defines the 64K spec-limit, but this 181 * define specifies the maximum number of queues this driver will actually 182 * try to configure, if available. 183 */ 184 #define DEFAULT_MAX_IO_QUEUES (1024) 185 #define DEFAULT_ADMIN_QUEUE_SIZE (32) 186 #define DEFAULT_IO_QUEUE_SIZE (256) 187 #define DEFAULT_IO_QUEUE_SIZE_FOR_QUIRK (1024) /* Matches Linux kernel driver */ 188 189 #define DEFAULT_IO_QUEUE_REQUESTS (512) 190 191 #define SPDK_NVME_DEFAULT_RETRY_COUNT (4) 192 193 #define SPDK_NVME_TRANSPORT_ACK_TIMEOUT_DISABLED (0) 194 #define SPDK_NVME_DEFAULT_TRANSPORT_ACK_TIMEOUT SPDK_NVME_TRANSPORT_ACK_TIMEOUT_DISABLED 195 196 #define MIN_KEEP_ALIVE_TIMEOUT_IN_MS (10000) 197 198 /* We want to fit submission and completion rings each in a single 2MB 199 * hugepage to ensure physical address contiguity. 200 */ 201 #define MAX_IO_QUEUE_ENTRIES (VALUE_2MB / spdk_max( \ 202 sizeof(struct spdk_nvme_cmd), \ 203 sizeof(struct spdk_nvme_cpl))) 204 205 /* Default timeout for fabrics connect commands. */ 206 #ifdef DEBUG 207 #define NVME_FABRIC_CONNECT_COMMAND_TIMEOUT 0 208 #else 209 /* 500 millisecond timeout. */ 210 #define NVME_FABRIC_CONNECT_COMMAND_TIMEOUT 500000 211 #endif 212 213 /* This value indicates that a read from a PCIe register is invalid. This can happen when a device is no longer present */ 214 #define SPDK_NVME_INVALID_REGISTER_VALUE 0xFFFFFFFFu 215 216 enum nvme_payload_type { 217 NVME_PAYLOAD_TYPE_INVALID = 0, 218 219 /** nvme_request::u.payload.contig_buffer is valid for this request */ 220 NVME_PAYLOAD_TYPE_CONTIG, 221 222 /** nvme_request::u.sgl is valid for this request */ 223 NVME_PAYLOAD_TYPE_SGL, 224 }; 225 226 /** Boot partition write states */ 227 enum nvme_bp_write_state { 228 SPDK_NVME_BP_WS_DOWNLOADING = 0x0, 229 SPDK_NVME_BP_WS_DOWNLOADED = 0x1, 230 SPDK_NVME_BP_WS_REPLACE = 0x2, 231 SPDK_NVME_BP_WS_ACTIVATE = 0x3, 232 }; 233 234 /** 235 * Descriptor for a request data payload. 236 */ 237 struct nvme_payload { 238 /** 239 * Functions for retrieving physical addresses for scattered payloads. 240 */ 241 spdk_nvme_req_reset_sgl_cb reset_sgl_fn; 242 spdk_nvme_req_next_sge_cb next_sge_fn; 243 244 /** 245 * Extended IO options passed by the user 246 */ 247 struct spdk_nvme_ns_cmd_ext_io_opts *opts; 248 /** 249 * If reset_sgl_fn == NULL, this is a contig payload, and contig_or_cb_arg contains the 250 * virtual memory address of a single virtually contiguous buffer. 251 * 252 * If reset_sgl_fn != NULL, this is a SGL payload, and contig_or_cb_arg contains the 253 * cb_arg that will be passed to the SGL callback functions. 254 */ 255 void *contig_or_cb_arg; 256 257 /** Virtual memory address of a single virtually contiguous metadata buffer */ 258 void *md; 259 }; 260 261 #define NVME_PAYLOAD_CONTIG(contig_, md_) \ 262 (struct nvme_payload) { \ 263 .reset_sgl_fn = NULL, \ 264 .next_sge_fn = NULL, \ 265 .contig_or_cb_arg = (contig_), \ 266 .md = (md_), \ 267 } 268 269 #define NVME_PAYLOAD_SGL(reset_sgl_fn_, next_sge_fn_, cb_arg_, md_) \ 270 (struct nvme_payload) { \ 271 .reset_sgl_fn = (reset_sgl_fn_), \ 272 .next_sge_fn = (next_sge_fn_), \ 273 .contig_or_cb_arg = (cb_arg_), \ 274 .md = (md_), \ 275 } 276 277 static inline enum nvme_payload_type 278 nvme_payload_type(const struct nvme_payload *payload) { 279 return payload->reset_sgl_fn ? NVME_PAYLOAD_TYPE_SGL : NVME_PAYLOAD_TYPE_CONTIG; 280 } 281 282 struct nvme_error_cmd { 283 bool do_not_submit; 284 uint64_t timeout_tsc; 285 uint32_t err_count; 286 uint8_t opc; 287 struct spdk_nvme_status status; 288 TAILQ_ENTRY(nvme_error_cmd) link; 289 }; 290 291 struct nvme_request { 292 struct spdk_nvme_cmd cmd; 293 294 uint8_t retries; 295 296 uint8_t timed_out : 1; 297 298 /** 299 * True if the request is in the queued_req list. 300 */ 301 uint8_t queued : 1; 302 uint8_t reserved : 6; 303 304 /** 305 * Number of children requests still outstanding for this 306 * request which was split into multiple child requests. 307 */ 308 uint16_t num_children; 309 310 /** 311 * Offset in bytes from the beginning of payload for this request. 312 * This is used for I/O commands that are split into multiple requests. 313 */ 314 uint32_t payload_offset; 315 uint32_t md_offset; 316 317 uint32_t payload_size; 318 319 /** 320 * Timeout ticks for error injection requests, can be extended in future 321 * to support per-request timeout feature. 322 */ 323 uint64_t timeout_tsc; 324 325 /** 326 * Data payload for this request's command. 327 */ 328 struct nvme_payload payload; 329 330 spdk_nvme_cmd_cb cb_fn; 331 void *cb_arg; 332 STAILQ_ENTRY(nvme_request) stailq; 333 334 struct spdk_nvme_qpair *qpair; 335 336 /* 337 * The value of spdk_get_ticks() when the request was submitted to the hardware. 338 * Only set if ctrlr->timeout_enabled is true. 339 */ 340 uint64_t submit_tick; 341 342 /** 343 * The active admin request can be moved to a per process pending 344 * list based on the saved pid to tell which process it belongs 345 * to. The cpl saves the original completion information which 346 * is used in the completion callback. 347 * NOTE: these below two fields are only used for admin request. 348 */ 349 pid_t pid; 350 struct spdk_nvme_cpl cpl; 351 352 uint32_t md_size; 353 354 /** 355 * The following members should not be reordered with members 356 * above. These members are only needed when splitting 357 * requests which is done rarely, and the driver is careful 358 * to not touch the following fields until a split operation is 359 * needed, to avoid touching an extra cacheline. 360 */ 361 362 /** 363 * Points to the outstanding child requests for a parent request. 364 * Only valid if a request was split into multiple children 365 * requests, and is not initialized for non-split requests. 366 */ 367 TAILQ_HEAD(, nvme_request) children; 368 369 /** 370 * Linked-list pointers for a child request in its parent's list. 371 */ 372 TAILQ_ENTRY(nvme_request) child_tailq; 373 374 /** 375 * Points to a parent request if part of a split request, 376 * NULL otherwise. 377 */ 378 struct nvme_request *parent; 379 380 /** 381 * Completion status for a parent request. Initialized to all 0's 382 * (SUCCESS) before child requests are submitted. If a child 383 * request completes with error, the error status is copied here, 384 * to ensure that the parent request is also completed with error 385 * status once all child requests are completed. 386 */ 387 struct spdk_nvme_cpl parent_status; 388 389 /** 390 * The user_cb_fn and user_cb_arg fields are used for holding the original 391 * callback data when using nvme_allocate_request_user_copy. 392 */ 393 spdk_nvme_cmd_cb user_cb_fn; 394 void *user_cb_arg; 395 void *user_buffer; 396 }; 397 398 struct nvme_completion_poll_status { 399 struct spdk_nvme_cpl cpl; 400 uint64_t timeout_tsc; 401 /** 402 * DMA buffer retained throughout the duration of the command. It'll be released 403 * automatically if the command times out, otherwise the user is responsible for freeing it. 404 */ 405 void *dma_data; 406 bool done; 407 /* This flag indicates that the request has been timed out and the memory 408 must be freed in a completion callback */ 409 bool timed_out; 410 }; 411 412 struct nvme_async_event_request { 413 struct spdk_nvme_ctrlr *ctrlr; 414 struct nvme_request *req; 415 struct spdk_nvme_cpl cpl; 416 }; 417 418 enum nvme_qpair_state { 419 NVME_QPAIR_DISCONNECTED, 420 NVME_QPAIR_DISCONNECTING, 421 NVME_QPAIR_CONNECTING, 422 NVME_QPAIR_CONNECTED, 423 NVME_QPAIR_ENABLING, 424 NVME_QPAIR_ENABLED, 425 NVME_QPAIR_DESTROYING, 426 }; 427 428 struct spdk_nvme_qpair { 429 struct spdk_nvme_ctrlr *ctrlr; 430 431 uint16_t id; 432 433 uint8_t qprio; 434 435 uint8_t state : 3; 436 437 uint8_t async: 1; 438 439 uint8_t is_new_qpair: 1; 440 441 /* 442 * Members for handling IO qpair deletion inside of a completion context. 443 * These are specifically defined as single bits, so that they do not 444 * push this data structure out to another cacheline. 445 */ 446 uint8_t in_completion_context : 1; 447 uint8_t delete_after_completion_context: 1; 448 449 /* 450 * Set when no deletion notification is needed. For example, the process 451 * which allocated this qpair exited unexpectedly. 452 */ 453 uint8_t no_deletion_notification_needed: 1; 454 455 uint8_t last_fuse: 2; 456 457 uint8_t transport_failure_reason: 2; 458 uint8_t last_transport_failure_reason: 2; 459 460 enum spdk_nvme_transport_type trtype; 461 462 /* request object used only for this qpair's FABRICS/CONNECT command (if needed) */ 463 struct nvme_request *reserved_req; 464 465 STAILQ_HEAD(, nvme_request) free_req; 466 STAILQ_HEAD(, nvme_request) queued_req; 467 468 /* List entry for spdk_nvme_transport_poll_group::qpairs */ 469 STAILQ_ENTRY(spdk_nvme_qpair) poll_group_stailq; 470 471 /** Commands opcode in this list will return error */ 472 TAILQ_HEAD(, nvme_error_cmd) err_cmd_head; 473 /** Requests in this list will return error */ 474 STAILQ_HEAD(, nvme_request) err_req_head; 475 476 struct spdk_nvme_ctrlr_process *active_proc; 477 478 struct spdk_nvme_transport_poll_group *poll_group; 479 480 void *poll_group_tailq_head; 481 482 const struct spdk_nvme_transport *transport; 483 484 /* Entries below here are not touched in the main I/O path. */ 485 486 struct nvme_completion_poll_status *poll_status; 487 488 /* List entry for spdk_nvme_ctrlr::active_io_qpairs */ 489 TAILQ_ENTRY(spdk_nvme_qpair) tailq; 490 491 /* List entry for spdk_nvme_ctrlr_process::allocated_io_qpairs */ 492 TAILQ_ENTRY(spdk_nvme_qpair) per_process_tailq; 493 494 STAILQ_HEAD(, nvme_request) aborting_queued_req; 495 496 void *req_buf; 497 }; 498 499 struct spdk_nvme_poll_group { 500 void *ctx; 501 struct spdk_nvme_accel_fn_table accel_fn_table; 502 STAILQ_HEAD(, spdk_nvme_transport_poll_group) tgroups; 503 }; 504 505 struct spdk_nvme_transport_poll_group { 506 struct spdk_nvme_poll_group *group; 507 const struct spdk_nvme_transport *transport; 508 STAILQ_HEAD(, spdk_nvme_qpair) connected_qpairs; 509 STAILQ_HEAD(, spdk_nvme_qpair) disconnected_qpairs; 510 STAILQ_ENTRY(spdk_nvme_transport_poll_group) link; 511 }; 512 513 struct spdk_nvme_ns { 514 struct spdk_nvme_ctrlr *ctrlr; 515 uint32_t sector_size; 516 517 /* 518 * Size of data transferred as part of each block, 519 * including metadata if FLBAS indicates the metadata is transferred 520 * as part of the data buffer at the end of each LBA. 521 */ 522 uint32_t extended_lba_size; 523 524 uint32_t md_size; 525 uint32_t pi_type; 526 uint32_t sectors_per_max_io; 527 uint32_t sectors_per_max_io_no_md; 528 uint32_t sectors_per_stripe; 529 uint32_t id; 530 uint16_t flags; 531 bool active; 532 533 /* Command Set Identifier */ 534 enum spdk_nvme_csi csi; 535 536 /* Namespace Identification Descriptor List (CNS = 03h) */ 537 uint8_t id_desc_list[4096]; 538 539 uint32_t ana_group_id; 540 enum spdk_nvme_ana_state ana_state; 541 542 /* Identify Namespace data. */ 543 struct spdk_nvme_ns_data nsdata; 544 545 /* Zoned Namespace Command Set Specific Identify Namespace data. */ 546 struct spdk_nvme_zns_ns_data *nsdata_zns; 547 548 RB_ENTRY(spdk_nvme_ns) node; 549 }; 550 551 /** 552 * State of struct spdk_nvme_ctrlr (in particular, during initialization). 553 */ 554 enum nvme_ctrlr_state { 555 /** 556 * Wait before initializing the controller. 557 */ 558 NVME_CTRLR_STATE_INIT_DELAY, 559 560 /** 561 * Connect the admin queue. 562 */ 563 NVME_CTRLR_STATE_CONNECT_ADMINQ, 564 565 /** 566 * Controller has not started initialized yet. 567 */ 568 NVME_CTRLR_STATE_INIT = NVME_CTRLR_STATE_CONNECT_ADMINQ, 569 570 /** 571 * Waiting for admin queue to connect. 572 */ 573 NVME_CTRLR_STATE_WAIT_FOR_CONNECT_ADMINQ, 574 575 /** 576 * Read Version (VS) register. 577 */ 578 NVME_CTRLR_STATE_READ_VS, 579 580 /** 581 * Waiting for Version (VS) register to be read. 582 */ 583 NVME_CTRLR_STATE_READ_VS_WAIT_FOR_VS, 584 585 /** 586 * Read Capabilities (CAP) register. 587 */ 588 NVME_CTRLR_STATE_READ_CAP, 589 590 /** 591 * Waiting for Capabilities (CAP) register to be read. 592 */ 593 NVME_CTRLR_STATE_READ_CAP_WAIT_FOR_CAP, 594 595 /** 596 * Check EN to prepare for controller initialization. 597 */ 598 NVME_CTRLR_STATE_CHECK_EN, 599 600 /** 601 * Waiting for CC to be read as part of EN check. 602 */ 603 NVME_CTRLR_STATE_CHECK_EN_WAIT_FOR_CC, 604 605 /** 606 * Waiting for CSTS.RDY to transition from 0 to 1 so that CC.EN may be set to 0. 607 */ 608 NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1, 609 610 /** 611 * Waiting for CSTS register to be read as part of waiting for CSTS.RDY = 1. 612 */ 613 NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS, 614 615 /** 616 * Disabling the controller by setting CC.EN to 0. 617 */ 618 NVME_CTRLR_STATE_SET_EN_0, 619 620 /** 621 * Waiting for the CC register to be read as part of disabling the controller. 622 */ 623 NVME_CTRLR_STATE_SET_EN_0_WAIT_FOR_CC, 624 625 /** 626 * Waiting for CSTS.RDY to transition from 1 to 0 so that CC.EN may be set to 1. 627 */ 628 NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0, 629 630 /** 631 * Waiting for CSTS register to be read as part of waiting for CSTS.RDY = 0. 632 */ 633 NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0_WAIT_FOR_CSTS, 634 635 /** 636 * Enable the controller by writing CC.EN to 1 637 */ 638 NVME_CTRLR_STATE_ENABLE, 639 640 /** 641 * Waiting for CC register to be written as part of enabling the controller. 642 */ 643 NVME_CTRLR_STATE_ENABLE_WAIT_FOR_CC, 644 645 /** 646 * Waiting for CSTS.RDY to transition from 0 to 1 after enabling the controller. 647 */ 648 NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1, 649 650 /** 651 * Waiting for CSTS register to be read as part of waiting for CSTS.RDY = 1. 652 */ 653 NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS, 654 655 /** 656 * Reset the Admin queue of the controller. 657 */ 658 NVME_CTRLR_STATE_RESET_ADMIN_QUEUE, 659 660 /** 661 * Identify Controller command will be sent to then controller. 662 */ 663 NVME_CTRLR_STATE_IDENTIFY, 664 665 /** 666 * Waiting for Identify Controller command be completed. 667 */ 668 NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY, 669 670 /** 671 * Configure AER of the controller. 672 */ 673 NVME_CTRLR_STATE_CONFIGURE_AER, 674 675 /** 676 * Waiting for the Configure AER to be completed. 677 */ 678 NVME_CTRLR_STATE_WAIT_FOR_CONFIGURE_AER, 679 680 /** 681 * Set Keep Alive Timeout of the controller. 682 */ 683 NVME_CTRLR_STATE_SET_KEEP_ALIVE_TIMEOUT, 684 685 /** 686 * Waiting for Set Keep Alive Timeout to be completed. 687 */ 688 NVME_CTRLR_STATE_WAIT_FOR_KEEP_ALIVE_TIMEOUT, 689 690 /** 691 * Get Identify I/O Command Set Specific Controller data structure. 692 */ 693 NVME_CTRLR_STATE_IDENTIFY_IOCS_SPECIFIC, 694 695 /** 696 * Waiting for Identify I/O Command Set Specific Controller command to be completed. 697 */ 698 NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_IOCS_SPECIFIC, 699 700 /** 701 * Get Commands Supported and Effects log page for the Zoned Namespace Command Set. 702 */ 703 NVME_CTRLR_STATE_GET_ZNS_CMD_EFFECTS_LOG, 704 705 /** 706 * Waiting for the Get Log Page command to be completed. 707 */ 708 NVME_CTRLR_STATE_WAIT_FOR_GET_ZNS_CMD_EFFECTS_LOG, 709 710 /** 711 * Set Number of Queues of the controller. 712 */ 713 NVME_CTRLR_STATE_SET_NUM_QUEUES, 714 715 /** 716 * Waiting for Set Num of Queues command to be completed. 717 */ 718 NVME_CTRLR_STATE_WAIT_FOR_SET_NUM_QUEUES, 719 720 /** 721 * Get active Namespace list of the controller. 722 */ 723 NVME_CTRLR_STATE_IDENTIFY_ACTIVE_NS, 724 725 /** 726 * Waiting for the Identify Active Namespace commands to be completed. 727 */ 728 NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ACTIVE_NS, 729 730 /** 731 * Get Identify Namespace Data structure for each NS. 732 */ 733 NVME_CTRLR_STATE_IDENTIFY_NS, 734 735 /** 736 * Waiting for the Identify Namespace commands to be completed. 737 */ 738 NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS, 739 740 /** 741 * Get Identify Namespace Identification Descriptors. 742 */ 743 NVME_CTRLR_STATE_IDENTIFY_ID_DESCS, 744 745 /** 746 * Get Identify I/O Command Set Specific Namespace data structure for each NS. 747 */ 748 NVME_CTRLR_STATE_IDENTIFY_NS_IOCS_SPECIFIC, 749 750 /** 751 * Waiting for the Identify I/O Command Set Specific Namespace commands to be completed. 752 */ 753 NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS_IOCS_SPECIFIC, 754 755 /** 756 * Waiting for the Identify Namespace Identification 757 * Descriptors to be completed. 758 */ 759 NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ID_DESCS, 760 761 /** 762 * Set supported log pages of the controller. 763 */ 764 NVME_CTRLR_STATE_SET_SUPPORTED_LOG_PAGES, 765 766 /** 767 * Set supported log pages of INTEL controller. 768 */ 769 NVME_CTRLR_STATE_SET_SUPPORTED_INTEL_LOG_PAGES, 770 771 /** 772 * Waiting for supported log pages of INTEL controller. 773 */ 774 NVME_CTRLR_STATE_WAIT_FOR_SUPPORTED_INTEL_LOG_PAGES, 775 776 /** 777 * Set supported features of the controller. 778 */ 779 NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES, 780 781 /** 782 * Set Doorbell Buffer Config of the controller. 783 */ 784 NVME_CTRLR_STATE_SET_DB_BUF_CFG, 785 786 /** 787 * Waiting for Doorbell Buffer Config to be completed. 788 */ 789 NVME_CTRLR_STATE_WAIT_FOR_DB_BUF_CFG, 790 791 /** 792 * Set Host ID of the controller. 793 */ 794 NVME_CTRLR_STATE_SET_HOST_ID, 795 796 /** 797 * Waiting for Set Host ID to be completed. 798 */ 799 NVME_CTRLR_STATE_WAIT_FOR_HOST_ID, 800 801 /** 802 * Controller initialization has completed and the controller is ready. 803 */ 804 NVME_CTRLR_STATE_READY, 805 806 /** 807 * Controller initialization has an error. 808 */ 809 NVME_CTRLR_STATE_ERROR 810 }; 811 812 #define NVME_TIMEOUT_INFINITE 0 813 #define NVME_TIMEOUT_KEEP_EXISTING UINT64_MAX 814 815 struct spdk_nvme_ctrlr_aer_completion_list { 816 struct spdk_nvme_cpl cpl; 817 STAILQ_ENTRY(spdk_nvme_ctrlr_aer_completion_list) link; 818 }; 819 820 /* 821 * Used to track properties for all processes accessing the controller. 822 */ 823 struct spdk_nvme_ctrlr_process { 824 /** Whether it is the primary process */ 825 bool is_primary; 826 827 /** Process ID */ 828 pid_t pid; 829 830 /** Active admin requests to be completed */ 831 STAILQ_HEAD(, nvme_request) active_reqs; 832 833 TAILQ_ENTRY(spdk_nvme_ctrlr_process) tailq; 834 835 /** Per process PCI device handle */ 836 struct spdk_pci_device *devhandle; 837 838 /** Reference to track the number of attachment to this controller. */ 839 int ref; 840 841 /** Allocated IO qpairs */ 842 TAILQ_HEAD(, spdk_nvme_qpair) allocated_io_qpairs; 843 844 spdk_nvme_aer_cb aer_cb_fn; 845 void *aer_cb_arg; 846 847 /** 848 * A function pointer to timeout callback function 849 */ 850 spdk_nvme_timeout_cb timeout_cb_fn; 851 void *timeout_cb_arg; 852 /** separate timeout values for io vs. admin reqs */ 853 uint64_t timeout_io_ticks; 854 uint64_t timeout_admin_ticks; 855 856 /** List to publish AENs to all procs in multiprocess setup */ 857 STAILQ_HEAD(, spdk_nvme_ctrlr_aer_completion_list) async_events; 858 }; 859 860 struct nvme_register_completion { 861 struct spdk_nvme_cpl cpl; 862 uint64_t value; 863 spdk_nvme_reg_cb cb_fn; 864 void *cb_ctx; 865 STAILQ_ENTRY(nvme_register_completion) stailq; 866 }; 867 868 /* 869 * One of these per allocated PCI device. 870 */ 871 struct spdk_nvme_ctrlr { 872 /* Hot data (accessed in I/O path) starts here. */ 873 874 /* Tree of namespaces */ 875 RB_HEAD(nvme_ns_tree, spdk_nvme_ns) ns; 876 877 /* The number of active namespaces */ 878 uint32_t active_ns_count; 879 880 bool is_removed; 881 882 bool is_resetting; 883 884 bool is_failed; 885 886 bool is_destructed; 887 888 bool timeout_enabled; 889 890 /* The application is preparing to reset the controller. Transports 891 * can use this to skip unnecessary parts of the qpair deletion process 892 * for example, like the DELETE_SQ/CQ commands. 893 */ 894 bool prepare_for_reset; 895 896 bool is_disconnecting; 897 898 uint16_t max_sges; 899 900 uint16_t cntlid; 901 902 /** Controller support flags */ 903 uint64_t flags; 904 905 /** NVMEoF in-capsule data size in bytes */ 906 uint32_t ioccsz_bytes; 907 908 /** NVMEoF in-capsule data offset in 16 byte units */ 909 uint16_t icdoff; 910 911 /* Cold data (not accessed in normal I/O path) is after this point. */ 912 913 struct spdk_nvme_transport_id trid; 914 915 union spdk_nvme_cap_register cap; 916 union spdk_nvme_vs_register vs; 917 918 int state; 919 uint64_t state_timeout_tsc; 920 921 uint64_t next_keep_alive_tick; 922 uint64_t keep_alive_interval_ticks; 923 924 TAILQ_ENTRY(spdk_nvme_ctrlr) tailq; 925 926 /** All the log pages supported */ 927 bool log_page_supported[256]; 928 929 /** All the features supported */ 930 bool feature_supported[256]; 931 932 /** maximum i/o size in bytes */ 933 uint32_t max_xfer_size; 934 935 /** minimum page size supported by this controller in bytes */ 936 uint32_t min_page_size; 937 938 /** selected memory page size for this controller in bytes */ 939 uint32_t page_size; 940 941 uint32_t num_aers; 942 struct nvme_async_event_request aer[NVME_MAX_ASYNC_EVENTS]; 943 944 /** guards access to the controller itself, including admin queues */ 945 pthread_mutex_t ctrlr_lock; 946 947 struct spdk_nvme_qpair *adminq; 948 949 /** shadow doorbell buffer */ 950 uint32_t *shadow_doorbell; 951 /** eventidx buffer */ 952 uint32_t *eventidx; 953 954 /** 955 * Identify Controller data. 956 */ 957 struct spdk_nvme_ctrlr_data cdata; 958 959 /** 960 * Zoned Namespace Command Set Specific Identify Controller data. 961 */ 962 struct spdk_nvme_zns_ctrlr_data *cdata_zns; 963 964 struct spdk_bit_array *free_io_qids; 965 TAILQ_HEAD(, spdk_nvme_qpair) active_io_qpairs; 966 967 struct spdk_nvme_ctrlr_opts opts; 968 969 uint64_t quirks; 970 971 /* Extra sleep time during controller initialization */ 972 uint64_t sleep_timeout_tsc; 973 974 /** Track all the processes manage this controller */ 975 TAILQ_HEAD(, spdk_nvme_ctrlr_process) active_procs; 976 977 978 STAILQ_HEAD(, nvme_request) queued_aborts; 979 uint32_t outstanding_aborts; 980 981 /* CB to notify the user when the ctrlr is removed/failed. */ 982 spdk_nvme_remove_cb remove_cb; 983 void *cb_ctx; 984 985 struct spdk_nvme_qpair *external_io_msgs_qpair; 986 pthread_mutex_t external_io_msgs_lock; 987 struct spdk_ring *external_io_msgs; 988 989 STAILQ_HEAD(, nvme_io_msg_producer) io_producers; 990 991 struct spdk_nvme_ana_page *ana_log_page; 992 struct spdk_nvme_ana_group_descriptor *copied_ana_desc; 993 uint32_t ana_log_page_size; 994 995 /* scratchpad pointer that can be used to send data between two NVME_CTRLR_STATEs */ 996 void *tmp_ptr; 997 998 /* maximum zone append size in bytes */ 999 uint32_t max_zone_append_size; 1000 1001 /* PMR size in bytes */ 1002 uint64_t pmr_size; 1003 1004 /* Boot Partition Info */ 1005 enum nvme_bp_write_state bp_ws; 1006 uint32_t bpid; 1007 spdk_nvme_cmd_cb bp_write_cb_fn; 1008 void *bp_write_cb_arg; 1009 1010 /* Firmware Download */ 1011 void *fw_payload; 1012 unsigned int fw_size_remaining; 1013 unsigned int fw_offset; 1014 unsigned int fw_transfer_size; 1015 1016 /* Completed register operations */ 1017 STAILQ_HEAD(, nvme_register_completion) register_operations; 1018 1019 union spdk_nvme_cc_register process_init_cc; 1020 }; 1021 1022 struct spdk_nvme_probe_ctx { 1023 struct spdk_nvme_transport_id trid; 1024 void *cb_ctx; 1025 spdk_nvme_probe_cb probe_cb; 1026 spdk_nvme_attach_cb attach_cb; 1027 spdk_nvme_remove_cb remove_cb; 1028 TAILQ_HEAD(, spdk_nvme_ctrlr) init_ctrlrs; 1029 }; 1030 1031 typedef void (*nvme_ctrlr_detach_cb)(struct spdk_nvme_ctrlr *ctrlr); 1032 1033 enum nvme_ctrlr_detach_state { 1034 NVME_CTRLR_DETACH_SET_CC, 1035 NVME_CTRLR_DETACH_CHECK_CSTS, 1036 NVME_CTRLR_DETACH_GET_CSTS, 1037 NVME_CTRLR_DETACH_GET_CSTS_DONE, 1038 }; 1039 1040 struct nvme_ctrlr_detach_ctx { 1041 struct spdk_nvme_ctrlr *ctrlr; 1042 nvme_ctrlr_detach_cb cb_fn; 1043 uint64_t shutdown_start_tsc; 1044 uint32_t shutdown_timeout_ms; 1045 bool shutdown_complete; 1046 enum nvme_ctrlr_detach_state state; 1047 union spdk_nvme_csts_register csts; 1048 TAILQ_ENTRY(nvme_ctrlr_detach_ctx) link; 1049 }; 1050 1051 struct spdk_nvme_detach_ctx { 1052 TAILQ_HEAD(, nvme_ctrlr_detach_ctx) head; 1053 }; 1054 1055 struct nvme_driver { 1056 pthread_mutex_t lock; 1057 1058 /** Multi-process shared attached controller list */ 1059 TAILQ_HEAD(, spdk_nvme_ctrlr) shared_attached_ctrlrs; 1060 1061 bool initialized; 1062 struct spdk_uuid default_extended_host_id; 1063 1064 /** netlink socket fd for hotplug messages */ 1065 int hotplug_fd; 1066 }; 1067 1068 extern struct nvme_driver *g_spdk_nvme_driver; 1069 1070 int nvme_driver_init(void); 1071 1072 #define nvme_delay usleep 1073 1074 static inline bool 1075 nvme_qpair_is_admin_queue(struct spdk_nvme_qpair *qpair) 1076 { 1077 return qpair->id == 0; 1078 } 1079 1080 static inline bool 1081 nvme_qpair_is_io_queue(struct spdk_nvme_qpair *qpair) 1082 { 1083 return qpair->id != 0; 1084 } 1085 1086 static inline int 1087 nvme_robust_mutex_lock(pthread_mutex_t *mtx) 1088 { 1089 int rc = pthread_mutex_lock(mtx); 1090 1091 #ifndef __FreeBSD__ 1092 if (rc == EOWNERDEAD) { 1093 rc = pthread_mutex_consistent(mtx); 1094 } 1095 #endif 1096 1097 return rc; 1098 } 1099 1100 static inline int 1101 nvme_robust_mutex_unlock(pthread_mutex_t *mtx) 1102 { 1103 return pthread_mutex_unlock(mtx); 1104 } 1105 1106 /* Poll group management functions. */ 1107 int nvme_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair); 1108 int nvme_poll_group_disconnect_qpair(struct spdk_nvme_qpair *qpair); 1109 1110 /* Admin functions */ 1111 int nvme_ctrlr_cmd_identify(struct spdk_nvme_ctrlr *ctrlr, 1112 uint8_t cns, uint16_t cntid, uint32_t nsid, 1113 uint8_t csi, void *payload, size_t payload_size, 1114 spdk_nvme_cmd_cb cb_fn, void *cb_arg); 1115 int nvme_ctrlr_cmd_set_num_queues(struct spdk_nvme_ctrlr *ctrlr, 1116 uint32_t num_queues, spdk_nvme_cmd_cb cb_fn, 1117 void *cb_arg); 1118 int nvme_ctrlr_cmd_get_num_queues(struct spdk_nvme_ctrlr *ctrlr, 1119 spdk_nvme_cmd_cb cb_fn, void *cb_arg); 1120 int nvme_ctrlr_cmd_set_async_event_config(struct spdk_nvme_ctrlr *ctrlr, 1121 union spdk_nvme_feat_async_event_configuration config, 1122 spdk_nvme_cmd_cb cb_fn, void *cb_arg); 1123 int nvme_ctrlr_cmd_set_host_id(struct spdk_nvme_ctrlr *ctrlr, void *host_id, uint32_t host_id_size, 1124 spdk_nvme_cmd_cb cb_fn, void *cb_arg); 1125 int nvme_ctrlr_cmd_attach_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, 1126 struct spdk_nvme_ctrlr_list *payload, spdk_nvme_cmd_cb cb_fn, void *cb_arg); 1127 int nvme_ctrlr_cmd_detach_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, 1128 struct spdk_nvme_ctrlr_list *payload, spdk_nvme_cmd_cb cb_fn, void *cb_arg); 1129 int nvme_ctrlr_cmd_create_ns(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns_data *payload, 1130 spdk_nvme_cmd_cb cb_fn, void *cb_arg); 1131 int nvme_ctrlr_cmd_doorbell_buffer_config(struct spdk_nvme_ctrlr *ctrlr, 1132 uint64_t prp1, uint64_t prp2, 1133 spdk_nvme_cmd_cb cb_fn, void *cb_arg); 1134 int nvme_ctrlr_cmd_delete_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, spdk_nvme_cmd_cb cb_fn, 1135 void *cb_arg); 1136 int nvme_ctrlr_cmd_format(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, 1137 struct spdk_nvme_format *format, spdk_nvme_cmd_cb cb_fn, void *cb_arg); 1138 int nvme_ctrlr_cmd_fw_commit(struct spdk_nvme_ctrlr *ctrlr, 1139 const struct spdk_nvme_fw_commit *fw_commit, 1140 spdk_nvme_cmd_cb cb_fn, void *cb_arg); 1141 int nvme_ctrlr_cmd_fw_image_download(struct spdk_nvme_ctrlr *ctrlr, 1142 uint32_t size, uint32_t offset, void *payload, 1143 spdk_nvme_cmd_cb cb_fn, void *cb_arg); 1144 int nvme_ctrlr_cmd_sanitize(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, 1145 struct spdk_nvme_sanitize *sanitize, uint32_t cdw11, 1146 spdk_nvme_cmd_cb cb_fn, void *cb_arg); 1147 void nvme_completion_poll_cb(void *arg, const struct spdk_nvme_cpl *cpl); 1148 int nvme_wait_for_completion(struct spdk_nvme_qpair *qpair, 1149 struct nvme_completion_poll_status *status); 1150 int nvme_wait_for_completion_robust_lock(struct spdk_nvme_qpair *qpair, 1151 struct nvme_completion_poll_status *status, 1152 pthread_mutex_t *robust_mutex); 1153 int nvme_wait_for_completion_timeout(struct spdk_nvme_qpair *qpair, 1154 struct nvme_completion_poll_status *status, 1155 uint64_t timeout_in_usecs); 1156 int nvme_wait_for_completion_robust_lock_timeout(struct spdk_nvme_qpair *qpair, 1157 struct nvme_completion_poll_status *status, 1158 pthread_mutex_t *robust_mutex, 1159 uint64_t timeout_in_usecs); 1160 int nvme_wait_for_completion_robust_lock_timeout_poll(struct spdk_nvme_qpair *qpair, 1161 struct nvme_completion_poll_status *status, 1162 pthread_mutex_t *robust_mutex); 1163 1164 struct spdk_nvme_ctrlr_process *nvme_ctrlr_get_process(struct spdk_nvme_ctrlr *ctrlr, 1165 pid_t pid); 1166 struct spdk_nvme_ctrlr_process *nvme_ctrlr_get_current_process(struct spdk_nvme_ctrlr *ctrlr); 1167 int nvme_ctrlr_add_process(struct spdk_nvme_ctrlr *ctrlr, void *devhandle); 1168 void nvme_ctrlr_free_processes(struct spdk_nvme_ctrlr *ctrlr); 1169 struct spdk_pci_device *nvme_ctrlr_proc_get_devhandle(struct spdk_nvme_ctrlr *ctrlr); 1170 1171 int nvme_ctrlr_probe(const struct spdk_nvme_transport_id *trid, 1172 struct spdk_nvme_probe_ctx *probe_ctx, void *devhandle); 1173 1174 int nvme_ctrlr_construct(struct spdk_nvme_ctrlr *ctrlr); 1175 void nvme_ctrlr_destruct_finish(struct spdk_nvme_ctrlr *ctrlr); 1176 void nvme_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr); 1177 void nvme_ctrlr_destruct_async(struct spdk_nvme_ctrlr *ctrlr, 1178 struct nvme_ctrlr_detach_ctx *ctx); 1179 int nvme_ctrlr_destruct_poll_async(struct spdk_nvme_ctrlr *ctrlr, 1180 struct nvme_ctrlr_detach_ctx *ctx); 1181 void nvme_ctrlr_fail(struct spdk_nvme_ctrlr *ctrlr, bool hot_remove); 1182 int nvme_ctrlr_process_init(struct spdk_nvme_ctrlr *ctrlr); 1183 void nvme_ctrlr_connected(struct spdk_nvme_probe_ctx *probe_ctx, 1184 struct spdk_nvme_ctrlr *ctrlr); 1185 1186 int nvme_ctrlr_submit_admin_request(struct spdk_nvme_ctrlr *ctrlr, 1187 struct nvme_request *req); 1188 int nvme_ctrlr_get_cap(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_cap_register *cap); 1189 int nvme_ctrlr_get_vs(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_vs_register *vs); 1190 int nvme_ctrlr_get_cmbsz(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_cmbsz_register *cmbsz); 1191 int nvme_ctrlr_get_pmrcap(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_pmrcap_register *pmrcap); 1192 int nvme_ctrlr_get_bpinfo(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_bpinfo_register *bpinfo); 1193 int nvme_ctrlr_set_bprsel(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_bprsel_register *bprsel); 1194 int nvme_ctrlr_set_bpmbl(struct spdk_nvme_ctrlr *ctrlr, uint64_t bpmbl_value); 1195 bool nvme_ctrlr_multi_iocs_enabled(struct spdk_nvme_ctrlr *ctrlr); 1196 void nvme_ctrlr_process_async_event(struct spdk_nvme_ctrlr *ctrlr, 1197 const struct spdk_nvme_cpl *cpl); 1198 void nvme_ctrlr_disconnect_qpair(struct spdk_nvme_qpair *qpair); 1199 void nvme_ctrlr_complete_queued_async_events(struct spdk_nvme_ctrlr *ctrlr); 1200 int nvme_qpair_init(struct spdk_nvme_qpair *qpair, uint16_t id, 1201 struct spdk_nvme_ctrlr *ctrlr, 1202 enum spdk_nvme_qprio qprio, 1203 uint32_t num_requests, bool async); 1204 void nvme_qpair_deinit(struct spdk_nvme_qpair *qpair); 1205 void nvme_qpair_complete_error_reqs(struct spdk_nvme_qpair *qpair); 1206 int nvme_qpair_submit_request(struct spdk_nvme_qpair *qpair, 1207 struct nvme_request *req); 1208 void nvme_qpair_abort_all_queued_reqs(struct spdk_nvme_qpair *qpair, uint32_t dnr); 1209 uint32_t nvme_qpair_abort_queued_reqs_with_cbarg(struct spdk_nvme_qpair *qpair, void *cmd_cb_arg); 1210 void nvme_qpair_abort_queued_reqs(struct spdk_nvme_qpair *qpair, uint32_t dnr); 1211 void nvme_qpair_resubmit_requests(struct spdk_nvme_qpair *qpair, uint32_t num_requests); 1212 int nvme_ctrlr_identify_active_ns(struct spdk_nvme_ctrlr *ctrlr); 1213 void nvme_ns_set_identify_data(struct spdk_nvme_ns *ns); 1214 void nvme_ns_set_id_desc_list_data(struct spdk_nvme_ns *ns); 1215 void nvme_ns_free_zns_specific_data(struct spdk_nvme_ns *ns); 1216 void nvme_ns_free_iocs_specific_data(struct spdk_nvme_ns *ns); 1217 bool nvme_ns_has_supported_iocs_specific_data(struct spdk_nvme_ns *ns); 1218 int nvme_ns_construct(struct spdk_nvme_ns *ns, uint32_t id, 1219 struct spdk_nvme_ctrlr *ctrlr); 1220 void nvme_ns_destruct(struct spdk_nvme_ns *ns); 1221 int nvme_ns_cmd_zone_append_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, 1222 void *buffer, void *metadata, uint64_t zslba, 1223 uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, void *cb_arg, 1224 uint32_t io_flags, uint16_t apptag_mask, uint16_t apptag); 1225 int nvme_ns_cmd_zone_appendv_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, 1226 uint64_t zslba, uint32_t lba_count, 1227 spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t io_flags, 1228 spdk_nvme_req_reset_sgl_cb reset_sgl_fn, 1229 spdk_nvme_req_next_sge_cb next_sge_fn, void *metadata, 1230 uint16_t apptag_mask, uint16_t apptag); 1231 1232 int nvme_fabric_ctrlr_set_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t value); 1233 int nvme_fabric_ctrlr_set_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t value); 1234 int nvme_fabric_ctrlr_get_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t *value); 1235 int nvme_fabric_ctrlr_get_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t *value); 1236 int nvme_fabric_ctrlr_set_reg_4_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, 1237 uint32_t value, spdk_nvme_reg_cb cb_fn, void *cb_arg); 1238 int nvme_fabric_ctrlr_set_reg_8_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, 1239 uint64_t value, spdk_nvme_reg_cb cb_fn, void *cb_arg); 1240 int nvme_fabric_ctrlr_get_reg_4_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, 1241 spdk_nvme_reg_cb cb_fn, void *cb_arg); 1242 int nvme_fabric_ctrlr_get_reg_8_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, 1243 spdk_nvme_reg_cb cb_fn, void *cb_arg); 1244 int nvme_fabric_ctrlr_scan(struct spdk_nvme_probe_ctx *probe_ctx, bool direct_connect); 1245 int nvme_fabric_ctrlr_discover(struct spdk_nvme_ctrlr *ctrlr, 1246 struct spdk_nvme_probe_ctx *probe_ctx); 1247 int nvme_fabric_qpair_connect(struct spdk_nvme_qpair *qpair, uint32_t num_entries); 1248 int nvme_fabric_qpair_connect_async(struct spdk_nvme_qpair *qpair, uint32_t num_entries); 1249 int nvme_fabric_qpair_connect_poll(struct spdk_nvme_qpair *qpair); 1250 1251 typedef int (*spdk_nvme_parse_ana_log_page_cb)( 1252 const struct spdk_nvme_ana_group_descriptor *desc, void *cb_arg); 1253 int nvme_ctrlr_parse_ana_log_page(struct spdk_nvme_ctrlr *ctrlr, 1254 spdk_nvme_parse_ana_log_page_cb cb_fn, void *cb_arg); 1255 1256 #define NVME_INIT_REQUEST(req, _cb_fn, _cb_arg, _payload, _payload_size, _md_size) \ 1257 do { \ 1258 req->cb_fn = _cb_fn; \ 1259 req->cb_arg = _cb_arg; \ 1260 req->payload = _payload; \ 1261 req->payload_size = _payload_size; \ 1262 req->md_size = _md_size; \ 1263 req->pid = g_spdk_nvme_pid; \ 1264 req->submit_tick = 0; \ 1265 } while (0); 1266 1267 static inline struct nvme_request * 1268 nvme_allocate_request(struct spdk_nvme_qpair *qpair, 1269 const struct nvme_payload *payload, uint32_t payload_size, uint32_t md_size, 1270 spdk_nvme_cmd_cb cb_fn, void *cb_arg) 1271 { 1272 struct nvme_request *req; 1273 1274 req = STAILQ_FIRST(&qpair->free_req); 1275 if (req == NULL) { 1276 return req; 1277 } 1278 1279 STAILQ_REMOVE_HEAD(&qpair->free_req, stailq); 1280 1281 /* 1282 * Only memset/zero fields that need it. All other fields 1283 * will be initialized appropriately either later in this 1284 * function, or before they are needed later in the 1285 * submission patch. For example, the children 1286 * TAILQ_ENTRY and following members are 1287 * only used as part of I/O splitting so we avoid 1288 * memsetting them until it is actually needed. 1289 * They will be initialized in nvme_request_add_child() 1290 * if the request is split. 1291 */ 1292 memset(req, 0, offsetof(struct nvme_request, payload_size)); 1293 1294 NVME_INIT_REQUEST(req, cb_fn, cb_arg, *payload, payload_size, md_size); 1295 1296 return req; 1297 } 1298 1299 static inline struct nvme_request * 1300 nvme_allocate_request_contig(struct spdk_nvme_qpair *qpair, 1301 void *buffer, uint32_t payload_size, 1302 spdk_nvme_cmd_cb cb_fn, void *cb_arg) 1303 { 1304 struct nvme_payload payload; 1305 1306 payload = NVME_PAYLOAD_CONTIG(buffer, NULL); 1307 1308 return nvme_allocate_request(qpair, &payload, payload_size, 0, cb_fn, cb_arg); 1309 } 1310 1311 static inline struct nvme_request * 1312 nvme_allocate_request_null(struct spdk_nvme_qpair *qpair, spdk_nvme_cmd_cb cb_fn, void *cb_arg) 1313 { 1314 return nvme_allocate_request_contig(qpair, NULL, 0, cb_fn, cb_arg); 1315 } 1316 1317 struct nvme_request *nvme_allocate_request_user_copy(struct spdk_nvme_qpair *qpair, 1318 void *buffer, uint32_t payload_size, 1319 spdk_nvme_cmd_cb cb_fn, void *cb_arg, bool host_to_controller); 1320 1321 static inline void 1322 nvme_complete_request(spdk_nvme_cmd_cb cb_fn, void *cb_arg, struct spdk_nvme_qpair *qpair, 1323 struct nvme_request *req, struct spdk_nvme_cpl *cpl) 1324 { 1325 struct spdk_nvme_cpl err_cpl; 1326 struct nvme_error_cmd *cmd; 1327 1328 /* error injection at completion path, 1329 * only inject for successful completed commands 1330 */ 1331 if (spdk_unlikely(!TAILQ_EMPTY(&qpair->err_cmd_head) && 1332 !spdk_nvme_cpl_is_error(cpl))) { 1333 TAILQ_FOREACH(cmd, &qpair->err_cmd_head, link) { 1334 1335 if (cmd->do_not_submit) { 1336 continue; 1337 } 1338 1339 if ((cmd->opc == req->cmd.opc) && cmd->err_count) { 1340 1341 err_cpl = *cpl; 1342 err_cpl.status.sct = cmd->status.sct; 1343 err_cpl.status.sc = cmd->status.sc; 1344 1345 cpl = &err_cpl; 1346 cmd->err_count--; 1347 break; 1348 } 1349 } 1350 } 1351 1352 if (cb_fn) { 1353 cb_fn(cb_arg, cpl); 1354 } 1355 } 1356 1357 static inline void 1358 nvme_free_request(struct nvme_request *req) 1359 { 1360 assert(req != NULL); 1361 assert(req->num_children == 0); 1362 assert(req->qpair != NULL); 1363 1364 /* The reserved_req does not go in the free_req STAILQ - it is 1365 * saved only for use with a FABRICS/CONNECT command. 1366 */ 1367 if (spdk_likely(req->qpair->reserved_req != req)) { 1368 STAILQ_INSERT_HEAD(&req->qpair->free_req, req, stailq); 1369 } 1370 } 1371 1372 static inline void 1373 nvme_qpair_set_state(struct spdk_nvme_qpair *qpair, enum nvme_qpair_state state) 1374 { 1375 qpair->state = state; 1376 if (state == NVME_QPAIR_ENABLED) { 1377 qpair->is_new_qpair = false; 1378 } 1379 } 1380 1381 static inline enum nvme_qpair_state 1382 nvme_qpair_get_state(struct spdk_nvme_qpair *qpair) { 1383 return qpair->state; 1384 } 1385 1386 static inline void 1387 nvme_qpair_free_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req) 1388 { 1389 assert(req != NULL); 1390 assert(req->num_children == 0); 1391 1392 STAILQ_INSERT_HEAD(&qpair->free_req, req, stailq); 1393 } 1394 1395 static inline void 1396 nvme_request_remove_child(struct nvme_request *parent, struct nvme_request *child) 1397 { 1398 assert(parent != NULL); 1399 assert(child != NULL); 1400 assert(child->parent == parent); 1401 assert(parent->num_children != 0); 1402 1403 parent->num_children--; 1404 child->parent = NULL; 1405 TAILQ_REMOVE(&parent->children, child, child_tailq); 1406 } 1407 1408 static inline void 1409 nvme_cb_complete_child(void *child_arg, const struct spdk_nvme_cpl *cpl) 1410 { 1411 struct nvme_request *child = child_arg; 1412 struct nvme_request *parent = child->parent; 1413 1414 nvme_request_remove_child(parent, child); 1415 1416 if (spdk_nvme_cpl_is_error(cpl)) { 1417 memcpy(&parent->parent_status, cpl, sizeof(*cpl)); 1418 } 1419 1420 if (parent->num_children == 0) { 1421 nvme_complete_request(parent->cb_fn, parent->cb_arg, parent->qpair, 1422 parent, &parent->parent_status); 1423 nvme_free_request(parent); 1424 } 1425 } 1426 1427 static inline void 1428 nvme_request_add_child(struct nvme_request *parent, struct nvme_request *child) 1429 { 1430 assert(parent->num_children != UINT16_MAX); 1431 1432 if (parent->num_children == 0) { 1433 /* 1434 * Defer initialization of the children TAILQ since it falls 1435 * on a separate cacheline. This ensures we do not touch this 1436 * cacheline except on request splitting cases, which are 1437 * relatively rare. 1438 */ 1439 TAILQ_INIT(&parent->children); 1440 parent->parent = NULL; 1441 memset(&parent->parent_status, 0, sizeof(struct spdk_nvme_cpl)); 1442 } 1443 1444 parent->num_children++; 1445 TAILQ_INSERT_TAIL(&parent->children, child, child_tailq); 1446 child->parent = parent; 1447 child->cb_fn = nvme_cb_complete_child; 1448 child->cb_arg = child; 1449 } 1450 1451 static inline void 1452 nvme_request_free_children(struct nvme_request *req) 1453 { 1454 struct nvme_request *child, *tmp; 1455 1456 if (req->num_children == 0) { 1457 return; 1458 } 1459 1460 /* free all child nvme_request */ 1461 TAILQ_FOREACH_SAFE(child, &req->children, child_tailq, tmp) { 1462 nvme_request_remove_child(req, child); 1463 nvme_request_free_children(child); 1464 nvme_free_request(child); 1465 } 1466 } 1467 1468 int nvme_request_check_timeout(struct nvme_request *req, uint16_t cid, 1469 struct spdk_nvme_ctrlr_process *active_proc, uint64_t now_tick); 1470 uint64_t nvme_get_quirks(const struct spdk_pci_id *id); 1471 1472 int nvme_robust_mutex_init_shared(pthread_mutex_t *mtx); 1473 int nvme_robust_mutex_init_recursive_shared(pthread_mutex_t *mtx); 1474 1475 bool nvme_completion_is_retry(const struct spdk_nvme_cpl *cpl); 1476 1477 struct spdk_nvme_ctrlr *nvme_get_ctrlr_by_trid_unsafe( 1478 const struct spdk_nvme_transport_id *trid); 1479 1480 const struct spdk_nvme_transport *nvme_get_transport(const char *transport_name); 1481 const struct spdk_nvme_transport *nvme_get_first_transport(void); 1482 const struct spdk_nvme_transport *nvme_get_next_transport(const struct spdk_nvme_transport 1483 *transport); 1484 void nvme_ctrlr_update_namespaces(struct spdk_nvme_ctrlr *ctrlr); 1485 1486 /* Transport specific functions */ 1487 struct spdk_nvme_ctrlr *nvme_transport_ctrlr_construct(const struct spdk_nvme_transport_id *trid, 1488 const struct spdk_nvme_ctrlr_opts *opts, 1489 void *devhandle); 1490 int nvme_transport_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr); 1491 int nvme_transport_ctrlr_scan(struct spdk_nvme_probe_ctx *probe_ctx, bool direct_connect); 1492 int nvme_transport_ctrlr_enable(struct spdk_nvme_ctrlr *ctrlr); 1493 int nvme_transport_ctrlr_set_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t value); 1494 int nvme_transport_ctrlr_set_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t value); 1495 int nvme_transport_ctrlr_get_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t *value); 1496 int nvme_transport_ctrlr_get_reg_8(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint64_t *value); 1497 int nvme_transport_ctrlr_set_reg_4_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, 1498 uint32_t value, spdk_nvme_reg_cb cb_fn, void *cb_arg); 1499 int nvme_transport_ctrlr_set_reg_8_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, 1500 uint64_t value, spdk_nvme_reg_cb cb_fn, void *cb_arg); 1501 int nvme_transport_ctrlr_get_reg_4_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, 1502 spdk_nvme_reg_cb cb_fn, void *cb_arg); 1503 int nvme_transport_ctrlr_get_reg_8_async(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, 1504 spdk_nvme_reg_cb cb_fn, void *cb_arg); 1505 uint32_t nvme_transport_ctrlr_get_max_xfer_size(struct spdk_nvme_ctrlr *ctrlr); 1506 uint16_t nvme_transport_ctrlr_get_max_sges(struct spdk_nvme_ctrlr *ctrlr); 1507 struct spdk_nvme_qpair *nvme_transport_ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr, 1508 uint16_t qid, const struct spdk_nvme_io_qpair_opts *opts); 1509 int nvme_transport_ctrlr_reserve_cmb(struct spdk_nvme_ctrlr *ctrlr); 1510 void *nvme_transport_ctrlr_map_cmb(struct spdk_nvme_ctrlr *ctrlr, size_t *size); 1511 int nvme_transport_ctrlr_unmap_cmb(struct spdk_nvme_ctrlr *ctrlr); 1512 int nvme_transport_ctrlr_enable_pmr(struct spdk_nvme_ctrlr *ctrlr); 1513 int nvme_transport_ctrlr_disable_pmr(struct spdk_nvme_ctrlr *ctrlr); 1514 void *nvme_transport_ctrlr_map_pmr(struct spdk_nvme_ctrlr *ctrlr, size_t *size); 1515 int nvme_transport_ctrlr_unmap_pmr(struct spdk_nvme_ctrlr *ctrlr); 1516 void nvme_transport_ctrlr_delete_io_qpair(struct spdk_nvme_ctrlr *ctrlr, 1517 struct spdk_nvme_qpair *qpair); 1518 int nvme_transport_ctrlr_connect_qpair(struct spdk_nvme_ctrlr *ctrlr, 1519 struct spdk_nvme_qpair *qpair); 1520 void nvme_transport_ctrlr_disconnect_qpair(struct spdk_nvme_ctrlr *ctrlr, 1521 struct spdk_nvme_qpair *qpair); 1522 void nvme_transport_ctrlr_disconnect_qpair_done(struct spdk_nvme_qpair *qpair); 1523 int nvme_transport_ctrlr_get_memory_domains(const struct spdk_nvme_ctrlr *ctrlr, 1524 struct spdk_memory_domain **domains, int array_size); 1525 void nvme_transport_qpair_abort_reqs(struct spdk_nvme_qpair *qpair, uint32_t dnr); 1526 int nvme_transport_qpair_reset(struct spdk_nvme_qpair *qpair); 1527 int nvme_transport_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req); 1528 int32_t nvme_transport_qpair_process_completions(struct spdk_nvme_qpair *qpair, 1529 uint32_t max_completions); 1530 void nvme_transport_admin_qpair_abort_aers(struct spdk_nvme_qpair *qpair); 1531 int nvme_transport_qpair_iterate_requests(struct spdk_nvme_qpair *qpair, 1532 int (*iter_fn)(struct nvme_request *req, void *arg), 1533 void *arg); 1534 1535 struct spdk_nvme_transport_poll_group *nvme_transport_poll_group_create( 1536 const struct spdk_nvme_transport *transport); 1537 struct spdk_nvme_transport_poll_group *nvme_transport_qpair_get_optimal_poll_group( 1538 const struct spdk_nvme_transport *transport, 1539 struct spdk_nvme_qpair *qpair); 1540 int nvme_transport_poll_group_add(struct spdk_nvme_transport_poll_group *tgroup, 1541 struct spdk_nvme_qpair *qpair); 1542 int nvme_transport_poll_group_remove(struct spdk_nvme_transport_poll_group *tgroup, 1543 struct spdk_nvme_qpair *qpair); 1544 int nvme_transport_poll_group_disconnect_qpair(struct spdk_nvme_qpair *qpair); 1545 int nvme_transport_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair); 1546 int64_t nvme_transport_poll_group_process_completions(struct spdk_nvme_transport_poll_group *tgroup, 1547 uint32_t completions_per_qpair, spdk_nvme_disconnected_qpair_cb disconnected_qpair_cb); 1548 int nvme_transport_poll_group_destroy(struct spdk_nvme_transport_poll_group *tgroup); 1549 int nvme_transport_poll_group_get_stats(struct spdk_nvme_transport_poll_group *tgroup, 1550 struct spdk_nvme_transport_poll_group_stat **stats); 1551 void nvme_transport_poll_group_free_stats(struct spdk_nvme_transport_poll_group *tgroup, 1552 struct spdk_nvme_transport_poll_group_stat *stats); 1553 enum spdk_nvme_transport_type nvme_transport_get_trtype(const struct spdk_nvme_transport 1554 *transport); 1555 /* 1556 * Below ref related functions must be called with the global 1557 * driver lock held for the multi-process condition. 1558 * Within these functions, the per ctrlr ctrlr_lock is also 1559 * acquired for the multi-thread condition. 1560 */ 1561 void nvme_ctrlr_proc_get_ref(struct spdk_nvme_ctrlr *ctrlr); 1562 void nvme_ctrlr_proc_put_ref(struct spdk_nvme_ctrlr *ctrlr); 1563 int nvme_ctrlr_get_ref_count(struct spdk_nvme_ctrlr *ctrlr); 1564 1565 static inline bool 1566 _is_page_aligned(uint64_t address, uint64_t page_size) 1567 { 1568 return (address & (page_size - 1)) == 0; 1569 } 1570 1571 #endif /* __NVME_INTERNAL_H__ */ 1572