1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "nvme_internal.h" 35 36 static inline struct nvme_request *_nvme_ns_cmd_rw(struct spdk_nvme_ns *ns, 37 struct spdk_nvme_qpair *qpair, 38 const struct nvme_payload *payload, uint32_t payload_offset, uint32_t md_offset, 39 uint64_t lba, uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, 40 void *cb_arg, uint32_t opc, uint32_t io_flags, 41 uint16_t apptag_mask, uint16_t apptag, bool check_sgl); 42 43 44 static bool 45 spdk_nvme_ns_check_request_length(uint32_t lba_count, uint32_t sectors_per_max_io, 46 uint32_t sectors_per_stripe, uint32_t qdepth) 47 { 48 uint32_t child_per_io = UINT32_MAX; 49 50 /* After a namespace is destroyed(e.g. hotplug), all the fields associated with the 51 * namespace will be cleared to zero, the function will return TRUE for this case, 52 * and -EINVAL will be returned to caller. 53 */ 54 if (sectors_per_stripe > 0) { 55 child_per_io = (lba_count + sectors_per_stripe - 1) / sectors_per_stripe; 56 } else if (sectors_per_max_io > 0) { 57 child_per_io = (lba_count + sectors_per_max_io - 1) / sectors_per_max_io; 58 } 59 60 SPDK_DEBUGLOG(SPDK_LOG_NVME, "checking maximum i/o length %d\n", child_per_io); 61 62 return child_per_io >= qdepth; 63 } 64 65 static struct nvme_request * 66 _nvme_add_child_request(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, 67 const struct nvme_payload *payload, 68 uint32_t payload_offset, uint32_t md_offset, 69 uint64_t lba, uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t opc, 70 uint32_t io_flags, uint16_t apptag_mask, uint16_t apptag, 71 struct nvme_request *parent, bool check_sgl) 72 { 73 struct nvme_request *child; 74 75 child = _nvme_ns_cmd_rw(ns, qpair, payload, payload_offset, md_offset, lba, lba_count, cb_fn, 76 cb_arg, opc, io_flags, apptag_mask, apptag, check_sgl); 77 if (child == NULL) { 78 nvme_request_free_children(parent); 79 nvme_free_request(parent); 80 return NULL; 81 } 82 83 nvme_request_add_child(parent, child); 84 return child; 85 } 86 87 static struct nvme_request * 88 _nvme_ns_cmd_split_request(struct spdk_nvme_ns *ns, 89 struct spdk_nvme_qpair *qpair, 90 const struct nvme_payload *payload, 91 uint32_t payload_offset, uint32_t md_offset, 92 uint64_t lba, uint32_t lba_count, 93 spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t opc, 94 uint32_t io_flags, struct nvme_request *req, 95 uint32_t sectors_per_max_io, uint32_t sector_mask, 96 uint16_t apptag_mask, uint16_t apptag) 97 { 98 uint32_t sector_size; 99 uint32_t md_size = ns->md_size; 100 uint32_t remaining_lba_count = lba_count; 101 struct nvme_request *child; 102 103 sector_size = ns->extended_lba_size; 104 105 if ((io_flags & SPDK_NVME_IO_FLAGS_PRACT) && 106 (ns->flags & SPDK_NVME_NS_EXTENDED_LBA_SUPPORTED) && 107 (ns->flags & SPDK_NVME_NS_DPS_PI_SUPPORTED) && 108 (md_size == 8)) { 109 sector_size -= 8; 110 } 111 112 while (remaining_lba_count > 0) { 113 lba_count = sectors_per_max_io - (lba & sector_mask); 114 lba_count = spdk_min(remaining_lba_count, lba_count); 115 116 child = _nvme_add_child_request(ns, qpair, payload, payload_offset, md_offset, 117 lba, lba_count, cb_fn, cb_arg, opc, 118 io_flags, apptag_mask, apptag, req, true); 119 if (child == NULL) { 120 return NULL; 121 } 122 123 remaining_lba_count -= lba_count; 124 lba += lba_count; 125 payload_offset += lba_count * sector_size; 126 md_offset += lba_count * md_size; 127 } 128 129 return req; 130 } 131 132 static void 133 _nvme_ns_cmd_setup_request(struct spdk_nvme_ns *ns, struct nvme_request *req, 134 uint32_t opc, uint64_t lba, uint32_t lba_count, 135 uint32_t io_flags, uint16_t apptag_mask, uint16_t apptag) 136 { 137 struct spdk_nvme_cmd *cmd; 138 139 cmd = &req->cmd; 140 cmd->opc = opc; 141 cmd->nsid = ns->id; 142 143 *(uint64_t *)&cmd->cdw10 = lba; 144 145 if (ns->flags & SPDK_NVME_NS_DPS_PI_SUPPORTED) { 146 switch (ns->pi_type) { 147 case SPDK_NVME_FMT_NVM_PROTECTION_TYPE1: 148 case SPDK_NVME_FMT_NVM_PROTECTION_TYPE2: 149 cmd->cdw14 = (uint32_t)lba; 150 break; 151 } 152 } 153 154 cmd->cdw12 = lba_count - 1; 155 cmd->cdw12 |= io_flags; 156 157 cmd->cdw15 = apptag_mask; 158 cmd->cdw15 = (cmd->cdw15 << 16 | apptag); 159 } 160 161 static struct nvme_request * 162 _nvme_ns_cmd_split_request_prp(struct spdk_nvme_ns *ns, 163 struct spdk_nvme_qpair *qpair, 164 const struct nvme_payload *payload, 165 uint32_t payload_offset, uint32_t md_offset, 166 uint64_t lba, uint32_t lba_count, 167 spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t opc, 168 uint32_t io_flags, struct nvme_request *req, 169 uint16_t apptag_mask, uint16_t apptag) 170 { 171 spdk_nvme_req_reset_sgl_cb reset_sgl_fn = req->payload.reset_sgl_fn; 172 spdk_nvme_req_next_sge_cb next_sge_fn = req->payload.next_sge_fn; 173 void *sgl_cb_arg = req->payload.contig_or_cb_arg; 174 bool start_valid, end_valid, last_sge, child_equals_parent; 175 uint64_t child_lba = lba; 176 uint32_t req_current_length = 0; 177 uint32_t child_length = 0; 178 uint32_t sge_length; 179 uint32_t page_size = qpair->ctrlr->page_size; 180 uintptr_t address; 181 182 reset_sgl_fn(sgl_cb_arg, payload_offset); 183 next_sge_fn(sgl_cb_arg, (void **)&address, &sge_length); 184 while (req_current_length < req->payload_size) { 185 186 if (sge_length == 0) { 187 continue; 188 } else if (req_current_length + sge_length > req->payload_size) { 189 sge_length = req->payload_size - req_current_length; 190 } 191 192 /* 193 * The start of the SGE is invalid if the start address is not page aligned, 194 * unless it is the first SGE in the child request. 195 */ 196 start_valid = child_length == 0 || _is_page_aligned(address, page_size); 197 198 /* Boolean for whether this is the last SGE in the parent request. */ 199 last_sge = (req_current_length + sge_length == req->payload_size); 200 201 /* 202 * The end of the SGE is invalid if the end address is not page aligned, 203 * unless it is the last SGE in the parent request. 204 */ 205 end_valid = last_sge || _is_page_aligned(address + sge_length, page_size); 206 207 /* 208 * This child request equals the parent request, meaning that no splitting 209 * was required for the parent request (the one passed into this function). 210 * In this case, we do not create a child request at all - we just send 211 * the original request as a single request at the end of this function. 212 */ 213 child_equals_parent = (child_length + sge_length == req->payload_size); 214 215 if (start_valid) { 216 /* 217 * The start of the SGE is valid, so advance the length parameters, 218 * to include this SGE with previous SGEs for this child request 219 * (if any). If it is not valid, we do not advance the length 220 * parameters nor get the next SGE, because we must send what has 221 * been collected before this SGE as a child request. 222 */ 223 child_length += sge_length; 224 req_current_length += sge_length; 225 if (req_current_length < req->payload_size) { 226 next_sge_fn(sgl_cb_arg, (void **)&address, &sge_length); 227 } 228 /* 229 * If the next SGE is not page aligned, we will need to create a child 230 * request for what we have so far, and then start a new child request for 231 * the next SGE. 232 */ 233 start_valid = _is_page_aligned(address, page_size); 234 } 235 236 if (start_valid && end_valid && !last_sge) { 237 continue; 238 } 239 240 /* 241 * We need to create a split here. Send what we have accumulated so far as a child 242 * request. Checking if child_equals_parent allows us to *not* create a child request 243 * when no splitting is required - in that case we will fall-through and just create 244 * a single request with no children for the entire I/O. 245 */ 246 if (!child_equals_parent) { 247 struct nvme_request *child; 248 uint32_t child_lba_count; 249 250 if ((child_length % ns->extended_lba_size) != 0) { 251 SPDK_ERRLOG("child_length %u not even multiple of lba_size %u\n", 252 child_length, ns->extended_lba_size); 253 return NULL; 254 } 255 child_lba_count = child_length / ns->extended_lba_size; 256 /* 257 * Note the last parameter is set to "false" - this tells the recursive 258 * call to _nvme_ns_cmd_rw() to not bother with checking for SGL splitting 259 * since we have already verified it here. 260 */ 261 child = _nvme_add_child_request(ns, qpair, payload, payload_offset, md_offset, 262 child_lba, child_lba_count, 263 cb_fn, cb_arg, opc, io_flags, 264 apptag_mask, apptag, req, false); 265 if (child == NULL) { 266 return NULL; 267 } 268 payload_offset += child_length; 269 md_offset += child_lba_count * ns->md_size; 270 child_lba += child_lba_count; 271 child_length = 0; 272 } 273 } 274 275 if (child_length == req->payload_size) { 276 /* No splitting was required, so setup the whole payload as one request. */ 277 _nvme_ns_cmd_setup_request(ns, req, opc, lba, lba_count, io_flags, apptag_mask, apptag); 278 } 279 280 return req; 281 } 282 283 static struct nvme_request * 284 _nvme_ns_cmd_split_request_sgl(struct spdk_nvme_ns *ns, 285 struct spdk_nvme_qpair *qpair, 286 const struct nvme_payload *payload, 287 uint32_t payload_offset, uint32_t md_offset, 288 uint64_t lba, uint32_t lba_count, 289 spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t opc, 290 uint32_t io_flags, struct nvme_request *req, 291 uint16_t apptag_mask, uint16_t apptag) 292 { 293 spdk_nvme_req_reset_sgl_cb reset_sgl_fn = req->payload.reset_sgl_fn; 294 spdk_nvme_req_next_sge_cb next_sge_fn = req->payload.next_sge_fn; 295 void *sgl_cb_arg = req->payload.contig_or_cb_arg; 296 uint64_t child_lba = lba; 297 uint32_t req_current_length = 0; 298 uint32_t child_length = 0; 299 uint32_t sge_length; 300 uint16_t max_sges, num_sges; 301 uintptr_t address; 302 303 max_sges = ns->ctrlr->max_sges; 304 305 reset_sgl_fn(sgl_cb_arg, payload_offset); 306 num_sges = 0; 307 308 while (req_current_length < req->payload_size) { 309 next_sge_fn(sgl_cb_arg, (void **)&address, &sge_length); 310 311 if (req_current_length + sge_length > req->payload_size) { 312 sge_length = req->payload_size - req_current_length; 313 } 314 315 child_length += sge_length; 316 req_current_length += sge_length; 317 num_sges++; 318 319 if (num_sges < max_sges && req_current_length < req->payload_size) { 320 continue; 321 } 322 323 /* 324 * We need to create a split here. Send what we have accumulated so far as a child 325 * request. Checking if the child equals the full payload allows us to *not* 326 * create a child request when no splitting is required - in that case we will 327 * fall-through and just create a single request with no children for the entire I/O. 328 */ 329 if (child_length != req->payload_size) { 330 struct nvme_request *child; 331 uint32_t child_lba_count; 332 333 if ((child_length % ns->extended_lba_size) != 0) { 334 SPDK_ERRLOG("child_length %u not even multiple of lba_size %u\n", 335 child_length, ns->extended_lba_size); 336 return NULL; 337 } 338 child_lba_count = child_length / ns->extended_lba_size; 339 /* 340 * Note the last parameter is set to "false" - this tells the recursive 341 * call to _nvme_ns_cmd_rw() to not bother with checking for SGL splitting 342 * since we have already verified it here. 343 */ 344 child = _nvme_add_child_request(ns, qpair, payload, payload_offset, md_offset, 345 child_lba, child_lba_count, 346 cb_fn, cb_arg, opc, io_flags, 347 apptag_mask, apptag, req, false); 348 if (child == NULL) { 349 return NULL; 350 } 351 payload_offset += child_length; 352 md_offset += child_lba_count * ns->md_size; 353 child_lba += child_lba_count; 354 child_length = 0; 355 num_sges = 0; 356 } 357 } 358 359 if (child_length == req->payload_size) { 360 /* No splitting was required, so setup the whole payload as one request. */ 361 _nvme_ns_cmd_setup_request(ns, req, opc, lba, lba_count, io_flags, apptag_mask, apptag); 362 } 363 364 return req; 365 } 366 367 static inline struct nvme_request * 368 _nvme_ns_cmd_rw(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, 369 const struct nvme_payload *payload, uint32_t payload_offset, uint32_t md_offset, 370 uint64_t lba, uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t opc, 371 uint32_t io_flags, uint16_t apptag_mask, uint16_t apptag, bool check_sgl) 372 { 373 struct nvme_request *req; 374 uint32_t sector_size; 375 uint32_t sectors_per_max_io; 376 uint32_t sectors_per_stripe; 377 378 if (io_flags & 0xFFFF) { 379 /* The bottom 16 bits must be empty */ 380 SPDK_ERRLOG("io_flags 0x%x bottom 16 bits is not empty\n", 381 io_flags); 382 return NULL; 383 } 384 385 sector_size = ns->extended_lba_size; 386 sectors_per_max_io = ns->sectors_per_max_io; 387 sectors_per_stripe = ns->sectors_per_stripe; 388 389 if ((io_flags & SPDK_NVME_IO_FLAGS_PRACT) && 390 (ns->flags & SPDK_NVME_NS_EXTENDED_LBA_SUPPORTED) && 391 (ns->flags & SPDK_NVME_NS_DPS_PI_SUPPORTED) && 392 (ns->md_size == 8)) { 393 sector_size -= 8; 394 } 395 396 req = nvme_allocate_request(qpair, payload, lba_count * sector_size, cb_fn, cb_arg); 397 if (req == NULL) { 398 return NULL; 399 } 400 401 req->payload_offset = payload_offset; 402 req->md_offset = md_offset; 403 404 /* 405 * Intel DC P3*00 NVMe controllers benefit from driver-assisted striping. 406 * If this controller defines a stripe boundary and this I/O spans a stripe 407 * boundary, split the request into multiple requests and submit each 408 * separately to hardware. 409 */ 410 if (sectors_per_stripe > 0 && 411 (((lba & (sectors_per_stripe - 1)) + lba_count) > sectors_per_stripe)) { 412 413 return _nvme_ns_cmd_split_request(ns, qpair, payload, payload_offset, md_offset, lba, lba_count, 414 cb_fn, 415 cb_arg, opc, 416 io_flags, req, sectors_per_stripe, sectors_per_stripe - 1, apptag_mask, apptag); 417 } else if (lba_count > sectors_per_max_io) { 418 return _nvme_ns_cmd_split_request(ns, qpair, payload, payload_offset, md_offset, lba, lba_count, 419 cb_fn, 420 cb_arg, opc, 421 io_flags, req, sectors_per_max_io, 0, apptag_mask, apptag); 422 } else if (nvme_payload_type(&req->payload) == NVME_PAYLOAD_TYPE_SGL && check_sgl) { 423 if (ns->ctrlr->flags & SPDK_NVME_CTRLR_SGL_SUPPORTED) { 424 return _nvme_ns_cmd_split_request_sgl(ns, qpair, payload, payload_offset, md_offset, 425 lba, lba_count, cb_fn, cb_arg, opc, io_flags, 426 req, apptag_mask, apptag); 427 } else { 428 return _nvme_ns_cmd_split_request_prp(ns, qpair, payload, payload_offset, md_offset, 429 lba, lba_count, cb_fn, cb_arg, opc, io_flags, 430 req, apptag_mask, apptag); 431 } 432 } 433 434 _nvme_ns_cmd_setup_request(ns, req, opc, lba, lba_count, io_flags, apptag_mask, apptag); 435 return req; 436 } 437 438 int 439 spdk_nvme_ns_cmd_compare(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, void *buffer, 440 uint64_t lba, 441 uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, void *cb_arg, 442 uint32_t io_flags) 443 { 444 struct nvme_request *req; 445 struct nvme_payload payload; 446 447 payload = NVME_PAYLOAD_CONTIG(buffer, NULL); 448 449 req = _nvme_ns_cmd_rw(ns, qpair, &payload, 0, 0, lba, lba_count, cb_fn, cb_arg, 450 SPDK_NVME_OPC_COMPARE, 451 io_flags, 0, 452 0, true); 453 if (req != NULL) { 454 return nvme_qpair_submit_request(qpair, req); 455 } else if (spdk_nvme_ns_check_request_length(lba_count, 456 ns->sectors_per_max_io, 457 ns->sectors_per_stripe, 458 qpair->ctrlr->opts.io_queue_requests)) { 459 return -EINVAL; 460 } else { 461 return -ENOMEM; 462 } 463 } 464 465 int 466 spdk_nvme_ns_cmd_compare_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, 467 void *buffer, 468 void *metadata, 469 uint64_t lba, 470 uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, void *cb_arg, 471 uint32_t io_flags, uint16_t apptag_mask, uint16_t apptag) 472 { 473 struct nvme_request *req; 474 struct nvme_payload payload; 475 476 payload = NVME_PAYLOAD_CONTIG(buffer, metadata); 477 478 req = _nvme_ns_cmd_rw(ns, qpair, &payload, 0, 0, lba, lba_count, cb_fn, cb_arg, 479 SPDK_NVME_OPC_COMPARE, 480 io_flags, 481 apptag_mask, apptag, true); 482 if (req != NULL) { 483 return nvme_qpair_submit_request(qpair, req); 484 } else if (spdk_nvme_ns_check_request_length(lba_count, 485 ns->sectors_per_max_io, 486 ns->sectors_per_stripe, 487 qpair->ctrlr->opts.io_queue_requests)) { 488 return -EINVAL; 489 } else { 490 return -ENOMEM; 491 } 492 } 493 494 int 495 spdk_nvme_ns_cmd_comparev(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, 496 uint64_t lba, uint32_t lba_count, 497 spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t io_flags, 498 spdk_nvme_req_reset_sgl_cb reset_sgl_fn, 499 spdk_nvme_req_next_sge_cb next_sge_fn) 500 { 501 struct nvme_request *req; 502 struct nvme_payload payload; 503 504 if (reset_sgl_fn == NULL || next_sge_fn == NULL) { 505 return -EINVAL; 506 } 507 508 payload = NVME_PAYLOAD_SGL(reset_sgl_fn, next_sge_fn, cb_arg, NULL); 509 510 req = _nvme_ns_cmd_rw(ns, qpair, &payload, 0, 0, lba, lba_count, cb_fn, cb_arg, 511 SPDK_NVME_OPC_COMPARE, 512 io_flags, 0, 0, true); 513 if (req != NULL) { 514 return nvme_qpair_submit_request(qpair, req); 515 } else if (spdk_nvme_ns_check_request_length(lba_count, 516 ns->sectors_per_max_io, 517 ns->sectors_per_stripe, 518 qpair->ctrlr->opts.io_queue_requests)) { 519 return -EINVAL; 520 } else { 521 return -ENOMEM; 522 } 523 } 524 525 int 526 spdk_nvme_ns_cmd_read(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, void *buffer, 527 uint64_t lba, 528 uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, void *cb_arg, 529 uint32_t io_flags) 530 { 531 struct nvme_request *req; 532 struct nvme_payload payload; 533 534 payload = NVME_PAYLOAD_CONTIG(buffer, NULL); 535 536 req = _nvme_ns_cmd_rw(ns, qpair, &payload, 0, 0, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_READ, 537 io_flags, 0, 538 0, true); 539 if (req != NULL) { 540 return nvme_qpair_submit_request(qpair, req); 541 } else if (spdk_nvme_ns_check_request_length(lba_count, 542 ns->sectors_per_max_io, 543 ns->sectors_per_stripe, 544 qpair->ctrlr->opts.io_queue_requests)) { 545 return -EINVAL; 546 } else { 547 return -ENOMEM; 548 } 549 } 550 551 int 552 spdk_nvme_ns_cmd_read_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, void *buffer, 553 void *metadata, 554 uint64_t lba, 555 uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, void *cb_arg, 556 uint32_t io_flags, uint16_t apptag_mask, uint16_t apptag) 557 { 558 struct nvme_request *req; 559 struct nvme_payload payload; 560 561 payload = NVME_PAYLOAD_CONTIG(buffer, metadata); 562 563 req = _nvme_ns_cmd_rw(ns, qpair, &payload, 0, 0, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_READ, 564 io_flags, 565 apptag_mask, apptag, true); 566 if (req != NULL) { 567 return nvme_qpair_submit_request(qpair, req); 568 } else if (spdk_nvme_ns_check_request_length(lba_count, 569 ns->sectors_per_max_io, 570 ns->sectors_per_stripe, 571 qpair->ctrlr->opts.io_queue_requests)) { 572 return -EINVAL; 573 } else { 574 return -ENOMEM; 575 } 576 } 577 578 int 579 spdk_nvme_ns_cmd_readv(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, 580 uint64_t lba, uint32_t lba_count, 581 spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t io_flags, 582 spdk_nvme_req_reset_sgl_cb reset_sgl_fn, 583 spdk_nvme_req_next_sge_cb next_sge_fn) 584 { 585 struct nvme_request *req; 586 struct nvme_payload payload; 587 588 if (reset_sgl_fn == NULL || next_sge_fn == NULL) { 589 return -EINVAL; 590 } 591 592 payload = NVME_PAYLOAD_SGL(reset_sgl_fn, next_sge_fn, cb_arg, NULL); 593 594 req = _nvme_ns_cmd_rw(ns, qpair, &payload, 0, 0, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_READ, 595 io_flags, 0, 0, true); 596 if (req != NULL) { 597 return nvme_qpair_submit_request(qpair, req); 598 } else if (spdk_nvme_ns_check_request_length(lba_count, 599 ns->sectors_per_max_io, 600 ns->sectors_per_stripe, 601 qpair->ctrlr->opts.io_queue_requests)) { 602 return -EINVAL; 603 } else { 604 return -ENOMEM; 605 } 606 } 607 608 int 609 spdk_nvme_ns_cmd_readv_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, 610 uint64_t lba, uint32_t lba_count, 611 spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t io_flags, 612 spdk_nvme_req_reset_sgl_cb reset_sgl_fn, 613 spdk_nvme_req_next_sge_cb next_sge_fn, void *metadata, 614 uint16_t apptag_mask, uint16_t apptag) 615 { 616 struct nvme_request *req; 617 struct nvme_payload payload; 618 619 if (reset_sgl_fn == NULL || next_sge_fn == NULL) { 620 return -EINVAL; 621 } 622 623 payload = NVME_PAYLOAD_SGL(reset_sgl_fn, next_sge_fn, cb_arg, metadata); 624 625 req = _nvme_ns_cmd_rw(ns, qpair, &payload, 0, 0, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_READ, 626 io_flags, apptag_mask, apptag, true); 627 if (req != NULL) { 628 return nvme_qpair_submit_request(qpair, req); 629 } else if (spdk_nvme_ns_check_request_length(lba_count, 630 ns->sectors_per_max_io, 631 ns->sectors_per_stripe, 632 qpair->ctrlr->opts.io_queue_requests)) { 633 return -EINVAL; 634 } else { 635 return -ENOMEM; 636 } 637 } 638 639 int 640 spdk_nvme_ns_cmd_write(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, 641 void *buffer, uint64_t lba, 642 uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, void *cb_arg, 643 uint32_t io_flags) 644 { 645 struct nvme_request *req; 646 struct nvme_payload payload; 647 648 payload = NVME_PAYLOAD_CONTIG(buffer, NULL); 649 650 req = _nvme_ns_cmd_rw(ns, qpair, &payload, 0, 0, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_WRITE, 651 io_flags, 0, 0, true); 652 if (req != NULL) { 653 return nvme_qpair_submit_request(qpair, req); 654 } else if (spdk_nvme_ns_check_request_length(lba_count, 655 ns->sectors_per_max_io, 656 ns->sectors_per_stripe, 657 qpair->ctrlr->opts.io_queue_requests)) { 658 return -EINVAL; 659 } else { 660 return -ENOMEM; 661 } 662 } 663 664 int 665 spdk_nvme_ns_cmd_write_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, 666 void *buffer, void *metadata, uint64_t lba, 667 uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, void *cb_arg, 668 uint32_t io_flags, uint16_t apptag_mask, uint16_t apptag) 669 { 670 struct nvme_request *req; 671 struct nvme_payload payload; 672 673 payload = NVME_PAYLOAD_CONTIG(buffer, metadata); 674 675 req = _nvme_ns_cmd_rw(ns, qpair, &payload, 0, 0, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_WRITE, 676 io_flags, apptag_mask, apptag, true); 677 if (req != NULL) { 678 return nvme_qpair_submit_request(qpair, req); 679 } else if (spdk_nvme_ns_check_request_length(lba_count, 680 ns->sectors_per_max_io, 681 ns->sectors_per_stripe, 682 qpair->ctrlr->opts.io_queue_requests)) { 683 return -EINVAL; 684 } else { 685 return -ENOMEM; 686 } 687 } 688 689 int 690 spdk_nvme_ns_cmd_writev(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, 691 uint64_t lba, uint32_t lba_count, 692 spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t io_flags, 693 spdk_nvme_req_reset_sgl_cb reset_sgl_fn, 694 spdk_nvme_req_next_sge_cb next_sge_fn) 695 { 696 struct nvme_request *req; 697 struct nvme_payload payload; 698 699 if (reset_sgl_fn == NULL || next_sge_fn == NULL) { 700 return -EINVAL; 701 } 702 703 payload = NVME_PAYLOAD_SGL(reset_sgl_fn, next_sge_fn, cb_arg, NULL); 704 705 req = _nvme_ns_cmd_rw(ns, qpair, &payload, 0, 0, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_WRITE, 706 io_flags, 0, 0, true); 707 if (req != NULL) { 708 return nvme_qpair_submit_request(qpair, req); 709 } else if (spdk_nvme_ns_check_request_length(lba_count, 710 ns->sectors_per_max_io, 711 ns->sectors_per_stripe, 712 qpair->ctrlr->opts.io_queue_requests)) { 713 return -EINVAL; 714 } else { 715 return -ENOMEM; 716 } 717 } 718 719 int 720 spdk_nvme_ns_cmd_writev_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, 721 uint64_t lba, uint32_t lba_count, 722 spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t io_flags, 723 spdk_nvme_req_reset_sgl_cb reset_sgl_fn, 724 spdk_nvme_req_next_sge_cb next_sge_fn, void *metadata, 725 uint16_t apptag_mask, uint16_t apptag) 726 { 727 struct nvme_request *req; 728 struct nvme_payload payload; 729 730 if (reset_sgl_fn == NULL || next_sge_fn == NULL) { 731 return -EINVAL; 732 } 733 734 payload = NVME_PAYLOAD_SGL(reset_sgl_fn, next_sge_fn, cb_arg, metadata); 735 736 req = _nvme_ns_cmd_rw(ns, qpair, &payload, 0, 0, lba, lba_count, cb_fn, cb_arg, SPDK_NVME_OPC_WRITE, 737 io_flags, apptag_mask, apptag, true); 738 if (req != NULL) { 739 return nvme_qpair_submit_request(qpair, req); 740 } else if (spdk_nvme_ns_check_request_length(lba_count, 741 ns->sectors_per_max_io, 742 ns->sectors_per_stripe, 743 qpair->ctrlr->opts.io_queue_requests)) { 744 return -EINVAL; 745 } else { 746 return -ENOMEM; 747 } 748 } 749 750 int 751 spdk_nvme_ns_cmd_write_zeroes(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, 752 uint64_t lba, uint32_t lba_count, 753 spdk_nvme_cmd_cb cb_fn, void *cb_arg, 754 uint32_t io_flags) 755 { 756 struct nvme_request *req; 757 struct spdk_nvme_cmd *cmd; 758 uint64_t *tmp_lba; 759 760 if (lba_count == 0 || lba_count > UINT16_MAX + 1) { 761 return -EINVAL; 762 } 763 764 req = nvme_allocate_request_null(qpair, cb_fn, cb_arg); 765 if (req == NULL) { 766 return -ENOMEM; 767 } 768 769 cmd = &req->cmd; 770 cmd->opc = SPDK_NVME_OPC_WRITE_ZEROES; 771 cmd->nsid = ns->id; 772 773 tmp_lba = (uint64_t *)&cmd->cdw10; 774 *tmp_lba = lba; 775 cmd->cdw12 = lba_count - 1; 776 cmd->cdw12 |= io_flags; 777 778 return nvme_qpair_submit_request(qpair, req); 779 } 780 781 int 782 spdk_nvme_ns_cmd_dataset_management(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, 783 uint32_t type, 784 const struct spdk_nvme_dsm_range *ranges, uint16_t num_ranges, 785 spdk_nvme_cmd_cb cb_fn, void *cb_arg) 786 { 787 struct nvme_request *req; 788 struct spdk_nvme_cmd *cmd; 789 790 if (num_ranges == 0 || num_ranges > SPDK_NVME_DATASET_MANAGEMENT_MAX_RANGES) { 791 return -EINVAL; 792 } 793 794 if (ranges == NULL) { 795 return -EINVAL; 796 } 797 798 req = nvme_allocate_request_user_copy(qpair, (void *)ranges, 799 num_ranges * sizeof(struct spdk_nvme_dsm_range), 800 cb_fn, cb_arg, true); 801 if (req == NULL) { 802 return -ENOMEM; 803 } 804 805 cmd = &req->cmd; 806 cmd->opc = SPDK_NVME_OPC_DATASET_MANAGEMENT; 807 cmd->nsid = ns->id; 808 809 cmd->cdw10 = num_ranges - 1; 810 cmd->cdw11 = type; 811 812 return nvme_qpair_submit_request(qpair, req); 813 } 814 815 int 816 spdk_nvme_ns_cmd_flush(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, 817 spdk_nvme_cmd_cb cb_fn, void *cb_arg) 818 { 819 struct nvme_request *req; 820 struct spdk_nvme_cmd *cmd; 821 822 req = nvme_allocate_request_null(qpair, cb_fn, cb_arg); 823 if (req == NULL) { 824 return -ENOMEM; 825 } 826 827 cmd = &req->cmd; 828 cmd->opc = SPDK_NVME_OPC_FLUSH; 829 cmd->nsid = ns->id; 830 831 return nvme_qpair_submit_request(qpair, req); 832 } 833 834 int 835 spdk_nvme_ns_cmd_reservation_register(struct spdk_nvme_ns *ns, 836 struct spdk_nvme_qpair *qpair, 837 struct spdk_nvme_reservation_register_data *payload, 838 bool ignore_key, 839 enum spdk_nvme_reservation_register_action action, 840 enum spdk_nvme_reservation_register_cptpl cptpl, 841 spdk_nvme_cmd_cb cb_fn, void *cb_arg) 842 { 843 struct nvme_request *req; 844 struct spdk_nvme_cmd *cmd; 845 846 req = nvme_allocate_request_user_copy(qpair, 847 payload, sizeof(struct spdk_nvme_reservation_register_data), 848 cb_fn, cb_arg, true); 849 if (req == NULL) { 850 return -ENOMEM; 851 } 852 853 cmd = &req->cmd; 854 cmd->opc = SPDK_NVME_OPC_RESERVATION_REGISTER; 855 cmd->nsid = ns->id; 856 857 /* Bits 0-2 */ 858 cmd->cdw10 = action; 859 /* Bit 3 */ 860 cmd->cdw10 |= ignore_key ? 1 << 3 : 0; 861 /* Bits 30-31 */ 862 cmd->cdw10 |= (uint32_t)cptpl << 30; 863 864 return nvme_qpair_submit_request(qpair, req); 865 } 866 867 int 868 spdk_nvme_ns_cmd_reservation_release(struct spdk_nvme_ns *ns, 869 struct spdk_nvme_qpair *qpair, 870 struct spdk_nvme_reservation_key_data *payload, 871 bool ignore_key, 872 enum spdk_nvme_reservation_release_action action, 873 enum spdk_nvme_reservation_type type, 874 spdk_nvme_cmd_cb cb_fn, void *cb_arg) 875 { 876 struct nvme_request *req; 877 struct spdk_nvme_cmd *cmd; 878 879 req = nvme_allocate_request_user_copy(qpair, 880 payload, sizeof(struct spdk_nvme_reservation_key_data), cb_fn, 881 cb_arg, true); 882 if (req == NULL) { 883 return -ENOMEM; 884 } 885 886 cmd = &req->cmd; 887 cmd->opc = SPDK_NVME_OPC_RESERVATION_RELEASE; 888 cmd->nsid = ns->id; 889 890 /* Bits 0-2 */ 891 cmd->cdw10 = action; 892 /* Bit 3 */ 893 cmd->cdw10 |= ignore_key ? 1 << 3 : 0; 894 /* Bits 8-15 */ 895 cmd->cdw10 |= (uint32_t)type << 8; 896 897 return nvme_qpair_submit_request(qpair, req); 898 } 899 900 int 901 spdk_nvme_ns_cmd_reservation_acquire(struct spdk_nvme_ns *ns, 902 struct spdk_nvme_qpair *qpair, 903 struct spdk_nvme_reservation_acquire_data *payload, 904 bool ignore_key, 905 enum spdk_nvme_reservation_acquire_action action, 906 enum spdk_nvme_reservation_type type, 907 spdk_nvme_cmd_cb cb_fn, void *cb_arg) 908 { 909 struct nvme_request *req; 910 struct spdk_nvme_cmd *cmd; 911 912 req = nvme_allocate_request_user_copy(qpair, 913 payload, sizeof(struct spdk_nvme_reservation_acquire_data), 914 cb_fn, cb_arg, true); 915 if (req == NULL) { 916 return -ENOMEM; 917 } 918 919 cmd = &req->cmd; 920 cmd->opc = SPDK_NVME_OPC_RESERVATION_ACQUIRE; 921 cmd->nsid = ns->id; 922 923 /* Bits 0-2 */ 924 cmd->cdw10 = action; 925 /* Bit 3 */ 926 cmd->cdw10 |= ignore_key ? 1 << 3 : 0; 927 /* Bits 8-15 */ 928 cmd->cdw10 |= (uint32_t)type << 8; 929 930 return nvme_qpair_submit_request(qpair, req); 931 } 932 933 int 934 spdk_nvme_ns_cmd_reservation_report(struct spdk_nvme_ns *ns, 935 struct spdk_nvme_qpair *qpair, 936 void *payload, uint32_t len, 937 spdk_nvme_cmd_cb cb_fn, void *cb_arg) 938 { 939 uint32_t num_dwords; 940 struct nvme_request *req; 941 struct spdk_nvme_cmd *cmd; 942 943 if (len % 4) { 944 return -EINVAL; 945 } 946 num_dwords = len / 4; 947 948 req = nvme_allocate_request_user_copy(qpair, payload, len, cb_fn, cb_arg, false); 949 if (req == NULL) { 950 return -ENOMEM; 951 } 952 953 cmd = &req->cmd; 954 cmd->opc = SPDK_NVME_OPC_RESERVATION_REPORT; 955 cmd->nsid = ns->id; 956 957 cmd->cdw10 = num_dwords; 958 959 return nvme_qpair_submit_request(qpair, req); 960 } 961