1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2017 Intel Corporation. All rights reserved. 3 * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #include "nvmf_internal.h" 10 11 #include "spdk/bdev.h" 12 #include "spdk/endian.h" 13 #include "spdk/thread.h" 14 #include "spdk/likely.h" 15 #include "spdk/nvme.h" 16 #include "spdk/nvmf_cmd.h" 17 #include "spdk/nvmf_spec.h" 18 #include "spdk/trace.h" 19 #include "spdk/scsi_spec.h" 20 #include "spdk/string.h" 21 #include "spdk/util.h" 22 23 #include "spdk/log.h" 24 25 static bool 26 nvmf_subsystem_bdev_io_type_supported(struct spdk_nvmf_subsystem *subsystem, 27 enum spdk_bdev_io_type io_type) 28 { 29 struct spdk_nvmf_ns *ns; 30 31 for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL; 32 ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) { 33 if (ns->bdev == NULL) { 34 continue; 35 } 36 37 if (!spdk_bdev_io_type_supported(ns->bdev, io_type)) { 38 SPDK_DEBUGLOG(nvmf, 39 "Subsystem %s namespace %u (%s) does not support io_type %d\n", 40 spdk_nvmf_subsystem_get_nqn(subsystem), 41 ns->opts.nsid, spdk_bdev_get_name(ns->bdev), (int)io_type); 42 return false; 43 } 44 } 45 46 SPDK_DEBUGLOG(nvmf, "All devices in Subsystem %s support io_type %d\n", 47 spdk_nvmf_subsystem_get_nqn(subsystem), (int)io_type); 48 return true; 49 } 50 51 bool 52 nvmf_ctrlr_dsm_supported(struct spdk_nvmf_ctrlr *ctrlr) 53 { 54 return nvmf_subsystem_bdev_io_type_supported(ctrlr->subsys, SPDK_BDEV_IO_TYPE_UNMAP); 55 } 56 57 bool 58 nvmf_ctrlr_write_zeroes_supported(struct spdk_nvmf_ctrlr *ctrlr) 59 { 60 return nvmf_subsystem_bdev_io_type_supported(ctrlr->subsys, SPDK_BDEV_IO_TYPE_WRITE_ZEROES); 61 } 62 63 bool 64 nvmf_ctrlr_copy_supported(struct spdk_nvmf_ctrlr *ctrlr) 65 { 66 return nvmf_subsystem_bdev_io_type_supported(ctrlr->subsys, SPDK_BDEV_IO_TYPE_COPY); 67 } 68 69 static void 70 nvmf_bdev_ctrlr_complete_cmd(struct spdk_bdev_io *bdev_io, bool success, 71 void *cb_arg) 72 { 73 struct spdk_nvmf_request *req = cb_arg; 74 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 75 int sc = 0, sct = 0; 76 uint32_t cdw0 = 0; 77 78 if (spdk_unlikely(req->first_fused)) { 79 struct spdk_nvmf_request *first_req = req->first_fused_req; 80 struct spdk_nvme_cpl *first_response = &first_req->rsp->nvme_cpl; 81 int first_sc = 0, first_sct = 0; 82 83 /* get status for both operations */ 84 spdk_bdev_io_get_nvme_fused_status(bdev_io, &cdw0, &first_sct, &first_sc, &sct, &sc); 85 first_response->cdw0 = cdw0; 86 first_response->status.sc = first_sc; 87 first_response->status.sct = first_sct; 88 89 /* first request should be completed */ 90 spdk_nvmf_request_complete(first_req); 91 req->first_fused_req = NULL; 92 req->first_fused = false; 93 } else { 94 spdk_bdev_io_get_nvme_status(bdev_io, &cdw0, &sct, &sc); 95 } 96 97 response->cdw0 = cdw0; 98 response->status.sc = sc; 99 response->status.sct = sct; 100 101 spdk_nvmf_request_complete(req); 102 spdk_bdev_free_io(bdev_io); 103 } 104 105 static void 106 nvmf_bdev_ctrlr_complete_admin_cmd(struct spdk_bdev_io *bdev_io, bool success, 107 void *cb_arg) 108 { 109 struct spdk_nvmf_request *req = cb_arg; 110 111 if (req->cmd_cb_fn) { 112 req->cmd_cb_fn(req); 113 } 114 115 nvmf_bdev_ctrlr_complete_cmd(bdev_io, success, req); 116 } 117 118 void 119 nvmf_bdev_ctrlr_identify_ns(struct spdk_nvmf_ns *ns, struct spdk_nvme_ns_data *nsdata, 120 bool dif_insert_or_strip) 121 { 122 struct spdk_bdev *bdev = ns->bdev; 123 uint64_t num_blocks; 124 uint32_t phys_blocklen; 125 uint32_t max_copy; 126 127 num_blocks = spdk_bdev_get_num_blocks(bdev); 128 129 nsdata->nsze = num_blocks; 130 nsdata->ncap = num_blocks; 131 nsdata->nuse = num_blocks; 132 nsdata->nlbaf = 0; 133 nsdata->flbas.format = 0; 134 nsdata->flbas.msb_format = 0; 135 nsdata->nacwu = spdk_bdev_get_acwu(bdev) - 1; /* nacwu is 0-based */ 136 if (!dif_insert_or_strip) { 137 nsdata->lbaf[0].ms = spdk_bdev_get_md_size(bdev); 138 nsdata->lbaf[0].lbads = spdk_u32log2(spdk_bdev_get_block_size(bdev)); 139 if (nsdata->lbaf[0].ms != 0) { 140 nsdata->flbas.extended = 1; 141 nsdata->mc.extended = 1; 142 nsdata->mc.pointer = 0; 143 nsdata->dps.md_start = spdk_bdev_is_dif_head_of_md(bdev); 144 /* NVMf library doesn't process PRACT and PRCHK flags, we 145 * leave the use of extended LBA buffer to users. 146 */ 147 nsdata->dps.pit = SPDK_NVME_FMT_NVM_PROTECTION_DISABLE; 148 } 149 } else { 150 nsdata->lbaf[0].ms = 0; 151 nsdata->lbaf[0].lbads = spdk_u32log2(spdk_bdev_get_data_block_size(bdev)); 152 } 153 154 phys_blocklen = spdk_bdev_get_physical_block_size(bdev); 155 assert(phys_blocklen > 0); 156 /* Linux driver uses min(nawupf, npwg) to set physical_block_size */ 157 nsdata->nsfeat.optperf = 1; 158 nsdata->nsfeat.ns_atomic_write_unit = 1; 159 nsdata->npwg = (phys_blocklen >> nsdata->lbaf[0].lbads) - 1; 160 nsdata->nawupf = nsdata->npwg; 161 nsdata->npwa = nsdata->npwg; 162 nsdata->npdg = nsdata->npwg; 163 nsdata->npda = nsdata->npwg; 164 165 if (spdk_bdev_get_write_unit_size(bdev) == 1) { 166 nsdata->noiob = spdk_bdev_get_optimal_io_boundary(bdev); 167 } 168 nsdata->nmic.can_share = 1; 169 if (nvmf_ns_is_ptpl_capable(ns)) { 170 nsdata->nsrescap.rescap.persist = 1; 171 } 172 nsdata->nsrescap.rescap.write_exclusive = 1; 173 nsdata->nsrescap.rescap.exclusive_access = 1; 174 nsdata->nsrescap.rescap.write_exclusive_reg_only = 1; 175 nsdata->nsrescap.rescap.exclusive_access_reg_only = 1; 176 nsdata->nsrescap.rescap.write_exclusive_all_reg = 1; 177 nsdata->nsrescap.rescap.exclusive_access_all_reg = 1; 178 nsdata->nsrescap.rescap.ignore_existing_key = 1; 179 180 SPDK_STATIC_ASSERT(sizeof(nsdata->nguid) == sizeof(ns->opts.nguid), "size mismatch"); 181 memcpy(nsdata->nguid, ns->opts.nguid, sizeof(nsdata->nguid)); 182 183 SPDK_STATIC_ASSERT(sizeof(nsdata->eui64) == sizeof(ns->opts.eui64), "size mismatch"); 184 memcpy(&nsdata->eui64, ns->opts.eui64, sizeof(nsdata->eui64)); 185 186 /* For now we support just one source range for copy command */ 187 nsdata->msrc = 0; 188 189 max_copy = spdk_bdev_get_max_copy(bdev); 190 if (max_copy == 0 || max_copy > UINT16_MAX) { 191 /* Zero means copy size is unlimited */ 192 nsdata->mcl = UINT16_MAX; 193 nsdata->mssrl = UINT16_MAX; 194 } else { 195 nsdata->mcl = max_copy; 196 nsdata->mssrl = max_copy; 197 } 198 } 199 200 void 201 nvmf_bdev_ctrlr_identify_iocs_nvm(struct spdk_nvmf_ns *ns, 202 struct spdk_nvme_nvm_ns_data *nsdata_nvm) 203 { 204 struct spdk_bdev *bdev = ns->bdev; 205 uint8_t _16bpists; 206 uint32_t sts, pif; 207 208 if (spdk_bdev_get_dif_type(bdev) == SPDK_DIF_DISABLE) { 209 return; 210 } 211 212 pif = spdk_bdev_get_dif_pi_format(bdev); 213 214 /* 215 * 16BPISTS shall be 1 for 32/64b Guard PI. 216 * STCRS shall be 1 if 16BPISTS is 1. 217 * 16 is the minimum value of STS for 32b Guard PI. 218 */ 219 switch (pif) { 220 case SPDK_DIF_PI_FORMAT_16: 221 _16bpists = 0; 222 sts = 0; 223 break; 224 case SPDK_DIF_PI_FORMAT_32: 225 _16bpists = 1; 226 sts = 16; 227 break; 228 case SPDK_DIF_PI_FORMAT_64: 229 _16bpists = 1; 230 sts = 0; 231 break; 232 default: 233 SPDK_WARNLOG("PI format %u is not supported\n", pif); 234 return; 235 } 236 237 /* For 16b Guard PI, Storage Tag is not available because we set STS to 0. 238 * In this case, we do not have to set 16BPISTM to 1. For simplicity, 239 * set 16BPISTM to 0 and set LBSTM to all zeroes. 240 * 241 * We will revisit here when we find any OS uses Storage Tag. 242 */ 243 nsdata_nvm->lbstm = 0; 244 nsdata_nvm->pic._16bpistm = 0; 245 246 nsdata_nvm->pic._16bpists = _16bpists; 247 nsdata_nvm->pic.stcrs = 0; 248 nsdata_nvm->elbaf[0].sts = sts; 249 nsdata_nvm->elbaf[0].pif = pif; 250 } 251 252 static void 253 nvmf_bdev_ctrlr_get_rw_params(const struct spdk_nvme_cmd *cmd, uint64_t *start_lba, 254 uint64_t *num_blocks) 255 { 256 /* SLBA: CDW10 and CDW11 */ 257 *start_lba = from_le64(&cmd->cdw10); 258 259 /* NLB: CDW12 bits 15:00, 0's based */ 260 *num_blocks = (from_le32(&cmd->cdw12) & 0xFFFFu) + 1; 261 } 262 263 static void 264 nvmf_bdev_ctrlr_get_rw_ext_params(const struct spdk_nvme_cmd *cmd, 265 struct spdk_bdev_ext_io_opts *opts) 266 { 267 /* Get CDW12 values */ 268 opts->nvme_cdw12.raw = from_le32(&cmd->cdw12); 269 270 /* Get CDW13 values */ 271 opts->nvme_cdw13.raw = from_le32(&cmd->cdw13); 272 } 273 274 static bool 275 nvmf_bdev_ctrlr_lba_in_range(uint64_t bdev_num_blocks, uint64_t io_start_lba, 276 uint64_t io_num_blocks) 277 { 278 if (io_start_lba + io_num_blocks > bdev_num_blocks || 279 io_start_lba + io_num_blocks < io_start_lba) { 280 return false; 281 } 282 283 return true; 284 } 285 286 static void 287 nvmf_ctrlr_process_io_cmd_resubmit(void *arg) 288 { 289 struct spdk_nvmf_request *req = arg; 290 int rc; 291 292 rc = nvmf_ctrlr_process_io_cmd(req); 293 if (rc == SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) { 294 spdk_nvmf_request_complete(req); 295 } 296 } 297 298 static void 299 nvmf_ctrlr_process_admin_cmd_resubmit(void *arg) 300 { 301 struct spdk_nvmf_request *req = arg; 302 int rc; 303 304 rc = nvmf_ctrlr_process_admin_cmd(req); 305 if (rc == SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) { 306 spdk_nvmf_request_complete(req); 307 } 308 } 309 310 static void 311 nvmf_bdev_ctrl_queue_io(struct spdk_nvmf_request *req, struct spdk_bdev *bdev, 312 struct spdk_io_channel *ch, spdk_bdev_io_wait_cb cb_fn, void *cb_arg) 313 { 314 int rc; 315 316 req->bdev_io_wait.bdev = bdev; 317 req->bdev_io_wait.cb_fn = cb_fn; 318 req->bdev_io_wait.cb_arg = cb_arg; 319 320 rc = spdk_bdev_queue_io_wait(bdev, ch, &req->bdev_io_wait); 321 if (rc != 0) { 322 assert(false); 323 } 324 req->qpair->group->stat.pending_bdev_io++; 325 } 326 327 bool 328 nvmf_bdev_zcopy_enabled(struct spdk_bdev *bdev) 329 { 330 return spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_ZCOPY); 331 } 332 333 int 334 nvmf_bdev_ctrlr_read_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 335 struct spdk_io_channel *ch, struct spdk_nvmf_request *req) 336 { 337 struct spdk_bdev_ext_io_opts opts = { 338 .size = SPDK_SIZEOF(&opts, accel_sequence), 339 .memory_domain = req->memory_domain, 340 .memory_domain_ctx = req->memory_domain_ctx, 341 .accel_sequence = req->accel_sequence, 342 }; 343 uint64_t bdev_num_blocks = spdk_bdev_get_num_blocks(bdev); 344 uint32_t block_size = spdk_bdev_get_block_size(bdev); 345 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 346 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 347 uint64_t start_lba; 348 uint64_t num_blocks; 349 int rc; 350 351 nvmf_bdev_ctrlr_get_rw_params(cmd, &start_lba, &num_blocks); 352 353 if (spdk_unlikely(!nvmf_bdev_ctrlr_lba_in_range(bdev_num_blocks, start_lba, num_blocks))) { 354 SPDK_ERRLOG("end of media\n"); 355 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 356 rsp->status.sc = SPDK_NVME_SC_LBA_OUT_OF_RANGE; 357 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 358 } 359 360 if (spdk_unlikely(num_blocks * block_size > req->length)) { 361 SPDK_ERRLOG("Read NLB %" PRIu64 " * block size %" PRIu32 " > SGL length %" PRIu32 "\n", 362 num_blocks, block_size, req->length); 363 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 364 rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID; 365 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 366 } 367 368 assert(!spdk_nvmf_request_using_zcopy(req)); 369 370 rc = spdk_bdev_readv_blocks_ext(desc, ch, req->iov, req->iovcnt, start_lba, num_blocks, 371 nvmf_bdev_ctrlr_complete_cmd, req, &opts); 372 if (spdk_unlikely(rc)) { 373 if (rc == -ENOMEM) { 374 nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req); 375 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 376 } 377 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 378 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 379 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 380 } 381 382 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 383 } 384 385 int 386 nvmf_bdev_ctrlr_write_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 387 struct spdk_io_channel *ch, struct spdk_nvmf_request *req) 388 { 389 struct spdk_bdev_ext_io_opts opts = { 390 .size = SPDK_SIZEOF(&opts, nvme_cdw13), 391 .memory_domain = req->memory_domain, 392 .memory_domain_ctx = req->memory_domain_ctx, 393 .accel_sequence = req->accel_sequence, 394 }; 395 uint64_t bdev_num_blocks = spdk_bdev_get_num_blocks(bdev); 396 uint32_t block_size = spdk_bdev_get_block_size(bdev); 397 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 398 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 399 uint64_t start_lba; 400 uint64_t num_blocks; 401 int rc; 402 403 nvmf_bdev_ctrlr_get_rw_params(cmd, &start_lba, &num_blocks); 404 nvmf_bdev_ctrlr_get_rw_ext_params(cmd, &opts); 405 406 if (spdk_unlikely(!nvmf_bdev_ctrlr_lba_in_range(bdev_num_blocks, start_lba, num_blocks))) { 407 SPDK_ERRLOG("end of media\n"); 408 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 409 rsp->status.sc = SPDK_NVME_SC_LBA_OUT_OF_RANGE; 410 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 411 } 412 413 if (spdk_unlikely(num_blocks * block_size > req->length)) { 414 SPDK_ERRLOG("Write NLB %" PRIu64 " * block size %" PRIu32 " > SGL length %" PRIu32 "\n", 415 num_blocks, block_size, req->length); 416 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 417 rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID; 418 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 419 } 420 421 assert(!spdk_nvmf_request_using_zcopy(req)); 422 423 rc = spdk_bdev_writev_blocks_ext(desc, ch, req->iov, req->iovcnt, start_lba, num_blocks, 424 nvmf_bdev_ctrlr_complete_cmd, req, &opts); 425 if (spdk_unlikely(rc)) { 426 if (rc == -ENOMEM) { 427 nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req); 428 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 429 } 430 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 431 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 432 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 433 } 434 435 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 436 } 437 438 int 439 nvmf_bdev_ctrlr_compare_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 440 struct spdk_io_channel *ch, struct spdk_nvmf_request *req) 441 { 442 uint64_t bdev_num_blocks = spdk_bdev_get_num_blocks(bdev); 443 uint32_t block_size = spdk_bdev_get_block_size(bdev); 444 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 445 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 446 uint64_t start_lba; 447 uint64_t num_blocks; 448 int rc; 449 450 nvmf_bdev_ctrlr_get_rw_params(cmd, &start_lba, &num_blocks); 451 452 if (spdk_unlikely(!nvmf_bdev_ctrlr_lba_in_range(bdev_num_blocks, start_lba, num_blocks))) { 453 SPDK_ERRLOG("end of media\n"); 454 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 455 rsp->status.sc = SPDK_NVME_SC_LBA_OUT_OF_RANGE; 456 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 457 } 458 459 if (spdk_unlikely(num_blocks * block_size > req->length)) { 460 SPDK_ERRLOG("Compare NLB %" PRIu64 " * block size %" PRIu32 " > SGL length %" PRIu32 "\n", 461 num_blocks, block_size, req->length); 462 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 463 rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID; 464 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 465 } 466 467 rc = spdk_bdev_comparev_blocks(desc, ch, req->iov, req->iovcnt, start_lba, num_blocks, 468 nvmf_bdev_ctrlr_complete_cmd, req); 469 if (spdk_unlikely(rc)) { 470 if (rc == -ENOMEM) { 471 nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req); 472 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 473 } 474 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 475 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 476 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 477 } 478 479 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 480 } 481 482 int 483 nvmf_bdev_ctrlr_compare_and_write_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 484 struct spdk_io_channel *ch, struct spdk_nvmf_request *cmp_req, struct spdk_nvmf_request *write_req) 485 { 486 uint64_t bdev_num_blocks = spdk_bdev_get_num_blocks(bdev); 487 uint32_t block_size = spdk_bdev_get_block_size(bdev); 488 struct spdk_nvme_cmd *cmp_cmd = &cmp_req->cmd->nvme_cmd; 489 struct spdk_nvme_cmd *write_cmd = &write_req->cmd->nvme_cmd; 490 struct spdk_nvme_cpl *rsp = &write_req->rsp->nvme_cpl; 491 uint64_t write_start_lba, cmp_start_lba; 492 uint64_t write_num_blocks, cmp_num_blocks; 493 int rc; 494 495 nvmf_bdev_ctrlr_get_rw_params(cmp_cmd, &cmp_start_lba, &cmp_num_blocks); 496 nvmf_bdev_ctrlr_get_rw_params(write_cmd, &write_start_lba, &write_num_blocks); 497 498 if (spdk_unlikely(write_start_lba != cmp_start_lba || write_num_blocks != cmp_num_blocks)) { 499 SPDK_ERRLOG("Fused command start lba / num blocks mismatch\n"); 500 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 501 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 502 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 503 } 504 505 if (spdk_unlikely(!nvmf_bdev_ctrlr_lba_in_range(bdev_num_blocks, write_start_lba, 506 write_num_blocks))) { 507 SPDK_ERRLOG("end of media\n"); 508 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 509 rsp->status.sc = SPDK_NVME_SC_LBA_OUT_OF_RANGE; 510 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 511 } 512 513 if (spdk_unlikely(write_num_blocks * block_size > write_req->length)) { 514 SPDK_ERRLOG("Write NLB %" PRIu64 " * block size %" PRIu32 " > SGL length %" PRIu32 "\n", 515 write_num_blocks, block_size, write_req->length); 516 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 517 rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID; 518 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 519 } 520 521 rc = spdk_bdev_comparev_and_writev_blocks(desc, ch, cmp_req->iov, cmp_req->iovcnt, write_req->iov, 522 write_req->iovcnt, write_start_lba, write_num_blocks, nvmf_bdev_ctrlr_complete_cmd, write_req); 523 if (spdk_unlikely(rc)) { 524 if (rc == -ENOMEM) { 525 nvmf_bdev_ctrl_queue_io(cmp_req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, cmp_req); 526 nvmf_bdev_ctrl_queue_io(write_req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, write_req); 527 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 528 } 529 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 530 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 531 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 532 } 533 534 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 535 } 536 537 int 538 nvmf_bdev_ctrlr_write_zeroes_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 539 struct spdk_io_channel *ch, struct spdk_nvmf_request *req) 540 { 541 uint64_t bdev_num_blocks = spdk_bdev_get_num_blocks(bdev); 542 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 543 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 544 uint64_t max_write_zeroes_size = req->qpair->ctrlr->subsys->max_write_zeroes_size_kib; 545 uint64_t start_lba; 546 uint64_t num_blocks; 547 int rc; 548 549 nvmf_bdev_ctrlr_get_rw_params(cmd, &start_lba, &num_blocks); 550 if (spdk_unlikely(max_write_zeroes_size > 0 && 551 num_blocks > (max_write_zeroes_size << 10) / spdk_bdev_get_block_size(bdev))) { 552 SPDK_ERRLOG("invalid write zeroes size, should not exceed %" PRIu64 "Kib\n", max_write_zeroes_size); 553 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 554 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 555 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 556 } 557 558 if (spdk_unlikely(!nvmf_bdev_ctrlr_lba_in_range(bdev_num_blocks, start_lba, num_blocks))) { 559 SPDK_ERRLOG("end of media\n"); 560 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 561 rsp->status.sc = SPDK_NVME_SC_LBA_OUT_OF_RANGE; 562 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 563 } 564 565 if (spdk_unlikely(cmd->cdw12_bits.write_zeroes.deac)) { 566 SPDK_ERRLOG("Write Zeroes Deallocate is not supported\n"); 567 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 568 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 569 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 570 } 571 572 rc = spdk_bdev_write_zeroes_blocks(desc, ch, start_lba, num_blocks, 573 nvmf_bdev_ctrlr_complete_cmd, req); 574 if (spdk_unlikely(rc)) { 575 if (rc == -ENOMEM) { 576 nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req); 577 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 578 } 579 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 580 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 581 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 582 } 583 584 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 585 } 586 587 int 588 nvmf_bdev_ctrlr_flush_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 589 struct spdk_io_channel *ch, struct spdk_nvmf_request *req) 590 { 591 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 592 int rc; 593 594 /* As for NVMeoF controller, SPDK always set volatile write 595 * cache bit to 1, return success for those block devices 596 * which can't support FLUSH command. 597 */ 598 if (!spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_FLUSH)) { 599 response->status.sct = SPDK_NVME_SCT_GENERIC; 600 response->status.sc = SPDK_NVME_SC_SUCCESS; 601 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 602 } 603 604 rc = spdk_bdev_flush_blocks(desc, ch, 0, spdk_bdev_get_num_blocks(bdev), 605 nvmf_bdev_ctrlr_complete_cmd, req); 606 if (spdk_unlikely(rc)) { 607 if (rc == -ENOMEM) { 608 nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req); 609 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 610 } 611 response->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 612 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 613 } 614 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 615 } 616 617 struct nvmf_bdev_ctrlr_unmap { 618 struct spdk_nvmf_request *req; 619 uint32_t count; 620 struct spdk_bdev_desc *desc; 621 struct spdk_bdev *bdev; 622 struct spdk_io_channel *ch; 623 uint32_t range_index; 624 }; 625 626 static void 627 nvmf_bdev_ctrlr_unmap_cpl(struct spdk_bdev_io *bdev_io, bool success, 628 void *cb_arg) 629 { 630 struct nvmf_bdev_ctrlr_unmap *unmap_ctx = cb_arg; 631 struct spdk_nvmf_request *req = unmap_ctx->req; 632 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 633 int sc, sct; 634 uint32_t cdw0; 635 636 unmap_ctx->count--; 637 638 if (response->status.sct == SPDK_NVME_SCT_GENERIC && 639 response->status.sc == SPDK_NVME_SC_SUCCESS) { 640 spdk_bdev_io_get_nvme_status(bdev_io, &cdw0, &sct, &sc); 641 response->cdw0 = cdw0; 642 response->status.sc = sc; 643 response->status.sct = sct; 644 } 645 646 if (unmap_ctx->count == 0) { 647 spdk_nvmf_request_complete(req); 648 free(unmap_ctx); 649 } 650 spdk_bdev_free_io(bdev_io); 651 } 652 653 static int nvmf_bdev_ctrlr_unmap(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 654 struct spdk_io_channel *ch, struct spdk_nvmf_request *req, 655 struct nvmf_bdev_ctrlr_unmap *unmap_ctx); 656 static void 657 nvmf_bdev_ctrlr_unmap_resubmit(void *arg) 658 { 659 struct nvmf_bdev_ctrlr_unmap *unmap_ctx = arg; 660 struct spdk_nvmf_request *req = unmap_ctx->req; 661 struct spdk_bdev_desc *desc = unmap_ctx->desc; 662 struct spdk_bdev *bdev = unmap_ctx->bdev; 663 struct spdk_io_channel *ch = unmap_ctx->ch; 664 665 nvmf_bdev_ctrlr_unmap(bdev, desc, ch, req, unmap_ctx); 666 } 667 668 static int 669 nvmf_bdev_ctrlr_unmap(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 670 struct spdk_io_channel *ch, struct spdk_nvmf_request *req, 671 struct nvmf_bdev_ctrlr_unmap *unmap_ctx) 672 { 673 uint16_t nr, i; 674 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 675 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 676 uint64_t max_discard_size = req->qpair->ctrlr->subsys->max_discard_size_kib; 677 uint32_t block_size = spdk_bdev_get_block_size(bdev); 678 struct spdk_iov_xfer ix; 679 uint64_t lba; 680 uint32_t lba_count; 681 int rc; 682 683 nr = cmd->cdw10_bits.dsm.nr + 1; 684 if (nr * sizeof(struct spdk_nvme_dsm_range) > req->length) { 685 SPDK_ERRLOG("Dataset Management number of ranges > SGL length\n"); 686 response->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID; 687 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 688 } 689 690 if (unmap_ctx == NULL) { 691 unmap_ctx = calloc(1, sizeof(*unmap_ctx)); 692 if (!unmap_ctx) { 693 response->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 694 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 695 } 696 697 unmap_ctx->req = req; 698 unmap_ctx->desc = desc; 699 unmap_ctx->ch = ch; 700 unmap_ctx->bdev = bdev; 701 702 response->status.sct = SPDK_NVME_SCT_GENERIC; 703 response->status.sc = SPDK_NVME_SC_SUCCESS; 704 } else { 705 unmap_ctx->count--; /* dequeued */ 706 } 707 708 spdk_iov_xfer_init(&ix, req->iov, req->iovcnt); 709 710 for (i = unmap_ctx->range_index; i < nr; i++) { 711 struct spdk_nvme_dsm_range dsm_range = { 0 }; 712 713 spdk_iov_xfer_to_buf(&ix, &dsm_range, sizeof(dsm_range)); 714 715 lba = dsm_range.starting_lba; 716 lba_count = dsm_range.length; 717 if (max_discard_size > 0 && lba_count > (max_discard_size << 10) / block_size) { 718 SPDK_ERRLOG("invalid unmap size, should not exceed %" PRIu64 "Kib\n", max_discard_size); 719 response->status.sct = SPDK_NVME_SCT_GENERIC; 720 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 721 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 722 } 723 724 unmap_ctx->count++; 725 726 rc = spdk_bdev_unmap_blocks(desc, ch, lba, lba_count, 727 nvmf_bdev_ctrlr_unmap_cpl, unmap_ctx); 728 if (rc) { 729 if (rc == -ENOMEM) { 730 nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_bdev_ctrlr_unmap_resubmit, unmap_ctx); 731 /* Unmap was not yet submitted to bdev */ 732 /* unmap_ctx->count will be decremented when the request is dequeued */ 733 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 734 } 735 response->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 736 unmap_ctx->count--; 737 /* We can't return here - we may have to wait for any other 738 * unmaps already sent to complete */ 739 break; 740 } 741 unmap_ctx->range_index++; 742 } 743 744 if (unmap_ctx->count == 0) { 745 free(unmap_ctx); 746 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 747 } 748 749 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 750 } 751 752 int 753 nvmf_bdev_ctrlr_dsm_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 754 struct spdk_io_channel *ch, struct spdk_nvmf_request *req) 755 { 756 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 757 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 758 759 if (cmd->cdw11_bits.dsm.ad) { 760 return nvmf_bdev_ctrlr_unmap(bdev, desc, ch, req, NULL); 761 } 762 763 response->status.sct = SPDK_NVME_SCT_GENERIC; 764 response->status.sc = SPDK_NVME_SC_SUCCESS; 765 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 766 } 767 768 int 769 nvmf_bdev_ctrlr_copy_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 770 struct spdk_io_channel *ch, struct spdk_nvmf_request *req) 771 { 772 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 773 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 774 uint64_t sdlba = ((uint64_t)cmd->cdw11 << 32) + cmd->cdw10; 775 struct spdk_nvme_scc_source_range range = { 0 }; 776 struct spdk_iov_xfer ix; 777 int rc; 778 779 SPDK_DEBUGLOG(nvmf, "Copy command: SDLBA %lu, NR %u, desc format %u, PRINFOR %u, " 780 "DTYPE %u, STCW %u, PRINFOW %u, FUA %u, LR %u\n", 781 sdlba, 782 cmd->cdw12_bits.copy.nr, 783 cmd->cdw12_bits.copy.df, 784 cmd->cdw12_bits.copy.prinfor, 785 cmd->cdw12_bits.copy.dtype, 786 cmd->cdw12_bits.copy.stcw, 787 cmd->cdw12_bits.copy.prinfow, 788 cmd->cdw12_bits.copy.fua, 789 cmd->cdw12_bits.copy.lr); 790 791 if (spdk_unlikely(req->length != (cmd->cdw12_bits.copy.nr + 1) * 792 sizeof(struct spdk_nvme_scc_source_range))) { 793 response->status.sct = SPDK_NVME_SCT_GENERIC; 794 response->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID; 795 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 796 } 797 798 /* 799 * We support only one source range, and rely on this with the xfer 800 * below. 801 */ 802 if (cmd->cdw12_bits.copy.nr > 0) { 803 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 804 response->status.sc = SPDK_NVME_SC_CMD_SIZE_LIMIT_SIZE_EXCEEDED; 805 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 806 } 807 808 if (cmd->cdw12_bits.copy.df != 0) { 809 response->status.sct = SPDK_NVME_SCT_GENERIC; 810 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 811 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 812 } 813 814 spdk_iov_xfer_init(&ix, req->iov, req->iovcnt); 815 spdk_iov_xfer_to_buf(&ix, &range, sizeof(range)); 816 817 rc = spdk_bdev_copy_blocks(desc, ch, sdlba, range.slba, range.nlb + 1, 818 nvmf_bdev_ctrlr_complete_cmd, req); 819 if (spdk_unlikely(rc)) { 820 if (rc == -ENOMEM) { 821 nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req); 822 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 823 } 824 825 response->status.sct = SPDK_NVME_SCT_GENERIC; 826 response->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 827 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 828 } 829 830 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 831 } 832 833 int 834 nvmf_bdev_ctrlr_nvme_passthru_io(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 835 struct spdk_io_channel *ch, struct spdk_nvmf_request *req) 836 { 837 int rc; 838 839 rc = spdk_bdev_nvme_iov_passthru_md(desc, ch, &req->cmd->nvme_cmd, req->iov, req->iovcnt, 840 req->length, NULL, 0, nvmf_bdev_ctrlr_complete_cmd, req); 841 842 if (spdk_unlikely(rc)) { 843 if (rc == -ENOMEM) { 844 nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req); 845 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 846 } 847 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 848 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INVALID_OPCODE; 849 req->rsp->nvme_cpl.status.dnr = 1; 850 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 851 } 852 853 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 854 } 855 856 int 857 spdk_nvmf_bdev_ctrlr_nvme_passthru_admin(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 858 struct spdk_io_channel *ch, struct spdk_nvmf_request *req, 859 spdk_nvmf_nvme_passthru_cmd_cb cb_fn) 860 { 861 int rc; 862 863 if (spdk_unlikely(req->iovcnt > 1)) { 864 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 865 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 866 req->rsp->nvme_cpl.status.dnr = 1; 867 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 868 } 869 870 req->cmd_cb_fn = cb_fn; 871 872 rc = spdk_bdev_nvme_admin_passthru(desc, ch, &req->cmd->nvme_cmd, req->iov[0].iov_base, req->length, 873 nvmf_bdev_ctrlr_complete_admin_cmd, req); 874 if (spdk_unlikely(rc)) { 875 if (rc == -ENOMEM) { 876 nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_admin_cmd_resubmit, req); 877 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 878 } 879 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 880 if (rc == -ENOTSUP) { 881 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INVALID_OPCODE; 882 } else { 883 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 884 } 885 886 req->rsp->nvme_cpl.status.dnr = 1; 887 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 888 } 889 890 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 891 } 892 893 static void 894 nvmf_bdev_ctrlr_complete_abort_cmd(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 895 { 896 struct spdk_nvmf_request *req = cb_arg; 897 898 if (success) { 899 req->rsp->nvme_cpl.cdw0 &= ~1U; 900 } 901 902 spdk_nvmf_request_complete(req); 903 spdk_bdev_free_io(bdev_io); 904 } 905 906 int 907 spdk_nvmf_bdev_ctrlr_abort_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 908 struct spdk_io_channel *ch, struct spdk_nvmf_request *req, 909 struct spdk_nvmf_request *req_to_abort) 910 { 911 int rc; 912 913 assert((req->rsp->nvme_cpl.cdw0 & 1U) != 0); 914 915 rc = spdk_bdev_abort(desc, ch, req_to_abort, nvmf_bdev_ctrlr_complete_abort_cmd, req); 916 if (spdk_likely(rc == 0)) { 917 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 918 } else if (rc == -ENOMEM) { 919 nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_admin_cmd_resubmit, req); 920 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 921 } else { 922 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 923 } 924 } 925 926 bool 927 nvmf_bdev_ctrlr_get_dif_ctx(struct spdk_bdev *bdev, struct spdk_nvme_cmd *cmd, 928 struct spdk_dif_ctx *dif_ctx) 929 { 930 uint32_t init_ref_tag, dif_check_flags = 0; 931 int rc; 932 struct spdk_dif_ctx_init_ext_opts dif_opts; 933 934 if (spdk_bdev_get_md_size(bdev) == 0) { 935 return false; 936 } 937 938 /* Initial Reference Tag is the lower 32 bits of the start LBA. */ 939 init_ref_tag = (uint32_t)from_le64(&cmd->cdw10); 940 941 if (spdk_bdev_is_dif_check_enabled(bdev, SPDK_DIF_CHECK_TYPE_REFTAG)) { 942 dif_check_flags |= SPDK_DIF_FLAGS_REFTAG_CHECK; 943 } 944 945 if (spdk_bdev_is_dif_check_enabled(bdev, SPDK_DIF_CHECK_TYPE_GUARD)) { 946 dif_check_flags |= SPDK_DIF_FLAGS_GUARD_CHECK; 947 } 948 949 dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format); 950 dif_opts.dif_pi_format = SPDK_DIF_PI_FORMAT_16; 951 rc = spdk_dif_ctx_init(dif_ctx, 952 spdk_bdev_get_block_size(bdev), 953 spdk_bdev_get_md_size(bdev), 954 spdk_bdev_is_md_interleaved(bdev), 955 spdk_bdev_is_dif_head_of_md(bdev), 956 spdk_bdev_get_dif_type(bdev), 957 dif_check_flags, 958 init_ref_tag, 0, 0, 0, 0, &dif_opts); 959 960 return (rc == 0) ? true : false; 961 } 962 963 static void 964 nvmf_bdev_ctrlr_zcopy_start_complete(struct spdk_bdev_io *bdev_io, bool success, 965 void *cb_arg) 966 { 967 struct spdk_nvmf_request *req = cb_arg; 968 struct iovec *iov; 969 int iovcnt = 0; 970 971 if (spdk_unlikely(!success)) { 972 int sc = 0, sct = 0; 973 uint32_t cdw0 = 0; 974 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 975 spdk_bdev_io_get_nvme_status(bdev_io, &cdw0, &sct, &sc); 976 977 response->cdw0 = cdw0; 978 response->status.sc = sc; 979 response->status.sct = sct; 980 981 spdk_bdev_free_io(bdev_io); 982 spdk_nvmf_request_complete(req); 983 return; 984 } 985 986 spdk_bdev_io_get_iovec(bdev_io, &iov, &iovcnt); 987 988 assert(iovcnt <= NVMF_REQ_MAX_BUFFERS); 989 assert(iovcnt > 0); 990 991 req->iovcnt = iovcnt; 992 993 assert(req->iov == iov); 994 995 req->zcopy_bdev_io = bdev_io; /* Preserve the bdev_io for the end zcopy */ 996 997 spdk_nvmf_request_complete(req); 998 /* Don't free the bdev_io here as it is needed for the END ZCOPY */ 999 } 1000 1001 int 1002 nvmf_bdev_ctrlr_zcopy_start(struct spdk_bdev *bdev, 1003 struct spdk_bdev_desc *desc, 1004 struct spdk_io_channel *ch, 1005 struct spdk_nvmf_request *req) 1006 { 1007 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1008 uint64_t bdev_num_blocks = spdk_bdev_get_num_blocks(bdev); 1009 uint32_t block_size = spdk_bdev_get_block_size(bdev); 1010 uint64_t start_lba; 1011 uint64_t num_blocks; 1012 int rc; 1013 1014 nvmf_bdev_ctrlr_get_rw_params(&req->cmd->nvme_cmd, &start_lba, &num_blocks); 1015 1016 if (spdk_unlikely(!nvmf_bdev_ctrlr_lba_in_range(bdev_num_blocks, start_lba, num_blocks))) { 1017 SPDK_ERRLOG("end of media\n"); 1018 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 1019 rsp->status.sc = SPDK_NVME_SC_LBA_OUT_OF_RANGE; 1020 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1021 } 1022 1023 if (spdk_unlikely(num_blocks * block_size > req->length)) { 1024 SPDK_ERRLOG("Read NLB %" PRIu64 " * block size %" PRIu32 " > SGL length %" PRIu32 "\n", 1025 num_blocks, block_size, req->length); 1026 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 1027 rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID; 1028 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1029 } 1030 1031 bool populate = (req->cmd->nvme_cmd.opc == SPDK_NVME_OPC_READ) ? true : false; 1032 1033 rc = spdk_bdev_zcopy_start(desc, ch, req->iov, req->iovcnt, start_lba, 1034 num_blocks, populate, nvmf_bdev_ctrlr_zcopy_start_complete, req); 1035 if (spdk_unlikely(rc != 0)) { 1036 if (rc == -ENOMEM) { 1037 nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req); 1038 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 1039 } 1040 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 1041 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 1042 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1043 } 1044 1045 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 1046 } 1047 1048 static void 1049 nvmf_bdev_ctrlr_zcopy_end_complete(struct spdk_bdev_io *bdev_io, bool success, 1050 void *cb_arg) 1051 { 1052 struct spdk_nvmf_request *req = cb_arg; 1053 1054 if (spdk_unlikely(!success)) { 1055 int sc = 0, sct = 0; 1056 uint32_t cdw0 = 0; 1057 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 1058 spdk_bdev_io_get_nvme_status(bdev_io, &cdw0, &sct, &sc); 1059 1060 response->cdw0 = cdw0; 1061 response->status.sc = sc; 1062 response->status.sct = sct; 1063 } 1064 1065 spdk_bdev_free_io(bdev_io); 1066 req->zcopy_bdev_io = NULL; 1067 spdk_nvmf_request_complete(req); 1068 } 1069 1070 void 1071 nvmf_bdev_ctrlr_zcopy_end(struct spdk_nvmf_request *req, bool commit) 1072 { 1073 int rc __attribute__((unused)); 1074 1075 rc = spdk_bdev_zcopy_end(req->zcopy_bdev_io, commit, nvmf_bdev_ctrlr_zcopy_end_complete, req); 1076 1077 /* The only way spdk_bdev_zcopy_end() can fail is if we pass a bdev_io type that isn't ZCOPY */ 1078 assert(rc == 0); 1079 } 1080