1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2017 Intel Corporation. All rights reserved. 3 * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #include "nvmf_internal.h" 10 11 #include "spdk/bdev.h" 12 #include "spdk/endian.h" 13 #include "spdk/thread.h" 14 #include "spdk/likely.h" 15 #include "spdk/nvme.h" 16 #include "spdk/nvmf_cmd.h" 17 #include "spdk/nvmf_spec.h" 18 #include "spdk/trace.h" 19 #include "spdk/scsi_spec.h" 20 #include "spdk/string.h" 21 #include "spdk/util.h" 22 23 #include "spdk/log.h" 24 25 static bool 26 nvmf_subsystem_bdev_io_type_supported(struct spdk_nvmf_subsystem *subsystem, 27 enum spdk_bdev_io_type io_type) 28 { 29 struct spdk_nvmf_ns *ns; 30 31 for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL; 32 ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) { 33 if (ns->bdev == NULL) { 34 continue; 35 } 36 37 if (!spdk_bdev_io_type_supported(ns->bdev, io_type)) { 38 SPDK_DEBUGLOG(nvmf, 39 "Subsystem %s namespace %u (%s) does not support io_type %d\n", 40 spdk_nvmf_subsystem_get_nqn(subsystem), 41 ns->opts.nsid, spdk_bdev_get_name(ns->bdev), (int)io_type); 42 return false; 43 } 44 } 45 46 SPDK_DEBUGLOG(nvmf, "All devices in Subsystem %s support io_type %d\n", 47 spdk_nvmf_subsystem_get_nqn(subsystem), (int)io_type); 48 return true; 49 } 50 51 bool 52 nvmf_ctrlr_dsm_supported(struct spdk_nvmf_ctrlr *ctrlr) 53 { 54 return nvmf_subsystem_bdev_io_type_supported(ctrlr->subsys, SPDK_BDEV_IO_TYPE_UNMAP); 55 } 56 57 bool 58 nvmf_ctrlr_write_zeroes_supported(struct spdk_nvmf_ctrlr *ctrlr) 59 { 60 return nvmf_subsystem_bdev_io_type_supported(ctrlr->subsys, SPDK_BDEV_IO_TYPE_WRITE_ZEROES); 61 } 62 63 bool 64 nvmf_ctrlr_copy_supported(struct spdk_nvmf_ctrlr *ctrlr) 65 { 66 return nvmf_subsystem_bdev_io_type_supported(ctrlr->subsys, SPDK_BDEV_IO_TYPE_COPY); 67 } 68 69 static void 70 nvmf_bdev_ctrlr_complete_cmd(struct spdk_bdev_io *bdev_io, bool success, 71 void *cb_arg) 72 { 73 struct spdk_nvmf_request *req = cb_arg; 74 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 75 int sc = 0, sct = 0; 76 uint32_t cdw0 = 0; 77 78 if (spdk_unlikely(req->first_fused)) { 79 struct spdk_nvmf_request *first_req = req->first_fused_req; 80 struct spdk_nvme_cpl *first_response = &first_req->rsp->nvme_cpl; 81 int first_sc = 0, first_sct = 0; 82 83 /* get status for both operations */ 84 spdk_bdev_io_get_nvme_fused_status(bdev_io, &cdw0, &first_sct, &first_sc, &sct, &sc); 85 first_response->cdw0 = cdw0; 86 first_response->status.sc = first_sc; 87 first_response->status.sct = first_sct; 88 89 /* first request should be completed */ 90 spdk_nvmf_request_complete(first_req); 91 req->first_fused_req = NULL; 92 req->first_fused = false; 93 } else { 94 spdk_bdev_io_get_nvme_status(bdev_io, &cdw0, &sct, &sc); 95 } 96 97 response->cdw0 = cdw0; 98 response->status.sc = sc; 99 response->status.sct = sct; 100 101 spdk_nvmf_request_complete(req); 102 spdk_bdev_free_io(bdev_io); 103 } 104 105 static void 106 nvmf_bdev_ctrlr_complete_admin_cmd(struct spdk_bdev_io *bdev_io, bool success, 107 void *cb_arg) 108 { 109 struct spdk_nvmf_request *req = cb_arg; 110 111 if (req->cmd_cb_fn) { 112 req->cmd_cb_fn(req); 113 } 114 115 nvmf_bdev_ctrlr_complete_cmd(bdev_io, success, req); 116 } 117 118 void 119 nvmf_bdev_ctrlr_identify_ns(struct spdk_nvmf_ns *ns, struct spdk_nvme_ns_data *nsdata, 120 bool dif_insert_or_strip) 121 { 122 struct spdk_bdev *bdev = ns->bdev; 123 struct spdk_bdev_desc *desc = ns->desc; 124 uint64_t num_blocks; 125 uint32_t phys_blocklen; 126 uint32_t max_copy; 127 128 num_blocks = spdk_bdev_get_num_blocks(bdev); 129 130 nsdata->nsze = num_blocks; 131 nsdata->ncap = num_blocks; 132 nsdata->nuse = num_blocks; 133 nsdata->nlbaf = 0; 134 nsdata->flbas.format = 0; 135 nsdata->flbas.msb_format = 0; 136 nsdata->nacwu = spdk_bdev_get_acwu(bdev) - 1; /* nacwu is 0-based */ 137 if (!dif_insert_or_strip) { 138 nsdata->lbaf[0].ms = spdk_bdev_desc_get_md_size(desc); 139 nsdata->lbaf[0].lbads = spdk_u32log2(spdk_bdev_desc_get_block_size(desc)); 140 if (nsdata->lbaf[0].ms != 0) { 141 nsdata->flbas.extended = 1; 142 nsdata->mc.extended = 1; 143 nsdata->mc.pointer = 0; 144 nsdata->dps.md_start = spdk_bdev_desc_is_dif_head_of_md(desc); 145 /* NVMf library doesn't process PRACT and PRCHK flags, we 146 * leave the use of extended LBA buffer to users. 147 */ 148 nsdata->dps.pit = SPDK_NVME_FMT_NVM_PROTECTION_DISABLE; 149 } 150 } else { 151 nsdata->lbaf[0].ms = 0; 152 nsdata->lbaf[0].lbads = spdk_u32log2(spdk_bdev_get_data_block_size(bdev)); 153 } 154 155 phys_blocklen = spdk_bdev_get_physical_block_size(bdev); 156 assert(phys_blocklen > 0); 157 /* Linux driver uses min(nawupf, npwg) to set physical_block_size */ 158 nsdata->nsfeat.optperf = 1; 159 nsdata->nsfeat.ns_atomic_write_unit = 1; 160 nsdata->npwg = (phys_blocklen >> nsdata->lbaf[0].lbads) - 1; 161 nsdata->nawupf = nsdata->npwg; 162 nsdata->npwa = nsdata->npwg; 163 nsdata->npdg = nsdata->npwg; 164 nsdata->npda = nsdata->npwg; 165 166 if (spdk_bdev_get_write_unit_size(bdev) == 1) { 167 nsdata->noiob = spdk_bdev_get_optimal_io_boundary(bdev); 168 } 169 nsdata->nmic.can_share = 1; 170 if (nvmf_ns_is_ptpl_capable(ns)) { 171 nsdata->nsrescap.rescap.persist = 1; 172 } 173 nsdata->nsrescap.rescap.write_exclusive = 1; 174 nsdata->nsrescap.rescap.exclusive_access = 1; 175 nsdata->nsrescap.rescap.write_exclusive_reg_only = 1; 176 nsdata->nsrescap.rescap.exclusive_access_reg_only = 1; 177 nsdata->nsrescap.rescap.write_exclusive_all_reg = 1; 178 nsdata->nsrescap.rescap.exclusive_access_all_reg = 1; 179 nsdata->nsrescap.rescap.ignore_existing_key = 1; 180 181 SPDK_STATIC_ASSERT(sizeof(nsdata->nguid) == sizeof(ns->opts.nguid), "size mismatch"); 182 memcpy(nsdata->nguid, ns->opts.nguid, sizeof(nsdata->nguid)); 183 184 SPDK_STATIC_ASSERT(sizeof(nsdata->eui64) == sizeof(ns->opts.eui64), "size mismatch"); 185 memcpy(&nsdata->eui64, ns->opts.eui64, sizeof(nsdata->eui64)); 186 187 /* For now we support just one source range for copy command */ 188 nsdata->msrc = 0; 189 190 max_copy = spdk_bdev_get_max_copy(bdev); 191 if (max_copy == 0 || max_copy > UINT16_MAX) { 192 /* Zero means copy size is unlimited */ 193 nsdata->mcl = UINT16_MAX; 194 nsdata->mssrl = UINT16_MAX; 195 } else { 196 nsdata->mcl = max_copy; 197 nsdata->mssrl = max_copy; 198 } 199 } 200 201 void 202 nvmf_bdev_ctrlr_identify_iocs_nvm(struct spdk_nvmf_ns *ns, 203 struct spdk_nvme_nvm_ns_data *nsdata_nvm) 204 { 205 struct spdk_bdev_desc *desc = ns->desc; 206 uint8_t _16bpists; 207 uint32_t sts, pif; 208 209 if (spdk_bdev_desc_get_dif_type(desc) == SPDK_DIF_DISABLE) { 210 return; 211 } 212 213 pif = spdk_bdev_desc_get_dif_pi_format(desc); 214 215 /* 216 * 16BPISTS shall be 1 for 32/64b Guard PI. 217 * STCRS shall be 1 if 16BPISTS is 1. 218 * 16 is the minimum value of STS for 32b Guard PI. 219 */ 220 switch (pif) { 221 case SPDK_DIF_PI_FORMAT_16: 222 _16bpists = 0; 223 sts = 0; 224 break; 225 case SPDK_DIF_PI_FORMAT_32: 226 _16bpists = 1; 227 sts = 16; 228 break; 229 case SPDK_DIF_PI_FORMAT_64: 230 _16bpists = 1; 231 sts = 0; 232 break; 233 default: 234 SPDK_WARNLOG("PI format %u is not supported\n", pif); 235 return; 236 } 237 238 /* For 16b Guard PI, Storage Tag is not available because we set STS to 0. 239 * In this case, we do not have to set 16BPISTM to 1. For simplicity, 240 * set 16BPISTM to 0 and set LBSTM to all zeroes. 241 * 242 * We will revisit here when we find any OS uses Storage Tag. 243 */ 244 nsdata_nvm->lbstm = 0; 245 nsdata_nvm->pic._16bpistm = 0; 246 247 nsdata_nvm->pic._16bpists = _16bpists; 248 nsdata_nvm->pic.stcrs = 0; 249 nsdata_nvm->elbaf[0].sts = sts; 250 nsdata_nvm->elbaf[0].pif = pif; 251 } 252 253 static void 254 nvmf_bdev_ctrlr_get_rw_params(const struct spdk_nvme_cmd *cmd, uint64_t *start_lba, 255 uint64_t *num_blocks) 256 { 257 /* SLBA: CDW10 and CDW11 */ 258 *start_lba = from_le64(&cmd->cdw10); 259 260 /* NLB: CDW12 bits 15:00, 0's based */ 261 *num_blocks = (from_le32(&cmd->cdw12) & 0xFFFFu) + 1; 262 } 263 264 static void 265 nvmf_bdev_ctrlr_get_rw_ext_params(const struct spdk_nvme_cmd *cmd, 266 struct spdk_bdev_ext_io_opts *opts) 267 { 268 /* Get CDW12 values */ 269 opts->nvme_cdw12.raw = from_le32(&cmd->cdw12); 270 271 /* Get CDW13 values */ 272 opts->nvme_cdw13.raw = from_le32(&cmd->cdw13); 273 } 274 275 static bool 276 nvmf_bdev_ctrlr_lba_in_range(uint64_t bdev_num_blocks, uint64_t io_start_lba, 277 uint64_t io_num_blocks) 278 { 279 if (io_start_lba + io_num_blocks > bdev_num_blocks || 280 io_start_lba + io_num_blocks < io_start_lba) { 281 return false; 282 } 283 284 return true; 285 } 286 287 static void 288 nvmf_ctrlr_process_io_cmd_resubmit(void *arg) 289 { 290 struct spdk_nvmf_request *req = arg; 291 int rc; 292 293 rc = nvmf_ctrlr_process_io_cmd(req); 294 if (rc == SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) { 295 spdk_nvmf_request_complete(req); 296 } 297 } 298 299 static void 300 nvmf_ctrlr_process_admin_cmd_resubmit(void *arg) 301 { 302 struct spdk_nvmf_request *req = arg; 303 int rc; 304 305 rc = nvmf_ctrlr_process_admin_cmd(req); 306 if (rc == SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) { 307 spdk_nvmf_request_complete(req); 308 } 309 } 310 311 static void 312 nvmf_bdev_ctrl_queue_io(struct spdk_nvmf_request *req, struct spdk_bdev *bdev, 313 struct spdk_io_channel *ch, spdk_bdev_io_wait_cb cb_fn, void *cb_arg) 314 { 315 int rc; 316 317 req->bdev_io_wait.bdev = bdev; 318 req->bdev_io_wait.cb_fn = cb_fn; 319 req->bdev_io_wait.cb_arg = cb_arg; 320 321 rc = spdk_bdev_queue_io_wait(bdev, ch, &req->bdev_io_wait); 322 if (rc != 0) { 323 assert(false); 324 } 325 req->qpair->group->stat.pending_bdev_io++; 326 } 327 328 bool 329 nvmf_bdev_zcopy_enabled(struct spdk_bdev *bdev) 330 { 331 return spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_ZCOPY); 332 } 333 334 int 335 nvmf_bdev_ctrlr_read_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 336 struct spdk_io_channel *ch, struct spdk_nvmf_request *req) 337 { 338 struct spdk_bdev_ext_io_opts opts = { 339 .size = SPDK_SIZEOF(&opts, accel_sequence), 340 .memory_domain = req->memory_domain, 341 .memory_domain_ctx = req->memory_domain_ctx, 342 .accel_sequence = req->accel_sequence, 343 }; 344 uint64_t bdev_num_blocks = spdk_bdev_get_num_blocks(bdev); 345 uint32_t block_size = spdk_bdev_desc_get_block_size(desc); 346 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 347 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 348 uint64_t start_lba; 349 uint64_t num_blocks; 350 int rc; 351 352 nvmf_bdev_ctrlr_get_rw_params(cmd, &start_lba, &num_blocks); 353 354 if (spdk_unlikely(!nvmf_bdev_ctrlr_lba_in_range(bdev_num_blocks, start_lba, num_blocks))) { 355 SPDK_ERRLOG("end of media\n"); 356 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 357 rsp->status.sc = SPDK_NVME_SC_LBA_OUT_OF_RANGE; 358 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 359 } 360 361 if (spdk_unlikely(num_blocks * block_size > req->length)) { 362 SPDK_ERRLOG("Read NLB %" PRIu64 " * block size %" PRIu32 " > SGL length %" PRIu32 "\n", 363 num_blocks, block_size, req->length); 364 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 365 rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID; 366 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 367 } 368 369 assert(!spdk_nvmf_request_using_zcopy(req)); 370 371 rc = spdk_bdev_readv_blocks_ext(desc, ch, req->iov, req->iovcnt, start_lba, num_blocks, 372 nvmf_bdev_ctrlr_complete_cmd, req, &opts); 373 if (spdk_unlikely(rc)) { 374 if (rc == -ENOMEM) { 375 nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req); 376 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 377 } 378 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 379 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 380 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 381 } 382 383 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 384 } 385 386 int 387 nvmf_bdev_ctrlr_write_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 388 struct spdk_io_channel *ch, struct spdk_nvmf_request *req) 389 { 390 struct spdk_bdev_ext_io_opts opts = { 391 .size = SPDK_SIZEOF(&opts, nvme_cdw13), 392 .memory_domain = req->memory_domain, 393 .memory_domain_ctx = req->memory_domain_ctx, 394 .accel_sequence = req->accel_sequence, 395 }; 396 uint64_t bdev_num_blocks = spdk_bdev_get_num_blocks(bdev); 397 uint32_t block_size = spdk_bdev_desc_get_block_size(desc); 398 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 399 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 400 uint64_t start_lba; 401 uint64_t num_blocks; 402 int rc; 403 404 nvmf_bdev_ctrlr_get_rw_params(cmd, &start_lba, &num_blocks); 405 nvmf_bdev_ctrlr_get_rw_ext_params(cmd, &opts); 406 407 if (spdk_unlikely(!nvmf_bdev_ctrlr_lba_in_range(bdev_num_blocks, start_lba, num_blocks))) { 408 SPDK_ERRLOG("end of media\n"); 409 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 410 rsp->status.sc = SPDK_NVME_SC_LBA_OUT_OF_RANGE; 411 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 412 } 413 414 if (spdk_unlikely(num_blocks * block_size > req->length)) { 415 SPDK_ERRLOG("Write NLB %" PRIu64 " * block size %" PRIu32 " > SGL length %" PRIu32 "\n", 416 num_blocks, block_size, req->length); 417 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 418 rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID; 419 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 420 } 421 422 assert(!spdk_nvmf_request_using_zcopy(req)); 423 424 rc = spdk_bdev_writev_blocks_ext(desc, ch, req->iov, req->iovcnt, start_lba, num_blocks, 425 nvmf_bdev_ctrlr_complete_cmd, req, &opts); 426 if (spdk_unlikely(rc)) { 427 if (rc == -ENOMEM) { 428 nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req); 429 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 430 } 431 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 432 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 433 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 434 } 435 436 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 437 } 438 439 int 440 nvmf_bdev_ctrlr_compare_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 441 struct spdk_io_channel *ch, struct spdk_nvmf_request *req) 442 { 443 uint64_t bdev_num_blocks = spdk_bdev_get_num_blocks(bdev); 444 uint32_t block_size = spdk_bdev_desc_get_block_size(desc); 445 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 446 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 447 uint64_t start_lba; 448 uint64_t num_blocks; 449 int rc; 450 451 nvmf_bdev_ctrlr_get_rw_params(cmd, &start_lba, &num_blocks); 452 453 if (spdk_unlikely(!nvmf_bdev_ctrlr_lba_in_range(bdev_num_blocks, start_lba, num_blocks))) { 454 SPDK_ERRLOG("end of media\n"); 455 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 456 rsp->status.sc = SPDK_NVME_SC_LBA_OUT_OF_RANGE; 457 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 458 } 459 460 if (spdk_unlikely(num_blocks * block_size > req->length)) { 461 SPDK_ERRLOG("Compare NLB %" PRIu64 " * block size %" PRIu32 " > SGL length %" PRIu32 "\n", 462 num_blocks, block_size, req->length); 463 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 464 rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID; 465 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 466 } 467 468 rc = spdk_bdev_comparev_blocks(desc, ch, req->iov, req->iovcnt, start_lba, num_blocks, 469 nvmf_bdev_ctrlr_complete_cmd, req); 470 if (spdk_unlikely(rc)) { 471 if (rc == -ENOMEM) { 472 nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req); 473 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 474 } 475 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 476 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 477 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 478 } 479 480 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 481 } 482 483 int 484 nvmf_bdev_ctrlr_compare_and_write_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 485 struct spdk_io_channel *ch, struct spdk_nvmf_request *cmp_req, struct spdk_nvmf_request *write_req) 486 { 487 uint64_t bdev_num_blocks = spdk_bdev_get_num_blocks(bdev); 488 uint32_t block_size = spdk_bdev_desc_get_block_size(desc); 489 struct spdk_nvme_cmd *cmp_cmd = &cmp_req->cmd->nvme_cmd; 490 struct spdk_nvme_cmd *write_cmd = &write_req->cmd->nvme_cmd; 491 struct spdk_nvme_cpl *rsp = &write_req->rsp->nvme_cpl; 492 uint64_t write_start_lba, cmp_start_lba; 493 uint64_t write_num_blocks, cmp_num_blocks; 494 int rc; 495 496 nvmf_bdev_ctrlr_get_rw_params(cmp_cmd, &cmp_start_lba, &cmp_num_blocks); 497 nvmf_bdev_ctrlr_get_rw_params(write_cmd, &write_start_lba, &write_num_blocks); 498 499 if (spdk_unlikely(write_start_lba != cmp_start_lba || write_num_blocks != cmp_num_blocks)) { 500 SPDK_ERRLOG("Fused command start lba / num blocks mismatch\n"); 501 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 502 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 503 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 504 } 505 506 if (spdk_unlikely(!nvmf_bdev_ctrlr_lba_in_range(bdev_num_blocks, write_start_lba, 507 write_num_blocks))) { 508 SPDK_ERRLOG("end of media\n"); 509 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 510 rsp->status.sc = SPDK_NVME_SC_LBA_OUT_OF_RANGE; 511 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 512 } 513 514 if (spdk_unlikely(write_num_blocks * block_size > write_req->length)) { 515 SPDK_ERRLOG("Write NLB %" PRIu64 " * block size %" PRIu32 " > SGL length %" PRIu32 "\n", 516 write_num_blocks, block_size, write_req->length); 517 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 518 rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID; 519 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 520 } 521 522 rc = spdk_bdev_comparev_and_writev_blocks(desc, ch, cmp_req->iov, cmp_req->iovcnt, write_req->iov, 523 write_req->iovcnt, write_start_lba, write_num_blocks, nvmf_bdev_ctrlr_complete_cmd, write_req); 524 if (spdk_unlikely(rc)) { 525 if (rc == -ENOMEM) { 526 nvmf_bdev_ctrl_queue_io(cmp_req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, cmp_req); 527 nvmf_bdev_ctrl_queue_io(write_req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, write_req); 528 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 529 } 530 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 531 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 532 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 533 } 534 535 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 536 } 537 538 int 539 nvmf_bdev_ctrlr_write_zeroes_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 540 struct spdk_io_channel *ch, struct spdk_nvmf_request *req) 541 { 542 uint64_t bdev_num_blocks = spdk_bdev_get_num_blocks(bdev); 543 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 544 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 545 uint64_t max_write_zeroes_size = req->qpair->ctrlr->subsys->max_write_zeroes_size_kib; 546 uint64_t start_lba; 547 uint64_t num_blocks; 548 int rc; 549 550 nvmf_bdev_ctrlr_get_rw_params(cmd, &start_lba, &num_blocks); 551 if (spdk_unlikely(max_write_zeroes_size > 0 && 552 num_blocks > (max_write_zeroes_size << 10) / spdk_bdev_desc_get_block_size(desc))) { 553 SPDK_ERRLOG("invalid write zeroes size, should not exceed %" PRIu64 "Kib\n", max_write_zeroes_size); 554 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 555 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 556 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 557 } 558 559 if (spdk_unlikely(!nvmf_bdev_ctrlr_lba_in_range(bdev_num_blocks, start_lba, num_blocks))) { 560 SPDK_ERRLOG("end of media\n"); 561 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 562 rsp->status.sc = SPDK_NVME_SC_LBA_OUT_OF_RANGE; 563 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 564 } 565 566 if (spdk_unlikely(cmd->cdw12_bits.write_zeroes.deac)) { 567 SPDK_ERRLOG("Write Zeroes Deallocate is not supported\n"); 568 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 569 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 570 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 571 } 572 573 rc = spdk_bdev_write_zeroes_blocks(desc, ch, start_lba, num_blocks, 574 nvmf_bdev_ctrlr_complete_cmd, req); 575 if (spdk_unlikely(rc)) { 576 if (rc == -ENOMEM) { 577 nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req); 578 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 579 } 580 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 581 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 582 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 583 } 584 585 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 586 } 587 588 int 589 nvmf_bdev_ctrlr_flush_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 590 struct spdk_io_channel *ch, struct spdk_nvmf_request *req) 591 { 592 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 593 int rc; 594 595 /* As for NVMeoF controller, SPDK always set volatile write 596 * cache bit to 1, return success for those block devices 597 * which can't support FLUSH command. 598 */ 599 if (!spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_FLUSH)) { 600 response->status.sct = SPDK_NVME_SCT_GENERIC; 601 response->status.sc = SPDK_NVME_SC_SUCCESS; 602 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 603 } 604 605 rc = spdk_bdev_flush_blocks(desc, ch, 0, spdk_bdev_get_num_blocks(bdev), 606 nvmf_bdev_ctrlr_complete_cmd, req); 607 if (spdk_unlikely(rc)) { 608 if (rc == -ENOMEM) { 609 nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req); 610 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 611 } 612 response->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 613 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 614 } 615 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 616 } 617 618 struct nvmf_bdev_ctrlr_unmap { 619 struct spdk_nvmf_request *req; 620 uint32_t count; 621 struct spdk_bdev_desc *desc; 622 struct spdk_bdev *bdev; 623 struct spdk_io_channel *ch; 624 uint32_t range_index; 625 }; 626 627 static void 628 nvmf_bdev_ctrlr_unmap_cpl(struct spdk_bdev_io *bdev_io, bool success, 629 void *cb_arg) 630 { 631 struct nvmf_bdev_ctrlr_unmap *unmap_ctx = cb_arg; 632 struct spdk_nvmf_request *req = unmap_ctx->req; 633 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 634 int sc, sct; 635 uint32_t cdw0; 636 637 unmap_ctx->count--; 638 639 if (response->status.sct == SPDK_NVME_SCT_GENERIC && 640 response->status.sc == SPDK_NVME_SC_SUCCESS) { 641 spdk_bdev_io_get_nvme_status(bdev_io, &cdw0, &sct, &sc); 642 response->cdw0 = cdw0; 643 response->status.sc = sc; 644 response->status.sct = sct; 645 } 646 647 if (unmap_ctx->count == 0) { 648 spdk_nvmf_request_complete(req); 649 free(unmap_ctx); 650 } 651 spdk_bdev_free_io(bdev_io); 652 } 653 654 static int nvmf_bdev_ctrlr_unmap(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 655 struct spdk_io_channel *ch, struct spdk_nvmf_request *req, 656 struct nvmf_bdev_ctrlr_unmap *unmap_ctx); 657 static void 658 nvmf_bdev_ctrlr_unmap_resubmit(void *arg) 659 { 660 struct nvmf_bdev_ctrlr_unmap *unmap_ctx = arg; 661 struct spdk_nvmf_request *req = unmap_ctx->req; 662 struct spdk_bdev_desc *desc = unmap_ctx->desc; 663 struct spdk_bdev *bdev = unmap_ctx->bdev; 664 struct spdk_io_channel *ch = unmap_ctx->ch; 665 666 nvmf_bdev_ctrlr_unmap(bdev, desc, ch, req, unmap_ctx); 667 } 668 669 static int 670 nvmf_bdev_ctrlr_unmap(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 671 struct spdk_io_channel *ch, struct spdk_nvmf_request *req, 672 struct nvmf_bdev_ctrlr_unmap *unmap_ctx) 673 { 674 uint16_t nr, i; 675 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 676 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 677 uint64_t max_discard_size = req->qpair->ctrlr->subsys->max_discard_size_kib; 678 uint32_t block_size = spdk_bdev_desc_get_block_size(desc); 679 struct spdk_iov_xfer ix; 680 uint64_t lba; 681 uint32_t lba_count; 682 int rc; 683 684 nr = cmd->cdw10_bits.dsm.nr + 1; 685 if (nr * sizeof(struct spdk_nvme_dsm_range) > req->length) { 686 SPDK_ERRLOG("Dataset Management number of ranges > SGL length\n"); 687 response->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID; 688 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 689 } 690 691 if (unmap_ctx == NULL) { 692 unmap_ctx = calloc(1, sizeof(*unmap_ctx)); 693 if (!unmap_ctx) { 694 response->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 695 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 696 } 697 698 unmap_ctx->req = req; 699 unmap_ctx->desc = desc; 700 unmap_ctx->ch = ch; 701 unmap_ctx->bdev = bdev; 702 703 response->status.sct = SPDK_NVME_SCT_GENERIC; 704 response->status.sc = SPDK_NVME_SC_SUCCESS; 705 } else { 706 unmap_ctx->count--; /* dequeued */ 707 } 708 709 spdk_iov_xfer_init(&ix, req->iov, req->iovcnt); 710 711 for (i = unmap_ctx->range_index; i < nr; i++) { 712 struct spdk_nvme_dsm_range dsm_range = { 0 }; 713 714 spdk_iov_xfer_to_buf(&ix, &dsm_range, sizeof(dsm_range)); 715 716 lba = dsm_range.starting_lba; 717 lba_count = dsm_range.length; 718 if (max_discard_size > 0 && lba_count > (max_discard_size << 10) / block_size) { 719 SPDK_ERRLOG("invalid unmap size %" PRIu32 " blocks, should not exceed %" PRIu64 " blocks\n", 720 lba_count, max_discard_size << 1); 721 response->status.sct = SPDK_NVME_SCT_GENERIC; 722 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 723 break; 724 } 725 726 unmap_ctx->count++; 727 728 rc = spdk_bdev_unmap_blocks(desc, ch, lba, lba_count, 729 nvmf_bdev_ctrlr_unmap_cpl, unmap_ctx); 730 if (rc) { 731 if (rc == -ENOMEM) { 732 nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_bdev_ctrlr_unmap_resubmit, unmap_ctx); 733 /* Unmap was not yet submitted to bdev */ 734 /* unmap_ctx->count will be decremented when the request is dequeued */ 735 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 736 } 737 response->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 738 unmap_ctx->count--; 739 /* We can't return here - we may have to wait for any other 740 * unmaps already sent to complete */ 741 break; 742 } 743 unmap_ctx->range_index++; 744 } 745 746 if (unmap_ctx->count == 0) { 747 free(unmap_ctx); 748 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 749 } 750 751 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 752 } 753 754 int 755 nvmf_bdev_ctrlr_dsm_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 756 struct spdk_io_channel *ch, struct spdk_nvmf_request *req) 757 { 758 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 759 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 760 761 if (cmd->cdw11_bits.dsm.ad) { 762 return nvmf_bdev_ctrlr_unmap(bdev, desc, ch, req, NULL); 763 } 764 765 response->status.sct = SPDK_NVME_SCT_GENERIC; 766 response->status.sc = SPDK_NVME_SC_SUCCESS; 767 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 768 } 769 770 int 771 nvmf_bdev_ctrlr_copy_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 772 struct spdk_io_channel *ch, struct spdk_nvmf_request *req) 773 { 774 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 775 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 776 uint64_t sdlba = ((uint64_t)cmd->cdw11 << 32) + cmd->cdw10; 777 struct spdk_nvme_scc_source_range range = { 0 }; 778 struct spdk_iov_xfer ix; 779 int rc; 780 781 SPDK_DEBUGLOG(nvmf, "Copy command: SDLBA %lu, NR %u, desc format %u, PRINFOR %u, " 782 "DTYPE %u, STCW %u, PRINFOW %u, FUA %u, LR %u\n", 783 sdlba, 784 cmd->cdw12_bits.copy.nr, 785 cmd->cdw12_bits.copy.df, 786 cmd->cdw12_bits.copy.prinfor, 787 cmd->cdw12_bits.copy.dtype, 788 cmd->cdw12_bits.copy.stcw, 789 cmd->cdw12_bits.copy.prinfow, 790 cmd->cdw12_bits.copy.fua, 791 cmd->cdw12_bits.copy.lr); 792 793 if (spdk_unlikely(req->length != (cmd->cdw12_bits.copy.nr + 1) * 794 sizeof(struct spdk_nvme_scc_source_range))) { 795 response->status.sct = SPDK_NVME_SCT_GENERIC; 796 response->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID; 797 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 798 } 799 800 /* 801 * We support only one source range, and rely on this with the xfer 802 * below. 803 */ 804 if (cmd->cdw12_bits.copy.nr > 0) { 805 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 806 response->status.sc = SPDK_NVME_SC_CMD_SIZE_LIMIT_SIZE_EXCEEDED; 807 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 808 } 809 810 if (cmd->cdw12_bits.copy.df != 0) { 811 response->status.sct = SPDK_NVME_SCT_GENERIC; 812 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 813 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 814 } 815 816 spdk_iov_xfer_init(&ix, req->iov, req->iovcnt); 817 spdk_iov_xfer_to_buf(&ix, &range, sizeof(range)); 818 819 rc = spdk_bdev_copy_blocks(desc, ch, sdlba, range.slba, range.nlb + 1, 820 nvmf_bdev_ctrlr_complete_cmd, req); 821 if (spdk_unlikely(rc)) { 822 if (rc == -ENOMEM) { 823 nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req); 824 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 825 } 826 827 response->status.sct = SPDK_NVME_SCT_GENERIC; 828 response->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 829 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 830 } 831 832 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 833 } 834 835 int 836 nvmf_bdev_ctrlr_nvme_passthru_io(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 837 struct spdk_io_channel *ch, struct spdk_nvmf_request *req) 838 { 839 int rc; 840 841 rc = spdk_bdev_nvme_iov_passthru_md(desc, ch, &req->cmd->nvme_cmd, req->iov, req->iovcnt, 842 req->length, NULL, 0, nvmf_bdev_ctrlr_complete_cmd, req); 843 844 if (spdk_unlikely(rc)) { 845 if (rc == -ENOMEM) { 846 nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req); 847 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 848 } 849 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 850 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INVALID_OPCODE; 851 req->rsp->nvme_cpl.status.dnr = 1; 852 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 853 } 854 855 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 856 } 857 858 int 859 spdk_nvmf_bdev_ctrlr_nvme_passthru_admin(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 860 struct spdk_io_channel *ch, struct spdk_nvmf_request *req, 861 spdk_nvmf_nvme_passthru_cmd_cb cb_fn) 862 { 863 int rc; 864 865 if (spdk_unlikely(req->iovcnt > 1)) { 866 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 867 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 868 req->rsp->nvme_cpl.status.dnr = 1; 869 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 870 } 871 872 req->cmd_cb_fn = cb_fn; 873 874 rc = spdk_bdev_nvme_admin_passthru(desc, ch, &req->cmd->nvme_cmd, req->iov[0].iov_base, req->length, 875 nvmf_bdev_ctrlr_complete_admin_cmd, req); 876 if (spdk_unlikely(rc)) { 877 if (rc == -ENOMEM) { 878 nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_admin_cmd_resubmit, req); 879 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 880 } 881 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 882 if (rc == -ENOTSUP) { 883 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INVALID_OPCODE; 884 } else { 885 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 886 } 887 888 req->rsp->nvme_cpl.status.dnr = 1; 889 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 890 } 891 892 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 893 } 894 895 static void 896 nvmf_bdev_ctrlr_complete_abort_cmd(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 897 { 898 struct spdk_nvmf_request *req = cb_arg; 899 900 if (success) { 901 req->rsp->nvme_cpl.cdw0 &= ~1U; 902 } 903 904 spdk_nvmf_request_complete(req); 905 spdk_bdev_free_io(bdev_io); 906 } 907 908 int 909 spdk_nvmf_bdev_ctrlr_abort_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 910 struct spdk_io_channel *ch, struct spdk_nvmf_request *req, 911 struct spdk_nvmf_request *req_to_abort) 912 { 913 int rc; 914 915 assert((req->rsp->nvme_cpl.cdw0 & 1U) != 0); 916 917 rc = spdk_bdev_abort(desc, ch, req_to_abort, nvmf_bdev_ctrlr_complete_abort_cmd, req); 918 if (spdk_likely(rc == 0)) { 919 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 920 } else if (rc == -ENOMEM) { 921 nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_admin_cmd_resubmit, req); 922 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 923 } else { 924 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 925 } 926 } 927 928 bool 929 nvmf_bdev_ctrlr_get_dif_ctx(struct spdk_bdev_desc *desc, struct spdk_nvme_cmd *cmd, 930 struct spdk_dif_ctx *dif_ctx) 931 { 932 uint32_t init_ref_tag, dif_check_flags = 0; 933 int rc; 934 struct spdk_dif_ctx_init_ext_opts dif_opts; 935 936 if (spdk_bdev_desc_get_md_size(desc) == 0) { 937 return false; 938 } 939 940 /* Initial Reference Tag is the lower 32 bits of the start LBA. */ 941 init_ref_tag = (uint32_t)from_le64(&cmd->cdw10); 942 943 if (spdk_bdev_desc_is_dif_check_enabled(desc, SPDK_DIF_CHECK_TYPE_REFTAG)) { 944 dif_check_flags |= SPDK_DIF_FLAGS_REFTAG_CHECK; 945 } 946 947 if (spdk_bdev_desc_is_dif_check_enabled(desc, SPDK_DIF_CHECK_TYPE_GUARD)) { 948 dif_check_flags |= SPDK_DIF_FLAGS_GUARD_CHECK; 949 } 950 951 dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format); 952 dif_opts.dif_pi_format = SPDK_DIF_PI_FORMAT_16; 953 rc = spdk_dif_ctx_init(dif_ctx, 954 spdk_bdev_desc_get_block_size(desc), 955 spdk_bdev_desc_get_md_size(desc), 956 spdk_bdev_desc_is_md_interleaved(desc), 957 spdk_bdev_desc_is_dif_head_of_md(desc), 958 spdk_bdev_desc_get_dif_type(desc), 959 dif_check_flags, 960 init_ref_tag, 0, 0, 0, 0, &dif_opts); 961 962 return (rc == 0) ? true : false; 963 } 964 965 static void 966 nvmf_bdev_ctrlr_zcopy_start_complete(struct spdk_bdev_io *bdev_io, bool success, 967 void *cb_arg) 968 { 969 struct spdk_nvmf_request *req = cb_arg; 970 struct iovec *iov; 971 int iovcnt = 0; 972 973 if (spdk_unlikely(!success)) { 974 int sc = 0, sct = 0; 975 uint32_t cdw0 = 0; 976 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 977 spdk_bdev_io_get_nvme_status(bdev_io, &cdw0, &sct, &sc); 978 979 response->cdw0 = cdw0; 980 response->status.sc = sc; 981 response->status.sct = sct; 982 983 spdk_bdev_free_io(bdev_io); 984 spdk_nvmf_request_complete(req); 985 return; 986 } 987 988 spdk_bdev_io_get_iovec(bdev_io, &iov, &iovcnt); 989 990 assert(iovcnt <= NVMF_REQ_MAX_BUFFERS); 991 assert(iovcnt > 0); 992 993 req->iovcnt = iovcnt; 994 995 assert(req->iov == iov); 996 997 req->zcopy_bdev_io = bdev_io; /* Preserve the bdev_io for the end zcopy */ 998 999 spdk_nvmf_request_complete(req); 1000 /* Don't free the bdev_io here as it is needed for the END ZCOPY */ 1001 } 1002 1003 int 1004 nvmf_bdev_ctrlr_zcopy_start(struct spdk_bdev *bdev, 1005 struct spdk_bdev_desc *desc, 1006 struct spdk_io_channel *ch, 1007 struct spdk_nvmf_request *req) 1008 { 1009 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 1010 uint64_t bdev_num_blocks = spdk_bdev_get_num_blocks(bdev); 1011 uint32_t block_size = spdk_bdev_desc_get_block_size(desc); 1012 uint64_t start_lba; 1013 uint64_t num_blocks; 1014 int rc; 1015 1016 nvmf_bdev_ctrlr_get_rw_params(&req->cmd->nvme_cmd, &start_lba, &num_blocks); 1017 1018 if (spdk_unlikely(!nvmf_bdev_ctrlr_lba_in_range(bdev_num_blocks, start_lba, num_blocks))) { 1019 SPDK_ERRLOG("end of media\n"); 1020 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 1021 rsp->status.sc = SPDK_NVME_SC_LBA_OUT_OF_RANGE; 1022 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1023 } 1024 1025 if (spdk_unlikely(num_blocks * block_size > req->length)) { 1026 SPDK_ERRLOG("Read NLB %" PRIu64 " * block size %" PRIu32 " > SGL length %" PRIu32 "\n", 1027 num_blocks, block_size, req->length); 1028 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 1029 rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID; 1030 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1031 } 1032 1033 bool populate = (req->cmd->nvme_cmd.opc == SPDK_NVME_OPC_READ) ? true : false; 1034 1035 rc = spdk_bdev_zcopy_start(desc, ch, req->iov, req->iovcnt, start_lba, 1036 num_blocks, populate, nvmf_bdev_ctrlr_zcopy_start_complete, req); 1037 if (spdk_unlikely(rc != 0)) { 1038 if (rc == -ENOMEM) { 1039 nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req); 1040 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 1041 } 1042 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 1043 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 1044 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 1045 } 1046 1047 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 1048 } 1049 1050 static void 1051 nvmf_bdev_ctrlr_zcopy_end_complete(struct spdk_bdev_io *bdev_io, bool success, 1052 void *cb_arg) 1053 { 1054 struct spdk_nvmf_request *req = cb_arg; 1055 1056 if (spdk_unlikely(!success)) { 1057 int sc = 0, sct = 0; 1058 uint32_t cdw0 = 0; 1059 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 1060 spdk_bdev_io_get_nvme_status(bdev_io, &cdw0, &sct, &sc); 1061 1062 response->cdw0 = cdw0; 1063 response->status.sc = sc; 1064 response->status.sct = sct; 1065 } 1066 1067 spdk_bdev_free_io(bdev_io); 1068 req->zcopy_bdev_io = NULL; 1069 spdk_nvmf_request_complete(req); 1070 } 1071 1072 void 1073 nvmf_bdev_ctrlr_zcopy_end(struct spdk_nvmf_request *req, bool commit) 1074 { 1075 int rc __attribute__((unused)); 1076 1077 rc = spdk_bdev_zcopy_end(req->zcopy_bdev_io, commit, nvmf_bdev_ctrlr_zcopy_end_complete, req); 1078 1079 /* The only way spdk_bdev_zcopy_end() can fail is if we pass a bdev_io type that isn't ZCOPY */ 1080 assert(rc == 0); 1081 } 1082