1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2017 Intel Corporation. All rights reserved. 3 * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #include "nvmf_internal.h" 10 11 #include "spdk/bdev.h" 12 #include "spdk/endian.h" 13 #include "spdk/thread.h" 14 #include "spdk/likely.h" 15 #include "spdk/nvme.h" 16 #include "spdk/nvmf_cmd.h" 17 #include "spdk/nvmf_spec.h" 18 #include "spdk/trace.h" 19 #include "spdk/scsi_spec.h" 20 #include "spdk/string.h" 21 #include "spdk/util.h" 22 23 #include "spdk/log.h" 24 25 static bool 26 nvmf_subsystem_bdev_io_type_supported(struct spdk_nvmf_subsystem *subsystem, 27 enum spdk_bdev_io_type io_type) 28 { 29 struct spdk_nvmf_ns *ns; 30 31 for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL; 32 ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) { 33 if (ns->bdev == NULL) { 34 continue; 35 } 36 37 if (!spdk_bdev_io_type_supported(ns->bdev, io_type)) { 38 SPDK_DEBUGLOG(nvmf, 39 "Subsystem %s namespace %u (%s) does not support io_type %d\n", 40 spdk_nvmf_subsystem_get_nqn(subsystem), 41 ns->opts.nsid, spdk_bdev_get_name(ns->bdev), (int)io_type); 42 return false; 43 } 44 } 45 46 SPDK_DEBUGLOG(nvmf, "All devices in Subsystem %s support io_type %d\n", 47 spdk_nvmf_subsystem_get_nqn(subsystem), (int)io_type); 48 return true; 49 } 50 51 bool 52 nvmf_ctrlr_dsm_supported(struct spdk_nvmf_ctrlr *ctrlr) 53 { 54 return nvmf_subsystem_bdev_io_type_supported(ctrlr->subsys, SPDK_BDEV_IO_TYPE_UNMAP); 55 } 56 57 bool 58 nvmf_ctrlr_write_zeroes_supported(struct spdk_nvmf_ctrlr *ctrlr) 59 { 60 return nvmf_subsystem_bdev_io_type_supported(ctrlr->subsys, SPDK_BDEV_IO_TYPE_WRITE_ZEROES); 61 } 62 63 bool 64 nvmf_ctrlr_copy_supported(struct spdk_nvmf_ctrlr *ctrlr) 65 { 66 return nvmf_subsystem_bdev_io_type_supported(ctrlr->subsys, SPDK_BDEV_IO_TYPE_COPY); 67 } 68 69 static void 70 nvmf_bdev_ctrlr_complete_cmd(struct spdk_bdev_io *bdev_io, bool success, 71 void *cb_arg) 72 { 73 struct spdk_nvmf_request *req = cb_arg; 74 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 75 int first_sc = 0, first_sct = 0, sc = 0, sct = 0; 76 uint32_t cdw0 = 0; 77 struct spdk_nvmf_request *first_req = req->first_fused_req; 78 79 if (spdk_unlikely(first_req != NULL)) { 80 /* fused commands - get status for both operations */ 81 struct spdk_nvme_cpl *first_response = &first_req->rsp->nvme_cpl; 82 83 spdk_bdev_io_get_nvme_fused_status(bdev_io, &cdw0, &first_sct, &first_sc, &sct, &sc); 84 first_response->cdw0 = cdw0; 85 first_response->status.sc = first_sc; 86 first_response->status.sct = first_sct; 87 88 /* first request should be completed */ 89 spdk_nvmf_request_complete(first_req); 90 req->first_fused_req = NULL; 91 } else { 92 spdk_bdev_io_get_nvme_status(bdev_io, &cdw0, &sct, &sc); 93 } 94 95 response->cdw0 = cdw0; 96 response->status.sc = sc; 97 response->status.sct = sct; 98 99 spdk_nvmf_request_complete(req); 100 spdk_bdev_free_io(bdev_io); 101 } 102 103 static void 104 nvmf_bdev_ctrlr_complete_admin_cmd(struct spdk_bdev_io *bdev_io, bool success, 105 void *cb_arg) 106 { 107 struct spdk_nvmf_request *req = cb_arg; 108 109 if (req->cmd_cb_fn) { 110 req->cmd_cb_fn(req); 111 } 112 113 nvmf_bdev_ctrlr_complete_cmd(bdev_io, success, req); 114 } 115 116 void 117 nvmf_bdev_ctrlr_identify_ns(struct spdk_nvmf_ns *ns, struct spdk_nvme_ns_data *nsdata, 118 bool dif_insert_or_strip) 119 { 120 struct spdk_bdev *bdev = ns->bdev; 121 uint64_t num_blocks; 122 uint32_t phys_blocklen; 123 uint32_t max_copy; 124 125 num_blocks = spdk_bdev_get_num_blocks(bdev); 126 127 nsdata->nsze = num_blocks; 128 nsdata->ncap = num_blocks; 129 nsdata->nuse = num_blocks; 130 nsdata->nlbaf = 0; 131 nsdata->flbas.format = 0; 132 nsdata->flbas.msb_format = 0; 133 nsdata->nacwu = spdk_bdev_get_acwu(bdev) - 1; /* nacwu is 0-based */ 134 if (!dif_insert_or_strip) { 135 nsdata->lbaf[0].ms = spdk_bdev_get_md_size(bdev); 136 nsdata->lbaf[0].lbads = spdk_u32log2(spdk_bdev_get_block_size(bdev)); 137 if (nsdata->lbaf[0].ms != 0) { 138 nsdata->flbas.extended = 1; 139 nsdata->mc.extended = 1; 140 nsdata->mc.pointer = 0; 141 nsdata->dps.md_start = spdk_bdev_is_dif_head_of_md(bdev); 142 /* NVMf library doesn't process PRACT and PRCHK flags, we 143 * leave the use of extended LBA buffer to users. 144 */ 145 nsdata->dps.pit = SPDK_NVME_FMT_NVM_PROTECTION_DISABLE; 146 } 147 } else { 148 nsdata->lbaf[0].ms = 0; 149 nsdata->lbaf[0].lbads = spdk_u32log2(spdk_bdev_get_data_block_size(bdev)); 150 } 151 152 phys_blocklen = spdk_bdev_get_physical_block_size(bdev); 153 assert(phys_blocklen > 0); 154 /* Linux driver uses min(nawupf, npwg) to set physical_block_size */ 155 nsdata->nsfeat.optperf = 1; 156 nsdata->nsfeat.ns_atomic_write_unit = 1; 157 nsdata->npwg = (phys_blocklen >> nsdata->lbaf[0].lbads) - 1; 158 nsdata->nawupf = nsdata->npwg; 159 nsdata->npwa = nsdata->npwg; 160 nsdata->npdg = nsdata->npwg; 161 nsdata->npda = nsdata->npwg; 162 163 if (spdk_bdev_get_write_unit_size(bdev) == 1) { 164 nsdata->noiob = spdk_bdev_get_optimal_io_boundary(bdev); 165 } 166 nsdata->nmic.can_share = 1; 167 if (ns->ptpl_file != NULL) { 168 nsdata->nsrescap.rescap.persist = 1; 169 } 170 nsdata->nsrescap.rescap.write_exclusive = 1; 171 nsdata->nsrescap.rescap.exclusive_access = 1; 172 nsdata->nsrescap.rescap.write_exclusive_reg_only = 1; 173 nsdata->nsrescap.rescap.exclusive_access_reg_only = 1; 174 nsdata->nsrescap.rescap.write_exclusive_all_reg = 1; 175 nsdata->nsrescap.rescap.exclusive_access_all_reg = 1; 176 nsdata->nsrescap.rescap.ignore_existing_key = 1; 177 178 SPDK_STATIC_ASSERT(sizeof(nsdata->nguid) == sizeof(ns->opts.nguid), "size mismatch"); 179 memcpy(nsdata->nguid, ns->opts.nguid, sizeof(nsdata->nguid)); 180 181 SPDK_STATIC_ASSERT(sizeof(nsdata->eui64) == sizeof(ns->opts.eui64), "size mismatch"); 182 memcpy(&nsdata->eui64, ns->opts.eui64, sizeof(nsdata->eui64)); 183 184 if (spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_COPY)) { 185 max_copy = spdk_bdev_get_max_copy(bdev); 186 if (max_copy == 0 || max_copy > UINT16_MAX) { 187 /* Zero means copy size is unlimited */ 188 nsdata->mcl = UINT16_MAX; 189 nsdata->mssrl = UINT16_MAX; 190 } else { 191 nsdata->mcl = max_copy; 192 nsdata->mssrl = max_copy; 193 } 194 195 /* For now we support just one source range */ 196 nsdata->msrc = 0; 197 } 198 } 199 200 static void 201 nvmf_bdev_ctrlr_get_rw_params(const struct spdk_nvme_cmd *cmd, uint64_t *start_lba, 202 uint64_t *num_blocks) 203 { 204 /* SLBA: CDW10 and CDW11 */ 205 *start_lba = from_le64(&cmd->cdw10); 206 207 /* NLB: CDW12 bits 15:00, 0's based */ 208 *num_blocks = (from_le32(&cmd->cdw12) & 0xFFFFu) + 1; 209 } 210 211 static bool 212 nvmf_bdev_ctrlr_lba_in_range(uint64_t bdev_num_blocks, uint64_t io_start_lba, 213 uint64_t io_num_blocks) 214 { 215 if (io_start_lba + io_num_blocks > bdev_num_blocks || 216 io_start_lba + io_num_blocks < io_start_lba) { 217 return false; 218 } 219 220 return true; 221 } 222 223 static void 224 nvmf_ctrlr_process_io_cmd_resubmit(void *arg) 225 { 226 struct spdk_nvmf_request *req = arg; 227 int rc; 228 229 rc = nvmf_ctrlr_process_io_cmd(req); 230 if (rc == SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) { 231 spdk_nvmf_request_complete(req); 232 } 233 } 234 235 static void 236 nvmf_ctrlr_process_admin_cmd_resubmit(void *arg) 237 { 238 struct spdk_nvmf_request *req = arg; 239 int rc; 240 241 rc = nvmf_ctrlr_process_admin_cmd(req); 242 if (rc == SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) { 243 spdk_nvmf_request_complete(req); 244 } 245 } 246 247 static void 248 nvmf_bdev_ctrl_queue_io(struct spdk_nvmf_request *req, struct spdk_bdev *bdev, 249 struct spdk_io_channel *ch, spdk_bdev_io_wait_cb cb_fn, void *cb_arg) 250 { 251 int rc; 252 253 req->bdev_io_wait.bdev = bdev; 254 req->bdev_io_wait.cb_fn = cb_fn; 255 req->bdev_io_wait.cb_arg = cb_arg; 256 257 rc = spdk_bdev_queue_io_wait(bdev, ch, &req->bdev_io_wait); 258 if (rc != 0) { 259 assert(false); 260 } 261 req->qpair->group->stat.pending_bdev_io++; 262 } 263 264 bool 265 nvmf_bdev_zcopy_enabled(struct spdk_bdev *bdev) 266 { 267 return spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_ZCOPY); 268 } 269 270 int 271 nvmf_bdev_ctrlr_read_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 272 struct spdk_io_channel *ch, struct spdk_nvmf_request *req) 273 { 274 uint64_t bdev_num_blocks = spdk_bdev_get_num_blocks(bdev); 275 uint32_t block_size = spdk_bdev_get_block_size(bdev); 276 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 277 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 278 uint64_t start_lba; 279 uint64_t num_blocks; 280 int rc; 281 282 nvmf_bdev_ctrlr_get_rw_params(cmd, &start_lba, &num_blocks); 283 284 if (spdk_unlikely(!nvmf_bdev_ctrlr_lba_in_range(bdev_num_blocks, start_lba, num_blocks))) { 285 SPDK_ERRLOG("end of media\n"); 286 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 287 rsp->status.sc = SPDK_NVME_SC_LBA_OUT_OF_RANGE; 288 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 289 } 290 291 if (spdk_unlikely(num_blocks * block_size > req->length)) { 292 SPDK_ERRLOG("Read NLB %" PRIu64 " * block size %" PRIu32 " > SGL length %" PRIu32 "\n", 293 num_blocks, block_size, req->length); 294 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 295 rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID; 296 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 297 } 298 299 assert(!spdk_nvmf_request_using_zcopy(req)); 300 301 rc = spdk_bdev_readv_blocks(desc, ch, req->iov, req->iovcnt, start_lba, num_blocks, 302 nvmf_bdev_ctrlr_complete_cmd, req); 303 if (spdk_unlikely(rc)) { 304 if (rc == -ENOMEM) { 305 nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req); 306 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 307 } 308 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 309 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 310 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 311 } 312 313 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 314 } 315 316 int 317 nvmf_bdev_ctrlr_write_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 318 struct spdk_io_channel *ch, struct spdk_nvmf_request *req) 319 { 320 uint64_t bdev_num_blocks = spdk_bdev_get_num_blocks(bdev); 321 uint32_t block_size = spdk_bdev_get_block_size(bdev); 322 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 323 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 324 uint64_t start_lba; 325 uint64_t num_blocks; 326 int rc; 327 328 nvmf_bdev_ctrlr_get_rw_params(cmd, &start_lba, &num_blocks); 329 330 if (spdk_unlikely(!nvmf_bdev_ctrlr_lba_in_range(bdev_num_blocks, start_lba, num_blocks))) { 331 SPDK_ERRLOG("end of media\n"); 332 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 333 rsp->status.sc = SPDK_NVME_SC_LBA_OUT_OF_RANGE; 334 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 335 } 336 337 if (spdk_unlikely(num_blocks * block_size > req->length)) { 338 SPDK_ERRLOG("Write NLB %" PRIu64 " * block size %" PRIu32 " > SGL length %" PRIu32 "\n", 339 num_blocks, block_size, req->length); 340 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 341 rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID; 342 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 343 } 344 345 assert(!spdk_nvmf_request_using_zcopy(req)); 346 347 rc = spdk_bdev_writev_blocks(desc, ch, req->iov, req->iovcnt, start_lba, num_blocks, 348 nvmf_bdev_ctrlr_complete_cmd, req); 349 if (spdk_unlikely(rc)) { 350 if (rc == -ENOMEM) { 351 nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req); 352 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 353 } 354 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 355 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 356 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 357 } 358 359 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 360 } 361 362 int 363 nvmf_bdev_ctrlr_compare_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 364 struct spdk_io_channel *ch, struct spdk_nvmf_request *req) 365 { 366 uint64_t bdev_num_blocks = spdk_bdev_get_num_blocks(bdev); 367 uint32_t block_size = spdk_bdev_get_block_size(bdev); 368 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 369 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 370 uint64_t start_lba; 371 uint64_t num_blocks; 372 int rc; 373 374 nvmf_bdev_ctrlr_get_rw_params(cmd, &start_lba, &num_blocks); 375 376 if (spdk_unlikely(!nvmf_bdev_ctrlr_lba_in_range(bdev_num_blocks, start_lba, num_blocks))) { 377 SPDK_ERRLOG("end of media\n"); 378 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 379 rsp->status.sc = SPDK_NVME_SC_LBA_OUT_OF_RANGE; 380 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 381 } 382 383 if (spdk_unlikely(num_blocks * block_size > req->length)) { 384 SPDK_ERRLOG("Compare NLB %" PRIu64 " * block size %" PRIu32 " > SGL length %" PRIu32 "\n", 385 num_blocks, block_size, req->length); 386 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 387 rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID; 388 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 389 } 390 391 rc = spdk_bdev_comparev_blocks(desc, ch, req->iov, req->iovcnt, start_lba, num_blocks, 392 nvmf_bdev_ctrlr_complete_cmd, req); 393 if (spdk_unlikely(rc)) { 394 if (rc == -ENOMEM) { 395 nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req); 396 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 397 } 398 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 399 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 400 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 401 } 402 403 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 404 } 405 406 int 407 nvmf_bdev_ctrlr_compare_and_write_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 408 struct spdk_io_channel *ch, struct spdk_nvmf_request *cmp_req, struct spdk_nvmf_request *write_req) 409 { 410 uint64_t bdev_num_blocks = spdk_bdev_get_num_blocks(bdev); 411 uint32_t block_size = spdk_bdev_get_block_size(bdev); 412 struct spdk_nvme_cmd *cmp_cmd = &cmp_req->cmd->nvme_cmd; 413 struct spdk_nvme_cmd *write_cmd = &write_req->cmd->nvme_cmd; 414 struct spdk_nvme_cpl *rsp = &write_req->rsp->nvme_cpl; 415 uint64_t write_start_lba, cmp_start_lba; 416 uint64_t write_num_blocks, cmp_num_blocks; 417 int rc; 418 419 nvmf_bdev_ctrlr_get_rw_params(cmp_cmd, &cmp_start_lba, &cmp_num_blocks); 420 nvmf_bdev_ctrlr_get_rw_params(write_cmd, &write_start_lba, &write_num_blocks); 421 422 if (spdk_unlikely(write_start_lba != cmp_start_lba || write_num_blocks != cmp_num_blocks)) { 423 SPDK_ERRLOG("Fused command start lba / num blocks mismatch\n"); 424 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 425 rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD; 426 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 427 } 428 429 if (spdk_unlikely(!nvmf_bdev_ctrlr_lba_in_range(bdev_num_blocks, write_start_lba, 430 write_num_blocks))) { 431 SPDK_ERRLOG("end of media\n"); 432 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 433 rsp->status.sc = SPDK_NVME_SC_LBA_OUT_OF_RANGE; 434 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 435 } 436 437 if (spdk_unlikely(write_num_blocks * block_size > write_req->length)) { 438 SPDK_ERRLOG("Write NLB %" PRIu64 " * block size %" PRIu32 " > SGL length %" PRIu32 "\n", 439 write_num_blocks, block_size, write_req->length); 440 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 441 rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID; 442 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 443 } 444 445 rc = spdk_bdev_comparev_and_writev_blocks(desc, ch, cmp_req->iov, cmp_req->iovcnt, write_req->iov, 446 write_req->iovcnt, write_start_lba, write_num_blocks, nvmf_bdev_ctrlr_complete_cmd, write_req); 447 if (spdk_unlikely(rc)) { 448 if (rc == -ENOMEM) { 449 nvmf_bdev_ctrl_queue_io(cmp_req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, cmp_req); 450 nvmf_bdev_ctrl_queue_io(write_req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, write_req); 451 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 452 } 453 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 454 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 455 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 456 } 457 458 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 459 } 460 461 int 462 nvmf_bdev_ctrlr_write_zeroes_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 463 struct spdk_io_channel *ch, struct spdk_nvmf_request *req) 464 { 465 uint64_t bdev_num_blocks = spdk_bdev_get_num_blocks(bdev); 466 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 467 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 468 uint64_t start_lba; 469 uint64_t num_blocks; 470 int rc; 471 472 nvmf_bdev_ctrlr_get_rw_params(cmd, &start_lba, &num_blocks); 473 474 if (spdk_unlikely(!nvmf_bdev_ctrlr_lba_in_range(bdev_num_blocks, start_lba, num_blocks))) { 475 SPDK_ERRLOG("end of media\n"); 476 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 477 rsp->status.sc = SPDK_NVME_SC_LBA_OUT_OF_RANGE; 478 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 479 } 480 481 rc = spdk_bdev_write_zeroes_blocks(desc, ch, start_lba, num_blocks, 482 nvmf_bdev_ctrlr_complete_cmd, req); 483 if (spdk_unlikely(rc)) { 484 if (rc == -ENOMEM) { 485 nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req); 486 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 487 } 488 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 489 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 490 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 491 } 492 493 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 494 } 495 496 int 497 nvmf_bdev_ctrlr_flush_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 498 struct spdk_io_channel *ch, struct spdk_nvmf_request *req) 499 { 500 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 501 int rc; 502 503 /* As for NVMeoF controller, SPDK always set volatile write 504 * cache bit to 1, return success for those block devices 505 * which can't support FLUSH command. 506 */ 507 if (!spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_FLUSH)) { 508 response->status.sct = SPDK_NVME_SCT_GENERIC; 509 response->status.sc = SPDK_NVME_SC_SUCCESS; 510 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 511 } 512 513 rc = spdk_bdev_flush_blocks(desc, ch, 0, spdk_bdev_get_num_blocks(bdev), 514 nvmf_bdev_ctrlr_complete_cmd, req); 515 if (spdk_unlikely(rc)) { 516 if (rc == -ENOMEM) { 517 nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req); 518 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 519 } 520 response->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 521 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 522 } 523 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 524 } 525 526 struct nvmf_bdev_ctrlr_unmap { 527 struct spdk_nvmf_request *req; 528 uint32_t count; 529 struct spdk_bdev_desc *desc; 530 struct spdk_bdev *bdev; 531 struct spdk_io_channel *ch; 532 uint32_t range_index; 533 }; 534 535 static void 536 nvmf_bdev_ctrlr_unmap_cpl(struct spdk_bdev_io *bdev_io, bool success, 537 void *cb_arg) 538 { 539 struct nvmf_bdev_ctrlr_unmap *unmap_ctx = cb_arg; 540 struct spdk_nvmf_request *req = unmap_ctx->req; 541 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 542 int sc, sct; 543 uint32_t cdw0; 544 545 unmap_ctx->count--; 546 547 if (response->status.sct == SPDK_NVME_SCT_GENERIC && 548 response->status.sc == SPDK_NVME_SC_SUCCESS) { 549 spdk_bdev_io_get_nvme_status(bdev_io, &cdw0, &sct, &sc); 550 response->cdw0 = cdw0; 551 response->status.sc = sc; 552 response->status.sct = sct; 553 } 554 555 if (unmap_ctx->count == 0) { 556 spdk_nvmf_request_complete(req); 557 free(unmap_ctx); 558 } 559 spdk_bdev_free_io(bdev_io); 560 } 561 562 static int nvmf_bdev_ctrlr_unmap(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 563 struct spdk_io_channel *ch, struct spdk_nvmf_request *req, 564 struct nvmf_bdev_ctrlr_unmap *unmap_ctx); 565 static void 566 nvmf_bdev_ctrlr_unmap_resubmit(void *arg) 567 { 568 struct nvmf_bdev_ctrlr_unmap *unmap_ctx = arg; 569 struct spdk_nvmf_request *req = unmap_ctx->req; 570 struct spdk_bdev_desc *desc = unmap_ctx->desc; 571 struct spdk_bdev *bdev = unmap_ctx->bdev; 572 struct spdk_io_channel *ch = unmap_ctx->ch; 573 574 nvmf_bdev_ctrlr_unmap(bdev, desc, ch, req, unmap_ctx); 575 } 576 577 static int 578 nvmf_bdev_ctrlr_unmap(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 579 struct spdk_io_channel *ch, struct spdk_nvmf_request *req, 580 struct nvmf_bdev_ctrlr_unmap *unmap_ctx) 581 { 582 uint16_t nr, i; 583 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 584 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 585 struct spdk_iov_xfer ix; 586 uint64_t lba; 587 uint32_t lba_count; 588 int rc; 589 590 nr = cmd->cdw10_bits.dsm.nr + 1; 591 if (nr * sizeof(struct spdk_nvme_dsm_range) > req->length) { 592 SPDK_ERRLOG("Dataset Management number of ranges > SGL length\n"); 593 response->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID; 594 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 595 } 596 597 if (unmap_ctx == NULL) { 598 unmap_ctx = calloc(1, sizeof(*unmap_ctx)); 599 if (!unmap_ctx) { 600 response->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 601 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 602 } 603 604 unmap_ctx->req = req; 605 unmap_ctx->desc = desc; 606 unmap_ctx->ch = ch; 607 unmap_ctx->bdev = bdev; 608 609 response->status.sct = SPDK_NVME_SCT_GENERIC; 610 response->status.sc = SPDK_NVME_SC_SUCCESS; 611 } else { 612 unmap_ctx->count--; /* dequeued */ 613 } 614 615 spdk_iov_xfer_init(&ix, req->iov, req->iovcnt); 616 617 for (i = unmap_ctx->range_index; i < nr; i++) { 618 struct spdk_nvme_dsm_range dsm_range = { 0 }; 619 620 spdk_iov_xfer_to_buf(&ix, &dsm_range, sizeof(dsm_range)); 621 622 lba = dsm_range.starting_lba; 623 lba_count = dsm_range.length; 624 625 unmap_ctx->count++; 626 627 rc = spdk_bdev_unmap_blocks(desc, ch, lba, lba_count, 628 nvmf_bdev_ctrlr_unmap_cpl, unmap_ctx); 629 if (rc) { 630 if (rc == -ENOMEM) { 631 nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_bdev_ctrlr_unmap_resubmit, unmap_ctx); 632 /* Unmap was not yet submitted to bdev */ 633 /* unmap_ctx->count will be decremented when the request is dequeued */ 634 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 635 } 636 response->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 637 unmap_ctx->count--; 638 /* We can't return here - we may have to wait for any other 639 * unmaps already sent to complete */ 640 break; 641 } 642 unmap_ctx->range_index++; 643 } 644 645 if (unmap_ctx->count == 0) { 646 free(unmap_ctx); 647 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 648 } 649 650 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 651 } 652 653 int 654 nvmf_bdev_ctrlr_dsm_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 655 struct spdk_io_channel *ch, struct spdk_nvmf_request *req) 656 { 657 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 658 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 659 660 if (cmd->cdw11_bits.dsm.ad) { 661 return nvmf_bdev_ctrlr_unmap(bdev, desc, ch, req, NULL); 662 } 663 664 response->status.sct = SPDK_NVME_SCT_GENERIC; 665 response->status.sc = SPDK_NVME_SC_SUCCESS; 666 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 667 } 668 669 int 670 nvmf_bdev_ctrlr_copy_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 671 struct spdk_io_channel *ch, struct spdk_nvmf_request *req) 672 { 673 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 674 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 675 uint64_t sdlba = ((uint64_t)cmd->cdw11 << 32) + cmd->cdw10; 676 struct spdk_nvme_scc_source_range range = { 0 }; 677 struct spdk_iov_xfer ix; 678 int rc; 679 680 SPDK_DEBUGLOG(nvmf, "Copy command: SDLBA %lu, NR %u, desc format %u, PRINFOR %u, " 681 "DTYPE %u, STCW %u, PRINFOW %u, FUA %u, LR %u\n", 682 sdlba, 683 cmd->cdw12_bits.copy.nr, 684 cmd->cdw12_bits.copy.df, 685 cmd->cdw12_bits.copy.prinfor, 686 cmd->cdw12_bits.copy.dtype, 687 cmd->cdw12_bits.copy.stcw, 688 cmd->cdw12_bits.copy.prinfow, 689 cmd->cdw12_bits.copy.fua, 690 cmd->cdw12_bits.copy.lr); 691 692 if (spdk_unlikely(req->length != (cmd->cdw12_bits.copy.nr + 1) * 693 sizeof(struct spdk_nvme_scc_source_range))) { 694 response->status.sct = SPDK_NVME_SCT_GENERIC; 695 response->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID; 696 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 697 } 698 699 if (!spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_COPY)) { 700 SPDK_NOTICELOG("Copy command not supported by bdev\n"); 701 response->status.sct = SPDK_NVME_SCT_GENERIC; 702 response->status.sc = SPDK_NVME_SC_INVALID_OPCODE; 703 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 704 } 705 706 /* 707 * We support only one source range, and rely on this with the xfer 708 * below. 709 */ 710 if (cmd->cdw12_bits.copy.nr > 0) { 711 response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC; 712 response->status.sc = SPDK_NVME_SC_CMD_SIZE_LIMIT_SIZE_EXCEEDED; 713 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 714 } 715 716 if (cmd->cdw12_bits.copy.df != 0) { 717 response->status.sct = SPDK_NVME_SCT_GENERIC; 718 response->status.sc = SPDK_NVME_SC_INVALID_FIELD; 719 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 720 } 721 722 spdk_iov_xfer_init(&ix, req->iov, req->iovcnt); 723 spdk_iov_xfer_to_buf(&ix, &range, sizeof(range)); 724 725 rc = spdk_bdev_copy_blocks(desc, ch, sdlba, range.slba, range.nlb + 1, 726 nvmf_bdev_ctrlr_complete_cmd, req); 727 if (spdk_unlikely(rc)) { 728 if (rc == -ENOMEM) { 729 nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req); 730 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 731 } 732 733 response->status.sct = SPDK_NVME_SCT_GENERIC; 734 response->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 735 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 736 } 737 738 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 739 } 740 741 int 742 nvmf_bdev_ctrlr_nvme_passthru_io(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 743 struct spdk_io_channel *ch, struct spdk_nvmf_request *req) 744 { 745 int rc; 746 747 if (spdk_unlikely(req->iovcnt != 1)) { 748 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 749 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 750 req->rsp->nvme_cpl.status.dnr = 1; 751 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 752 } 753 754 rc = spdk_bdev_nvme_io_passthru(desc, ch, &req->cmd->nvme_cmd, req->iov[0].iov_base, req->length, 755 nvmf_bdev_ctrlr_complete_cmd, req); 756 if (spdk_unlikely(rc)) { 757 if (rc == -ENOMEM) { 758 nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req); 759 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 760 } 761 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 762 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INVALID_OPCODE; 763 req->rsp->nvme_cpl.status.dnr = 1; 764 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 765 } 766 767 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 768 } 769 770 int 771 spdk_nvmf_bdev_ctrlr_nvme_passthru_admin(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 772 struct spdk_io_channel *ch, struct spdk_nvmf_request *req, 773 spdk_nvmf_nvme_passthru_cmd_cb cb_fn) 774 { 775 int rc; 776 777 if (spdk_unlikely(req->iovcnt != 1)) { 778 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 779 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 780 req->rsp->nvme_cpl.status.dnr = 1; 781 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 782 } 783 784 req->cmd_cb_fn = cb_fn; 785 786 rc = spdk_bdev_nvme_admin_passthru(desc, ch, &req->cmd->nvme_cmd, req->iov[0].iov_base, req->length, 787 nvmf_bdev_ctrlr_complete_admin_cmd, req); 788 if (spdk_unlikely(rc)) { 789 if (rc == -ENOMEM) { 790 nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_admin_cmd_resubmit, req); 791 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 792 } 793 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 794 if (rc == -ENOTSUP) { 795 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INVALID_OPCODE; 796 } else { 797 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 798 } 799 800 req->rsp->nvme_cpl.status.dnr = 1; 801 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 802 } 803 804 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 805 } 806 807 static void 808 nvmf_bdev_ctrlr_complete_abort_cmd(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 809 { 810 struct spdk_nvmf_request *req = cb_arg; 811 812 if (success) { 813 req->rsp->nvme_cpl.cdw0 &= ~1U; 814 } 815 816 spdk_nvmf_request_complete(req); 817 spdk_bdev_free_io(bdev_io); 818 } 819 820 int 821 spdk_nvmf_bdev_ctrlr_abort_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 822 struct spdk_io_channel *ch, struct spdk_nvmf_request *req, 823 struct spdk_nvmf_request *req_to_abort) 824 { 825 int rc; 826 827 assert((req->rsp->nvme_cpl.cdw0 & 1U) != 0); 828 829 rc = spdk_bdev_abort(desc, ch, req_to_abort, nvmf_bdev_ctrlr_complete_abort_cmd, req); 830 if (spdk_likely(rc == 0)) { 831 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 832 } else if (rc == -ENOMEM) { 833 nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_admin_cmd_resubmit, req); 834 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 835 } else { 836 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 837 } 838 } 839 840 bool 841 nvmf_bdev_ctrlr_get_dif_ctx(struct spdk_bdev *bdev, struct spdk_nvme_cmd *cmd, 842 struct spdk_dif_ctx *dif_ctx) 843 { 844 uint32_t init_ref_tag, dif_check_flags = 0; 845 int rc; 846 847 if (spdk_bdev_get_md_size(bdev) == 0) { 848 return false; 849 } 850 851 /* Initial Reference Tag is the lower 32 bits of the start LBA. */ 852 init_ref_tag = (uint32_t)from_le64(&cmd->cdw10); 853 854 if (spdk_bdev_is_dif_check_enabled(bdev, SPDK_DIF_CHECK_TYPE_REFTAG)) { 855 dif_check_flags |= SPDK_DIF_FLAGS_REFTAG_CHECK; 856 } 857 858 if (spdk_bdev_is_dif_check_enabled(bdev, SPDK_DIF_CHECK_TYPE_GUARD)) { 859 dif_check_flags |= SPDK_DIF_FLAGS_GUARD_CHECK; 860 } 861 862 rc = spdk_dif_ctx_init(dif_ctx, 863 spdk_bdev_get_block_size(bdev), 864 spdk_bdev_get_md_size(bdev), 865 spdk_bdev_is_md_interleaved(bdev), 866 spdk_bdev_is_dif_head_of_md(bdev), 867 spdk_bdev_get_dif_type(bdev), 868 dif_check_flags, 869 init_ref_tag, 0, 0, 0, 0); 870 871 return (rc == 0) ? true : false; 872 } 873 874 static void 875 nvmf_bdev_ctrlr_zcopy_start_complete(struct spdk_bdev_io *bdev_io, bool success, 876 void *cb_arg) 877 { 878 struct spdk_nvmf_request *req = cb_arg; 879 struct iovec *iov; 880 int iovcnt = 0; 881 882 if (spdk_unlikely(!success)) { 883 int sc = 0, sct = 0; 884 uint32_t cdw0 = 0; 885 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 886 spdk_bdev_io_get_nvme_status(bdev_io, &cdw0, &sct, &sc); 887 888 response->cdw0 = cdw0; 889 response->status.sc = sc; 890 response->status.sct = sct; 891 892 spdk_bdev_free_io(bdev_io); 893 spdk_nvmf_request_complete(req); 894 return; 895 } 896 897 spdk_bdev_io_get_iovec(bdev_io, &iov, &iovcnt); 898 899 assert(iovcnt <= NVMF_REQ_MAX_BUFFERS); 900 assert(iovcnt > 0); 901 902 req->iovcnt = iovcnt; 903 904 assert(req->iov == iov); 905 906 /* backward compatible */ 907 req->data = req->iov[0].iov_base; 908 909 req->zcopy_bdev_io = bdev_io; /* Preserve the bdev_io for the end zcopy */ 910 911 spdk_nvmf_request_complete(req); 912 /* Don't free the bdev_io here as it is needed for the END ZCOPY */ 913 } 914 915 int 916 nvmf_bdev_ctrlr_zcopy_start(struct spdk_bdev *bdev, 917 struct spdk_bdev_desc *desc, 918 struct spdk_io_channel *ch, 919 struct spdk_nvmf_request *req) 920 { 921 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 922 uint64_t bdev_num_blocks = spdk_bdev_get_num_blocks(bdev); 923 uint32_t block_size = spdk_bdev_get_block_size(bdev); 924 uint64_t start_lba; 925 uint64_t num_blocks; 926 int rc; 927 928 nvmf_bdev_ctrlr_get_rw_params(&req->cmd->nvme_cmd, &start_lba, &num_blocks); 929 930 if (spdk_unlikely(!nvmf_bdev_ctrlr_lba_in_range(bdev_num_blocks, start_lba, num_blocks))) { 931 SPDK_ERRLOG("end of media\n"); 932 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 933 rsp->status.sc = SPDK_NVME_SC_LBA_OUT_OF_RANGE; 934 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 935 } 936 937 if (spdk_unlikely(num_blocks * block_size > req->length)) { 938 SPDK_ERRLOG("Read NLB %" PRIu64 " * block size %" PRIu32 " > SGL length %" PRIu32 "\n", 939 num_blocks, block_size, req->length); 940 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 941 rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID; 942 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 943 } 944 945 bool populate = (req->cmd->nvme_cmd.opc == SPDK_NVME_OPC_READ) ? true : false; 946 947 rc = spdk_bdev_zcopy_start(desc, ch, req->iov, req->iovcnt, start_lba, 948 num_blocks, populate, nvmf_bdev_ctrlr_zcopy_start_complete, req); 949 if (spdk_unlikely(rc != 0)) { 950 if (rc == -ENOMEM) { 951 nvmf_bdev_ctrl_queue_io(req, bdev, ch, nvmf_ctrlr_process_io_cmd_resubmit, req); 952 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 953 } 954 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 955 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 956 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 957 } 958 959 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 960 } 961 962 static void 963 nvmf_bdev_ctrlr_zcopy_end_complete(struct spdk_bdev_io *bdev_io, bool success, 964 void *cb_arg) 965 { 966 struct spdk_nvmf_request *req = cb_arg; 967 968 if (spdk_unlikely(!success)) { 969 int sc = 0, sct = 0; 970 uint32_t cdw0 = 0; 971 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 972 spdk_bdev_io_get_nvme_status(bdev_io, &cdw0, &sct, &sc); 973 974 response->cdw0 = cdw0; 975 response->status.sc = sc; 976 response->status.sct = sct; 977 } 978 979 spdk_bdev_free_io(bdev_io); 980 req->zcopy_bdev_io = NULL; 981 spdk_nvmf_request_complete(req); 982 } 983 984 void 985 nvmf_bdev_ctrlr_zcopy_end(struct spdk_nvmf_request *req, bool commit) 986 { 987 int rc __attribute__((unused)); 988 989 rc = spdk_bdev_zcopy_end(req->zcopy_bdev_io, commit, nvmf_bdev_ctrlr_zcopy_end_complete, req); 990 991 /* The only way spdk_bdev_zcopy_end() can fail is if we pass a bdev_io type that isn't ZCOPY */ 992 assert(rc == 0); 993 } 994