1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 36 #include "nvmf_internal.h" 37 38 #include "spdk/bdev.h" 39 #include "spdk/endian.h" 40 #include "spdk/io_channel.h" 41 #include "spdk/likely.h" 42 #include "spdk/nvme.h" 43 #include "spdk/nvmf_spec.h" 44 #include "spdk/trace.h" 45 #include "spdk/scsi_spec.h" 46 #include "spdk/string.h" 47 #include "spdk/util.h" 48 49 #include "spdk_internal/log.h" 50 51 static bool 52 spdk_nvmf_subsystem_bdev_io_type_supported(struct spdk_nvmf_subsystem *subsystem, 53 enum spdk_bdev_io_type io_type) 54 { 55 struct spdk_nvmf_ns *ns; 56 57 for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL; 58 ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) { 59 if (ns->bdev == NULL) { 60 continue; 61 } 62 63 if (!spdk_bdev_io_type_supported(ns->bdev, io_type)) { 64 SPDK_DEBUGLOG(SPDK_TRACE_NVMF, 65 "Subsystem %s namespace %u (%s) does not support io_type %d\n", 66 spdk_nvmf_subsystem_get_nqn(subsystem), 67 ns->id, spdk_bdev_get_name(ns->bdev), (int)io_type); 68 return false; 69 } 70 } 71 72 SPDK_DEBUGLOG(SPDK_TRACE_NVMF, "All devices in Subsystem %s support io_type %d\n", 73 spdk_nvmf_subsystem_get_nqn(subsystem), (int)io_type); 74 return true; 75 } 76 77 bool 78 spdk_nvmf_ctrlr_dsm_supported(struct spdk_nvmf_ctrlr *ctrlr) 79 { 80 return spdk_nvmf_subsystem_bdev_io_type_supported(ctrlr->subsys, SPDK_BDEV_IO_TYPE_UNMAP); 81 } 82 83 bool 84 spdk_nvmf_ctrlr_write_zeroes_supported(struct spdk_nvmf_ctrlr *ctrlr) 85 { 86 return spdk_nvmf_subsystem_bdev_io_type_supported(ctrlr->subsys, SPDK_BDEV_IO_TYPE_WRITE_ZEROES); 87 } 88 89 static void 90 nvmf_bdev_ctrlr_complete_cmd(struct spdk_bdev_io *bdev_io, bool success, 91 void *cb_arg) 92 { 93 struct spdk_nvmf_request *req = cb_arg; 94 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 95 int sc, sct; 96 97 spdk_bdev_io_get_nvme_status(bdev_io, &sc, &sct); 98 response->status.sc = sc; 99 response->status.sct = sct; 100 101 spdk_nvmf_request_complete(req); 102 spdk_bdev_free_io(bdev_io); 103 } 104 105 int 106 spdk_nvmf_bdev_ctrlr_identify_ns(struct spdk_bdev *bdev, struct spdk_nvme_ns_data *nsdata) 107 { 108 uint64_t num_blocks; 109 110 num_blocks = spdk_bdev_get_num_blocks(bdev); 111 112 nsdata->nsze = num_blocks; 113 nsdata->ncap = num_blocks; 114 nsdata->nuse = num_blocks; 115 nsdata->nlbaf = 0; 116 nsdata->flbas.format = 0; 117 nsdata->lbaf[0].lbads = spdk_u32log2(spdk_bdev_get_block_size(bdev)); 118 nsdata->noiob = spdk_bdev_get_optimal_io_boundary(bdev); 119 120 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 121 } 122 123 static void 124 nvmf_bdev_ctrlr_get_rw_params(const struct spdk_nvme_cmd *cmd, uint64_t *start_lba, 125 uint64_t *num_blocks) 126 { 127 /* SLBA: CDW10 and CDW11 */ 128 *start_lba = from_le64(&cmd->cdw10); 129 130 /* NLB: CDW12 bits 15:00, 0's based */ 131 *num_blocks = (from_le32(&cmd->cdw12) & 0xFFFFu) + 1; 132 } 133 134 static bool 135 nvmf_bdev_ctrlr_lba_in_range(uint64_t bdev_num_blocks, uint64_t io_start_lba, 136 uint64_t io_num_blocks) 137 { 138 if (io_start_lba + io_num_blocks > bdev_num_blocks || 139 io_start_lba + io_num_blocks < io_start_lba) { 140 return false; 141 } 142 143 return true; 144 } 145 146 static int 147 nvmf_bdev_ctrlr_read_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 148 struct spdk_io_channel *ch, struct spdk_nvmf_request *req) 149 { 150 uint64_t bdev_num_blocks = spdk_bdev_get_num_blocks(bdev); 151 uint32_t block_size = spdk_bdev_get_block_size(bdev); 152 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 153 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 154 uint64_t start_lba; 155 uint64_t num_blocks; 156 157 nvmf_bdev_ctrlr_get_rw_params(cmd, &start_lba, &num_blocks); 158 159 if (spdk_unlikely(!nvmf_bdev_ctrlr_lba_in_range(bdev_num_blocks, start_lba, num_blocks))) { 160 SPDK_ERRLOG("end of media\n"); 161 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 162 rsp->status.sc = SPDK_NVME_SC_LBA_OUT_OF_RANGE; 163 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 164 } 165 166 if (spdk_unlikely(num_blocks * block_size > req->length)) { 167 SPDK_ERRLOG("Read NLB %" PRIu64 " * block size %" PRIu32 " > SGL length %" PRIu32 "\n", 168 num_blocks, block_size, req->length); 169 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 170 rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID; 171 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 172 } 173 174 spdk_trace_record(TRACE_NVMF_LIB_READ_START, 0, 0, (uint64_t)req, 0); 175 if (spdk_unlikely(spdk_bdev_read_blocks(desc, ch, req->data, start_lba, num_blocks, 176 nvmf_bdev_ctrlr_complete_cmd, req))) { 177 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 178 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 179 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 180 } 181 182 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 183 } 184 185 static int 186 nvmf_bdev_ctrlr_write_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 187 struct spdk_io_channel *ch, struct spdk_nvmf_request *req) 188 { 189 uint64_t bdev_num_blocks = spdk_bdev_get_num_blocks(bdev); 190 uint32_t block_size = spdk_bdev_get_block_size(bdev); 191 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 192 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 193 uint64_t start_lba; 194 uint64_t num_blocks; 195 196 nvmf_bdev_ctrlr_get_rw_params(cmd, &start_lba, &num_blocks); 197 198 if (spdk_unlikely(!nvmf_bdev_ctrlr_lba_in_range(bdev_num_blocks, start_lba, num_blocks))) { 199 SPDK_ERRLOG("end of media\n"); 200 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 201 rsp->status.sc = SPDK_NVME_SC_LBA_OUT_OF_RANGE; 202 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 203 } 204 205 if (spdk_unlikely(num_blocks * block_size > req->length)) { 206 SPDK_ERRLOG("Write NLB %" PRIu64 " * block size %" PRIu32 " > SGL length %" PRIu32 "\n", 207 num_blocks, block_size, req->length); 208 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 209 rsp->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID; 210 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 211 } 212 213 spdk_trace_record(TRACE_NVMF_LIB_WRITE_START, 0, 0, (uint64_t)req, 0); 214 if (spdk_unlikely(spdk_bdev_write_blocks(desc, ch, req->data, start_lba, num_blocks, 215 nvmf_bdev_ctrlr_complete_cmd, req))) { 216 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 217 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 218 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 219 } 220 221 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 222 } 223 224 static int 225 nvmf_bdev_ctrlr_write_zeroes_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 226 struct spdk_io_channel *ch, struct spdk_nvmf_request *req) 227 { 228 uint64_t bdev_num_blocks = spdk_bdev_get_num_blocks(bdev); 229 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 230 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl; 231 uint64_t start_lba; 232 uint64_t num_blocks; 233 234 nvmf_bdev_ctrlr_get_rw_params(cmd, &start_lba, &num_blocks); 235 236 if (spdk_unlikely(!nvmf_bdev_ctrlr_lba_in_range(bdev_num_blocks, start_lba, num_blocks))) { 237 SPDK_ERRLOG("end of media\n"); 238 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 239 rsp->status.sc = SPDK_NVME_SC_LBA_OUT_OF_RANGE; 240 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 241 } 242 243 spdk_trace_record(TRACE_NVMF_LIB_WRITE_START, 0, 0, (uint64_t)req, 0); 244 if (spdk_unlikely(spdk_bdev_write_zeroes_blocks(desc, ch, start_lba, num_blocks, 245 nvmf_bdev_ctrlr_complete_cmd, req))) { 246 rsp->status.sct = SPDK_NVME_SCT_GENERIC; 247 rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 248 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 249 } 250 251 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 252 } 253 254 static int 255 nvmf_bdev_ctrlr_flush_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 256 struct spdk_io_channel *ch, struct spdk_nvmf_request *req) 257 { 258 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 259 260 if (spdk_bdev_flush_blocks(desc, ch, 0, spdk_bdev_get_num_blocks(bdev), 261 nvmf_bdev_ctrlr_complete_cmd, req)) { 262 response->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 263 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 264 } 265 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 266 } 267 268 struct nvmf_virtual_ctrlr_unmap { 269 struct spdk_nvmf_request *req; 270 uint32_t count; 271 }; 272 273 static void 274 nvmf_virtual_ctrlr_dsm_cpl(struct spdk_bdev_io *bdev_io, bool success, 275 void *cb_arg) 276 { 277 struct nvmf_virtual_ctrlr_unmap *unmap_ctx = cb_arg; 278 struct spdk_nvmf_request *req = unmap_ctx->req; 279 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 280 int sc, sct; 281 282 unmap_ctx->count--; 283 284 if (response->status.sct == SPDK_NVME_SCT_GENERIC && 285 response->status.sc == SPDK_NVME_SC_SUCCESS) { 286 spdk_bdev_io_get_nvme_status(bdev_io, &sc, &sct); 287 response->status.sc = sc; 288 response->status.sct = sct; 289 } 290 291 if (unmap_ctx->count == 0) { 292 spdk_nvmf_request_complete(req); 293 spdk_bdev_free_io(bdev_io); 294 free(unmap_ctx); 295 } 296 } 297 298 static int 299 nvmf_bdev_ctrlr_dsm_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 300 struct spdk_io_channel *ch, struct spdk_nvmf_request *req) 301 { 302 uint32_t attribute; 303 uint16_t nr, i; 304 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 305 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 306 307 nr = ((cmd->cdw10 & 0x000000ff) + 1); 308 if (nr * sizeof(struct spdk_nvme_dsm_range) > req->length) { 309 SPDK_ERRLOG("Dataset Management number of ranges > SGL length\n"); 310 response->status.sc = SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID; 311 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 312 } 313 314 attribute = cmd->cdw11 & 0x00000007; 315 if (attribute & SPDK_NVME_DSM_ATTR_DEALLOCATE) { 316 struct nvmf_virtual_ctrlr_unmap *unmap_ctx; 317 struct spdk_nvme_dsm_range *dsm_range; 318 uint64_t lba; 319 uint32_t lba_count; 320 321 unmap_ctx = calloc(1, sizeof(*unmap_ctx)); 322 if (!unmap_ctx) { 323 response->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 324 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 325 } 326 327 unmap_ctx->req = req; 328 329 response->status.sct = SPDK_NVME_SCT_GENERIC; 330 response->status.sc = SPDK_NVME_SC_SUCCESS; 331 332 dsm_range = (struct spdk_nvme_dsm_range *)req->data; 333 for (i = 0; i < nr; i++) { 334 lba = dsm_range[i].starting_lba; 335 lba_count = dsm_range[i].length; 336 337 unmap_ctx->count++; 338 339 if (spdk_bdev_unmap_blocks(desc, ch, lba, lba_count, 340 nvmf_virtual_ctrlr_dsm_cpl, unmap_ctx)) { 341 response->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 342 unmap_ctx->count--; 343 /* We can't return here - we may have to wait for any other 344 * unmaps already sent to complete */ 345 break; 346 } 347 } 348 349 if (unmap_ctx->count == 0) { 350 free(unmap_ctx); 351 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 352 } 353 354 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 355 } 356 357 response->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; 358 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 359 } 360 361 static int 362 nvmf_bdev_ctrlr_nvme_passthru_io(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 363 struct spdk_io_channel *ch, struct spdk_nvmf_request *req) 364 { 365 if (spdk_bdev_nvme_io_passthru(desc, ch, &req->cmd->nvme_cmd, req->data, req->length, 366 nvmf_bdev_ctrlr_complete_cmd, req)) { 367 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC; 368 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INVALID_OPCODE; 369 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 370 } 371 372 return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; 373 } 374 375 int 376 spdk_nvmf_ctrlr_process_io_cmd(struct spdk_nvmf_request *req) 377 { 378 uint32_t nsid; 379 struct spdk_nvmf_ns *ns; 380 struct spdk_bdev *bdev; 381 struct spdk_bdev_desc *desc; 382 struct spdk_io_channel *ch; 383 struct spdk_nvmf_subsystem *subsystem = req->qpair->ctrlr->subsys; 384 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; 385 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; 386 387 /* pre-set response details for this command */ 388 response->status.sc = SPDK_NVME_SC_SUCCESS; 389 nsid = cmd->nsid; 390 391 ns = _spdk_nvmf_subsystem_get_ns(subsystem, nsid); 392 if (ns == NULL || ns->bdev == NULL) { 393 SPDK_ERRLOG("Unsuccessful query for nsid %u\n", cmd->nsid); 394 response->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; 395 return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; 396 } 397 398 bdev = ns->bdev; 399 desc = ns->desc; 400 ch = ns->ch; 401 switch (cmd->opc) { 402 case SPDK_NVME_OPC_READ: 403 return nvmf_bdev_ctrlr_read_cmd(bdev, desc, ch, req); 404 case SPDK_NVME_OPC_WRITE: 405 return nvmf_bdev_ctrlr_write_cmd(bdev, desc, ch, req); 406 case SPDK_NVME_OPC_WRITE_ZEROES: 407 return nvmf_bdev_ctrlr_write_zeroes_cmd(bdev, desc, ch, req); 408 case SPDK_NVME_OPC_FLUSH: 409 return nvmf_bdev_ctrlr_flush_cmd(bdev, desc, ch, req); 410 case SPDK_NVME_OPC_DATASET_MANAGEMENT: 411 return nvmf_bdev_ctrlr_dsm_cmd(bdev, desc, ch, req); 412 default: 413 return nvmf_bdev_ctrlr_nvme_passthru_io(bdev, desc, ch, req); 414 } 415 } 416 417 static int 418 spdk_nvmf_ns_bdev_attach(struct spdk_nvmf_ns *ns) 419 { 420 if (ns->bdev == NULL) { 421 return 0; 422 } 423 424 ns->ch = spdk_bdev_get_io_channel(ns->desc); 425 if (ns->ch == NULL) { 426 SPDK_ERRLOG("io_channel allocation failed\n"); 427 return -1; 428 } 429 430 return 0; 431 } 432 433 static void 434 spdk_nvmf_ns_bdev_detach(struct spdk_nvmf_ns *ns) 435 { 436 if (ns->bdev == NULL) { 437 return; 438 } 439 440 if (ns->ch) { 441 spdk_put_io_channel(ns->ch); 442 ns->ch = NULL; 443 } 444 if (ns->desc) { 445 spdk_bdev_close(ns->desc); 446 ns->desc = NULL; 447 } 448 ns->bdev = NULL; 449 } 450 451 int 452 spdk_nvmf_subsystem_bdev_attach(struct spdk_nvmf_subsystem *subsystem) 453 { 454 struct spdk_nvmf_ns *ns; 455 456 for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL; 457 ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) { 458 if (spdk_nvmf_ns_bdev_attach(ns)) { 459 return -1; 460 } 461 } 462 463 return 0; 464 } 465 466 void 467 spdk_nvmf_subsystem_bdev_detach(struct spdk_nvmf_subsystem *subsystem) 468 { 469 struct spdk_nvmf_ns *ns; 470 471 for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL; 472 ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) { 473 spdk_nvmf_ns_bdev_detach(ns); 474 } 475 subsystem->max_nsid = 0; 476 } 477