1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) Intel Corporation. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 36 #include <linux/virtio_scsi.h> 37 38 #include "spdk/env.h" 39 #include "spdk/thread.h" 40 #include "spdk/scsi.h" 41 #include "spdk/scsi_spec.h" 42 #include "spdk/conf.h" 43 #include "spdk/event.h" 44 #include "spdk/util.h" 45 #include "spdk/likely.h" 46 47 #include "spdk/vhost.h" 48 #include "vhost_internal.h" 49 50 /* Features supported by SPDK VHOST lib. */ 51 #define SPDK_VHOST_SCSI_FEATURES (SPDK_VHOST_FEATURES | \ 52 (1ULL << VIRTIO_SCSI_F_INOUT) | \ 53 (1ULL << VIRTIO_SCSI_F_HOTPLUG) | \ 54 (1ULL << VIRTIO_SCSI_F_CHANGE ) | \ 55 (1ULL << VIRTIO_SCSI_F_T10_PI )) 56 57 /* Features that are specified in VIRTIO SCSI but currently not supported: 58 * - Live migration not supported yet 59 * - T10 PI 60 */ 61 #define SPDK_VHOST_SCSI_DISABLED_FEATURES (SPDK_VHOST_DISABLED_FEATURES | \ 62 (1ULL << VIRTIO_SCSI_F_T10_PI )) 63 64 #define MGMT_POLL_PERIOD_US (1000 * 5) 65 66 #define VIRTIO_SCSI_CONTROLQ 0 67 #define VIRTIO_SCSI_EVENTQ 1 68 #define VIRTIO_SCSI_REQUESTQ 2 69 70 enum spdk_scsi_dev_vhost_status { 71 /* Target ID is empty. */ 72 VHOST_SCSI_DEV_EMPTY, 73 74 /* Target is still being added. */ 75 VHOST_SCSI_DEV_ADDING, 76 77 /* Target ID occupied. */ 78 VHOST_SCSI_DEV_PRESENT, 79 80 /* Target ID is occupied but removal is in progress. */ 81 VHOST_SCSI_DEV_REMOVING, 82 83 /* In session - device (SCSI target) seen but removed. */ 84 VHOST_SCSI_DEV_REMOVED, 85 }; 86 87 /** Context for a SCSI target in a vhost device */ 88 struct spdk_scsi_dev_vhost_state { 89 struct spdk_scsi_dev *dev; 90 enum spdk_scsi_dev_vhost_status status; 91 spdk_vhost_event_fn remove_cb; 92 void *remove_ctx; 93 }; 94 95 struct spdk_vhost_scsi_dev { 96 struct spdk_vhost_dev vdev; 97 struct spdk_scsi_dev_vhost_state scsi_dev_state[SPDK_VHOST_SCSI_CTRLR_MAX_DEVS]; 98 99 /* The CPU chosen to poll I/O of all active vhost sessions */ 100 int32_t lcore; 101 } __rte_cache_aligned; 102 103 /** Context for a SCSI target in a vhost session */ 104 struct spdk_scsi_dev_session_state { 105 struct spdk_scsi_dev *dev; 106 enum spdk_scsi_dev_vhost_status status; 107 }; 108 109 struct spdk_vhost_scsi_session { 110 struct spdk_vhost_session vsession; 111 112 struct spdk_vhost_scsi_dev *svdev; 113 /** Local copy of the device state */ 114 struct spdk_scsi_dev_session_state scsi_dev_state[SPDK_VHOST_SCSI_CTRLR_MAX_DEVS]; 115 struct spdk_poller *requestq_poller; 116 struct spdk_poller *mgmt_poller; 117 struct spdk_poller *stop_poller; 118 }; 119 120 struct spdk_vhost_scsi_task { 121 struct spdk_scsi_task scsi; 122 struct iovec iovs[SPDK_VHOST_IOVS_MAX]; 123 124 union { 125 struct virtio_scsi_cmd_resp *resp; 126 struct virtio_scsi_ctrl_tmf_resp *tmf_resp; 127 }; 128 129 struct spdk_vhost_scsi_session *svsession; 130 struct spdk_scsi_dev *scsi_dev; 131 132 /** Number of bytes that were written. */ 133 uint32_t used_len; 134 135 int req_idx; 136 137 /* If set, the task is currently used for I/O processing. */ 138 bool used; 139 140 struct spdk_vhost_virtqueue *vq; 141 }; 142 143 static int spdk_vhost_scsi_start(struct spdk_vhost_session *vsession); 144 static int spdk_vhost_scsi_stop(struct spdk_vhost_session *vsession); 145 static void spdk_vhost_scsi_dump_info_json(struct spdk_vhost_dev *vdev, 146 struct spdk_json_write_ctx *w); 147 static void spdk_vhost_scsi_write_config_json(struct spdk_vhost_dev *vdev, 148 struct spdk_json_write_ctx *w); 149 static int spdk_vhost_scsi_dev_remove(struct spdk_vhost_dev *vdev); 150 151 const struct spdk_vhost_dev_backend spdk_vhost_scsi_device_backend = { 152 .virtio_features = SPDK_VHOST_SCSI_FEATURES, 153 .disabled_features = SPDK_VHOST_SCSI_DISABLED_FEATURES, 154 .session_ctx_size = sizeof(struct spdk_vhost_scsi_session) - sizeof(struct spdk_vhost_session), 155 .start_session = spdk_vhost_scsi_start, 156 .stop_session = spdk_vhost_scsi_stop, 157 .dump_info_json = spdk_vhost_scsi_dump_info_json, 158 .write_config_json = spdk_vhost_scsi_write_config_json, 159 .remove_device = spdk_vhost_scsi_dev_remove, 160 }; 161 162 static void 163 spdk_vhost_scsi_task_put(struct spdk_vhost_scsi_task *task) 164 { 165 spdk_scsi_task_put(&task->scsi); 166 } 167 168 static void 169 spdk_vhost_scsi_task_free_cb(struct spdk_scsi_task *scsi_task) 170 { 171 struct spdk_vhost_scsi_task *task = SPDK_CONTAINEROF(scsi_task, struct spdk_vhost_scsi_task, scsi); 172 struct spdk_vhost_session *vsession = &task->svsession->vsession; 173 174 assert(vsession->task_cnt > 0); 175 vsession->task_cnt--; 176 task->used = false; 177 } 178 179 static int 180 remove_scsi_tgt(struct spdk_vhost_scsi_dev *svdev, 181 unsigned scsi_tgt_num) 182 { 183 struct spdk_scsi_dev_vhost_state *state; 184 struct spdk_scsi_dev *dev; 185 186 state = &svdev->scsi_dev_state[scsi_tgt_num]; 187 if (state->dev == NULL) { 188 /* we've been already removed in the meantime */ 189 return 0; 190 } 191 192 dev = state->dev; 193 state->dev = NULL; 194 assert(state->status == VHOST_SCSI_DEV_REMOVING); 195 state->status = VHOST_SCSI_DEV_EMPTY; 196 spdk_scsi_dev_destruct(dev); 197 if (state->remove_cb) { 198 state->remove_cb(&svdev->vdev, state->remove_ctx); 199 state->remove_cb = NULL; 200 } 201 SPDK_INFOLOG(SPDK_LOG_VHOST, "%s: removed target 'Target %u'\n", 202 svdev->vdev.name, scsi_tgt_num); 203 return 0; 204 } 205 206 static int 207 spdk_vhost_scsi_session_process_removed(struct spdk_vhost_dev *vdev, 208 struct spdk_vhost_session *vsession, void *ctx) 209 { 210 unsigned scsi_tgt_num = (unsigned)(uintptr_t)ctx; 211 struct spdk_vhost_scsi_session *svsession; 212 struct spdk_scsi_dev_session_state *state; 213 214 if (vsession == NULL) { 215 /* all sessions have already detached the device */ 216 struct spdk_vhost_scsi_dev *svdev = SPDK_CONTAINEROF(vdev, 217 struct spdk_vhost_scsi_dev, vdev); 218 219 return remove_scsi_tgt(svdev, scsi_tgt_num); 220 } 221 222 svsession = (struct spdk_vhost_scsi_session *)vsession; 223 state = &svsession->scsi_dev_state[scsi_tgt_num]; 224 225 if (state->dev != NULL) { 226 /* there's still a session that references this device, 227 * so abort our foreach chain here. We'll be called 228 * again from this session's management poller after it 229 * is removed in there 230 */ 231 return -1; 232 } 233 234 return 0; 235 } 236 237 static void 238 process_removed_devs(struct spdk_vhost_scsi_session *svsession) 239 { 240 struct spdk_scsi_dev *dev; 241 struct spdk_scsi_dev_session_state *state; 242 int i; 243 244 for (i = 0; i < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; ++i) { 245 state = &svsession->scsi_dev_state[i]; 246 dev = state->dev; 247 248 if (dev && state->status == VHOST_SCSI_DEV_REMOVING && !spdk_scsi_dev_has_pending_tasks(dev)) { 249 /* detach the device from this session */ 250 spdk_scsi_dev_free_io_channels(dev); 251 state->dev = NULL; 252 state->status = VHOST_SCSI_DEV_REMOVED; 253 /* try to detach it globally */ 254 spdk_vhost_lock(); 255 spdk_vhost_dev_foreach_session(&svsession->svdev->vdev, 256 spdk_vhost_scsi_session_process_removed, 257 (void *)(uintptr_t)i); 258 spdk_vhost_unlock(); 259 } 260 } 261 } 262 263 static void 264 eventq_enqueue(struct spdk_vhost_scsi_session *svsession, unsigned scsi_dev_num, 265 uint32_t event, uint32_t reason) 266 { 267 struct spdk_vhost_session *vsession = &svsession->vsession; 268 struct spdk_vhost_dev *vdev = vsession->vdev; 269 struct spdk_vhost_virtqueue *vq; 270 struct vring_desc *desc, *desc_table; 271 struct virtio_scsi_event *desc_ev; 272 uint32_t desc_table_size, req_size = 0; 273 uint16_t req; 274 int rc; 275 276 assert(scsi_dev_num < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS); 277 vq = &vsession->virtqueue[VIRTIO_SCSI_EVENTQ]; 278 279 if (spdk_vhost_vq_avail_ring_get(vq, &req, 1) != 1) { 280 SPDK_ERRLOG("Controller %s: Failed to send virtio event (no avail ring entries?).\n", 281 vdev->name); 282 return; 283 } 284 285 rc = spdk_vhost_vq_get_desc(vsession, vq, req, &desc, &desc_table, &desc_table_size); 286 if (rc != 0 || desc->len < sizeof(*desc_ev)) { 287 SPDK_ERRLOG("Controller %s: Invalid eventq descriptor at index %"PRIu16".\n", 288 vdev->name, req); 289 goto out; 290 } 291 292 desc_ev = spdk_vhost_gpa_to_vva(vsession, desc->addr, sizeof(*desc_ev)); 293 if (desc_ev == NULL) { 294 SPDK_ERRLOG("Controller %s: Eventq descriptor at index %"PRIu16" points to unmapped guest memory address %p.\n", 295 vdev->name, req, (void *)(uintptr_t)desc->addr); 296 goto out; 297 } 298 299 desc_ev->event = event; 300 desc_ev->lun[0] = 1; 301 desc_ev->lun[1] = scsi_dev_num; 302 /* virtio LUN id 0 can refer either to the entire device 303 * or actual LUN 0 (the only supported by vhost for now) 304 */ 305 desc_ev->lun[2] = 0 >> 8; 306 desc_ev->lun[3] = 0 & 0xFF; 307 /* virtio doesn't specify any strict format for LUN id (bytes 2 and 3) 308 * current implementation relies on linux kernel sources 309 */ 310 memset(&desc_ev->lun[4], 0, 4); 311 desc_ev->reason = reason; 312 req_size = sizeof(*desc_ev); 313 314 out: 315 spdk_vhost_vq_used_ring_enqueue(vsession, vq, req, req_size); 316 } 317 318 static void 319 submit_completion(struct spdk_vhost_scsi_task *task) 320 { 321 struct spdk_vhost_session *vsession = &task->svsession->vsession; 322 323 spdk_vhost_vq_used_ring_enqueue(vsession, task->vq, task->req_idx, 324 task->used_len); 325 SPDK_DEBUGLOG(SPDK_LOG_VHOST_SCSI, "Finished task (%p) req_idx=%d\n", task, task->req_idx); 326 327 spdk_vhost_scsi_task_put(task); 328 } 329 330 static void 331 spdk_vhost_scsi_task_mgmt_cpl(struct spdk_scsi_task *scsi_task) 332 { 333 struct spdk_vhost_scsi_task *task = SPDK_CONTAINEROF(scsi_task, struct spdk_vhost_scsi_task, scsi); 334 335 submit_completion(task); 336 } 337 338 static void 339 spdk_vhost_scsi_task_cpl(struct spdk_scsi_task *scsi_task) 340 { 341 struct spdk_vhost_scsi_task *task = SPDK_CONTAINEROF(scsi_task, struct spdk_vhost_scsi_task, scsi); 342 343 /* The SCSI task has completed. Do final processing and then post 344 notification to the virtqueue's "used" ring. 345 */ 346 task->resp->status = task->scsi.status; 347 348 if (task->scsi.status != SPDK_SCSI_STATUS_GOOD) { 349 memcpy(task->resp->sense, task->scsi.sense_data, task->scsi.sense_data_len); 350 task->resp->sense_len = task->scsi.sense_data_len; 351 SPDK_DEBUGLOG(SPDK_LOG_VHOST_SCSI, "Task (%p) req_idx=%d failed - status=%u\n", task, task->req_idx, 352 task->scsi.status); 353 } 354 assert(task->scsi.transfer_len == task->scsi.length); 355 task->resp->resid = task->scsi.length - task->scsi.data_transferred; 356 357 submit_completion(task); 358 } 359 360 static void 361 task_submit(struct spdk_vhost_scsi_task *task) 362 { 363 task->resp->response = VIRTIO_SCSI_S_OK; 364 spdk_scsi_dev_queue_task(task->scsi_dev, &task->scsi); 365 } 366 367 static void 368 mgmt_task_submit(struct spdk_vhost_scsi_task *task, enum spdk_scsi_task_func func) 369 { 370 task->tmf_resp->response = VIRTIO_SCSI_S_OK; 371 task->scsi.function = func; 372 spdk_scsi_dev_queue_mgmt_task(task->scsi_dev, &task->scsi); 373 } 374 375 static void 376 invalid_request(struct spdk_vhost_scsi_task *task) 377 { 378 struct spdk_vhost_session *vsession = &task->svsession->vsession; 379 380 spdk_vhost_vq_used_ring_enqueue(vsession, task->vq, task->req_idx, 381 task->used_len); 382 spdk_vhost_scsi_task_put(task); 383 384 SPDK_DEBUGLOG(SPDK_LOG_VHOST_SCSI, "Invalid request (status=%" PRIu8")\n", 385 task->resp ? task->resp->response : -1); 386 } 387 388 static int 389 spdk_vhost_scsi_task_init_target(struct spdk_vhost_scsi_task *task, const __u8 *lun) 390 { 391 struct spdk_vhost_scsi_session *svsession = task->svsession; 392 struct spdk_scsi_dev_session_state *state; 393 uint16_t lun_id = (((uint16_t)lun[2] << 8) | lun[3]) & 0x3FFF; 394 395 SPDK_LOGDUMP(SPDK_LOG_VHOST_SCSI_QUEUE, "LUN", lun, 8); 396 397 /* First byte must be 1 and second is target */ 398 if (lun[0] != 1 || lun[1] >= SPDK_VHOST_SCSI_CTRLR_MAX_DEVS) { 399 return -1; 400 } 401 402 state = &svsession->scsi_dev_state[lun[1]]; 403 task->scsi_dev = state->dev; 404 if (state->dev == NULL || state->status != VHOST_SCSI_DEV_PRESENT) { 405 /* If dev has been hotdetached, return 0 to allow sending 406 * additional hotremove event via sense codes. 407 */ 408 return state->status != VHOST_SCSI_DEV_EMPTY ? 0 : -1; 409 } 410 411 task->scsi.target_port = spdk_scsi_dev_find_port_by_id(task->scsi_dev, 0); 412 task->scsi.lun = spdk_scsi_dev_get_lun(state->dev, lun_id); 413 return 0; 414 } 415 416 static void 417 process_ctrl_request(struct spdk_vhost_scsi_task *task) 418 { 419 struct spdk_vhost_session *vsession = &task->svsession->vsession; 420 struct spdk_vhost_dev *vdev = vsession->vdev; 421 struct vring_desc *desc, *desc_table; 422 struct virtio_scsi_ctrl_tmf_req *ctrl_req; 423 struct virtio_scsi_ctrl_an_resp *an_resp; 424 uint32_t desc_table_size, used_len = 0; 425 int rc; 426 427 spdk_scsi_task_construct(&task->scsi, spdk_vhost_scsi_task_mgmt_cpl, spdk_vhost_scsi_task_free_cb); 428 rc = spdk_vhost_vq_get_desc(vsession, task->vq, task->req_idx, &desc, &desc_table, 429 &desc_table_size); 430 if (spdk_unlikely(rc != 0)) { 431 SPDK_ERRLOG("%s: Invalid controlq descriptor at index %d.\n", 432 vdev->name, task->req_idx); 433 goto out; 434 } 435 436 ctrl_req = spdk_vhost_gpa_to_vva(vsession, desc->addr, sizeof(*ctrl_req)); 437 if (ctrl_req == NULL) { 438 SPDK_ERRLOG("%s: Invalid task management request at index %d.\n", 439 vdev->name, task->req_idx); 440 goto out; 441 } 442 443 SPDK_DEBUGLOG(SPDK_LOG_VHOST_SCSI_QUEUE, 444 "Processing controlq descriptor: desc %d/%p, desc_addr %p, len %d, flags %d, last_used_idx %d; kickfd %d; size %d\n", 445 task->req_idx, desc, (void *)desc->addr, desc->len, desc->flags, task->vq->last_used_idx, 446 task->vq->vring.kickfd, task->vq->vring.size); 447 SPDK_LOGDUMP(SPDK_LOG_VHOST_SCSI_QUEUE, "Request descriptor", (uint8_t *)ctrl_req, desc->len); 448 449 spdk_vhost_scsi_task_init_target(task, ctrl_req->lun); 450 451 spdk_vhost_vring_desc_get_next(&desc, desc_table, desc_table_size); 452 if (spdk_unlikely(desc == NULL)) { 453 SPDK_ERRLOG("%s: No response descriptor for controlq request %d.\n", 454 vdev->name, task->req_idx); 455 goto out; 456 } 457 458 /* Process the TMF request */ 459 switch (ctrl_req->type) { 460 case VIRTIO_SCSI_T_TMF: 461 task->tmf_resp = spdk_vhost_gpa_to_vva(vsession, desc->addr, sizeof(*task->tmf_resp)); 462 if (spdk_unlikely(desc->len < sizeof(struct virtio_scsi_ctrl_tmf_resp) || task->tmf_resp == NULL)) { 463 SPDK_ERRLOG("%s: TMF response descriptor at index %d points to invalid guest memory region\n", 464 vdev->name, task->req_idx); 465 goto out; 466 } 467 468 /* Check if we are processing a valid request */ 469 if (task->scsi_dev == NULL) { 470 task->tmf_resp->response = VIRTIO_SCSI_S_BAD_TARGET; 471 break; 472 } 473 474 switch (ctrl_req->subtype) { 475 case VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET: 476 /* Handle LUN reset */ 477 SPDK_DEBUGLOG(SPDK_LOG_VHOST_SCSI_QUEUE, "LUN reset\n"); 478 479 mgmt_task_submit(task, SPDK_SCSI_TASK_FUNC_LUN_RESET); 480 return; 481 default: 482 task->tmf_resp->response = VIRTIO_SCSI_S_ABORTED; 483 /* Unsupported command */ 484 SPDK_DEBUGLOG(SPDK_LOG_VHOST_SCSI_QUEUE, "Unsupported TMF command %x\n", ctrl_req->subtype); 485 break; 486 } 487 break; 488 case VIRTIO_SCSI_T_AN_QUERY: 489 case VIRTIO_SCSI_T_AN_SUBSCRIBE: { 490 an_resp = spdk_vhost_gpa_to_vva(vsession, desc->addr, sizeof(*an_resp)); 491 if (spdk_unlikely(desc->len < sizeof(struct virtio_scsi_ctrl_an_resp) || an_resp == NULL)) { 492 SPDK_WARNLOG("%s: Asynchronous response descriptor points to invalid guest memory region\n", 493 vdev->name); 494 goto out; 495 } 496 497 an_resp->response = VIRTIO_SCSI_S_ABORTED; 498 break; 499 } 500 default: 501 SPDK_DEBUGLOG(SPDK_LOG_VHOST_SCSI_QUEUE, "Unsupported control command %x\n", ctrl_req->type); 502 break; 503 } 504 505 used_len = sizeof(struct virtio_scsi_ctrl_tmf_resp); 506 out: 507 spdk_vhost_vq_used_ring_enqueue(vsession, task->vq, task->req_idx, used_len); 508 spdk_vhost_scsi_task_put(task); 509 } 510 511 /* 512 * Process task's descriptor chain and setup data related fields. 513 * Return 514 * -1 if request is invalid and must be aborted, 515 * 0 if all data are set. 516 */ 517 static int 518 task_data_setup(struct spdk_vhost_scsi_task *task, 519 struct virtio_scsi_cmd_req **req) 520 { 521 struct spdk_vhost_session *vsession = &task->svsession->vsession; 522 struct spdk_vhost_dev *vdev = vsession->vdev; 523 struct vring_desc *desc, *desc_table; 524 struct iovec *iovs = task->iovs; 525 uint16_t iovcnt = 0; 526 uint32_t desc_table_len, len = 0; 527 int rc; 528 529 spdk_scsi_task_construct(&task->scsi, spdk_vhost_scsi_task_cpl, spdk_vhost_scsi_task_free_cb); 530 531 rc = spdk_vhost_vq_get_desc(vsession, task->vq, task->req_idx, &desc, &desc_table, &desc_table_len); 532 /* First descriptor must be readable */ 533 if (spdk_unlikely(rc != 0 || spdk_vhost_vring_desc_is_wr(desc) || 534 desc->len < sizeof(struct virtio_scsi_cmd_req))) { 535 SPDK_WARNLOG("%s: invalid first (request) descriptor at index %"PRIu16".\n", 536 vdev->name, task->req_idx); 537 goto invalid_task; 538 } 539 540 *req = spdk_vhost_gpa_to_vva(vsession, desc->addr, sizeof(**req)); 541 if (spdk_unlikely(*req == NULL)) { 542 SPDK_WARNLOG("%s: Request descriptor at index %d points to invalid guest memory region\n", 543 vdev->name, task->req_idx); 544 goto invalid_task; 545 } 546 547 /* Each request must have at least 2 descriptors (e.g. request and response) */ 548 spdk_vhost_vring_desc_get_next(&desc, desc_table, desc_table_len); 549 if (desc == NULL) { 550 SPDK_WARNLOG("%s: Descriptor chain at index %d contains neither payload nor response buffer.\n", 551 vdev->name, task->req_idx); 552 goto invalid_task; 553 } 554 task->scsi.dxfer_dir = spdk_vhost_vring_desc_is_wr(desc) ? SPDK_SCSI_DIR_FROM_DEV : 555 SPDK_SCSI_DIR_TO_DEV; 556 task->scsi.iovs = iovs; 557 558 if (task->scsi.dxfer_dir == SPDK_SCSI_DIR_FROM_DEV) { 559 /* 560 * FROM_DEV (READ): [RD_req][WR_resp][WR_buf0]...[WR_bufN] 561 */ 562 task->resp = spdk_vhost_gpa_to_vva(vsession, desc->addr, sizeof(*task->resp)); 563 if (spdk_unlikely(desc->len < sizeof(struct virtio_scsi_cmd_resp) || task->resp == NULL)) { 564 SPDK_WARNLOG("%s: Response descriptor at index %d points to invalid guest memory region\n", 565 vdev->name, task->req_idx); 566 goto invalid_task; 567 } 568 rc = spdk_vhost_vring_desc_get_next(&desc, desc_table, desc_table_len); 569 if (spdk_unlikely(rc != 0)) { 570 SPDK_WARNLOG("%s: invalid descriptor chain at request index %d (descriptor id overflow?).\n", 571 vdev->name, task->req_idx); 572 goto invalid_task; 573 } 574 575 if (desc == NULL) { 576 /* 577 * TEST UNIT READY command and some others might not contain any payload and this is not an error. 578 */ 579 SPDK_DEBUGLOG(SPDK_LOG_VHOST_SCSI_DATA, 580 "No payload descriptors for FROM DEV command req_idx=%"PRIu16".\n", task->req_idx); 581 SPDK_LOGDUMP(SPDK_LOG_VHOST_SCSI_DATA, "CDB=", (*req)->cdb, VIRTIO_SCSI_CDB_SIZE); 582 task->used_len = sizeof(struct virtio_scsi_cmd_resp); 583 task->scsi.iovcnt = 1; 584 task->scsi.iovs[0].iov_len = 0; 585 task->scsi.length = 0; 586 task->scsi.transfer_len = 0; 587 return 0; 588 } 589 590 /* All remaining descriptors are data. */ 591 while (desc) { 592 if (spdk_unlikely(!spdk_vhost_vring_desc_is_wr(desc))) { 593 SPDK_WARNLOG("FROM DEV cmd: descriptor nr %" PRIu16" in payload chain is read only.\n", iovcnt); 594 goto invalid_task; 595 } 596 597 if (spdk_unlikely(spdk_vhost_vring_desc_to_iov(vsession, iovs, &iovcnt, desc))) { 598 goto invalid_task; 599 } 600 len += desc->len; 601 602 rc = spdk_vhost_vring_desc_get_next(&desc, desc_table, desc_table_len); 603 if (spdk_unlikely(rc != 0)) { 604 SPDK_WARNLOG("%s: invalid payload in descriptor chain starting at index %d.\n", 605 vdev->name, task->req_idx); 606 goto invalid_task; 607 } 608 } 609 610 task->used_len = sizeof(struct virtio_scsi_cmd_resp) + len; 611 } else { 612 SPDK_DEBUGLOG(SPDK_LOG_VHOST_SCSI_DATA, "TO DEV"); 613 /* 614 * TO_DEV (WRITE):[RD_req][RD_buf0]...[RD_bufN][WR_resp] 615 * No need to check descriptor WR flag as this is done while setting scsi.dxfer_dir. 616 */ 617 618 /* Process descriptors up to response. */ 619 while (!spdk_vhost_vring_desc_is_wr(desc)) { 620 if (spdk_unlikely(spdk_vhost_vring_desc_to_iov(vsession, iovs, &iovcnt, desc))) { 621 goto invalid_task; 622 } 623 len += desc->len; 624 625 spdk_vhost_vring_desc_get_next(&desc, desc_table, desc_table_len); 626 if (spdk_unlikely(desc == NULL)) { 627 SPDK_WARNLOG("TO_DEV cmd: no response descriptor.\n"); 628 goto invalid_task; 629 } 630 } 631 632 task->resp = spdk_vhost_gpa_to_vva(vsession, desc->addr, sizeof(*task->resp)); 633 if (spdk_unlikely(desc->len < sizeof(struct virtio_scsi_cmd_resp) || task->resp == NULL)) { 634 SPDK_WARNLOG("%s: Response descriptor at index %d points to invalid guest memory region\n", 635 vdev->name, task->req_idx); 636 goto invalid_task; 637 } 638 639 task->used_len = sizeof(struct virtio_scsi_cmd_resp); 640 } 641 642 task->scsi.iovcnt = iovcnt; 643 task->scsi.length = len; 644 task->scsi.transfer_len = len; 645 return 0; 646 647 invalid_task: 648 SPDK_DEBUGLOG(SPDK_LOG_VHOST_SCSI_DATA, "%s: Invalid task at index %"PRIu16".\n", 649 vdev->name, task->req_idx); 650 return -1; 651 } 652 653 static int 654 process_request(struct spdk_vhost_scsi_task *task) 655 { 656 struct virtio_scsi_cmd_req *req; 657 int result; 658 659 result = task_data_setup(task, &req); 660 if (result) { 661 return result; 662 } 663 664 result = spdk_vhost_scsi_task_init_target(task, req->lun); 665 if (spdk_unlikely(result != 0)) { 666 task->resp->response = VIRTIO_SCSI_S_BAD_TARGET; 667 return -1; 668 } 669 670 task->scsi.cdb = req->cdb; 671 SPDK_LOGDUMP(SPDK_LOG_VHOST_SCSI_DATA, "request CDB", req->cdb, VIRTIO_SCSI_CDB_SIZE); 672 673 if (spdk_unlikely(task->scsi.lun == NULL)) { 674 spdk_scsi_task_process_null_lun(&task->scsi); 675 task->resp->response = VIRTIO_SCSI_S_OK; 676 return 1; 677 } 678 679 return 0; 680 } 681 682 static void 683 process_controlq(struct spdk_vhost_scsi_session *svsession, struct spdk_vhost_virtqueue *vq) 684 { 685 struct spdk_vhost_scsi_dev *svdev = svsession->svdev; 686 struct spdk_vhost_session *vsession = &svsession->vsession; 687 struct spdk_vhost_scsi_task *task; 688 uint16_t reqs[32]; 689 uint16_t reqs_cnt, i; 690 691 reqs_cnt = spdk_vhost_vq_avail_ring_get(vq, reqs, SPDK_COUNTOF(reqs)); 692 for (i = 0; i < reqs_cnt; i++) { 693 if (spdk_unlikely(reqs[i] >= vq->vring.size)) { 694 SPDK_ERRLOG("%s: invalid entry in avail ring. Buffer '%"PRIu16"' exceeds virtqueue size (%"PRIu16")\n", 695 svdev->vdev.name, reqs[i], vq->vring.size); 696 spdk_vhost_vq_used_ring_enqueue(vsession, vq, reqs[i], 0); 697 continue; 698 } 699 700 task = &((struct spdk_vhost_scsi_task *)vq->tasks)[reqs[i]]; 701 if (spdk_unlikely(task->used)) { 702 SPDK_ERRLOG("%s: invalid entry in avail ring. Buffer '%"PRIu16"' is still in use!\n", 703 svdev->vdev.name, reqs[i]); 704 spdk_vhost_vq_used_ring_enqueue(vsession, vq, reqs[i], 0); 705 continue; 706 } 707 708 vsession->task_cnt++; 709 memset(&task->scsi, 0, sizeof(task->scsi)); 710 task->tmf_resp = NULL; 711 task->used = true; 712 process_ctrl_request(task); 713 } 714 } 715 716 static void 717 process_requestq(struct spdk_vhost_scsi_session *svsession, struct spdk_vhost_virtqueue *vq) 718 { 719 struct spdk_vhost_session *vsession = &svsession->vsession; 720 struct spdk_vhost_dev *vdev = vsession->vdev; 721 struct spdk_vhost_scsi_task *task; 722 uint16_t reqs[32]; 723 uint16_t reqs_cnt, i; 724 int result; 725 726 reqs_cnt = spdk_vhost_vq_avail_ring_get(vq, reqs, SPDK_COUNTOF(reqs)); 727 assert(reqs_cnt <= 32); 728 729 for (i = 0; i < reqs_cnt; i++) { 730 SPDK_DEBUGLOG(SPDK_LOG_VHOST_SCSI, "====== Starting processing request idx %"PRIu16"======\n", 731 reqs[i]); 732 733 if (spdk_unlikely(reqs[i] >= vq->vring.size)) { 734 SPDK_ERRLOG("%s: request idx '%"PRIu16"' exceeds virtqueue size (%"PRIu16").\n", 735 vdev->name, reqs[i], vq->vring.size); 736 spdk_vhost_vq_used_ring_enqueue(vsession, vq, reqs[i], 0); 737 continue; 738 } 739 740 task = &((struct spdk_vhost_scsi_task *)vq->tasks)[reqs[i]]; 741 if (spdk_unlikely(task->used)) { 742 SPDK_ERRLOG("%s: request with idx '%"PRIu16"' is already pending.\n", 743 vdev->name, reqs[i]); 744 spdk_vhost_vq_used_ring_enqueue(vsession, vq, reqs[i], 0); 745 continue; 746 } 747 748 vsession->task_cnt++; 749 memset(&task->scsi, 0, sizeof(task->scsi)); 750 task->resp = NULL; 751 task->used = true; 752 task->used_len = 0; 753 result = process_request(task); 754 if (likely(result == 0)) { 755 task_submit(task); 756 SPDK_DEBUGLOG(SPDK_LOG_VHOST_SCSI, "====== Task %p req_idx %d submitted ======\n", task, 757 task->req_idx); 758 } else if (result > 0) { 759 spdk_vhost_scsi_task_cpl(&task->scsi); 760 SPDK_DEBUGLOG(SPDK_LOG_VHOST_SCSI, "====== Task %p req_idx %d finished early ======\n", task, 761 task->req_idx); 762 } else { 763 invalid_request(task); 764 SPDK_DEBUGLOG(SPDK_LOG_VHOST_SCSI, "====== Task %p req_idx %d failed ======\n", task, 765 task->req_idx); 766 } 767 } 768 } 769 770 static int 771 vdev_mgmt_worker(void *arg) 772 { 773 struct spdk_vhost_scsi_session *svsession = arg; 774 struct spdk_vhost_session *vsession = &svsession->vsession; 775 776 process_removed_devs(svsession); 777 spdk_vhost_vq_used_signal(vsession, &vsession->virtqueue[VIRTIO_SCSI_EVENTQ]); 778 779 process_controlq(svsession, &vsession->virtqueue[VIRTIO_SCSI_CONTROLQ]); 780 spdk_vhost_vq_used_signal(vsession, &vsession->virtqueue[VIRTIO_SCSI_CONTROLQ]); 781 782 return -1; 783 } 784 785 static int 786 vdev_worker(void *arg) 787 { 788 struct spdk_vhost_scsi_session *svsession = arg; 789 struct spdk_vhost_session *vsession = &svsession->vsession; 790 uint32_t q_idx; 791 792 for (q_idx = VIRTIO_SCSI_REQUESTQ; q_idx < vsession->max_queues; q_idx++) { 793 process_requestq(svsession, &vsession->virtqueue[q_idx]); 794 } 795 796 spdk_vhost_session_used_signal(vsession); 797 798 return -1; 799 } 800 801 static struct spdk_vhost_scsi_dev * 802 to_scsi_dev(struct spdk_vhost_dev *ctrlr) 803 { 804 if (ctrlr == NULL) { 805 return NULL; 806 } 807 808 if (ctrlr->backend != &spdk_vhost_scsi_device_backend) { 809 SPDK_ERRLOG("%s: not a vhost-scsi device.\n", ctrlr->name); 810 return NULL; 811 } 812 813 return SPDK_CONTAINEROF(ctrlr, struct spdk_vhost_scsi_dev, vdev); 814 } 815 816 static struct spdk_vhost_scsi_session * 817 to_scsi_session(struct spdk_vhost_session *vsession) 818 { 819 if (vsession == NULL) { 820 return NULL; 821 } 822 823 if (vsession->vdev->backend != &spdk_vhost_scsi_device_backend) { 824 SPDK_ERRLOG("%s: not a vhost-scsi device.\n", vsession->vdev->name); 825 return NULL; 826 } 827 828 return (struct spdk_vhost_scsi_session *)vsession; 829 } 830 831 int 832 spdk_vhost_scsi_dev_construct(const char *name, const char *cpumask) 833 { 834 struct spdk_vhost_scsi_dev *svdev = spdk_dma_zmalloc(sizeof(struct spdk_vhost_scsi_dev), 835 SPDK_CACHE_LINE_SIZE, NULL); 836 int rc; 837 838 if (svdev == NULL) { 839 return -ENOMEM; 840 } 841 842 spdk_vhost_lock(); 843 rc = spdk_vhost_dev_register(&svdev->vdev, name, cpumask, 844 &spdk_vhost_scsi_device_backend); 845 846 if (rc) { 847 spdk_dma_free(svdev); 848 } 849 850 spdk_vhost_unlock(); 851 return rc; 852 } 853 854 static int 855 spdk_vhost_scsi_dev_remove(struct spdk_vhost_dev *vdev) 856 { 857 struct spdk_vhost_scsi_dev *svdev = to_scsi_dev(vdev); 858 int rc, i; 859 860 if (svdev == NULL) { 861 return -EINVAL; 862 } 863 864 for (i = 0; i < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; ++i) { 865 if (svdev->scsi_dev_state[i].dev) { 866 if (vdev->registered) { 867 SPDK_ERRLOG("Trying to remove non-empty controller: %s.\n", vdev->name); 868 return -EBUSY; 869 } 870 871 rc = spdk_vhost_scsi_dev_remove_tgt(vdev, i, NULL, NULL); 872 if (rc != 0) { 873 SPDK_ERRLOG("%s: failed to force-remove target %d\n", vdev->name, i); 874 return rc; 875 } 876 } 877 } 878 879 rc = spdk_vhost_dev_unregister(vdev); 880 if (rc != 0) { 881 return rc; 882 } 883 884 spdk_dma_free(svdev); 885 return 0; 886 } 887 888 struct spdk_scsi_dev * 889 spdk_vhost_scsi_dev_get_tgt(struct spdk_vhost_dev *vdev, uint8_t num) 890 { 891 struct spdk_vhost_scsi_dev *svdev; 892 893 assert(num < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS); 894 svdev = to_scsi_dev(vdev); 895 if (svdev == NULL || svdev->scsi_dev_state[num].status != VHOST_SCSI_DEV_PRESENT) { 896 return NULL; 897 } 898 899 assert(svdev->scsi_dev_state[num].dev != NULL); 900 return svdev->scsi_dev_state[num].dev; 901 } 902 903 static void 904 spdk_vhost_scsi_lun_hotremove(const struct spdk_scsi_lun *lun, void *arg) 905 { 906 struct spdk_vhost_scsi_dev *svdev = arg; 907 const struct spdk_scsi_dev *scsi_dev; 908 unsigned scsi_dev_num; 909 910 assert(lun != NULL); 911 assert(svdev != NULL); 912 scsi_dev = spdk_scsi_lun_get_dev(lun); 913 for (scsi_dev_num = 0; scsi_dev_num < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; scsi_dev_num++) { 914 if (svdev->scsi_dev_state[scsi_dev_num].dev == scsi_dev) { 915 break; 916 } 917 } 918 919 if (scsi_dev_num == SPDK_VHOST_SCSI_CTRLR_MAX_DEVS) { 920 /* The entire device has been already removed. */ 921 return; 922 } 923 924 /* remove entire device */ 925 spdk_vhost_scsi_dev_remove_tgt(&svdev->vdev, scsi_dev_num, NULL, NULL); 926 } 927 928 static int 929 spdk_vhost_scsi_session_add_tgt(struct spdk_vhost_dev *vdev, 930 struct spdk_vhost_session *vsession, void *ctx) 931 { 932 unsigned scsi_tgt_num = (unsigned)(uintptr_t)ctx; 933 struct spdk_vhost_scsi_session *svsession; 934 struct spdk_scsi_dev_vhost_state *vhost_sdev; 935 struct spdk_scsi_dev_session_state *session_sdev; 936 int rc; 937 938 if (vsession == NULL) { 939 struct spdk_vhost_scsi_dev *svdev = SPDK_CONTAINEROF(vdev, 940 struct spdk_vhost_scsi_dev, vdev); 941 vhost_sdev = &svdev->scsi_dev_state[scsi_tgt_num]; 942 943 /* All sessions have added the target */ 944 assert(vhost_sdev->status == VHOST_SCSI_DEV_ADDING); 945 vhost_sdev->status = VHOST_SCSI_DEV_PRESENT; 946 return 0; 947 } 948 949 if (vsession->lcore == -1) { 950 /* Nothing to do. */ 951 return 0; 952 } 953 954 svsession = (struct spdk_vhost_scsi_session *)vsession; 955 vhost_sdev = &svsession->svdev->scsi_dev_state[scsi_tgt_num]; 956 session_sdev = &svsession->scsi_dev_state[scsi_tgt_num]; 957 958 session_sdev->dev = vhost_sdev->dev; 959 session_sdev->status = VHOST_SCSI_DEV_PRESENT; 960 961 rc = spdk_scsi_dev_allocate_io_channels(svsession->scsi_dev_state[scsi_tgt_num].dev); 962 if (rc != 0) { 963 SPDK_ERRLOG("Couldn't allocate io channnel for SCSI target %u in device %s\n", 964 scsi_tgt_num, vdev->name); 965 966 /* unset the SCSI target so that all I/O to it will be rejected */ 967 session_sdev->dev = NULL; 968 /* Set status to EMPTY so that we won't reply with SCSI hotremove 969 * sense codes - the device hasn't ever been added. 970 */ 971 session_sdev->status = VHOST_SCSI_DEV_EMPTY; 972 973 /* Return with no error. We'll continue allocating io_channels for 974 * other sessions on this device in hopes they succeed. The sessions 975 * that failed to allocate io_channels simply won't be able to 976 * detect the SCSI target, nor do any I/O to it. 977 */ 978 return 0; 979 } 980 981 if (spdk_vhost_dev_has_feature(vsession, VIRTIO_SCSI_F_HOTPLUG)) { 982 eventq_enqueue(svsession, scsi_tgt_num, 983 VIRTIO_SCSI_T_TRANSPORT_RESET, VIRTIO_SCSI_EVT_RESET_RESCAN); 984 } else { 985 SPDK_NOTICELOG("Device %s does not support hotplug. " 986 "Please restart the driver or perform a rescan.\n", 987 vdev->name); 988 } 989 990 return 0; 991 } 992 993 int 994 spdk_vhost_scsi_dev_add_tgt(struct spdk_vhost_dev *vdev, int scsi_tgt_num, 995 const char *bdev_name) 996 { 997 struct spdk_vhost_scsi_dev *svdev; 998 struct spdk_scsi_dev_vhost_state *state; 999 char target_name[SPDK_SCSI_DEV_MAX_NAME]; 1000 int lun_id_list[1]; 1001 const char *bdev_names_list[1]; 1002 1003 svdev = to_scsi_dev(vdev); 1004 if (svdev == NULL) { 1005 return -EINVAL; 1006 } 1007 1008 if (scsi_tgt_num < 0) { 1009 for (scsi_tgt_num = 0; scsi_tgt_num < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; scsi_tgt_num++) { 1010 if (svdev->scsi_dev_state[scsi_tgt_num].dev == NULL) { 1011 break; 1012 } 1013 } 1014 1015 if (scsi_tgt_num == SPDK_VHOST_SCSI_CTRLR_MAX_DEVS) { 1016 SPDK_ERRLOG("Controller %s - all targets already in use.\n", vdev->name); 1017 return -ENOSPC; 1018 } 1019 } else { 1020 if (scsi_tgt_num >= SPDK_VHOST_SCSI_CTRLR_MAX_DEVS) { 1021 SPDK_ERRLOG("Controller %s target %d number too big (max %d)\n", vdev->name, scsi_tgt_num, 1022 SPDK_VHOST_SCSI_CTRLR_MAX_DEVS); 1023 return -EINVAL; 1024 } 1025 } 1026 1027 if (bdev_name == NULL) { 1028 SPDK_ERRLOG("No lun name specified\n"); 1029 return -EINVAL; 1030 } 1031 1032 state = &svdev->scsi_dev_state[scsi_tgt_num]; 1033 if (state->dev != NULL) { 1034 SPDK_ERRLOG("Controller %s target %u already occupied\n", vdev->name, scsi_tgt_num); 1035 return -EEXIST; 1036 } 1037 1038 /* 1039 * At this stage only one LUN per target 1040 */ 1041 snprintf(target_name, sizeof(target_name), "Target %u", scsi_tgt_num); 1042 lun_id_list[0] = 0; 1043 bdev_names_list[0] = (char *)bdev_name; 1044 1045 state->status = VHOST_SCSI_DEV_ADDING; 1046 state->dev = spdk_scsi_dev_construct(target_name, bdev_names_list, lun_id_list, 1, 1047 SPDK_SPC_PROTOCOL_IDENTIFIER_SAS, 1048 spdk_vhost_scsi_lun_hotremove, svdev); 1049 1050 if (state->dev == NULL) { 1051 state->status = VHOST_SCSI_DEV_EMPTY; 1052 SPDK_ERRLOG("Couldn't create spdk SCSI target '%s' using bdev '%s' in controller: %s\n", 1053 target_name, bdev_name, vdev->name); 1054 return -EINVAL; 1055 } 1056 spdk_scsi_dev_add_port(state->dev, 0, "vhost"); 1057 1058 SPDK_INFOLOG(SPDK_LOG_VHOST, "Controller %s: defined target '%s' using bdev '%s'\n", 1059 vdev->name, target_name, bdev_name); 1060 1061 spdk_vhost_dev_foreach_session(vdev, spdk_vhost_scsi_session_add_tgt, 1062 (void *)(uintptr_t)scsi_tgt_num); 1063 return scsi_tgt_num; 1064 } 1065 1066 static int 1067 spdk_vhost_scsi_session_remove_tgt(struct spdk_vhost_dev *vdev, 1068 struct spdk_vhost_session *vsession, void *ctx) 1069 { 1070 unsigned scsi_tgt_num = (unsigned)(uintptr_t)ctx; 1071 struct spdk_vhost_scsi_session *svsession; 1072 struct spdk_scsi_dev_session_state *state; 1073 int rc = 0; 1074 1075 if (vsession == NULL) { 1076 struct spdk_vhost_scsi_dev *svdev = SPDK_CONTAINEROF(vdev, 1077 struct spdk_vhost_scsi_dev, vdev); 1078 1079 if (vdev->active_session_num == 0) { 1080 /* there aren't any active sessions, so remove the dev and exit */ 1081 rc = remove_scsi_tgt(svdev, scsi_tgt_num); 1082 } 1083 return rc; 1084 } 1085 1086 if (vsession->lcore == -1) { 1087 /* Nothing to do */ 1088 return 0; 1089 } 1090 1091 /* Mark the target for removal */ 1092 svsession = (struct spdk_vhost_scsi_session *)vsession; 1093 state = &svsession->scsi_dev_state[scsi_tgt_num]; 1094 assert(state->status == VHOST_SCSI_DEV_PRESENT); 1095 state->status = VHOST_SCSI_DEV_REMOVING; 1096 1097 /* Send a hotremove Virtio event and wait for the session's 1098 * management poller to remove the target after all its pending I/O 1099 * has finished. 1100 */ 1101 if (spdk_vhost_dev_has_feature(vsession, VIRTIO_SCSI_F_HOTPLUG)) { 1102 eventq_enqueue(svsession, scsi_tgt_num, 1103 VIRTIO_SCSI_T_TRANSPORT_RESET, VIRTIO_SCSI_EVT_RESET_REMOVED); 1104 } 1105 1106 return 0; 1107 } 1108 1109 int 1110 spdk_vhost_scsi_dev_remove_tgt(struct spdk_vhost_dev *vdev, unsigned scsi_tgt_num, 1111 spdk_vhost_event_fn cb_fn, void *cb_arg) 1112 { 1113 struct spdk_vhost_scsi_dev *svdev; 1114 struct spdk_scsi_dev_vhost_state *scsi_dev_state; 1115 1116 if (scsi_tgt_num >= SPDK_VHOST_SCSI_CTRLR_MAX_DEVS) { 1117 SPDK_ERRLOG("%s: invalid target number %d\n", vdev->name, scsi_tgt_num); 1118 return -EINVAL; 1119 } 1120 1121 svdev = to_scsi_dev(vdev); 1122 if (svdev == NULL) { 1123 return -ENODEV; 1124 } 1125 1126 scsi_dev_state = &svdev->scsi_dev_state[scsi_tgt_num]; 1127 if (scsi_dev_state->dev == NULL || scsi_dev_state->status == VHOST_SCSI_DEV_ADDING) { 1128 SPDK_ERRLOG("Controller %s target %u is not occupied\n", vdev->name, scsi_tgt_num); 1129 return -ENODEV; 1130 } 1131 1132 assert(scsi_dev_state->status != VHOST_SCSI_DEV_EMPTY); 1133 if (scsi_dev_state->status != VHOST_SCSI_DEV_PRESENT) { 1134 SPDK_WARNLOG("%s: 'Target %u' has been already marked to hotremove.\n", 1135 vdev->name, scsi_tgt_num); 1136 return -EBUSY; 1137 } 1138 1139 scsi_dev_state->remove_cb = cb_fn; 1140 scsi_dev_state->remove_ctx = cb_arg; 1141 scsi_dev_state->status = VHOST_SCSI_DEV_REMOVING; 1142 1143 spdk_vhost_dev_foreach_session(vdev, spdk_vhost_scsi_session_remove_tgt, 1144 (void *)(uintptr_t)scsi_tgt_num); 1145 return 0; 1146 } 1147 1148 int 1149 spdk_vhost_scsi_controller_construct(void) 1150 { 1151 struct spdk_conf_section *sp = spdk_conf_first_section(NULL); 1152 struct spdk_vhost_dev *vdev; 1153 int i, dev_num; 1154 unsigned ctrlr_num = 0; 1155 char *bdev_name, *tgt_num_str; 1156 char *cpumask; 1157 char *name; 1158 char *tgt = NULL; 1159 1160 while (sp != NULL) { 1161 if (!spdk_conf_section_match_prefix(sp, "VhostScsi")) { 1162 sp = spdk_conf_next_section(sp); 1163 continue; 1164 } 1165 1166 if (sscanf(spdk_conf_section_get_name(sp), "VhostScsi%u", &ctrlr_num) != 1) { 1167 SPDK_ERRLOG("Section '%s' has non-numeric suffix.\n", 1168 spdk_conf_section_get_name(sp)); 1169 return -1; 1170 } 1171 1172 name = spdk_conf_section_get_val(sp, "Name"); 1173 cpumask = spdk_conf_section_get_val(sp, "Cpumask"); 1174 1175 if (spdk_vhost_scsi_dev_construct(name, cpumask) < 0) { 1176 return -1; 1177 } 1178 1179 vdev = spdk_vhost_dev_find(name); 1180 assert(vdev); 1181 1182 for (i = 0; ; i++) { 1183 1184 tgt = spdk_conf_section_get_nval(sp, "Target", i); 1185 if (tgt == NULL) { 1186 break; 1187 } 1188 1189 tgt_num_str = spdk_conf_section_get_nmval(sp, "Target", i, 0); 1190 if (tgt_num_str == NULL) { 1191 SPDK_ERRLOG("%s: Invalid or missing target number\n", name); 1192 return -1; 1193 } 1194 1195 dev_num = (int)strtol(tgt_num_str, NULL, 10); 1196 bdev_name = spdk_conf_section_get_nmval(sp, "Target", i, 1); 1197 if (bdev_name == NULL) { 1198 SPDK_ERRLOG("%s: Invalid or missing bdev name for target %d\n", name, dev_num); 1199 return -1; 1200 } else if (spdk_conf_section_get_nmval(sp, "Target", i, 2)) { 1201 SPDK_ERRLOG("%s: Only one LUN per vhost SCSI device supported\n", name); 1202 return -1; 1203 } 1204 1205 if (spdk_vhost_scsi_dev_add_tgt(vdev, dev_num, bdev_name) < 0) { 1206 return -1; 1207 } 1208 } 1209 1210 sp = spdk_conf_next_section(sp); 1211 } 1212 1213 return 0; 1214 } 1215 1216 static void 1217 free_task_pool(struct spdk_vhost_scsi_session *svsession) 1218 { 1219 struct spdk_vhost_session *vsession = &svsession->vsession; 1220 struct spdk_vhost_virtqueue *vq; 1221 uint16_t i; 1222 1223 for (i = 0; i < vsession->max_queues; i++) { 1224 vq = &vsession->virtqueue[i]; 1225 if (vq->tasks == NULL) { 1226 continue; 1227 } 1228 1229 spdk_dma_free(vq->tasks); 1230 vq->tasks = NULL; 1231 } 1232 } 1233 1234 static int 1235 alloc_task_pool(struct spdk_vhost_scsi_session *svsession) 1236 { 1237 struct spdk_vhost_session *vsession = &svsession->vsession; 1238 struct spdk_vhost_scsi_dev *svdev = svsession->svdev; 1239 struct spdk_vhost_virtqueue *vq; 1240 struct spdk_vhost_scsi_task *task; 1241 uint32_t task_cnt; 1242 uint16_t i; 1243 uint32_t j; 1244 1245 for (i = 0; i < vsession->max_queues; i++) { 1246 vq = &vsession->virtqueue[i]; 1247 if (vq->vring.desc == NULL) { 1248 continue; 1249 } 1250 1251 task_cnt = vq->vring.size; 1252 if (task_cnt > SPDK_VHOST_MAX_VQ_SIZE) { 1253 /* sanity check */ 1254 SPDK_ERRLOG("Controller %s: virtuque %"PRIu16" is too big. (size = %"PRIu32", max = %"PRIu32")\n", 1255 svdev->vdev.name, i, task_cnt, SPDK_VHOST_MAX_VQ_SIZE); 1256 free_task_pool(svsession); 1257 return -1; 1258 } 1259 vq->tasks = spdk_zmalloc(sizeof(struct spdk_vhost_scsi_task) * task_cnt, 1260 SPDK_CACHE_LINE_SIZE, NULL, 1261 SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA); 1262 if (vq->tasks == NULL) { 1263 SPDK_ERRLOG("Controller %s: failed to allocate %"PRIu32" tasks for virtqueue %"PRIu16"\n", 1264 svdev->vdev.name, task_cnt, i); 1265 free_task_pool(svsession); 1266 return -1; 1267 } 1268 1269 for (j = 0; j < task_cnt; j++) { 1270 task = &((struct spdk_vhost_scsi_task *)vq->tasks)[j]; 1271 task->svsession = svsession; 1272 task->vq = vq; 1273 task->req_idx = j; 1274 } 1275 } 1276 1277 return 0; 1278 } 1279 1280 static int 1281 spdk_vhost_scsi_start_cb(struct spdk_vhost_dev *vdev, 1282 struct spdk_vhost_session *vsession, void *unused) 1283 { 1284 struct spdk_vhost_scsi_dev *svdev; 1285 struct spdk_vhost_scsi_session *svsession; 1286 struct spdk_scsi_dev_vhost_state *state; 1287 uint32_t i; 1288 int rc; 1289 1290 svsession = to_scsi_session(vsession); 1291 assert(svsession != NULL); 1292 svdev = svsession->svdev; 1293 1294 /* validate all I/O queues are in a contiguous index range */ 1295 for (i = VIRTIO_SCSI_REQUESTQ; i < vsession->max_queues; i++) { 1296 if (vsession->virtqueue[i].vring.desc == NULL) { 1297 SPDK_ERRLOG("%s: queue %"PRIu32" is empty\n", vsession->vdev->name, i); 1298 rc = -1; 1299 goto out; 1300 } 1301 } 1302 1303 rc = alloc_task_pool(svsession); 1304 if (rc != 0) { 1305 SPDK_ERRLOG("%s: failed to alloc task pool.\n", vdev->name); 1306 goto out; 1307 } 1308 1309 for (i = 0; i < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; i++) { 1310 state = &svdev->scsi_dev_state[i]; 1311 if (state->dev == NULL) { 1312 continue; 1313 } 1314 assert(svsession->scsi_dev_state[i].status == VHOST_SCSI_DEV_EMPTY); 1315 svsession->scsi_dev_state[i].dev = state->dev; 1316 svsession->scsi_dev_state[i].status = state->status; 1317 rc = spdk_scsi_dev_allocate_io_channels(state->dev); 1318 if (rc != 0) { 1319 SPDK_ERRLOG("%s: failed to alloc io_channel for SCSI target %"PRIu32"\n", vdev->name, i); 1320 /* unset the SCSI target so that all I/O to it will be rejected */ 1321 svsession->scsi_dev_state[i].dev = NULL; 1322 /* set EMPTY state so that we won't reply with SCSI hotremove 1323 * sense codes - the device hasn't ever been added. 1324 */ 1325 svsession->scsi_dev_state[i].status = VHOST_SCSI_DEV_EMPTY; 1326 continue; 1327 } 1328 } 1329 SPDK_INFOLOG(SPDK_LOG_VHOST, "Started poller for vhost controller %s on lcore %d\n", 1330 vdev->name, spdk_env_get_current_core()); 1331 1332 svsession->requestq_poller = spdk_poller_register(vdev_worker, svsession, 0); 1333 if (vsession->virtqueue[VIRTIO_SCSI_CONTROLQ].vring.desc && 1334 vsession->virtqueue[VIRTIO_SCSI_EVENTQ].vring.desc) { 1335 svsession->mgmt_poller = spdk_poller_register(vdev_mgmt_worker, svsession, 1336 MGMT_POLL_PERIOD_US); 1337 } 1338 out: 1339 spdk_vhost_session_start_done(vsession, rc); 1340 return rc; 1341 } 1342 1343 static int 1344 spdk_vhost_scsi_start(struct spdk_vhost_session *vsession) 1345 { 1346 struct spdk_vhost_scsi_session *svsession; 1347 struct spdk_vhost_scsi_dev *svdev; 1348 int rc; 1349 1350 svsession = to_scsi_session(vsession); 1351 if (svsession == NULL) { 1352 SPDK_ERRLOG("Trying to start non-scsi session as a scsi one.\n"); 1353 return -1; 1354 } 1355 1356 svdev = to_scsi_dev(vsession->vdev); 1357 assert(svdev != NULL); 1358 svsession->svdev = svdev; 1359 1360 if (svdev->vdev.active_session_num == 0) { 1361 svdev->lcore = spdk_vhost_allocate_reactor(svdev->vdev.cpumask); 1362 } 1363 1364 rc = spdk_vhost_session_send_event(svdev->lcore, vsession, spdk_vhost_scsi_start_cb, 1365 3, "start session"); 1366 if (rc != 0) { 1367 if (svdev->vdev.active_session_num == 0) { 1368 spdk_vhost_free_reactor(svdev->lcore); 1369 svdev->lcore = -1; 1370 } 1371 } 1372 1373 return rc; 1374 } 1375 1376 static int 1377 destroy_session_poller_cb(void *arg) 1378 { 1379 struct spdk_vhost_scsi_session *svsession = arg; 1380 struct spdk_vhost_session *vsession = &svsession->vsession; 1381 struct spdk_scsi_dev_session_state *state; 1382 uint32_t i; 1383 1384 if (vsession->task_cnt > 0) { 1385 return -1; 1386 } 1387 1388 if (spdk_vhost_trylock() != 0) { 1389 return -1; 1390 } 1391 1392 for (i = 0; i < vsession->max_queues; i++) { 1393 spdk_vhost_vq_used_signal(vsession, &vsession->virtqueue[i]); 1394 } 1395 1396 for (i = 0; i < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; i++) { 1397 enum spdk_scsi_dev_vhost_status prev_status; 1398 1399 state = &svsession->scsi_dev_state[i]; 1400 if (state->dev == NULL) { 1401 continue; 1402 } 1403 1404 spdk_scsi_dev_free_io_channels(state->dev); 1405 1406 prev_status = state->status; 1407 state->status = VHOST_SCSI_DEV_EMPTY; 1408 state->dev = NULL; 1409 1410 if (prev_status == VHOST_SCSI_DEV_REMOVING) { 1411 /* try to detach it globally */ 1412 spdk_vhost_dev_foreach_session(vsession->vdev, 1413 spdk_vhost_scsi_session_process_removed, 1414 (void *)(uintptr_t)i); 1415 } 1416 } 1417 1418 SPDK_INFOLOG(SPDK_LOG_VHOST, "Stopping poller for vhost controller %s\n", 1419 svsession->svdev->vdev.name); 1420 1421 free_task_pool(svsession); 1422 1423 spdk_poller_unregister(&svsession->stop_poller); 1424 spdk_vhost_session_stop_done(vsession, 0); 1425 1426 spdk_vhost_unlock(); 1427 return -1; 1428 } 1429 1430 static int 1431 spdk_vhost_scsi_stop_cb(struct spdk_vhost_dev *vdev, 1432 struct spdk_vhost_session *vsession, void *unused) 1433 { 1434 struct spdk_vhost_scsi_session *svsession; 1435 1436 svsession = to_scsi_session(vsession); 1437 assert(svsession != NULL); 1438 1439 /* Stop receiving new I/O requests */ 1440 spdk_poller_unregister(&svsession->requestq_poller); 1441 1442 /* Stop receiving controlq requests, also stop processing the 1443 * asynchronous hotremove events. All the remaining events 1444 * will be finalized by the stop_poller below. 1445 */ 1446 spdk_poller_unregister(&svsession->mgmt_poller); 1447 1448 /* Wait for all pending I/Os to complete, then process all the 1449 * remaining hotremove events one last time. 1450 */ 1451 svsession->stop_poller = spdk_poller_register(destroy_session_poller_cb, 1452 svsession, 1000); 1453 1454 return 0; 1455 } 1456 1457 static int 1458 spdk_vhost_scsi_stop(struct spdk_vhost_session *vsession) 1459 { 1460 struct spdk_vhost_scsi_session *svsession; 1461 int rc; 1462 1463 svsession = to_scsi_session(vsession); 1464 if (svsession == NULL) { 1465 SPDK_ERRLOG("Trying to stop non-scsi session as a scsi one.\n"); 1466 return -1; 1467 } 1468 rc = spdk_vhost_session_send_event(vsession->lcore, vsession, 1469 spdk_vhost_scsi_stop_cb, 3, "stop session"); 1470 if (rc != 0) { 1471 return rc; 1472 } 1473 1474 if (vsession->vdev->active_session_num == 0) { 1475 spdk_vhost_free_reactor(svsession->svdev->lcore); 1476 svsession->svdev->lcore = -1; 1477 } 1478 return 0; 1479 } 1480 1481 static void 1482 spdk_vhost_scsi_dump_info_json(struct spdk_vhost_dev *vdev, struct spdk_json_write_ctx *w) 1483 { 1484 struct spdk_scsi_dev *sdev; 1485 struct spdk_scsi_lun *lun; 1486 uint32_t dev_idx; 1487 uint32_t lun_idx; 1488 1489 assert(vdev != NULL); 1490 spdk_json_write_named_array_begin(w, "scsi"); 1491 for (dev_idx = 0; dev_idx < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; dev_idx++) { 1492 sdev = spdk_vhost_scsi_dev_get_tgt(vdev, dev_idx); 1493 if (!sdev) { 1494 continue; 1495 } 1496 1497 spdk_json_write_object_begin(w); 1498 1499 spdk_json_write_named_uint32(w, "scsi_dev_num", dev_idx); 1500 1501 spdk_json_write_named_uint32(w, "id", spdk_scsi_dev_get_id(sdev)); 1502 1503 spdk_json_write_named_string(w, "target_name", spdk_scsi_dev_get_name(sdev)); 1504 1505 spdk_json_write_named_array_begin(w, "luns"); 1506 1507 for (lun_idx = 0; lun_idx < SPDK_SCSI_DEV_MAX_LUN; lun_idx++) { 1508 lun = spdk_scsi_dev_get_lun(sdev, lun_idx); 1509 if (!lun) { 1510 continue; 1511 } 1512 1513 spdk_json_write_object_begin(w); 1514 1515 spdk_json_write_named_int32(w, "id", spdk_scsi_lun_get_id(lun)); 1516 1517 spdk_json_write_named_string(w, "bdev_name", spdk_scsi_lun_get_bdev_name(lun)); 1518 1519 spdk_json_write_object_end(w); 1520 } 1521 1522 spdk_json_write_array_end(w); 1523 spdk_json_write_object_end(w); 1524 } 1525 1526 spdk_json_write_array_end(w); 1527 } 1528 1529 static void 1530 spdk_vhost_scsi_write_config_json(struct spdk_vhost_dev *vdev, struct spdk_json_write_ctx *w) 1531 { 1532 struct spdk_scsi_dev *scsi_dev; 1533 struct spdk_scsi_lun *lun; 1534 uint32_t i; 1535 1536 if (to_scsi_dev(vdev) == NULL) { 1537 return; 1538 } 1539 1540 spdk_json_write_object_begin(w); 1541 spdk_json_write_named_string(w, "method", "construct_vhost_scsi_controller"); 1542 1543 spdk_json_write_named_object_begin(w, "params"); 1544 spdk_json_write_named_string(w, "ctrlr", vdev->name); 1545 spdk_json_write_named_string(w, "cpumask", spdk_cpuset_fmt(vdev->cpumask)); 1546 spdk_json_write_object_end(w); 1547 1548 spdk_json_write_object_end(w); 1549 1550 for (i = 0; i < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; i++) { 1551 scsi_dev = spdk_vhost_scsi_dev_get_tgt(vdev, i); 1552 if (scsi_dev == NULL) { 1553 continue; 1554 } 1555 1556 lun = spdk_scsi_dev_get_lun(scsi_dev, 0); 1557 1558 spdk_json_write_object_begin(w); 1559 spdk_json_write_named_string(w, "method", "add_vhost_scsi_lun"); 1560 1561 spdk_json_write_named_object_begin(w, "params"); 1562 spdk_json_write_named_string(w, "ctrlr", vdev->name); 1563 spdk_json_write_named_uint32(w, "scsi_target_num", i); 1564 1565 spdk_json_write_named_string(w, "bdev_name", spdk_scsi_lun_get_bdev_name(lun)); 1566 spdk_json_write_object_end(w); 1567 1568 spdk_json_write_object_end(w); 1569 } 1570 } 1571 1572 SPDK_LOG_REGISTER_COMPONENT("vhost_scsi", SPDK_LOG_VHOST_SCSI) 1573 SPDK_LOG_REGISTER_COMPONENT("vhost_scsi_queue", SPDK_LOG_VHOST_SCSI_QUEUE) 1574 SPDK_LOG_REGISTER_COMPONENT("vhost_scsi_data", SPDK_LOG_VHOST_SCSI_DATA) 1575