1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "nvme_internal.h" 35 36 struct nvme_driver _g_nvme_driver = { 37 .lock = PTHREAD_MUTEX_INITIALIZER, 38 .init_ctrlrs = TAILQ_HEAD_INITIALIZER(_g_nvme_driver.init_ctrlrs), 39 .attached_ctrlrs = TAILQ_HEAD_INITIALIZER(_g_nvme_driver.attached_ctrlrs), 40 .request_mempool = NULL, 41 }; 42 43 struct nvme_driver *g_spdk_nvme_driver = &_g_nvme_driver; 44 45 int32_t spdk_nvme_retry_count; 46 47 static struct spdk_nvme_ctrlr * 48 nvme_attach(void *devhandle) 49 { 50 enum spdk_nvme_transport transport; 51 struct spdk_nvme_ctrlr *ctrlr; 52 53 transport = SPDK_NVME_TRANSPORT_PCIE; 54 55 ctrlr = nvme_transport_ctrlr_construct(transport, devhandle); 56 57 return ctrlr; 58 } 59 60 int 61 spdk_nvme_detach(struct spdk_nvme_ctrlr *ctrlr) 62 { 63 pthread_mutex_lock(&g_spdk_nvme_driver->lock); 64 65 nvme_ctrlr_proc_put_ref(ctrlr); 66 67 if (nvme_ctrlr_get_ref_count(ctrlr) == 0) { 68 TAILQ_REMOVE(&g_spdk_nvme_driver->attached_ctrlrs, ctrlr, tailq); 69 nvme_ctrlr_destruct(ctrlr); 70 } 71 72 pthread_mutex_unlock(&g_spdk_nvme_driver->lock); 73 return 0; 74 } 75 76 void 77 nvme_completion_poll_cb(void *arg, const struct spdk_nvme_cpl *cpl) 78 { 79 struct nvme_completion_poll_status *status = arg; 80 81 /* 82 * Copy status into the argument passed by the caller, so that 83 * the caller can check the status to determine if the 84 * the request passed or failed. 85 */ 86 memcpy(&status->cpl, cpl, sizeof(*cpl)); 87 status->done = true; 88 } 89 90 struct nvme_request * 91 nvme_allocate_request(const struct nvme_payload *payload, uint32_t payload_size, 92 spdk_nvme_cmd_cb cb_fn, void *cb_arg) 93 { 94 struct nvme_request *req = NULL; 95 96 req = spdk_mempool_get(g_spdk_nvme_driver->request_mempool); 97 if (req == NULL) { 98 return req; 99 } 100 101 /* 102 * Only memset up to (but not including) the children 103 * TAILQ_ENTRY. children, and following members, are 104 * only used as part of I/O splitting so we avoid 105 * memsetting them until it is actually needed. 106 * They will be initialized in nvme_request_add_child() 107 * if the request is split. 108 */ 109 memset(req, 0, offsetof(struct nvme_request, children)); 110 req->cb_fn = cb_fn; 111 req->cb_arg = cb_arg; 112 req->payload = *payload; 113 req->payload_size = payload_size; 114 req->pid = getpid(); 115 116 return req; 117 } 118 119 struct nvme_request * 120 nvme_allocate_request_contig(void *buffer, uint32_t payload_size, spdk_nvme_cmd_cb cb_fn, 121 void *cb_arg) 122 { 123 struct nvme_payload payload; 124 125 payload.type = NVME_PAYLOAD_TYPE_CONTIG; 126 payload.u.contig = buffer; 127 payload.md = NULL; 128 129 return nvme_allocate_request(&payload, payload_size, cb_fn, cb_arg); 130 } 131 132 struct nvme_request * 133 nvme_allocate_request_null(spdk_nvme_cmd_cb cb_fn, void *cb_arg) 134 { 135 return nvme_allocate_request_contig(NULL, 0, cb_fn, cb_arg); 136 } 137 138 static void 139 nvme_user_copy_cmd_complete(void *arg, const struct spdk_nvme_cpl *cpl) 140 { 141 struct nvme_request *req = arg; 142 enum spdk_nvme_data_transfer xfer; 143 144 if (req->user_buffer && req->payload_size) { 145 /* Copy back to the user buffer and free the contig buffer */ 146 assert(req->payload.type == NVME_PAYLOAD_TYPE_CONTIG); 147 xfer = spdk_nvme_opc_get_data_transfer(req->cmd.opc); 148 if (xfer == SPDK_NVME_DATA_CONTROLLER_TO_HOST || 149 xfer == SPDK_NVME_DATA_BIDIRECTIONAL) { 150 assert(req->pid == getpid()); 151 memcpy(req->user_buffer, req->payload.u.contig, req->payload_size); 152 } 153 154 spdk_free(req->payload.u.contig); 155 } 156 157 /* Call the user's original callback now that the buffer has been copied */ 158 req->user_cb_fn(req->user_cb_arg, cpl); 159 } 160 161 /** 162 * Allocate a request as well as a physically contiguous buffer to copy to/from the user's buffer. 163 * 164 * This is intended for use in non-fast-path functions (admin commands, reservations, etc.) 165 * where the overhead of a copy is not a problem. 166 */ 167 struct nvme_request * 168 nvme_allocate_request_user_copy(void *buffer, uint32_t payload_size, spdk_nvme_cmd_cb cb_fn, 169 void *cb_arg, bool host_to_controller) 170 { 171 struct nvme_request *req; 172 void *contig_buffer = NULL; 173 uint64_t phys_addr; 174 175 if (buffer && payload_size) { 176 contig_buffer = spdk_zmalloc(payload_size, 4096, &phys_addr); 177 if (!contig_buffer) { 178 return NULL; 179 } 180 181 if (host_to_controller) { 182 memcpy(contig_buffer, buffer, payload_size); 183 } 184 } 185 186 req = nvme_allocate_request_contig(contig_buffer, payload_size, nvme_user_copy_cmd_complete, NULL); 187 if (!req) { 188 spdk_free(buffer); 189 return NULL; 190 } 191 192 req->user_cb_fn = cb_fn; 193 req->user_cb_arg = cb_arg; 194 req->user_buffer = buffer; 195 req->cb_arg = req; 196 197 return req; 198 } 199 200 void 201 nvme_free_request(struct nvme_request *req) 202 { 203 assert(req != NULL); 204 assert(req->num_children == 0); 205 206 spdk_mempool_put(g_spdk_nvme_driver->request_mempool, req); 207 } 208 209 int 210 nvme_mutex_init_shared(pthread_mutex_t *mtx) 211 { 212 pthread_mutexattr_t attr; 213 int rc = 0; 214 215 if (pthread_mutexattr_init(&attr)) { 216 return -1; 217 } 218 if (pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED) || 219 pthread_mutex_init(mtx, &attr)) { 220 rc = -1; 221 } 222 pthread_mutexattr_destroy(&attr); 223 return rc; 224 } 225 226 struct nvme_enum_ctx { 227 spdk_nvme_probe_cb probe_cb; 228 void *cb_ctx; 229 }; 230 231 /* This function must only be called while holding g_spdk_nvme_driver->lock */ 232 static int 233 nvme_enum_cb(void *ctx, struct spdk_pci_device *pci_dev) 234 { 235 struct nvme_enum_ctx *enum_ctx = ctx; 236 struct spdk_nvme_ctrlr *ctrlr; 237 struct spdk_nvme_ctrlr_opts opts; 238 struct spdk_nvme_probe_info probe_info; 239 240 probe_info.pci_addr = spdk_pci_device_get_addr(pci_dev); 241 probe_info.pci_id = spdk_pci_device_get_id(pci_dev); 242 243 /* Verify that this controller is not already attached */ 244 TAILQ_FOREACH(ctrlr, &g_spdk_nvme_driver->attached_ctrlrs, tailq) { 245 /* NOTE: In the case like multi-process environment where the device handle is 246 * different per each process, we compare by BDF to determine whether it is the 247 * same controller. 248 */ 249 if (spdk_pci_addr_compare(&probe_info.pci_addr, &ctrlr->probe_info.pci_addr) == 0) { 250 return 0; 251 } 252 } 253 254 spdk_nvme_ctrlr_opts_set_defaults(&opts); 255 256 if (enum_ctx->probe_cb(enum_ctx->cb_ctx, &probe_info, &opts)) { 257 ctrlr = nvme_attach(pci_dev); 258 if (ctrlr == NULL) { 259 SPDK_ERRLOG("nvme_attach() failed\n"); 260 return -1; 261 } 262 263 ctrlr->opts = opts; 264 ctrlr->probe_info = probe_info; 265 266 TAILQ_INSERT_TAIL(&g_spdk_nvme_driver->init_ctrlrs, ctrlr, tailq); 267 } 268 269 return 0; 270 } 271 272 int 273 spdk_nvme_probe(void *cb_ctx, spdk_nvme_probe_cb probe_cb, spdk_nvme_attach_cb attach_cb, 274 spdk_nvme_remove_cb remove_cb) 275 { 276 int rc, start_rc; 277 struct nvme_enum_ctx enum_ctx; 278 struct spdk_nvme_ctrlr *ctrlr, *ctrlr_tmp; 279 280 pthread_mutex_lock(&g_spdk_nvme_driver->lock); 281 282 if (g_spdk_nvme_driver->request_mempool == NULL) { 283 g_spdk_nvme_driver->request_mempool = spdk_mempool_create("nvme_request", 8192, 284 sizeof(struct nvme_request), -1); 285 if (g_spdk_nvme_driver->request_mempool == NULL) { 286 SPDK_ERRLOG("Unable to allocate pool of requests\n"); 287 pthread_mutex_unlock(&g_spdk_nvme_driver->lock); 288 return -1; 289 } 290 } 291 292 enum_ctx.probe_cb = probe_cb; 293 enum_ctx.cb_ctx = cb_ctx; 294 295 rc = spdk_pci_enumerate(SPDK_PCI_DEVICE_NVME, nvme_enum_cb, &enum_ctx); 296 /* 297 * Keep going even if one or more nvme_attach() calls failed, 298 * but maintain the value of rc to signal errors when we return. 299 */ 300 301 /* Initialize all new controllers in the init_ctrlrs list in parallel. */ 302 while (!TAILQ_EMPTY(&g_spdk_nvme_driver->init_ctrlrs)) { 303 TAILQ_FOREACH_SAFE(ctrlr, &g_spdk_nvme_driver->init_ctrlrs, tailq, ctrlr_tmp) { 304 /* Drop the driver lock while calling nvme_ctrlr_process_init() 305 * since it needs to acquire the driver lock internally when calling 306 * nvme_ctrlr_start(). 307 * 308 * TODO: Rethink the locking - maybe reset should take the lock so that start() and 309 * the functions it calls (in particular nvme_ctrlr_set_num_qpairs()) 310 * can assume it is held. 311 */ 312 pthread_mutex_unlock(&g_spdk_nvme_driver->lock); 313 start_rc = nvme_ctrlr_process_init(ctrlr); 314 pthread_mutex_lock(&g_spdk_nvme_driver->lock); 315 316 if (start_rc) { 317 /* Controller failed to initialize. */ 318 TAILQ_REMOVE(&g_spdk_nvme_driver->init_ctrlrs, ctrlr, tailq); 319 nvme_ctrlr_destruct(ctrlr); 320 spdk_free(ctrlr); 321 rc = -1; 322 break; 323 } 324 325 if (ctrlr->state == NVME_CTRLR_STATE_READY) { 326 /* 327 * Controller has been initialized. 328 * Move it to the attached_ctrlrs list. 329 */ 330 TAILQ_REMOVE(&g_spdk_nvme_driver->init_ctrlrs, ctrlr, tailq); 331 TAILQ_INSERT_TAIL(&g_spdk_nvme_driver->attached_ctrlrs, ctrlr, tailq); 332 333 /* 334 * Increase the ref count before calling attach_cb() as the user may 335 * call nvme_detach() immediately. 336 */ 337 nvme_ctrlr_proc_get_ref(ctrlr); 338 339 /* 340 * Unlock while calling attach_cb() so the user can call other functions 341 * that may take the driver lock, like nvme_detach(). 342 */ 343 pthread_mutex_unlock(&g_spdk_nvme_driver->lock); 344 attach_cb(cb_ctx, &ctrlr->probe_info, ctrlr, &ctrlr->opts); 345 pthread_mutex_lock(&g_spdk_nvme_driver->lock); 346 347 break; 348 } 349 } 350 } 351 352 pthread_mutex_unlock(&g_spdk_nvme_driver->lock); 353 return rc; 354 } 355 356 SPDK_LOG_REGISTER_TRACE_FLAG("nvme", SPDK_TRACE_NVME) 357