1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <stdio.h> 35 #include <stdlib.h> 36 #include <string.h> 37 #include <unistd.h> 38 39 #include "spdk/nvme.h" 40 #include "spdk/env.h" 41 42 struct ctrlr_entry { 43 struct spdk_nvme_ctrlr *ctrlr; 44 struct ctrlr_entry *next; 45 char name[1024]; 46 }; 47 48 struct ns_entry { 49 struct spdk_nvme_ctrlr *ctrlr; 50 struct spdk_nvme_ns *ns; 51 struct ns_entry *next; 52 struct spdk_nvme_qpair *qpair; 53 }; 54 55 static struct ctrlr_entry *g_controllers = NULL; 56 static struct ns_entry *g_namespaces = NULL; 57 58 static void 59 register_ns(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns *ns) 60 { 61 struct ns_entry *entry; 62 const struct spdk_nvme_ctrlr_data *cdata; 63 64 /* 65 * spdk_nvme_ctrlr is the logical abstraction in SPDK for an NVMe 66 * controller. During initialization, the IDENTIFY data for the 67 * controller is read using an NVMe admin command, and that data 68 * can be retrieved using spdk_nvme_ctrlr_get_data() to get 69 * detailed information on the controller. Refer to the NVMe 70 * specification for more details on IDENTIFY for NVMe controllers. 71 */ 72 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 73 74 if (!spdk_nvme_ns_is_active(ns)) { 75 printf("Controller %-20.20s (%-20.20s): Skipping inactive NS %u\n", 76 cdata->mn, cdata->sn, 77 spdk_nvme_ns_get_id(ns)); 78 return; 79 } 80 81 entry = malloc(sizeof(struct ns_entry)); 82 if (entry == NULL) { 83 perror("ns_entry malloc"); 84 exit(1); 85 } 86 87 entry->ctrlr = ctrlr; 88 entry->ns = ns; 89 entry->next = g_namespaces; 90 g_namespaces = entry; 91 92 printf(" Namespace ID: %d size: %juGB\n", spdk_nvme_ns_get_id(ns), 93 spdk_nvme_ns_get_size(ns) / 1000000000); 94 } 95 96 struct hello_world_sequence { 97 struct ns_entry *ns_entry; 98 char *buf; 99 int is_completed; 100 }; 101 102 static void 103 read_complete(void *arg, const struct spdk_nvme_cpl *completion) 104 { 105 struct hello_world_sequence *sequence = arg; 106 107 /* 108 * The read I/O has completed. Print the contents of the 109 * buffer, free the buffer, then mark the sequence as 110 * completed. This will trigger the hello_world() function 111 * to exit its polling loop. 112 */ 113 printf("%s", sequence->buf); 114 spdk_free(sequence->buf); 115 sequence->is_completed = 1; 116 } 117 118 static void 119 write_complete(void *arg, const struct spdk_nvme_cpl *completion) 120 { 121 struct hello_world_sequence *sequence = arg; 122 struct ns_entry *ns_entry = sequence->ns_entry; 123 int rc; 124 125 /* 126 * The write I/O has completed. Free the buffer associated with 127 * the write I/O and allocate a new zeroed buffer for reading 128 * the data back from the NVMe namespace. 129 */ 130 spdk_free(sequence->buf); 131 sequence->buf = spdk_zmalloc(0x1000, 0x1000, NULL); 132 133 rc = spdk_nvme_ns_cmd_read(ns_entry->ns, ns_entry->qpair, sequence->buf, 134 0, /* LBA start */ 135 1, /* number of LBAs */ 136 read_complete, (void *)sequence, 0); 137 if (rc != 0) { 138 fprintf(stderr, "starting read I/O failed\n"); 139 exit(1); 140 } 141 } 142 143 static void 144 hello_world(void) 145 { 146 struct ns_entry *ns_entry; 147 struct hello_world_sequence sequence; 148 int rc; 149 150 ns_entry = g_namespaces; 151 while (ns_entry != NULL) { 152 /* 153 * Allocate an I/O qpair that we can use to submit read/write requests 154 * to namespaces on the controller. NVMe controllers typically support 155 * many qpairs per controller. Any I/O qpair allocated for a controller 156 * can submit I/O to any namespace on that controller. 157 * 158 * The SPDK NVMe driver provides no synchronization for qpair accesses - 159 * the application must ensure only a single thread submits I/O to a 160 * qpair, and that same thread must also check for completions on that 161 * qpair. This enables extremely efficient I/O processing by making all 162 * I/O operations completely lockless. 163 */ 164 ns_entry->qpair = spdk_nvme_ctrlr_alloc_io_qpair(ns_entry->ctrlr, 0); 165 if (ns_entry->qpair == NULL) { 166 printf("ERROR: spdk_nvme_ctrlr_alloc_io_qpair() failed\n"); 167 return; 168 } 169 170 /* 171 * Use spdk_zmalloc to allocate a 4KB zeroed buffer. This memory 172 * will be pinned, which is required for data buffers used for SPDK NVMe 173 * I/O operations. 174 */ 175 sequence.buf = spdk_zmalloc(0x1000, 0x1000, NULL); 176 sequence.is_completed = 0; 177 sequence.ns_entry = ns_entry; 178 179 /* 180 * Print "Hello world!" to sequence.buf. We will write this data to LBA 181 * 0 on the namespace, and then later read it back into a separate buffer 182 * to demonstrate the full I/O path. 183 */ 184 snprintf(sequence.buf, 0x1000, "%s", "Hello world!\n"); 185 186 /* 187 * Write the data buffer to LBA 0 of this namespace. "write_complete" and 188 * "&sequence" are specified as the completion callback function and 189 * argument respectively. write_complete() will be called with the 190 * value of &sequence as a parameter when the write I/O is completed. 191 * This allows users to potentially specify different completion 192 * callback routines for each I/O, as well as pass a unique handle 193 * as an argument so the application knows which I/O has completed. 194 * 195 * Note that the SPDK NVMe driver will only check for completions 196 * when the application calls spdk_nvme_qpair_process_completions(). 197 * It is the responsibility of the application to trigger the polling 198 * process. 199 */ 200 rc = spdk_nvme_ns_cmd_write(ns_entry->ns, ns_entry->qpair, sequence.buf, 201 0, /* LBA start */ 202 1, /* number of LBAs */ 203 write_complete, &sequence, 0); 204 if (rc != 0) { 205 fprintf(stderr, "starting write I/O failed\n"); 206 exit(1); 207 } 208 209 /* 210 * Poll for completions. 0 here means process all available completions. 211 * In certain usage models, the caller may specify a positive integer 212 * instead of 0 to signify the maximum number of completions it should 213 * process. This function will never block - if there are no 214 * completions pending on the specified qpair, it will return immediately. 215 * 216 * When the write I/O completes, write_complete() will submit a new I/O 217 * to read LBA 0 into a separate buffer, specifying read_complete() as its 218 * completion routine. When the read I/O completes, read_complete() will 219 * print the buffer contents and set sequence.is_completed = 1. That will 220 * break this loop and then exit the program. 221 */ 222 while (!sequence.is_completed) { 223 spdk_nvme_qpair_process_completions(ns_entry->qpair, 0); 224 } 225 226 /* 227 * Free the I/O qpair. This typically is done when an application exits. 228 * But SPDK does support freeing and then reallocating qpairs during 229 * operation. It is the responsibility of the caller to ensure all 230 * pending I/O are completed before trying to free the qpair. 231 */ 232 spdk_nvme_ctrlr_free_io_qpair(ns_entry->qpair); 233 ns_entry = ns_entry->next; 234 } 235 } 236 237 static bool 238 probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 239 struct spdk_nvme_ctrlr_opts *opts) 240 { 241 printf("Attaching to %s\n", trid->traddr); 242 243 return true; 244 } 245 246 static void 247 attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 248 struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts) 249 { 250 int nsid, num_ns; 251 struct ctrlr_entry *entry; 252 struct spdk_nvme_ns *ns; 253 const struct spdk_nvme_ctrlr_data *cdata = spdk_nvme_ctrlr_get_data(ctrlr); 254 255 entry = malloc(sizeof(struct ctrlr_entry)); 256 if (entry == NULL) { 257 perror("ctrlr_entry malloc"); 258 exit(1); 259 } 260 261 printf("Attached to %s\n", trid->traddr); 262 263 snprintf(entry->name, sizeof(entry->name), "%-20.20s (%-20.20s)", cdata->mn, cdata->sn); 264 265 entry->ctrlr = ctrlr; 266 entry->next = g_controllers; 267 g_controllers = entry; 268 269 /* 270 * Each controller has one or more namespaces. An NVMe namespace is basically 271 * equivalent to a SCSI LUN. The controller's IDENTIFY data tells us how 272 * many namespaces exist on the controller. For Intel(R) P3X00 controllers, 273 * it will just be one namespace. 274 * 275 * Note that in NVMe, namespace IDs start at 1, not 0. 276 */ 277 num_ns = spdk_nvme_ctrlr_get_num_ns(ctrlr); 278 printf("Using controller %s with %d namespaces.\n", entry->name, num_ns); 279 for (nsid = 1; nsid <= num_ns; nsid++) { 280 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 281 if (ns == NULL) { 282 continue; 283 } 284 register_ns(ctrlr, ns); 285 } 286 } 287 288 static void 289 cleanup(void) 290 { 291 struct ns_entry *ns_entry = g_namespaces; 292 struct ctrlr_entry *ctrlr_entry = g_controllers; 293 294 while (ns_entry) { 295 struct ns_entry *next = ns_entry->next; 296 free(ns_entry); 297 ns_entry = next; 298 } 299 300 while (ctrlr_entry) { 301 struct ctrlr_entry *next = ctrlr_entry->next; 302 303 spdk_nvme_detach(ctrlr_entry->ctrlr); 304 free(ctrlr_entry); 305 ctrlr_entry = next; 306 } 307 } 308 309 int main(int argc, char **argv) 310 { 311 int rc; 312 struct spdk_env_opts opts; 313 314 /* 315 * SPDK relies on an abstraction around the local environment 316 * named env that handles memory allocation and PCI device operations. 317 * This library must be initialized first. 318 * 319 */ 320 spdk_env_opts_init(&opts); 321 opts.name = "hello_world"; 322 spdk_env_init(&opts); 323 324 printf("Initializing NVMe Controllers\n"); 325 326 /* 327 * Start the SPDK NVMe enumeration process. probe_cb will be called 328 * for each NVMe controller found, giving our application a choice on 329 * whether to attach to each controller. attach_cb will then be 330 * called for each controller after the SPDK NVMe driver has completed 331 * initializing the controller we chose to attach. 332 */ 333 rc = spdk_nvme_probe(NULL, NULL, probe_cb, attach_cb, NULL); 334 if (rc != 0) { 335 fprintf(stderr, "spdk_nvme_probe() failed\n"); 336 cleanup(); 337 return 1; 338 } 339 340 printf("Initialization complete.\n"); 341 hello_world(); 342 cleanup(); 343 return 0; 344 } 345