1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "nvme_internal.h" 35 #include "nvme_io_msg.h" 36 37 #define SPDK_NVME_MSG_IO_PROCESS_SIZE 8 38 39 /** 40 * Send message to IO queue. 41 */ 42 int 43 nvme_io_msg_send(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, spdk_nvme_io_msg_fn fn, 44 void *arg) 45 { 46 int rc; 47 struct spdk_nvme_io_msg *io; 48 49 /* Protect requests ring against preemptive producers */ 50 pthread_mutex_lock(&ctrlr->external_io_msgs_lock); 51 52 io = (struct spdk_nvme_io_msg *)calloc(1, sizeof(struct spdk_nvme_io_msg)); 53 if (!io) { 54 SPDK_ERRLOG("IO msg allocation failed."); 55 pthread_mutex_unlock(&ctrlr->external_io_msgs_lock); 56 return -ENOMEM; 57 } 58 59 io->ctrlr = ctrlr; 60 io->nsid = nsid; 61 io->fn = fn; 62 io->arg = arg; 63 64 rc = spdk_ring_enqueue(ctrlr->external_io_msgs, (void **)&io, 1, NULL); 65 if (rc != 1) { 66 assert(false); 67 free(io); 68 pthread_mutex_unlock(&ctrlr->external_io_msgs_lock); 69 return -ENOMEM; 70 } 71 72 pthread_mutex_unlock(&ctrlr->external_io_msgs_lock); 73 74 return 0; 75 } 76 77 int 78 nvme_io_msg_process(struct spdk_nvme_ctrlr *ctrlr) 79 { 80 int i; 81 int count; 82 struct spdk_nvme_io_msg *io; 83 void *requests[SPDK_NVME_MSG_IO_PROCESS_SIZE]; 84 85 if (!ctrlr->external_io_msgs || !ctrlr->external_io_msgs_qpair) { 86 /* Not ready or pending reset */ 87 return 0; 88 } 89 90 if (!spdk_process_is_primary()) { 91 return 0; 92 } 93 94 spdk_nvme_qpair_process_completions(ctrlr->external_io_msgs_qpair, 0); 95 96 count = spdk_ring_dequeue(ctrlr->external_io_msgs, requests, 97 SPDK_NVME_MSG_IO_PROCESS_SIZE); 98 if (count == 0) { 99 return 0; 100 } 101 102 for (i = 0; i < count; i++) { 103 io = requests[i]; 104 105 assert(io != NULL); 106 107 io->fn(io->ctrlr, io->nsid, io->arg); 108 free(io); 109 } 110 111 return count; 112 } 113 114 static bool 115 nvme_io_msg_is_producer_registered(struct spdk_nvme_ctrlr *ctrlr, 116 struct nvme_io_msg_producer *io_msg_producer) 117 { 118 struct nvme_io_msg_producer *tmp; 119 120 STAILQ_FOREACH(tmp, &ctrlr->io_producers, link) { 121 if (tmp == io_msg_producer) { 122 return true; 123 } 124 } 125 return false; 126 } 127 128 int 129 nvme_io_msg_ctrlr_register(struct spdk_nvme_ctrlr *ctrlr, 130 struct nvme_io_msg_producer *io_msg_producer) 131 { 132 if (io_msg_producer == NULL) { 133 SPDK_ERRLOG("io_msg_producer cannot be NULL\n"); 134 return -EINVAL; 135 } 136 137 if (nvme_io_msg_is_producer_registered(ctrlr, io_msg_producer)) { 138 return -EEXIST; 139 } 140 141 if (!STAILQ_EMPTY(&ctrlr->io_producers) || ctrlr->is_resetting) { 142 /* There are registered producers - IO messaging already started */ 143 STAILQ_INSERT_TAIL(&ctrlr->io_producers, io_msg_producer, link); 144 return 0; 145 } 146 147 pthread_mutex_init(&ctrlr->external_io_msgs_lock, NULL); 148 149 /** 150 * Initialize ring and qpair for controller 151 */ 152 ctrlr->external_io_msgs = spdk_ring_create(SPDK_RING_TYPE_MP_SC, 65536, SPDK_ENV_SOCKET_ID_ANY); 153 if (!ctrlr->external_io_msgs) { 154 SPDK_ERRLOG("Unable to allocate memory for message ring\n"); 155 return -ENOMEM; 156 } 157 158 ctrlr->external_io_msgs_qpair = spdk_nvme_ctrlr_alloc_io_qpair(ctrlr, NULL, 0); 159 if (ctrlr->external_io_msgs_qpair == NULL) { 160 SPDK_ERRLOG("spdk_nvme_ctrlr_alloc_io_qpair() failed\n"); 161 spdk_ring_free(ctrlr->external_io_msgs); 162 ctrlr->external_io_msgs = NULL; 163 return -ENOMEM; 164 } 165 166 STAILQ_INSERT_TAIL(&ctrlr->io_producers, io_msg_producer, link); 167 168 return 0; 169 } 170 171 void 172 nvme_io_msg_ctrlr_update(struct spdk_nvme_ctrlr *ctrlr) 173 { 174 struct nvme_io_msg_producer *io_msg_producer; 175 176 /* Update all producers */ 177 STAILQ_FOREACH(io_msg_producer, &ctrlr->io_producers, link) { 178 io_msg_producer->update(ctrlr); 179 } 180 } 181 182 void 183 nvme_io_msg_ctrlr_detach(struct spdk_nvme_ctrlr *ctrlr) 184 { 185 struct nvme_io_msg_producer *io_msg_producer, *tmp; 186 187 /* Stop all producers */ 188 STAILQ_FOREACH_SAFE(io_msg_producer, &ctrlr->io_producers, link, tmp) { 189 io_msg_producer->stop(ctrlr); 190 STAILQ_REMOVE(&ctrlr->io_producers, io_msg_producer, nvme_io_msg_producer, link); 191 } 192 193 if (ctrlr->external_io_msgs) { 194 spdk_ring_free(ctrlr->external_io_msgs); 195 ctrlr->external_io_msgs = NULL; 196 } 197 198 if (ctrlr->external_io_msgs_qpair) { 199 spdk_nvme_ctrlr_free_io_qpair(ctrlr->external_io_msgs_qpair); 200 ctrlr->external_io_msgs_qpair = NULL; 201 } 202 203 pthread_mutex_destroy(&ctrlr->external_io_msgs_lock); 204 } 205 206 void 207 nvme_io_msg_ctrlr_unregister(struct spdk_nvme_ctrlr *ctrlr, 208 struct nvme_io_msg_producer *io_msg_producer) 209 { 210 assert(io_msg_producer != NULL); 211 212 if (!nvme_io_msg_is_producer_registered(ctrlr, io_msg_producer)) { 213 return; 214 } 215 216 STAILQ_REMOVE(&ctrlr->io_producers, io_msg_producer, nvme_io_msg_producer, link); 217 if (STAILQ_EMPTY(&ctrlr->io_producers)) { 218 nvme_io_msg_ctrlr_detach(ctrlr); 219 } 220 } 221