1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2020 Intel Corporation. 3 * Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. 4 * All rights reserved. 5 */ 6 7 #include "accel_ioat.h" 8 9 #include "spdk/stdinc.h" 10 11 #include "spdk/accel_module.h" 12 #include "spdk/log.h" 13 #include "spdk/likely.h" 14 15 #include "spdk/env.h" 16 #include "spdk/event.h" 17 #include "spdk/thread.h" 18 #include "spdk/ioat.h" 19 20 static bool g_ioat_enable = false; 21 static bool g_ioat_initialized = false; 22 23 struct ioat_device { 24 struct spdk_ioat_chan *ioat; 25 bool is_allocated; 26 /** linked list pointer for device list */ 27 TAILQ_ENTRY(ioat_device) tailq; 28 }; 29 30 struct pci_device { 31 struct spdk_pci_device *pci_dev; 32 TAILQ_ENTRY(pci_device) tailq; 33 }; 34 35 static TAILQ_HEAD(, ioat_device) g_devices = TAILQ_HEAD_INITIALIZER(g_devices); 36 static pthread_mutex_t g_ioat_mutex = PTHREAD_MUTEX_INITIALIZER; 37 38 static TAILQ_HEAD(, pci_device) g_pci_devices = TAILQ_HEAD_INITIALIZER(g_pci_devices); 39 40 struct ioat_io_channel { 41 struct spdk_ioat_chan *ioat_ch; 42 struct ioat_device *ioat_dev; 43 struct spdk_poller *poller; 44 }; 45 46 static struct ioat_device * 47 ioat_allocate_device(void) 48 { 49 struct ioat_device *dev; 50 51 pthread_mutex_lock(&g_ioat_mutex); 52 TAILQ_FOREACH(dev, &g_devices, tailq) { 53 if (!dev->is_allocated) { 54 dev->is_allocated = true; 55 pthread_mutex_unlock(&g_ioat_mutex); 56 return dev; 57 } 58 } 59 pthread_mutex_unlock(&g_ioat_mutex); 60 61 return NULL; 62 } 63 64 static void 65 ioat_free_device(struct ioat_device *dev) 66 { 67 pthread_mutex_lock(&g_ioat_mutex); 68 dev->is_allocated = false; 69 pthread_mutex_unlock(&g_ioat_mutex); 70 } 71 72 static int accel_ioat_init(void); 73 static void accel_ioat_exit(void *ctx); 74 static bool ioat_supports_opcode(enum accel_opcode opc); 75 static struct spdk_io_channel *ioat_get_io_channel(void); 76 static int ioat_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *accel_task); 77 78 static size_t 79 accel_ioat_get_ctx_size(void) 80 { 81 return sizeof(struct spdk_accel_task); 82 } 83 84 static struct spdk_accel_module_if g_ioat_module = { 85 .module_init = accel_ioat_init, 86 .module_fini = accel_ioat_exit, 87 .write_config_json = NULL, 88 .get_ctx_size = accel_ioat_get_ctx_size, 89 .name = "ioat", 90 .supports_opcode = ioat_supports_opcode, 91 .get_io_channel = ioat_get_io_channel, 92 .submit_tasks = ioat_submit_tasks 93 }; 94 95 static void 96 ioat_done(void *cb_arg) 97 { 98 struct spdk_accel_task *accel_task = cb_arg; 99 100 spdk_accel_task_complete(accel_task, 0); 101 } 102 103 static int 104 ioat_poll(void *arg) 105 { 106 struct spdk_ioat_chan *chan = arg; 107 108 return spdk_ioat_process_events(chan) != 0 ? SPDK_POLLER_BUSY : 109 SPDK_POLLER_IDLE; 110 } 111 112 static struct spdk_io_channel *ioat_get_io_channel(void); 113 114 static bool 115 ioat_supports_opcode(enum accel_opcode opc) 116 { 117 if (!g_ioat_initialized) { 118 assert(0); 119 return false; 120 } 121 122 switch (opc) { 123 case ACCEL_OPC_COPY: 124 case ACCEL_OPC_FILL: 125 return true; 126 default: 127 return false; 128 } 129 } 130 131 static int 132 ioat_submit_fill(struct ioat_io_channel *ioat_ch, struct spdk_accel_task *task) 133 { 134 if (spdk_unlikely(task->d.iovcnt != 1)) { 135 return -EINVAL; 136 } 137 138 return spdk_ioat_build_fill(ioat_ch->ioat_ch, task, ioat_done, 139 task->d.iovs[0].iov_base, task->fill_pattern, 140 task->d.iovs[0].iov_len); 141 } 142 143 static int 144 ioat_submit_copy(struct ioat_io_channel *ioat_ch, struct spdk_accel_task *task) 145 { 146 if (spdk_unlikely(task->d.iovcnt != 1 || task->s.iovcnt != 1)) { 147 return -EINVAL; 148 } 149 150 if (spdk_unlikely(task->d.iovs[0].iov_len != task->s.iovs[0].iov_len)) { 151 return -EINVAL; 152 } 153 154 return spdk_ioat_build_copy(ioat_ch->ioat_ch, task, ioat_done, 155 task->d.iovs[0].iov_base, task->s.iovs[0].iov_base, 156 task->d.iovs[0].iov_len); 157 } 158 159 static int 160 ioat_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *accel_task) 161 { 162 struct ioat_io_channel *ioat_ch = spdk_io_channel_get_ctx(ch); 163 struct spdk_accel_task *tmp; 164 int rc = 0; 165 166 do { 167 switch (accel_task->op_code) { 168 case ACCEL_OPC_FILL: 169 rc = ioat_submit_fill(ioat_ch, accel_task); 170 break; 171 case ACCEL_OPC_COPY: 172 rc = ioat_submit_copy(ioat_ch, accel_task); 173 break; 174 default: 175 assert(false); 176 break; 177 } 178 179 tmp = TAILQ_NEXT(accel_task, link); 180 181 /* Report any build errors via the callback now. */ 182 if (rc) { 183 spdk_accel_task_complete(accel_task, rc); 184 } 185 186 accel_task = tmp; 187 } while (accel_task); 188 189 spdk_ioat_flush(ioat_ch->ioat_ch); 190 191 return 0; 192 } 193 194 static int 195 ioat_create_cb(void *io_device, void *ctx_buf) 196 { 197 struct ioat_io_channel *ch = ctx_buf; 198 struct ioat_device *ioat_dev; 199 200 ioat_dev = ioat_allocate_device(); 201 if (ioat_dev == NULL) { 202 return -1; 203 } 204 205 ch->ioat_dev = ioat_dev; 206 ch->ioat_ch = ioat_dev->ioat; 207 ch->poller = SPDK_POLLER_REGISTER(ioat_poll, ch->ioat_ch, 0); 208 209 return 0; 210 } 211 212 static void 213 ioat_destroy_cb(void *io_device, void *ctx_buf) 214 { 215 struct ioat_io_channel *ch = ctx_buf; 216 217 ioat_free_device(ch->ioat_dev); 218 spdk_poller_unregister(&ch->poller); 219 } 220 221 static struct spdk_io_channel * 222 ioat_get_io_channel(void) 223 { 224 return spdk_get_io_channel(&g_ioat_module); 225 } 226 227 static bool 228 probe_cb(void *cb_ctx, struct spdk_pci_device *pci_dev) 229 { 230 struct spdk_pci_addr pci_addr = spdk_pci_device_get_addr(pci_dev); 231 struct pci_device *pdev; 232 233 SPDK_INFOLOG(accel_ioat, 234 " Found matching device at %04x:%02x:%02x.%x vendor:0x%04x device:0x%04x\n", 235 pci_addr.domain, 236 pci_addr.bus, 237 pci_addr.dev, 238 pci_addr.func, 239 spdk_pci_device_get_vendor_id(pci_dev), 240 spdk_pci_device_get_device_id(pci_dev)); 241 242 pdev = calloc(1, sizeof(*pdev)); 243 if (pdev == NULL) { 244 return false; 245 } 246 pdev->pci_dev = pci_dev; 247 TAILQ_INSERT_TAIL(&g_pci_devices, pdev, tailq); 248 249 /* Claim the device in case conflict with other process */ 250 if (spdk_pci_device_claim(pci_dev) < 0) { 251 return false; 252 } 253 254 return true; 255 } 256 257 static void 258 attach_cb(void *cb_ctx, struct spdk_pci_device *pci_dev, struct spdk_ioat_chan *ioat) 259 { 260 struct ioat_device *dev; 261 262 dev = calloc(1, sizeof(*dev)); 263 if (dev == NULL) { 264 SPDK_ERRLOG("Failed to allocate device struct\n"); 265 return; 266 } 267 268 dev->ioat = ioat; 269 TAILQ_INSERT_TAIL(&g_devices, dev, tailq); 270 } 271 272 void 273 accel_ioat_enable_probe(void) 274 { 275 g_ioat_enable = true; 276 spdk_accel_module_list_add(&g_ioat_module); 277 } 278 279 static int 280 accel_ioat_init(void) 281 { 282 if (!g_ioat_enable) { 283 assert(0); 284 return 0; 285 } 286 287 if (spdk_ioat_probe(NULL, probe_cb, attach_cb) != 0) { 288 SPDK_ERRLOG("spdk_ioat_probe() failed\n"); 289 return -1; 290 } 291 292 if (TAILQ_EMPTY(&g_devices)) { 293 SPDK_NOTICELOG("No available ioat devices\n"); 294 return -1; 295 } 296 297 g_ioat_initialized = true; 298 spdk_io_device_register(&g_ioat_module, ioat_create_cb, ioat_destroy_cb, 299 sizeof(struct ioat_io_channel), "ioat_accel_module"); 300 return 0; 301 } 302 303 static void 304 _device_unregister_cb(void *io_device) 305 { 306 struct ioat_device *dev = io_device; 307 struct pci_device *pci_dev; 308 309 while (!TAILQ_EMPTY(&g_devices)) { 310 dev = TAILQ_FIRST(&g_devices); 311 TAILQ_REMOVE(&g_devices, dev, tailq); 312 spdk_ioat_detach(dev->ioat); 313 free(dev); 314 } 315 316 while (!TAILQ_EMPTY(&g_pci_devices)) { 317 pci_dev = TAILQ_FIRST(&g_pci_devices); 318 TAILQ_REMOVE(&g_pci_devices, pci_dev, tailq); 319 spdk_pci_device_detach(pci_dev->pci_dev); 320 free(pci_dev); 321 } 322 323 g_ioat_initialized = false; 324 325 spdk_accel_module_finish(); 326 } 327 328 static void 329 accel_ioat_exit(void *ctx) 330 { 331 if (g_ioat_initialized) { 332 spdk_io_device_unregister(&g_ioat_module, _device_unregister_cb); 333 } else { 334 spdk_accel_module_finish(); 335 } 336 } 337 338 SPDK_LOG_REGISTER_COMPONENT(accel_ioat) 339