1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 12 * * Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * * Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in 16 * the documentation and/or other materials provided with the 17 * distribution. 18 * * Neither the name of Intel Corporation nor the names of its 19 * contributors may be used to endorse or promote products derived 20 * from this software without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 23 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 24 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 25 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 26 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 27 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 28 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 32 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 */ 34 35 #include "spdk/stdinc.h" 36 37 #include <accel-config/libaccel_config.h> 38 39 #include "spdk/env.h" 40 #include "spdk/util.h" 41 #include "spdk/memory.h" 42 #include "spdk/likely.h" 43 44 #include "spdk/log.h" 45 #include "spdk_internal/idxd.h" 46 47 #include "idxd.h" 48 49 #define MAX_DSA_DEVICE_ID 16 50 51 struct device_config g_kernel_dev_cfg = {}; 52 53 struct spdk_wq_context { 54 struct accfg_wq *wq; 55 unsigned int max_batch_size; 56 unsigned int max_xfer_size; 57 unsigned int max_xfer_bits; 58 59 int fd; 60 int wq_idx; 61 void *wq_reg; 62 int wq_size; 63 int dedicated; 64 int bof; 65 66 unsigned int wq_max_batch_size; 67 unsigned long wq_max_xfer_size; 68 }; 69 70 struct spdk_kernel_idxd_device { 71 struct spdk_idxd_device idxd; 72 struct accfg_ctx *ctx; 73 struct spdk_wq_context *wq_ctx; 74 uint32_t wq_active_num; 75 }; 76 77 #define __kernel_idxd(idxd) SPDK_CONTAINEROF(idxd, struct spdk_kernel_idxd_device, idxd) 78 79 /* Bit scan reverse */ 80 static uint32_t bsr(uint32_t val) 81 { 82 uint32_t msb; 83 84 msb = (val == 0) ? 0 : 32 - __builtin_clz(val); 85 return msb - 1; 86 } 87 88 static void init_idxd_impl(struct spdk_idxd_device *idxd); 89 90 static int 91 dsa_setup_single_wq(struct spdk_kernel_idxd_device *kernel_idxd, struct accfg_wq *wq, int shared) 92 { 93 struct accfg_device *dev; 94 int major, minor; 95 char path[1024]; 96 struct spdk_wq_context *wq_ctx = &kernel_idxd->wq_ctx[kernel_idxd->wq_active_num]; 97 98 dev = accfg_wq_get_device(wq); 99 major = accfg_device_get_cdev_major(dev); 100 if (major < 0) { 101 return -ENODEV; 102 } 103 minor = accfg_wq_get_cdev_minor(wq); 104 if (minor < 0) { 105 return -ENODEV; 106 } 107 108 snprintf(path, sizeof(path), "/dev/char/%u:%u", major, minor); 109 wq_ctx->fd = open(path, O_RDWR); 110 if (wq_ctx->fd < 0) { 111 SPDK_ERRLOG("Can not open the Working queue file descriptor on path=%s\n", 112 path); 113 return -errno; 114 } 115 116 wq_ctx->wq_reg = mmap(NULL, 0x1000, PROT_WRITE, 117 MAP_SHARED | MAP_POPULATE, wq_ctx->fd, 0); 118 if (wq_ctx->wq_reg == MAP_FAILED) { 119 perror("mmap"); 120 return -errno; 121 } 122 123 wq_ctx->dedicated = !shared; 124 wq_ctx->wq_size = accfg_wq_get_size(wq); 125 wq_ctx->wq_idx = accfg_wq_get_id(wq); 126 wq_ctx->bof = accfg_wq_get_block_on_fault(wq); 127 wq_ctx->wq_max_batch_size = accfg_wq_get_max_batch_size(wq); 128 wq_ctx->wq_max_xfer_size = accfg_wq_get_max_transfer_size(wq); 129 130 wq_ctx->max_batch_size = accfg_device_get_max_batch_size(dev); 131 wq_ctx->max_xfer_size = accfg_device_get_max_transfer_size(dev); 132 wq_ctx->max_xfer_bits = bsr(wq_ctx->max_xfer_size); 133 134 SPDK_NOTICELOG("alloc wq %d shared %d size %d addr %p batch sz %#x xfer sz %#x\n", 135 wq_ctx->wq_idx, shared, wq_ctx->wq_size, wq_ctx->wq_reg, 136 wq_ctx->max_batch_size, wq_ctx->max_xfer_size); 137 138 wq_ctx->wq = wq; 139 140 /* Update the active_wq_num of the kernel device */ 141 kernel_idxd->wq_active_num++; 142 kernel_idxd->idxd.total_wq_size += wq_ctx->wq_size; 143 kernel_idxd->idxd.socket_id = accfg_device_get_numa_node(dev); 144 145 return 0; 146 } 147 148 static int 149 config_wqs(struct spdk_kernel_idxd_device *kernel_idxd, 150 int dev_id, int shared) 151 { 152 struct accfg_device *device; 153 struct accfg_wq *wq; 154 int rc; 155 156 accfg_device_foreach(kernel_idxd->ctx, device) { 157 enum accfg_device_state dstate; 158 159 /* Make sure that the device is enabled */ 160 dstate = accfg_device_get_state(device); 161 if (dstate != ACCFG_DEVICE_ENABLED) { 162 continue; 163 } 164 165 /* Match the device to the id requested */ 166 if (accfg_device_get_id(device) != dev_id && 167 dev_id != -1) { 168 continue; 169 } 170 171 accfg_wq_foreach(device, wq) { 172 enum accfg_wq_state wstate; 173 enum accfg_wq_mode mode; 174 enum accfg_wq_type type; 175 176 /* Get a workqueue that's enabled */ 177 wstate = accfg_wq_get_state(wq); 178 if (wstate != ACCFG_WQ_ENABLED) { 179 continue; 180 } 181 182 /* The wq type should be user */ 183 type = accfg_wq_get_type(wq); 184 if (type != ACCFG_WQT_USER) { 185 continue; 186 } 187 188 /* Make sure the mode is correct */ 189 mode = accfg_wq_get_mode(wq); 190 if ((mode == ACCFG_WQ_SHARED && !shared) 191 || (mode == ACCFG_WQ_DEDICATED && shared)) { 192 continue; 193 } 194 195 /* We already config enough work queues */ 196 if (kernel_idxd->wq_active_num == g_kernel_dev_cfg.total_wqs) { 197 break; 198 } 199 200 rc = dsa_setup_single_wq(kernel_idxd, wq, shared); 201 if (rc < 0) { 202 return -1; 203 } 204 } 205 } 206 207 if ((kernel_idxd->wq_active_num != 0) && 208 (kernel_idxd->wq_active_num != g_kernel_dev_cfg.total_wqs)) { 209 SPDK_ERRLOG("Failed to configure the expected wq nums=%d, and get the real wq nums=%d\n", 210 g_kernel_dev_cfg.total_wqs, kernel_idxd->wq_active_num); 211 return -1; 212 } 213 214 /* Spread the channels we allow per device based on the total number of WQE to try 215 * and achieve optimal performance for common cases. 216 */ 217 kernel_idxd->idxd.chan_per_device = (kernel_idxd->idxd.total_wq_size >= 128) ? 8 : 4; 218 return 0; 219 } 220 221 static void 222 kernel_idxd_device_destruct(struct spdk_idxd_device *idxd) 223 { 224 uint32_t i; 225 struct spdk_kernel_idxd_device *kernel_idxd = __kernel_idxd(idxd); 226 227 if (kernel_idxd->wq_ctx) { 228 for (i = 0; i < kernel_idxd->wq_active_num; i++) { 229 if (munmap(kernel_idxd->wq_ctx[i].wq_reg, 0x1000)) { 230 SPDK_ERRLOG("munmap failed %d on kernel_device=%p on dsa_context with wq_reg=%p\n", 231 errno, kernel_idxd, kernel_idxd->wq_ctx[i].wq_reg); 232 } 233 close(kernel_idxd->wq_ctx[i].fd); 234 } 235 free(kernel_idxd->wq_ctx); 236 } 237 238 accfg_unref(kernel_idxd->ctx); 239 free(idxd); 240 } 241 242 /* 243 * Build work queue (WQ) config based on getting info from the device combined 244 * with the defined configuration. Once built, it is written to the device. 245 */ 246 static int 247 kernel_idxd_wq_config(struct spdk_kernel_idxd_device *kernel_idxd) 248 { 249 uint32_t i; 250 struct idxd_wq *queue; 251 struct spdk_idxd_device *idxd = &kernel_idxd->idxd; 252 253 /* initialize the group */ 254 idxd->groups = calloc(g_kernel_dev_cfg.num_groups, sizeof(struct idxd_group)); 255 if (idxd->groups == NULL) { 256 SPDK_ERRLOG("Failed to allocate group memory\n"); 257 return -ENOMEM; 258 } 259 260 for (i = 0; i < g_kernel_dev_cfg.num_groups; i++) { 261 idxd->groups[i].idxd = idxd; 262 idxd->groups[i].id = i; 263 } 264 265 idxd->queues = calloc(g_kernel_dev_cfg.total_wqs, sizeof(struct idxd_wq)); 266 if (idxd->queues == NULL) { 267 SPDK_ERRLOG("Failed to allocate queue memory\n"); 268 return -ENOMEM; 269 } 270 271 for (i = 0; i < g_kernel_dev_cfg.total_wqs; i++) { 272 queue = &idxd->queues[i]; 273 queue->wqcfg.wq_size = kernel_idxd->wq_ctx[i].wq_size; 274 queue->wqcfg.mode = WQ_MODE_DEDICATED; 275 queue->wqcfg.max_batch_shift = LOG2_WQ_MAX_BATCH; 276 queue->wqcfg.max_xfer_shift = LOG2_WQ_MAX_XFER; 277 queue->wqcfg.wq_state = WQ_ENABLED; 278 queue->wqcfg.priority = WQ_PRIORITY_1; 279 280 /* Not part of the config struct */ 281 queue->idxd = idxd; 282 queue->group = &idxd->groups[i % g_kernel_dev_cfg.num_groups]; 283 } 284 285 return 0; 286 } 287 288 static int 289 _kernel_idxd_probe(void *cb_ctx, spdk_idxd_attach_cb attach_cb, int dev_id) 290 { 291 int rc; 292 struct spdk_kernel_idxd_device *kernel_idxd; 293 struct accfg_ctx *ctx; 294 295 kernel_idxd = calloc(1, sizeof(struct spdk_kernel_idxd_device)); 296 if (kernel_idxd == NULL) { 297 SPDK_ERRLOG("Failed to allocate memory for kernel_idxd device.\n"); 298 return -ENOMEM; 299 } 300 301 kernel_idxd->wq_ctx = calloc(g_kernel_dev_cfg.total_wqs, sizeof(struct spdk_wq_context)); 302 if (kernel_idxd->wq_ctx == NULL) { 303 rc = -ENOMEM; 304 SPDK_ERRLOG("Failed to allocate memory for the work queue contexts on kernel_idxd=%p.\n", 305 kernel_idxd); 306 goto end; 307 } 308 309 rc = accfg_new(&ctx); 310 if (rc < 0) { 311 SPDK_ERRLOG("Failed to allocate accfg context when probe kernel_idxd=%p\n", kernel_idxd); 312 goto end; 313 } 314 315 init_idxd_impl(&kernel_idxd->idxd); 316 kernel_idxd->ctx = ctx; 317 318 /* Supporting non-shared mode first. 319 * Todo: Add the shared mode support later. 320 */ 321 rc = config_wqs(kernel_idxd, dev_id, 0); 322 if (rc) { 323 SPDK_ERRLOG("Failed to probe requested wqs on kernel device context=%p\n", ctx); 324 return -ENODEV; 325 } 326 327 /* No active work queues */ 328 if (kernel_idxd->wq_active_num == 0) { 329 goto end; 330 } 331 332 kernel_idxd_wq_config(kernel_idxd); 333 334 attach_cb(cb_ctx, &kernel_idxd->idxd); 335 336 SPDK_NOTICELOG("Successfully got an kernel device=%p\n", kernel_idxd); 337 return 0; 338 339 end: 340 kernel_idxd_device_destruct(&kernel_idxd->idxd); 341 return rc; 342 } 343 344 static int 345 kernel_idxd_probe(void *cb_ctx, spdk_idxd_attach_cb attach_cb) 346 { 347 int i; 348 349 for (i = 0; i < MAX_DSA_DEVICE_ID; i++) { 350 _kernel_idxd_probe(cb_ctx, attach_cb, i); 351 } 352 353 return 0; 354 } 355 356 static void 357 kernel_idxd_dump_sw_error(struct spdk_idxd_device *idxd, void *portal) 358 { 359 /* Need to be enhanced later */ 360 } 361 362 static void 363 kernel_idxd_set_config(struct device_config *dev_cfg, uint32_t config_num) 364 { 365 g_kernel_dev_cfg = *dev_cfg; 366 } 367 368 static char * 369 kernel_idxd_portal_get_addr(struct spdk_idxd_device *idxd) 370 { 371 struct spdk_kernel_idxd_device *kernel_idxd = __kernel_idxd(idxd); 372 assert(idxd->wq_id <= (g_kernel_dev_cfg.total_wqs - 1)); 373 return (char *)kernel_idxd->wq_ctx[idxd->wq_id].wq_reg; 374 } 375 376 static struct spdk_idxd_impl g_kernel_idxd_impl = { 377 .name = "kernel", 378 .set_config = kernel_idxd_set_config, 379 .probe = kernel_idxd_probe, 380 .destruct = kernel_idxd_device_destruct, 381 .dump_sw_error = kernel_idxd_dump_sw_error, 382 .portal_get_addr = kernel_idxd_portal_get_addr, 383 }; 384 385 static void 386 init_idxd_impl(struct spdk_idxd_device *idxd) 387 { 388 idxd->impl = &g_kernel_idxd_impl; 389 } 390 391 SPDK_IDXD_IMPL_REGISTER(kernel, &g_kernel_idxd_impl); 392