1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2021 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 8 #include <accel-config/libaccel_config.h> 9 10 #include "spdk/env.h" 11 #include "spdk/util.h" 12 #include "spdk/memory.h" 13 #include "spdk/likely.h" 14 15 #include "spdk/log.h" 16 #include "spdk_internal/idxd.h" 17 18 #include "idxd_internal.h" 19 20 struct spdk_kernel_idxd_device { 21 struct spdk_idxd_device idxd; 22 struct accfg_ctx *ctx; 23 24 unsigned int max_batch_size; 25 unsigned int max_xfer_size; 26 unsigned int max_xfer_bits; 27 28 /* We only use a single WQ */ 29 struct accfg_wq *wq; 30 int fd; 31 void *portal; 32 }; 33 34 #define __kernel_idxd(idxd) SPDK_CONTAINEROF(idxd, struct spdk_kernel_idxd_device, idxd) 35 36 static void 37 kernel_idxd_device_destruct(struct spdk_idxd_device *idxd) 38 { 39 struct spdk_kernel_idxd_device *kernel_idxd = __kernel_idxd(idxd); 40 41 if (kernel_idxd->portal != NULL) { 42 munmap(kernel_idxd->portal, 0x1000); 43 } 44 45 if (kernel_idxd->fd >= 0) { 46 close(kernel_idxd->fd); 47 } 48 49 accfg_unref(kernel_idxd->ctx); 50 free(kernel_idxd); 51 } 52 53 static struct spdk_idxd_impl g_kernel_idxd_impl; 54 55 static int 56 kernel_idxd_probe(void *cb_ctx, spdk_idxd_attach_cb attach_cb, spdk_idxd_probe_cb probe_cb) 57 { 58 int rc; 59 struct accfg_ctx *ctx; 60 struct accfg_device *device; 61 62 rc = accfg_new(&ctx); 63 if (rc < 0) { 64 SPDK_ERRLOG("Unable to allocate accel-config context\n"); 65 return rc; 66 } 67 68 /* Loop over each IDXD device */ 69 accfg_device_foreach(ctx, device) { 70 enum accfg_device_state dstate; 71 struct spdk_kernel_idxd_device *kernel_idxd; 72 struct accfg_wq *wq; 73 bool pasid_enabled; 74 75 /* Make sure that the device is enabled */ 76 dstate = accfg_device_get_state(device); 77 if (dstate != ACCFG_DEVICE_ENABLED) { 78 continue; 79 } 80 81 pasid_enabled = accfg_device_get_pasid_enabled(device); 82 if (!pasid_enabled && spdk_iommu_is_enabled()) { 83 /* 84 * If the IOMMU is enabled but shared memory mode is not on, 85 * then we have no way to get the IOVA from userspace to use this 86 * device or any kernel device. Return an error. 87 */ 88 SPDK_ERRLOG("Found kernel IDXD device, but cannot use it when IOMMU is enabled but SM is disabled\n"); 89 return -ENOTSUP; 90 } 91 92 kernel_idxd = calloc(1, sizeof(struct spdk_kernel_idxd_device)); 93 if (kernel_idxd == NULL) { 94 SPDK_ERRLOG("Failed to allocate memory for kernel_idxd device.\n"); 95 /* TODO: Goto error cleanup */ 96 return -ENOMEM; 97 } 98 99 kernel_idxd->max_batch_size = accfg_device_get_max_batch_size(device); 100 kernel_idxd->max_xfer_size = accfg_device_get_max_transfer_size(device); 101 kernel_idxd->idxd.socket_id = accfg_device_get_numa_node(device); 102 kernel_idxd->idxd.impl = &g_kernel_idxd_impl; 103 kernel_idxd->fd = -1; 104 kernel_idxd->idxd.version = accfg_device_get_version(device); 105 kernel_idxd->idxd.pasid_enabled = pasid_enabled; 106 107 accfg_wq_foreach(device, wq) { 108 enum accfg_wq_state wstate; 109 enum accfg_wq_mode mode; 110 enum accfg_wq_type type; 111 int major, minor; 112 char path[1024]; 113 114 wstate = accfg_wq_get_state(wq); 115 if (wstate != ACCFG_WQ_ENABLED) { 116 continue; 117 } 118 119 type = accfg_wq_get_type(wq); 120 if (type != ACCFG_WQT_USER) { 121 continue; 122 } 123 124 /* TODO: For now, only support dedicated WQ */ 125 mode = accfg_wq_get_mode(wq); 126 if (mode != ACCFG_WQ_DEDICATED) { 127 continue; 128 } 129 130 major = accfg_device_get_cdev_major(device); 131 if (major < 0) { 132 continue; 133 } 134 135 minor = accfg_wq_get_cdev_minor(wq); 136 if (minor < 0) { 137 continue; 138 } 139 140 /* Map the portal */ 141 snprintf(path, sizeof(path), "/dev/char/%u:%u", major, minor); 142 kernel_idxd->fd = open(path, O_RDWR); 143 if (kernel_idxd->fd < 0) { 144 SPDK_ERRLOG("Can not open the WQ file descriptor on path=%s\n", 145 path); 146 continue; 147 } 148 149 kernel_idxd->portal = mmap(NULL, 0x1000, PROT_WRITE, 150 MAP_SHARED | MAP_POPULATE, kernel_idxd->fd, 0); 151 if (kernel_idxd->portal == MAP_FAILED) { 152 perror("mmap"); 153 continue; 154 } 155 156 kernel_idxd->wq = wq; 157 158 /* Since we only use a single WQ, the total size is the size of this WQ */ 159 kernel_idxd->idxd.total_wq_size = accfg_wq_get_size(wq); 160 kernel_idxd->idxd.chan_per_device = (kernel_idxd->idxd.total_wq_size >= 128) ? 8 : 4; 161 162 /* We only use a single WQ, so once we've found one we can stop looking. */ 163 break; 164 } 165 166 if (kernel_idxd->idxd.total_wq_size > 0) { 167 /* This device has at least 1 WQ available, so ask the user if they want to use it. */ 168 attach_cb(cb_ctx, &kernel_idxd->idxd); 169 } else { 170 kernel_idxd_device_destruct(&kernel_idxd->idxd); 171 } 172 } 173 174 return 0; 175 } 176 177 static void 178 kernel_idxd_dump_sw_error(struct spdk_idxd_device *idxd, void *portal) 179 { 180 /* Need to be enhanced later */ 181 } 182 183 static char * 184 kernel_idxd_portal_get_addr(struct spdk_idxd_device *idxd) 185 { 186 struct spdk_kernel_idxd_device *kernel_idxd = __kernel_idxd(idxd); 187 188 return kernel_idxd->portal; 189 } 190 191 static struct spdk_idxd_impl g_kernel_idxd_impl = { 192 .name = "kernel", 193 .probe = kernel_idxd_probe, 194 .destruct = kernel_idxd_device_destruct, 195 .dump_sw_error = kernel_idxd_dump_sw_error, 196 .portal_get_addr = kernel_idxd_portal_get_addr, 197 }; 198 199 SPDK_IDXD_IMPL_REGISTER(kernel, &g_kernel_idxd_impl); 200