1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2022 Intel Corporation 3 */ 4 5 #include <errno.h> 6 #include <stdio.h> 7 #include <stdint.h> 8 #include <stdlib.h> 9 #include <string.h> 10 #include <unistd.h> 11 #include <fcntl.h> 12 #include <poll.h> 13 #include <sys/eventfd.h> 14 15 #include <rte_eal.h> 16 #include <rte_malloc.h> 17 #include <rte_memzone.h> 18 #include <rte_rawdev_pmd.h> 19 20 #include "afu_pmd_core.h" 21 22 static struct rte_afu_uuid afu_pmd_uuid_map[AFU_RAWDEV_MAX_DRVS+1]; 23 TAILQ_HEAD(afu_drv_list, afu_rawdev_drv); 24 static struct afu_drv_list afu_pmd_list = TAILQ_HEAD_INITIALIZER(afu_pmd_list); 25 26 #define afu_rawdev_trylock(dev) rte_spinlock_trylock(&dev->sd->lock) 27 #define afu_rawdev_unlock(dev) rte_spinlock_unlock(&dev->sd->lock) 28 29 static int afu_rawdev_configure(const struct rte_rawdev *rawdev, 30 rte_rawdev_obj_t config, size_t config_size) 31 { 32 struct afu_rawdev *dev = NULL; 33 int ret = 0; 34 35 IFPGA_RAWDEV_PMD_FUNC_TRACE(); 36 37 dev = afu_rawdev_get_priv(rawdev); 38 if (!dev) 39 return -ENODEV; 40 41 if (dev->ops && dev->ops->config) 42 ret = (*dev->ops->config)(dev, config, config_size); 43 44 return ret; 45 } 46 47 static int afu_rawdev_start(struct rte_rawdev *rawdev) 48 { 49 struct afu_rawdev *dev = NULL; 50 int ret = 0; 51 52 IFPGA_RAWDEV_PMD_FUNC_TRACE(); 53 54 dev = afu_rawdev_get_priv(rawdev); 55 if (!dev) 56 return -ENODEV; 57 58 ret = afu_rawdev_trylock(dev); 59 if (!ret) { 60 IFPGA_RAWDEV_PMD_WARN("AFU is busy, please start it later"); 61 return ret; 62 } 63 64 if (dev->ops && dev->ops->start) 65 ret = (*dev->ops->start)(dev); 66 67 afu_rawdev_unlock(dev); 68 69 return ret; 70 } 71 72 static void afu_rawdev_stop(struct rte_rawdev *rawdev) 73 { 74 struct afu_rawdev *dev = NULL; 75 int ret = 0; 76 77 IFPGA_RAWDEV_PMD_FUNC_TRACE(); 78 79 dev = afu_rawdev_get_priv(rawdev); 80 if (!dev) 81 return; 82 83 ret = afu_rawdev_trylock(dev); 84 if (!ret) { 85 IFPGA_RAWDEV_PMD_WARN("AFU is busy, please stop it later"); 86 return; 87 } 88 89 if (dev->ops && dev->ops->stop) 90 ret = (*dev->ops->stop)(dev); 91 92 afu_rawdev_unlock(dev); 93 } 94 95 static int afu_rawdev_close(struct rte_rawdev *rawdev) 96 { 97 struct afu_rawdev *dev = NULL; 98 int ret = 0; 99 100 IFPGA_RAWDEV_PMD_FUNC_TRACE(); 101 102 dev = afu_rawdev_get_priv(rawdev); 103 if (!dev) 104 return -ENODEV; 105 106 if (dev->ops && dev->ops->close) 107 ret = (*dev->ops->close)(dev); 108 109 return ret; 110 } 111 112 static int afu_rawdev_reset(struct rte_rawdev *rawdev) 113 { 114 struct afu_rawdev *dev = NULL; 115 int ret = 0; 116 117 IFPGA_RAWDEV_PMD_FUNC_TRACE(); 118 119 dev = afu_rawdev_get_priv(rawdev); 120 if (!dev) 121 return -ENODEV; 122 123 ret = afu_rawdev_trylock(dev); 124 if (!ret) { 125 IFPGA_RAWDEV_PMD_WARN("AFU is busy, please reset it later"); 126 return ret; 127 } 128 129 if (dev->ops && dev->ops->reset) 130 ret = (*dev->ops->reset)(dev); 131 132 afu_rawdev_unlock(dev); 133 134 return ret; 135 } 136 137 static int afu_rawdev_selftest(uint16_t dev_id) 138 { 139 struct afu_rawdev *dev = NULL; 140 int ret = 0; 141 142 IFPGA_RAWDEV_PMD_FUNC_TRACE(); 143 144 if (!rte_rawdev_pmd_is_valid_dev(dev_id)) 145 return -ENODEV; 146 147 dev = afu_rawdev_get_priv(&rte_rawdevs[dev_id]); 148 if (!dev) 149 return -ENOENT; 150 151 ret = afu_rawdev_trylock(dev); 152 if (!ret) { 153 IFPGA_RAWDEV_PMD_WARN("AFU is busy, please test it later"); 154 return ret; 155 } 156 157 if (dev->ops && dev->ops->test) 158 ret = (*dev->ops->test)(dev); 159 160 afu_rawdev_unlock(dev); 161 162 return ret; 163 } 164 165 static int afu_rawdev_dump(struct rte_rawdev *rawdev, FILE *f) 166 { 167 struct afu_rawdev *dev = NULL; 168 int ret = 0; 169 170 IFPGA_RAWDEV_PMD_FUNC_TRACE(); 171 172 dev = afu_rawdev_get_priv(rawdev); 173 if (!dev) 174 return -ENODEV; 175 176 if (dev->ops && dev->ops->dump) 177 ret = (*dev->ops->dump)(dev, f); 178 179 return ret; 180 } 181 182 static const struct rte_rawdev_ops afu_rawdev_ops = { 183 .dev_info_get = NULL, 184 .dev_configure = afu_rawdev_configure, 185 .dev_start = afu_rawdev_start, 186 .dev_stop = afu_rawdev_stop, 187 .dev_close = afu_rawdev_close, 188 .dev_reset = afu_rawdev_reset, 189 190 .queue_def_conf = NULL, 191 .queue_setup = NULL, 192 .queue_release = NULL, 193 .queue_count = NULL, 194 195 .attr_get = NULL, 196 .attr_set = NULL, 197 198 .enqueue_bufs = NULL, 199 .dequeue_bufs = NULL, 200 201 .dump = afu_rawdev_dump, 202 203 .xstats_get = NULL, 204 .xstats_get_names = NULL, 205 .xstats_get_by_name = NULL, 206 .xstats_reset = NULL, 207 208 .firmware_status_get = NULL, 209 .firmware_version_get = NULL, 210 .firmware_load = NULL, 211 .firmware_unload = NULL, 212 213 .dev_selftest = afu_rawdev_selftest, 214 }; 215 216 static int afu_shared_data_alloc(const char *name, 217 struct afu_shared_data **data, int socket_id) 218 { 219 const struct rte_memzone *mz; 220 char mz_name[RTE_MEMZONE_NAMESIZE]; 221 struct afu_shared_data *sd = NULL; 222 int init_mz = 0; 223 224 if (!name || !data) 225 return -EINVAL; 226 227 /* name format is afu_?|??:??.? which is unique */ 228 snprintf(mz_name, sizeof(mz_name), "%s", name); 229 230 mz = rte_memzone_lookup(mz_name); 231 if (!mz) { 232 mz = rte_memzone_reserve(mz_name, sizeof(struct afu_shared_data), 233 socket_id, 0); 234 init_mz = 1; 235 } 236 237 if (!mz) { 238 IFPGA_RAWDEV_PMD_ERR("Allocate memory zone %s failed!", 239 mz_name); 240 return -ENOMEM; 241 } 242 243 sd = (struct afu_shared_data *)mz->addr; 244 245 if (init_mz) /* initialize memory zone on the first time */ 246 rte_spinlock_init(&sd->lock); 247 248 *data = sd; 249 250 return 0; 251 } 252 253 static int afu_rawdev_name_get(struct rte_afu_device *afu_dev, char *name, 254 size_t size) 255 { 256 int n = 0; 257 258 if (!afu_dev || !name || !size) 259 return -EINVAL; 260 261 n = snprintf(name, size, "afu_%s", afu_dev->device.name); 262 if (n >= (int)size) { 263 IFPGA_RAWDEV_PMD_ERR("Name of AFU device is too long!"); 264 return -ENAMETOOLONG; 265 } 266 267 return 0; 268 } 269 270 static struct afu_ops *afu_ops_get(struct rte_afu_uuid *afu_id) 271 { 272 struct afu_rawdev_drv *drv = NULL; 273 274 if (!afu_id) 275 return NULL; 276 277 TAILQ_FOREACH(drv, &afu_pmd_list, next) { 278 if ((drv->uuid.uuid_low == afu_id->uuid_low) && 279 (drv->uuid.uuid_high == afu_id->uuid_high)) 280 break; 281 } 282 283 return drv ? drv->ops : NULL; 284 } 285 286 static int afu_rawdev_create(struct rte_afu_device *afu_dev, int socket_id) 287 { 288 struct rte_rawdev *rawdev = NULL; 289 struct afu_rawdev *dev = NULL; 290 char name[RTE_RAWDEV_NAME_MAX_LEN] = {0}; 291 int ret = 0; 292 293 if (!afu_dev) 294 return -EINVAL; 295 296 ret = afu_rawdev_name_get(afu_dev, name, sizeof(name)); 297 if (ret) 298 return ret; 299 300 IFPGA_RAWDEV_PMD_INFO("Create raw device %s on NUMA node %d", 301 name, socket_id); 302 303 /* Allocate device structure */ 304 rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct afu_rawdev), 305 socket_id); 306 if (!rawdev) { 307 IFPGA_RAWDEV_PMD_ERR("Unable to allocate raw device"); 308 return -ENOMEM; 309 } 310 311 rawdev->dev_ops = &afu_rawdev_ops; 312 rawdev->device = &afu_dev->device; 313 rawdev->driver_name = afu_dev->driver->driver.name; 314 315 dev = afu_rawdev_get_priv(rawdev); 316 if (!dev) 317 goto cleanup; 318 319 dev->rawdev = rawdev; 320 dev->port = afu_dev->id.port; 321 dev->addr = afu_dev->mem_resource[0].addr; 322 dev->ops = afu_ops_get(&afu_dev->id.uuid); 323 if (dev->ops == NULL) { 324 IFPGA_RAWDEV_PMD_ERR("Unsupported AFU device"); 325 goto cleanup; 326 } 327 328 if (dev->ops->init) { 329 ret = (*dev->ops->init)(dev); 330 if (ret) { 331 IFPGA_RAWDEV_PMD_ERR("Failed to init %s", name); 332 goto cleanup; 333 } 334 } 335 336 ret = afu_shared_data_alloc(name, &dev->sd, socket_id); 337 if (ret) 338 goto cleanup; 339 340 return ret; 341 342 cleanup: 343 rte_rawdev_pmd_release(rawdev); 344 return ret; 345 } 346 347 static int afu_rawdev_destroy(struct rte_afu_device *afu_dev) 348 { 349 struct rte_rawdev *rawdev = NULL; 350 char name[RTE_RAWDEV_NAME_MAX_LEN] = {0}; 351 int ret = 0; 352 353 if (!afu_dev) 354 return -EINVAL; 355 356 ret = afu_rawdev_name_get(afu_dev, name, sizeof(name)); 357 if (ret) 358 return ret; 359 360 IFPGA_RAWDEV_PMD_INFO("Destroy raw device %s", name); 361 362 rawdev = rte_rawdev_pmd_get_named_dev(name); 363 if (!rawdev) { 364 IFPGA_RAWDEV_PMD_ERR("Raw device %s not found", name); 365 return -EINVAL; 366 } 367 368 /* rte_rawdev_close is called by pmd_release */ 369 ret = rte_rawdev_pmd_release(rawdev); 370 if (ret) 371 IFPGA_RAWDEV_PMD_DEBUG("Device cleanup failed"); 372 373 return 0; 374 } 375 376 static int afu_rawdev_probe(struct rte_afu_device *afu_dev) 377 { 378 IFPGA_RAWDEV_PMD_FUNC_TRACE(); 379 return afu_rawdev_create(afu_dev, rte_socket_id()); 380 } 381 382 static int afu_rawdev_remove(struct rte_afu_device *afu_dev) 383 { 384 IFPGA_RAWDEV_PMD_FUNC_TRACE(); 385 return afu_rawdev_destroy(afu_dev); 386 } 387 388 static struct rte_afu_driver afu_pmd = { 389 .id_table = afu_pmd_uuid_map, 390 .probe = afu_rawdev_probe, 391 .remove = afu_rawdev_remove 392 }; 393 394 RTE_PMD_REGISTER_AFU(afu_rawdev_driver, afu_pmd); 395 396 static void update_uuid_map(void) 397 { 398 int i = 0; 399 struct rte_afu_uuid *afu_id = afu_pmd_uuid_map; 400 struct afu_rawdev_drv *drv; 401 402 TAILQ_FOREACH(drv, &afu_pmd_list, next) { 403 if (i++ < AFU_RAWDEV_MAX_DRVS) { 404 afu_id->uuid_low = drv->uuid.uuid_low; 405 afu_id->uuid_high = drv->uuid.uuid_high; 406 afu_id++; 407 } 408 } 409 if (i <= AFU_RAWDEV_MAX_DRVS) { 410 afu_id->uuid_low = 0; 411 afu_id->uuid_high = 0; 412 } 413 } 414 415 void afu_pmd_register(struct afu_rawdev_drv *driver) 416 { 417 TAILQ_INSERT_TAIL(&afu_pmd_list, driver, next); 418 update_uuid_map(); 419 } 420 421 void afu_pmd_unregister(struct afu_rawdev_drv *driver) 422 { 423 TAILQ_REMOVE(&afu_pmd_list, driver, next); 424 update_uuid_map(); 425 } 426