1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2022 Intel Corporation 3 */ 4 5 #include <errno.h> 6 #include <stdio.h> 7 #include <stdint.h> 8 #include <stdlib.h> 9 #include <string.h> 10 #include <unistd.h> 11 #include <fcntl.h> 12 #include <poll.h> 13 #include <sys/eventfd.h> 14 15 #include <rte_eal.h> 16 #include <rte_malloc.h> 17 #include <rte_memzone.h> 18 #include <rte_rawdev_pmd.h> 19 20 #include "afu_pmd_core.h" 21 22 static struct rte_afu_uuid afu_pmd_uuid_map[AFU_RAWDEV_MAX_DRVS+1]; 23 TAILQ_HEAD(afu_drv_list, afu_rawdev_drv); 24 static struct afu_drv_list afu_pmd_list = TAILQ_HEAD_INITIALIZER(afu_pmd_list); 25 26 static inline int afu_rawdev_trylock(struct afu_rawdev *dev) 27 { 28 if (!dev || !dev->sd) 29 return 0; 30 31 return rte_spinlock_trylock(&dev->sd->lock); 32 } 33 34 static inline void afu_rawdev_unlock(struct afu_rawdev *dev) 35 { 36 if (!dev || !dev->sd) 37 return; 38 39 rte_spinlock_unlock(&dev->sd->lock); 40 } 41 42 static int afu_rawdev_configure(const struct rte_rawdev *rawdev, 43 rte_rawdev_obj_t config, size_t config_size) 44 { 45 struct afu_rawdev *dev = NULL; 46 int ret = 0; 47 48 IFPGA_RAWDEV_PMD_FUNC_TRACE(); 49 50 dev = afu_rawdev_get_priv(rawdev); 51 if (!dev) 52 return -ENODEV; 53 54 if (dev->ops && dev->ops->config) 55 ret = (*dev->ops->config)(dev, config, config_size); 56 57 return ret; 58 } 59 60 static int afu_rawdev_start(struct rte_rawdev *rawdev) 61 { 62 struct afu_rawdev *dev = NULL; 63 int ret = 0; 64 65 IFPGA_RAWDEV_PMD_FUNC_TRACE(); 66 67 dev = afu_rawdev_get_priv(rawdev); 68 if (!dev) 69 return -ENODEV; 70 71 ret = afu_rawdev_trylock(dev); 72 if (!ret) { 73 IFPGA_RAWDEV_PMD_WARN("AFU is busy, please start it later"); 74 return ret; 75 } 76 77 if (dev->ops && dev->ops->start) 78 ret = (*dev->ops->start)(dev); 79 80 afu_rawdev_unlock(dev); 81 82 return ret; 83 } 84 85 static void afu_rawdev_stop(struct rte_rawdev *rawdev) 86 { 87 struct afu_rawdev *dev = NULL; 88 int ret = 0; 89 90 IFPGA_RAWDEV_PMD_FUNC_TRACE(); 91 92 dev = afu_rawdev_get_priv(rawdev); 93 if (!dev) 94 return; 95 96 ret = afu_rawdev_trylock(dev); 97 if (!ret) { 98 IFPGA_RAWDEV_PMD_WARN("AFU is busy, please stop it later"); 99 return; 100 } 101 102 if (dev->ops && dev->ops->stop) 103 ret = (*dev->ops->stop)(dev); 104 105 afu_rawdev_unlock(dev); 106 } 107 108 static int afu_rawdev_close(struct rte_rawdev *rawdev) 109 { 110 struct afu_rawdev *dev = NULL; 111 int ret = 0; 112 113 IFPGA_RAWDEV_PMD_FUNC_TRACE(); 114 115 dev = afu_rawdev_get_priv(rawdev); 116 if (!dev) 117 return -ENODEV; 118 119 if (dev->ops && dev->ops->close) 120 ret = (*dev->ops->close)(dev); 121 122 return ret; 123 } 124 125 static int afu_rawdev_reset(struct rte_rawdev *rawdev) 126 { 127 struct afu_rawdev *dev = NULL; 128 int ret = 0; 129 130 IFPGA_RAWDEV_PMD_FUNC_TRACE(); 131 132 dev = afu_rawdev_get_priv(rawdev); 133 if (!dev) 134 return -ENODEV; 135 136 ret = afu_rawdev_trylock(dev); 137 if (!ret) { 138 IFPGA_RAWDEV_PMD_WARN("AFU is busy, please reset it later"); 139 return ret; 140 } 141 142 if (dev->ops && dev->ops->reset) 143 ret = (*dev->ops->reset)(dev); 144 145 afu_rawdev_unlock(dev); 146 147 return ret; 148 } 149 150 static int afu_rawdev_selftest(uint16_t dev_id) 151 { 152 struct afu_rawdev *dev = NULL; 153 int ret = 0; 154 155 IFPGA_RAWDEV_PMD_FUNC_TRACE(); 156 157 if (!rte_rawdev_pmd_is_valid_dev(dev_id)) 158 return -ENODEV; 159 160 dev = afu_rawdev_get_priv(&rte_rawdevs[dev_id]); 161 if (!dev) 162 return -ENOENT; 163 164 ret = afu_rawdev_trylock(dev); 165 if (!ret) { 166 IFPGA_RAWDEV_PMD_WARN("AFU is busy, please test it later"); 167 return ret; 168 } 169 170 if (dev->ops && dev->ops->test) 171 ret = (*dev->ops->test)(dev); 172 173 afu_rawdev_unlock(dev); 174 175 return ret; 176 } 177 178 static int afu_rawdev_dump(struct rte_rawdev *rawdev, FILE *f) 179 { 180 struct afu_rawdev *dev = NULL; 181 int ret = 0; 182 183 IFPGA_RAWDEV_PMD_FUNC_TRACE(); 184 185 dev = afu_rawdev_get_priv(rawdev); 186 if (!dev) 187 return -ENODEV; 188 189 if (dev->ops && dev->ops->dump) 190 ret = (*dev->ops->dump)(dev, f); 191 192 return ret; 193 } 194 195 static const struct rte_rawdev_ops afu_rawdev_ops = { 196 .dev_info_get = NULL, 197 .dev_configure = afu_rawdev_configure, 198 .dev_start = afu_rawdev_start, 199 .dev_stop = afu_rawdev_stop, 200 .dev_close = afu_rawdev_close, 201 .dev_reset = afu_rawdev_reset, 202 203 .queue_def_conf = NULL, 204 .queue_setup = NULL, 205 .queue_release = NULL, 206 .queue_count = NULL, 207 208 .attr_get = NULL, 209 .attr_set = NULL, 210 211 .enqueue_bufs = NULL, 212 .dequeue_bufs = NULL, 213 214 .dump = afu_rawdev_dump, 215 216 .xstats_get = NULL, 217 .xstats_get_names = NULL, 218 .xstats_get_by_name = NULL, 219 .xstats_reset = NULL, 220 221 .firmware_status_get = NULL, 222 .firmware_version_get = NULL, 223 .firmware_load = NULL, 224 .firmware_unload = NULL, 225 226 .dev_selftest = afu_rawdev_selftest, 227 }; 228 229 static int afu_shared_data_alloc(const char *name, 230 struct afu_shared_data **data, int socket_id) 231 { 232 const struct rte_memzone *mz; 233 char mz_name[RTE_MEMZONE_NAMESIZE]; 234 struct afu_shared_data *sd = NULL; 235 int init_mz = 0; 236 237 if (!name || !data) 238 return -EINVAL; 239 240 /* name format is afu_?|??:??.? which is unique */ 241 snprintf(mz_name, sizeof(mz_name), "%s", name); 242 243 mz = rte_memzone_lookup(mz_name); 244 if (!mz) { 245 mz = rte_memzone_reserve(mz_name, sizeof(struct afu_shared_data), 246 socket_id, 0); 247 init_mz = 1; 248 } 249 250 if (!mz) { 251 IFPGA_RAWDEV_PMD_ERR("Allocate memory zone %s failed!", 252 mz_name); 253 return -ENOMEM; 254 } 255 256 sd = (struct afu_shared_data *)mz->addr; 257 258 if (init_mz) /* initialize memory zone on the first time */ 259 rte_spinlock_init(&sd->lock); 260 261 *data = sd; 262 263 return 0; 264 } 265 266 static int afu_rawdev_name_get(struct rte_afu_device *afu_dev, char *name, 267 size_t size) 268 { 269 int n = 0; 270 271 if (!afu_dev || !name || !size) 272 return -EINVAL; 273 274 n = snprintf(name, size, "afu_%s", afu_dev->device.name); 275 if (n >= (int)size) { 276 IFPGA_RAWDEV_PMD_ERR("Name of AFU device is too long!"); 277 return -ENAMETOOLONG; 278 } 279 280 return 0; 281 } 282 283 static struct afu_ops *afu_ops_get(struct rte_afu_uuid *afu_id) 284 { 285 struct afu_rawdev_drv *drv = NULL; 286 287 if (!afu_id) 288 return NULL; 289 290 TAILQ_FOREACH(drv, &afu_pmd_list, next) { 291 if ((drv->uuid.uuid_low == afu_id->uuid_low) && 292 (drv->uuid.uuid_high == afu_id->uuid_high)) 293 break; 294 } 295 296 return drv ? drv->ops : NULL; 297 } 298 299 static int afu_rawdev_create(struct rte_afu_device *afu_dev, int socket_id) 300 { 301 struct rte_rawdev *rawdev = NULL; 302 struct afu_rawdev *dev = NULL; 303 char name[RTE_RAWDEV_NAME_MAX_LEN] = {0}; 304 int ret = 0; 305 306 if (!afu_dev) 307 return -EINVAL; 308 309 ret = afu_rawdev_name_get(afu_dev, name, sizeof(name)); 310 if (ret) 311 return ret; 312 313 IFPGA_RAWDEV_PMD_INFO("Create raw device %s on NUMA node %d", 314 name, socket_id); 315 316 /* Allocate device structure */ 317 rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct afu_rawdev), 318 socket_id); 319 if (!rawdev) { 320 IFPGA_RAWDEV_PMD_ERR("Unable to allocate raw device"); 321 return -ENOMEM; 322 } 323 324 rawdev->dev_ops = &afu_rawdev_ops; 325 rawdev->device = &afu_dev->device; 326 rawdev->driver_name = afu_dev->driver->driver.name; 327 328 dev = afu_rawdev_get_priv(rawdev); 329 if (!dev) 330 goto cleanup; 331 332 dev->rawdev = rawdev; 333 dev->port = afu_dev->id.port; 334 dev->addr = afu_dev->mem_resource[0].addr; 335 dev->ops = afu_ops_get(&afu_dev->id.uuid); 336 if (dev->ops == NULL) { 337 IFPGA_RAWDEV_PMD_ERR("Unsupported AFU device"); 338 goto cleanup; 339 } 340 341 if (dev->ops->init) { 342 ret = (*dev->ops->init)(dev); 343 if (ret) { 344 IFPGA_RAWDEV_PMD_ERR("Failed to init %s", name); 345 goto cleanup; 346 } 347 } 348 349 ret = afu_shared_data_alloc(name, &dev->sd, socket_id); 350 if (ret) 351 goto cleanup; 352 353 return ret; 354 355 cleanup: 356 rte_rawdev_pmd_release(rawdev); 357 return ret; 358 } 359 360 static int afu_rawdev_destroy(struct rte_afu_device *afu_dev) 361 { 362 struct rte_rawdev *rawdev = NULL; 363 char name[RTE_RAWDEV_NAME_MAX_LEN] = {0}; 364 int ret = 0; 365 366 if (!afu_dev) 367 return -EINVAL; 368 369 ret = afu_rawdev_name_get(afu_dev, name, sizeof(name)); 370 if (ret) 371 return ret; 372 373 IFPGA_RAWDEV_PMD_INFO("Destroy raw device %s", name); 374 375 rawdev = rte_rawdev_pmd_get_named_dev(name); 376 if (!rawdev) { 377 IFPGA_RAWDEV_PMD_ERR("Raw device %s not found", name); 378 return -EINVAL; 379 } 380 381 /* rte_rawdev_close is called by pmd_release */ 382 ret = rte_rawdev_pmd_release(rawdev); 383 if (ret) 384 IFPGA_RAWDEV_PMD_DEBUG("Device cleanup failed"); 385 386 return 0; 387 } 388 389 static int afu_rawdev_probe(struct rte_afu_device *afu_dev) 390 { 391 IFPGA_RAWDEV_PMD_FUNC_TRACE(); 392 return afu_rawdev_create(afu_dev, rte_socket_id()); 393 } 394 395 static int afu_rawdev_remove(struct rte_afu_device *afu_dev) 396 { 397 IFPGA_RAWDEV_PMD_FUNC_TRACE(); 398 return afu_rawdev_destroy(afu_dev); 399 } 400 401 static struct rte_afu_driver afu_pmd = { 402 .id_table = afu_pmd_uuid_map, 403 .probe = afu_rawdev_probe, 404 .remove = afu_rawdev_remove 405 }; 406 407 RTE_PMD_REGISTER_AFU(afu_rawdev_driver, afu_pmd); 408 409 static void update_uuid_map(void) 410 { 411 int i = 0; 412 struct rte_afu_uuid *afu_id = afu_pmd_uuid_map; 413 struct afu_rawdev_drv *drv; 414 415 TAILQ_FOREACH(drv, &afu_pmd_list, next) { 416 if (i++ < AFU_RAWDEV_MAX_DRVS) { 417 afu_id->uuid_low = drv->uuid.uuid_low; 418 afu_id->uuid_high = drv->uuid.uuid_high; 419 afu_id++; 420 } 421 } 422 if (i <= AFU_RAWDEV_MAX_DRVS) { 423 afu_id->uuid_low = 0; 424 afu_id->uuid_high = 0; 425 } 426 } 427 428 void afu_pmd_register(struct afu_rawdev_drv *driver) 429 { 430 TAILQ_INSERT_TAIL(&afu_pmd_list, driver, next); 431 update_uuid_map(); 432 } 433 434 void afu_pmd_unregister(struct afu_rawdev_drv *driver) 435 { 436 TAILQ_REMOVE(&afu_pmd_list, driver, next); 437 update_uuid_map(); 438 } 439