1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2018 Mellanox Technologies, Ltd 3 */ 4 5 /** 6 * @file 7 * Interrupts handling for failsafe driver. 8 */ 9 10 #if defined(LINUX) 11 #include <sys/epoll.h> 12 #endif 13 #include <unistd.h> 14 15 #include <rte_alarm.h> 16 #include <rte_config.h> 17 #include <rte_errno.h> 18 #include <rte_ethdev.h> 19 #include <rte_interrupts.h> 20 #include <rte_io.h> 21 #include <rte_service_component.h> 22 23 #include "failsafe_private.h" 24 25 #define NUM_RX_PROXIES (FAILSAFE_MAX_ETHPORTS * RTE_MAX_RXTX_INTR_VEC_ID) 26 27 28 /** 29 * Open an epoll file descriptor. 30 * 31 * @param flags 32 * Flags for defining epoll behavior. 33 * @return 34 * 0 on success, negative errno value otherwise. 35 */ 36 static int 37 fs_epoll_create1(int flags) 38 { 39 #if defined(LINUX) 40 return epoll_create1(flags); 41 #elif defined(BSD) 42 RTE_SET_USED(flags); 43 return -ENOTSUP; 44 #endif 45 } 46 47 /** 48 * Install failsafe Rx event proxy service. 49 * The Rx event proxy is the service that listens to Rx events from the 50 * subdevices and triggers failsafe Rx events accordingly. 51 * 52 * @param priv 53 * Pointer to failsafe private structure. 54 * @return 55 * 0 on success, negative errno value otherwise. 56 */ 57 static int 58 fs_rx_event_proxy_routine(void *data) 59 { 60 struct fs_priv *priv; 61 struct rxq *rxq; 62 struct rte_epoll_event *events; 63 uint64_t u64; 64 int i, n; 65 int rc = 0; 66 67 u64 = 1; 68 priv = data; 69 events = priv->rxp.evec; 70 n = rte_epoll_wait(priv->rxp.efd, events, NUM_RX_PROXIES, -1); 71 for (i = 0; i < n; i++) { 72 rxq = events[i].epdata.data; 73 if (rxq->enable_events && rxq->event_fd != -1) { 74 if (write(rxq->event_fd, &u64, sizeof(u64)) != 75 sizeof(u64)) { 76 ERROR("Failed to proxy Rx event to socket %d", 77 rxq->event_fd); 78 rc = -EIO; 79 } 80 } 81 } 82 return rc; 83 } 84 85 /** 86 * Uninstall failsafe Rx event proxy service. 87 * 88 * @param priv 89 * Pointer to failsafe private structure. 90 */ 91 static void 92 fs_rx_event_proxy_service_uninstall(struct fs_priv *priv) 93 { 94 /* Unregister the event service. */ 95 switch (priv->rxp.sstate) { 96 case SS_RUNNING: 97 rte_service_map_lcore_set(priv->rxp.sid, priv->rxp.scid, 0); 98 /* fall through */ 99 case SS_READY: 100 rte_service_runstate_set(priv->rxp.sid, 0); 101 rte_service_set_stats_enable(priv->rxp.sid, 0); 102 rte_service_component_runstate_set(priv->rxp.sid, 0); 103 /* fall through */ 104 case SS_REGISTERED: 105 rte_service_component_unregister(priv->rxp.sid); 106 /* fall through */ 107 default: 108 break; 109 } 110 } 111 112 /** 113 * Install the failsafe Rx event proxy service. 114 * 115 * @param priv 116 * Pointer to failsafe private structure. 117 * @return 118 * 0 on success, negative errno value otherwise. 119 */ 120 static int 121 fs_rx_event_proxy_service_install(struct fs_priv *priv) 122 { 123 struct rte_service_spec service; 124 int32_t num_service_cores; 125 int ret = 0; 126 127 num_service_cores = rte_service_lcore_count(); 128 if (num_service_cores <= 0) { 129 ERROR("Failed to install Rx interrupts, " 130 "no service core found"); 131 return -ENOTSUP; 132 } 133 /* prepare service info */ 134 memset(&service, 0, sizeof(struct rte_service_spec)); 135 snprintf(service.name, sizeof(service.name), "%s_Rx_service", 136 priv->data->name); 137 service.socket_id = priv->data->numa_node; 138 service.callback = fs_rx_event_proxy_routine; 139 service.callback_userdata = priv; 140 141 if (priv->rxp.sstate == SS_NO_SERVICE) { 142 uint32_t service_core_list[num_service_cores]; 143 144 /* get a service core to work with */ 145 ret = rte_service_lcore_list(service_core_list, 146 num_service_cores); 147 if (ret <= 0) { 148 ERROR("Failed to install Rx interrupts, " 149 "service core list empty or corrupted"); 150 return -ENOTSUP; 151 } 152 priv->rxp.scid = service_core_list[0]; 153 ret = rte_service_lcore_add(priv->rxp.scid); 154 if (ret && ret != -EALREADY) { 155 ERROR("Failed adding service core"); 156 return ret; 157 } 158 /* service core may be in "stopped" state, start it */ 159 ret = rte_service_lcore_start(priv->rxp.scid); 160 if (ret && (ret != -EALREADY)) { 161 ERROR("Failed to install Rx interrupts, " 162 "service core not started"); 163 return ret; 164 } 165 /* register our service */ 166 int32_t ret = rte_service_component_register(&service, 167 &priv->rxp.sid); 168 if (ret) { 169 ERROR("service register() failed"); 170 return -ENOEXEC; 171 } 172 priv->rxp.sstate = SS_REGISTERED; 173 /* run the service */ 174 ret = rte_service_component_runstate_set(priv->rxp.sid, 1); 175 if (ret < 0) { 176 ERROR("Failed Setting component runstate\n"); 177 return ret; 178 } 179 ret = rte_service_set_stats_enable(priv->rxp.sid, 1); 180 if (ret < 0) { 181 ERROR("Failed enabling stats\n"); 182 return ret; 183 } 184 ret = rte_service_runstate_set(priv->rxp.sid, 1); 185 if (ret < 0) { 186 ERROR("Failed to run service\n"); 187 return ret; 188 } 189 priv->rxp.sstate = SS_READY; 190 /* map the service with the service core */ 191 ret = rte_service_map_lcore_set(priv->rxp.sid, 192 priv->rxp.scid, 1); 193 if (ret) { 194 ERROR("Failed to install Rx interrupts, " 195 "could not map service core"); 196 return ret; 197 } 198 priv->rxp.sstate = SS_RUNNING; 199 } 200 return 0; 201 } 202 203 /** 204 * Install failsafe Rx event proxy subsystem. 205 * This is the way the failsafe PMD generates Rx events on behalf of its 206 * subdevices. 207 * 208 * @param priv 209 * Pointer to failsafe private structure. 210 * @return 211 * 0 on success, negative errno value otherwise and rte_errno is set. 212 */ 213 static int 214 fs_rx_event_proxy_install(struct fs_priv *priv) 215 { 216 int rc = 0; 217 218 /* 219 * Create the epoll fd and event vector for the proxy service to 220 * wait on for Rx events generated by the subdevices. 221 */ 222 priv->rxp.efd = fs_epoll_create1(0); 223 if (priv->rxp.efd < 0) { 224 rte_errno = errno; 225 ERROR("Failed to create epoll," 226 " Rx interrupts will not be supported"); 227 return -rte_errno; 228 } 229 priv->rxp.evec = calloc(NUM_RX_PROXIES, sizeof(*priv->rxp.evec)); 230 if (priv->rxp.evec == NULL) { 231 ERROR("Failed to allocate memory for event vectors," 232 " Rx interrupts will not be supported"); 233 rc = -ENOMEM; 234 goto error; 235 } 236 rc = fs_rx_event_proxy_service_install(priv); 237 if (rc < 0) 238 goto error; 239 return 0; 240 error: 241 if (priv->rxp.efd >= 0) { 242 close(priv->rxp.efd); 243 priv->rxp.efd = -1; 244 } 245 if (priv->rxp.evec != NULL) { 246 free(priv->rxp.evec); 247 priv->rxp.evec = NULL; 248 } 249 rte_errno = -rc; 250 return rc; 251 } 252 253 /** 254 * RX Interrupt control per subdevice. 255 * 256 * @param sdev 257 * Pointer to sub-device structure. 258 * @param op 259 * The operation be performed for the vector. 260 * Operation type of {RTE_INTR_EVENT_ADD, RTE_INTR_EVENT_DEL}. 261 * @return 262 * - On success, zero. 263 * - On failure, a negative value. 264 */ 265 static int 266 failsafe_eth_rx_intr_ctl_subdevice(struct sub_device *sdev, int op) 267 { 268 struct rte_eth_dev *dev; 269 struct rte_eth_dev *fsdev; 270 int epfd; 271 uint16_t pid; 272 uint16_t qid; 273 struct rxq *fsrxq; 274 int rc; 275 int ret = 0; 276 277 fsdev = fs_dev(sdev); 278 if (sdev == NULL || (ETH(sdev) == NULL) || 279 fsdev == NULL || (PRIV(fsdev) == NULL)) { 280 ERROR("Called with invalid arguments"); 281 return -EINVAL; 282 } 283 dev = ETH(sdev); 284 epfd = PRIV(fsdev)->rxp.efd; 285 pid = PORT_ID(sdev); 286 287 if (epfd <= 0) { 288 if (op == RTE_INTR_EVENT_ADD) { 289 ERROR("Proxy events are not initialized"); 290 return -EBADF; 291 } else { 292 return 0; 293 } 294 } 295 if (dev->data->nb_rx_queues > fsdev->data->nb_rx_queues) { 296 ERROR("subdevice has too many queues," 297 " Interrupts will not be enabled"); 298 return -E2BIG; 299 } 300 for (qid = 0; qid < dev->data->nb_rx_queues; qid++) { 301 fsrxq = fsdev->data->rx_queues[qid]; 302 rc = rte_eth_dev_rx_intr_ctl_q(pid, qid, epfd, 303 op, (void *)fsrxq); 304 if (rc) { 305 ERROR("rte_eth_dev_rx_intr_ctl_q failed for " 306 "port %d queue %d, epfd %d, error %d", 307 pid, qid, epfd, rc); 308 ret = rc; 309 } 310 } 311 return ret; 312 } 313 314 /** 315 * Install Rx interrupts subsystem for a subdevice. 316 * This is a support for dynamically adding subdevices. 317 * 318 * @param sdev 319 * Pointer to subdevice structure. 320 * 321 * @return 322 * 0 on success, negative errno value otherwise and rte_errno is set. 323 */ 324 int failsafe_rx_intr_install_subdevice(struct sub_device *sdev) 325 { 326 int rc; 327 int qid; 328 struct rte_eth_dev *fsdev; 329 struct rxq **rxq; 330 const struct rte_intr_conf *const intr_conf = 331 Ð(sdev)->data->dev_conf.intr_conf; 332 333 fsdev = fs_dev(sdev); 334 rxq = (struct rxq **)fsdev->data->rx_queues; 335 if (intr_conf->rxq == 0) 336 return 0; 337 rc = failsafe_eth_rx_intr_ctl_subdevice(sdev, RTE_INTR_EVENT_ADD); 338 if (rc) 339 return rc; 340 /* enable interrupts on already-enabled queues */ 341 for (qid = 0; qid < ETH(sdev)->data->nb_rx_queues; qid++) { 342 if (rxq[qid]->enable_events) { 343 int ret = rte_eth_dev_rx_intr_enable(PORT_ID(sdev), 344 qid); 345 if (ret && (ret != -ENOTSUP)) { 346 ERROR("Failed to enable interrupts on " 347 "port %d queue %d", PORT_ID(sdev), qid); 348 rc = ret; 349 } 350 } 351 } 352 return rc; 353 } 354 355 /** 356 * Uninstall Rx interrupts subsystem for a subdevice. 357 * This is a support for dynamically removing subdevices. 358 * 359 * @param sdev 360 * Pointer to subdevice structure. 361 * 362 * @return 363 * 0 on success, negative errno value otherwise and rte_errno is set. 364 */ 365 void failsafe_rx_intr_uninstall_subdevice(struct sub_device *sdev) 366 { 367 int qid; 368 struct rte_eth_dev *fsdev; 369 struct rxq *fsrxq; 370 371 fsdev = fs_dev(sdev); 372 for (qid = 0; qid < ETH(sdev)->data->nb_rx_queues; qid++) { 373 if (qid < fsdev->data->nb_rx_queues) { 374 fsrxq = fsdev->data->rx_queues[qid]; 375 if (fsrxq != NULL && fsrxq->enable_events) 376 rte_eth_dev_rx_intr_disable(PORT_ID(sdev), 377 qid); 378 } 379 } 380 failsafe_eth_rx_intr_ctl_subdevice(sdev, RTE_INTR_EVENT_DEL); 381 } 382 383 /** 384 * Uninstall failsafe Rx event proxy. 385 * 386 * @param priv 387 * Pointer to failsafe private structure. 388 */ 389 static void 390 fs_rx_event_proxy_uninstall(struct fs_priv *priv) 391 { 392 fs_rx_event_proxy_service_uninstall(priv); 393 if (priv->rxp.evec != NULL) { 394 free(priv->rxp.evec); 395 priv->rxp.evec = NULL; 396 } 397 if (priv->rxp.efd > 0) { 398 close(priv->rxp.efd); 399 priv->rxp.efd = -1; 400 } 401 } 402 403 /** 404 * Uninstall failsafe interrupt vector. 405 * 406 * @param priv 407 * Pointer to failsafe private structure. 408 */ 409 static void 410 fs_rx_intr_vec_uninstall(struct fs_priv *priv) 411 { 412 struct rte_intr_handle *intr_handle; 413 414 intr_handle = &priv->intr_handle; 415 if (intr_handle->intr_vec != NULL) { 416 free(intr_handle->intr_vec); 417 intr_handle->intr_vec = NULL; 418 } 419 intr_handle->nb_efd = 0; 420 } 421 422 /** 423 * Installs failsafe interrupt vector to be registered with EAL later on. 424 * 425 * @param priv 426 * Pointer to failsafe private structure. 427 * 428 * @return 429 * 0 on success, negative errno value otherwise and rte_errno is set. 430 */ 431 static int 432 fs_rx_intr_vec_install(struct fs_priv *priv) 433 { 434 unsigned int i; 435 unsigned int rxqs_n; 436 unsigned int n; 437 unsigned int count; 438 struct rte_intr_handle *intr_handle; 439 440 rxqs_n = priv->data->nb_rx_queues; 441 n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID); 442 count = 0; 443 intr_handle = &priv->intr_handle; 444 RTE_ASSERT(intr_handle->intr_vec == NULL); 445 /* Allocate the interrupt vector of the failsafe Rx proxy interrupts */ 446 intr_handle->intr_vec = malloc(n * sizeof(intr_handle->intr_vec[0])); 447 if (intr_handle->intr_vec == NULL) { 448 fs_rx_intr_vec_uninstall(priv); 449 rte_errno = ENOMEM; 450 ERROR("Failed to allocate memory for interrupt vector," 451 " Rx interrupts will not be supported"); 452 return -rte_errno; 453 } 454 for (i = 0; i < n; i++) { 455 struct rxq *rxq = priv->data->rx_queues[i]; 456 457 /* Skip queues that cannot request interrupts. */ 458 if (rxq == NULL || rxq->event_fd < 0) { 459 /* Use invalid intr_vec[] index to disable entry. */ 460 intr_handle->intr_vec[i] = 461 RTE_INTR_VEC_RXTX_OFFSET + 462 RTE_MAX_RXTX_INTR_VEC_ID; 463 continue; 464 } 465 if (count >= RTE_MAX_RXTX_INTR_VEC_ID) { 466 rte_errno = E2BIG; 467 ERROR("Too many Rx queues for interrupt vector size" 468 " (%d), Rx interrupts cannot be enabled", 469 RTE_MAX_RXTX_INTR_VEC_ID); 470 fs_rx_intr_vec_uninstall(priv); 471 return -rte_errno; 472 } 473 intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count; 474 intr_handle->efds[count] = rxq->event_fd; 475 count++; 476 } 477 if (count == 0) { 478 fs_rx_intr_vec_uninstall(priv); 479 } else { 480 intr_handle->nb_efd = count; 481 intr_handle->efd_counter_size = sizeof(uint64_t); 482 } 483 return 0; 484 } 485 486 487 /** 488 * Uninstall failsafe Rx interrupts subsystem. 489 * 490 * @param priv 491 * Pointer to private structure. 492 * 493 * @return 494 * 0 on success, negative errno value otherwise and rte_errno is set. 495 */ 496 void 497 failsafe_rx_intr_uninstall(struct rte_eth_dev *dev) 498 { 499 struct fs_priv *priv; 500 struct rte_intr_handle *intr_handle; 501 502 priv = PRIV(dev); 503 intr_handle = &priv->intr_handle; 504 rte_intr_free_epoll_fd(intr_handle); 505 fs_rx_event_proxy_uninstall(priv); 506 fs_rx_intr_vec_uninstall(priv); 507 dev->intr_handle = NULL; 508 } 509 510 /** 511 * Install failsafe Rx interrupts subsystem. 512 * 513 * @param priv 514 * Pointer to private structure. 515 * 516 * @return 517 * 0 on success, negative errno value otherwise and rte_errno is set. 518 */ 519 int 520 failsafe_rx_intr_install(struct rte_eth_dev *dev) 521 { 522 struct fs_priv *priv = PRIV(dev); 523 const struct rte_intr_conf *const intr_conf = 524 &priv->data->dev_conf.intr_conf; 525 526 if (intr_conf->rxq == 0 || dev->intr_handle != NULL) 527 return 0; 528 if (fs_rx_intr_vec_install(priv) < 0) 529 return -rte_errno; 530 if (fs_rx_event_proxy_install(priv) < 0) { 531 fs_rx_intr_vec_uninstall(priv); 532 return -rte_errno; 533 } 534 dev->intr_handle = &priv->intr_handle; 535 return 0; 536 } 537