1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2017 6WIND S.A. 3 * Copyright 2017 Mellanox Technologies, Ltd 4 */ 5 6 #ifndef _ETH_FAILSAFE_PRIVATE_H_ 7 #define _ETH_FAILSAFE_PRIVATE_H_ 8 9 #include <stdint.h> 10 #include <sys/queue.h> 11 #include <pthread.h> 12 13 #include <rte_atomic.h> 14 #include <dev_driver.h> 15 #include <ethdev_driver.h> 16 #include <rte_devargs.h> 17 #include <rte_flow.h> 18 #include <rte_interrupts.h> 19 20 #define FAILSAFE_DRIVER_NAME "Fail-safe PMD" 21 #define FAILSAFE_OWNER_NAME "Fail-safe" 22 23 #define PMD_FAILSAFE_MAC_KVARG "mac" 24 #define PMD_FAILSAFE_HOTPLUG_POLL_KVARG "hotplug_poll" 25 #define PMD_FAILSAFE_PARAM_STRING \ 26 "dev(<ifc>)," \ 27 "exec(<shell command>)," \ 28 "fd(<fd number>)," \ 29 "mac=mac_addr," \ 30 "hotplug_poll=u64" \ 31 "" 32 33 #define FAILSAFE_HOTPLUG_DEFAULT_TIMEOUT_MS 2000 34 35 #define FAILSAFE_MAX_ETHPORTS 2 36 #define FAILSAFE_MAX_ETHADDR 128 37 38 #define DEVARGS_MAXLEN 4096 39 40 enum rxp_service_state { 41 SS_NO_SERVICE = 0, 42 SS_REGISTERED, 43 SS_READY, 44 SS_RUNNING, 45 }; 46 47 /* TYPES */ 48 49 struct rx_proxy { 50 /* epoll file descriptor */ 51 int efd; 52 /* event vector to be used by epoll */ 53 struct rte_epoll_event *evec; 54 /* rte service id */ 55 uint32_t sid; 56 /* service core id */ 57 uint32_t scid; 58 enum rxp_service_state sstate; 59 }; 60 61 #define FS_RX_PROXY_INIT (struct rx_proxy){ \ 62 .efd = -1, \ 63 .evec = NULL, \ 64 .sid = 0, \ 65 .scid = 0, \ 66 .sstate = SS_NO_SERVICE, \ 67 } 68 69 struct rxq { 70 struct fs_priv *priv; 71 uint16_t qid; 72 /* next sub_device to poll */ 73 struct sub_device *sdev; 74 unsigned int socket_id; 75 int event_fd; 76 unsigned int enable_events:1; 77 struct rte_eth_rxq_info info; 78 rte_atomic64_t refcnt[]; 79 }; 80 81 struct txq { 82 struct fs_priv *priv; 83 uint16_t qid; 84 unsigned int socket_id; 85 struct rte_eth_txq_info info; 86 rte_atomic64_t refcnt[]; 87 }; 88 89 struct rte_flow { 90 TAILQ_ENTRY(rte_flow) next; 91 /* sub_flows */ 92 struct rte_flow *flows[FAILSAFE_MAX_ETHPORTS]; 93 /* flow description for synchronization */ 94 struct rte_flow_conv_rule rule; 95 uint8_t rule_data[]; 96 }; 97 98 enum dev_state { 99 DEV_UNDEFINED, 100 DEV_PARSED, 101 DEV_PROBED, 102 DEV_ACTIVE, 103 DEV_STARTED, 104 }; 105 106 struct fs_stats { 107 struct rte_eth_stats stats; 108 uint64_t timestamp; 109 }; 110 111 /* 112 * Allocated in shared memory. 113 */ 114 struct sub_device { 115 /* Exhaustive DPDK device description */ 116 struct sub_device *next; 117 struct rte_devargs devargs; 118 struct rte_bus *bus; /* for primary process only. */ 119 struct rte_device *dev; /* for primary process only. */ 120 uint8_t sid; 121 /* Device state machine */ 122 enum dev_state state; 123 /* Last stats snapshot passed to user */ 124 struct fs_stats stats_snapshot; 125 /* Some device are defined as a command line */ 126 char *cmdline; 127 /* Others are retrieved through a file descriptor */ 128 char *fd_str; 129 /* fail-safe device backreference */ 130 uint16_t fs_port_id; /* shared between processes */ 131 /* sub device port id*/ 132 uint16_t sdev_port_id; /* shared between processes */ 133 /* flag calling for recollection */ 134 volatile unsigned int remove:1; 135 /* flow isolation state */ 136 int flow_isolated:1; 137 /* RMV callback registration state */ 138 unsigned int rmv_callback:1; 139 /* LSC callback registration state */ 140 unsigned int lsc_callback:1; 141 }; 142 143 /* 144 * This is referenced by eth_dev->data->dev_private 145 * This is shared between processes. 146 */ 147 struct fs_priv { 148 struct rte_eth_dev_data *data; /* backreference to shared data. */ 149 /* 150 * Set of sub_devices. 151 * subs[0] is the preferred device 152 * any other is just another sub device 153 */ 154 struct sub_device *subs; /* shared between processes */ 155 uint8_t subs_head; /* if head == tail, no subs */ 156 uint8_t subs_tail; /* first invalid */ 157 uint8_t subs_tx; /* current emitting device */ 158 uint8_t current_probed; 159 /* flow mapping */ 160 TAILQ_HEAD(sub_flows, rte_flow) flow_list; 161 /* current number of mac_addr slots allocated. */ 162 uint32_t nb_mac_addr; 163 struct rte_ether_addr mac_addrs[FAILSAFE_MAX_ETHADDR]; 164 uint32_t mac_addr_pool[FAILSAFE_MAX_ETHADDR]; 165 uint32_t nb_mcast_addr; 166 struct rte_ether_addr *mcast_addrs; 167 /* current capabilities */ 168 struct rte_eth_dev_owner my_owner; /* Unique owner. */ 169 struct rte_intr_handle *intr_handle; /* Port interrupt handle. */ 170 /* 171 * Fail-safe state machine. 172 * This level will be tracking state of the EAL and eth 173 * layer at large as defined by the user application. 174 * It will then steer the sub_devices toward the same 175 * synchronized state. 176 */ 177 enum dev_state state; 178 struct rte_eth_stats stats_accumulator; 179 /* 180 * Rx interrupts/events proxy. 181 * The PMD issues Rx events to the EAL on behalf of its subdevices, 182 * it does that by registering an event-fd for each of its queues with 183 * the EAL. A PMD service thread listens to all the Rx events from the 184 * subdevices, when an Rx event is issued by a subdevice it will be 185 * caught by this service with will trigger an Rx event in the 186 * appropriate failsafe Rx queue. 187 */ 188 struct rx_proxy rxp; 189 pthread_mutex_t hotplug_mutex; 190 /* Hot-plug mutex is locked by the alarm mechanism. */ 191 volatile unsigned int alarm_lock:1; 192 unsigned int pending_alarm:1; /* An alarm is pending */ 193 /* flow isolation state */ 194 int flow_isolated:1; 195 }; 196 197 /* FAILSAFE_INTR */ 198 199 int failsafe_rx_intr_install(struct rte_eth_dev *dev); 200 void failsafe_rx_intr_uninstall(struct rte_eth_dev *dev); 201 int failsafe_rx_intr_install_subdevice(struct sub_device *sdev); 202 void failsafe_rx_intr_uninstall_subdevice(struct sub_device *sdev); 203 204 /* MISC */ 205 206 int failsafe_hotplug_alarm_install(struct rte_eth_dev *dev); 207 int failsafe_hotplug_alarm_cancel(struct rte_eth_dev *dev); 208 209 /* RX / TX */ 210 211 void failsafe_set_burst_fn(struct rte_eth_dev *dev, int force_safe); 212 213 uint16_t failsafe_rx_burst(void *rxq, 214 struct rte_mbuf **rx_pkts, uint16_t nb_pkts); 215 uint16_t failsafe_tx_burst(void *txq, 216 struct rte_mbuf **tx_pkts, uint16_t nb_pkts); 217 218 uint16_t failsafe_rx_burst_fast(void *rxq, 219 struct rte_mbuf **rx_pkts, uint16_t nb_pkts); 220 uint16_t failsafe_tx_burst_fast(void *txq, 221 struct rte_mbuf **tx_pkts, uint16_t nb_pkts); 222 223 /* ARGS */ 224 225 int failsafe_args_parse(struct rte_eth_dev *dev, const char *params); 226 void failsafe_args_free(struct rte_eth_dev *dev); 227 int failsafe_args_count_subdevice(struct rte_eth_dev *dev, const char *params); 228 int failsafe_args_parse_subs(struct rte_eth_dev *dev); 229 230 /* EAL */ 231 232 int failsafe_eal_init(struct rte_eth_dev *dev); 233 int failsafe_eal_uninit(struct rte_eth_dev *dev); 234 235 /* ETH_DEV */ 236 237 int failsafe_eth_dev_state_sync(struct rte_eth_dev *dev); 238 void failsafe_eth_dev_unregister_callbacks(struct sub_device *sdev); 239 int failsafe_eth_dev_close(struct rte_eth_dev *dev); 240 void failsafe_dev_remove(struct rte_eth_dev *dev); 241 void failsafe_stats_increment(struct rte_eth_stats *to, 242 struct rte_eth_stats *from); 243 int failsafe_eth_rmv_event_callback(uint16_t port_id, 244 enum rte_eth_event_type type, 245 void *arg, void *out); 246 int failsafe_eth_lsc_event_callback(uint16_t port_id, 247 enum rte_eth_event_type event, 248 void *cb_arg, void *out); 249 int failsafe_eth_new_event_callback(uint16_t port_id, 250 enum rte_eth_event_type event, 251 void *cb_arg, void *out); 252 253 /* GLOBALS */ 254 255 extern const char pmd_failsafe_driver_name[]; 256 extern const struct eth_dev_ops failsafe_ops; 257 extern const struct rte_flow_ops fs_flow_ops; 258 extern uint64_t failsafe_hotplug_poll; 259 extern int failsafe_mac_from_arg; 260 261 /* HELPERS */ 262 263 /* dev: (struct rte_eth_dev *) fail-safe device */ 264 #define PRIV(dev) \ 265 ((struct fs_priv *)(dev)->data->dev_private) 266 267 /* sdev: (struct sub_device *) */ 268 #define ETH(sdev) \ 269 ((sdev)->sdev_port_id == RTE_MAX_ETHPORTS ? \ 270 NULL : &rte_eth_devices[(sdev)->sdev_port_id]) 271 272 /* sdev: (struct sub_device *) */ 273 #define PORT_ID(sdev) \ 274 ((sdev)->sdev_port_id) 275 276 /* sdev: (struct sub_device *) */ 277 #define SUB_ID(sdev) \ 278 ((sdev)->sid) 279 280 /** 281 * Stateful iterator construct over fail-safe sub-devices: 282 * s: (struct sub_device *), iterator 283 * i: (uint8_t), increment 284 * dev: (struct rte_eth_dev *), fail-safe ethdev 285 * state: (enum dev_state), minimum acceptable device state 286 */ 287 #define FOREACH_SUBDEV_STATE(s, i, dev, state) \ 288 for (s = fs_find_next((dev), 0, state, &i); \ 289 s != NULL; \ 290 s = fs_find_next((dev), i + 1, state, &i)) 291 292 /** 293 * Iterator construct over fail-safe sub-devices: 294 * s: (struct sub_device *), iterator 295 * i: (uint8_t), increment 296 * dev: (struct rte_eth_dev *), fail-safe ethdev 297 */ 298 #define FOREACH_SUBDEV(s, i, dev) \ 299 FOREACH_SUBDEV_STATE(s, i, dev, DEV_UNDEFINED) 300 301 /* dev: (struct rte_eth_dev *) fail-safe device */ 302 #define PREFERRED_SUBDEV(dev) \ 303 (&PRIV(dev)->subs[0]) 304 305 /* dev: (struct rte_eth_dev *) fail-safe device */ 306 #define TX_SUBDEV(dev) \ 307 (PRIV(dev)->subs_tx >= PRIV(dev)->subs_tail ? NULL \ 308 : (PRIV(dev)->subs[PRIV(dev)->subs_tx].state < DEV_PROBED ? NULL \ 309 : &PRIV(dev)->subs[PRIV(dev)->subs_tx])) 310 311 /** 312 * s: (struct sub_device *) 313 * ops: (struct eth_dev_ops) member 314 */ 315 #define SUBOPS(s, ops) \ 316 (ETH(s)->dev_ops->ops) 317 318 /** 319 * Atomic guard 320 */ 321 322 /** 323 * a: (rte_atomic64_t) 324 */ 325 #define FS_ATOMIC_P(a) \ 326 rte_atomic64_set(&(a), 1) 327 328 /** 329 * a: (rte_atomic64_t) 330 */ 331 #define FS_ATOMIC_V(a) \ 332 rte_atomic64_set(&(a), 0) 333 334 /** 335 * s: (struct sub_device *) 336 * i: uint16_t qid 337 */ 338 #define FS_ATOMIC_RX(s, i) \ 339 rte_atomic64_read( \ 340 &((struct rxq *) \ 341 (fs_dev(s)->data->rx_queues[i]))->refcnt[(s)->sid]) 342 /** 343 * s: (struct sub_device *) 344 * i: uint16_t qid 345 */ 346 #define FS_ATOMIC_TX(s, i) \ 347 rte_atomic64_read( \ 348 &((struct txq *) \ 349 (fs_dev(s)->data->tx_queues[i]))->refcnt[(s)->sid]) 350 351 #ifdef RTE_EXEC_ENV_FREEBSD 352 #define FS_THREADID_TYPE void* 353 #define FS_THREADID_FMT "p" 354 #else 355 #define FS_THREADID_TYPE unsigned long 356 #define FS_THREADID_FMT "lu" 357 #endif 358 359 extern int failsafe_logtype; 360 #define RTE_LOGTYPE_NET_FAILSAFE failsafe_logtype 361 362 #define LOG_(l, ...) RTE_LOG_LINE(l, NET_FAILSAFE, __VA_ARGS__) 363 #define DEBUG(...) LOG_(DEBUG, __VA_ARGS__) 364 #define INFO(...) LOG_(INFO, __VA_ARGS__) 365 #define WARN(...) LOG_(WARNING, __VA_ARGS__) 366 #define ERROR(...) LOG_(ERR, __VA_ARGS__) 367 368 /* inlined functions */ 369 370 static inline struct sub_device * 371 fs_find_next(struct rte_eth_dev *dev, 372 uint8_t sid, 373 enum dev_state min_state, 374 uint8_t *sid_out) 375 { 376 struct sub_device *subs; 377 uint8_t tail; 378 379 subs = PRIV(dev)->subs; 380 tail = PRIV(dev)->subs_tail; 381 while (sid < tail) { 382 if (subs[sid].state >= min_state) 383 break; 384 sid++; 385 } 386 *sid_out = sid; 387 if (sid >= tail) 388 return NULL; 389 return &subs[sid]; 390 } 391 392 static inline struct rte_eth_dev * 393 fs_dev(struct sub_device *sdev) { 394 return &rte_eth_devices[sdev->fs_port_id]; 395 } 396 397 /* 398 * Lock hot-plug mutex. 399 * is_alarm means that the caller is, for sure, the hot-plug alarm mechanism. 400 */ 401 static inline int 402 fs_lock(struct rte_eth_dev *dev, unsigned int is_alarm) 403 { 404 int ret; 405 406 if (is_alarm) { 407 ret = pthread_mutex_trylock(&PRIV(dev)->hotplug_mutex); 408 if (ret) { 409 DEBUG("Hot-plug mutex lock trying failed(%s), will try" 410 " again later...", strerror(ret)); 411 return ret; 412 } 413 PRIV(dev)->alarm_lock = 1; 414 } else { 415 ret = pthread_mutex_lock(&PRIV(dev)->hotplug_mutex); 416 if (ret) { 417 ERROR("Cannot lock mutex(%s)", strerror(ret)); 418 return ret; 419 } 420 } 421 return ret; 422 } 423 424 /* 425 * Unlock hot-plug mutex. 426 * is_alarm means that the caller is, for sure, the hot-plug alarm mechanism. 427 */ 428 static inline void 429 fs_unlock(struct rte_eth_dev *dev, unsigned int is_alarm) 430 { 431 int ret; 432 433 if (is_alarm) { 434 RTE_ASSERT(PRIV(dev)->alarm_lock == 1); 435 PRIV(dev)->alarm_lock = 0; 436 } 437 ret = pthread_mutex_unlock(&PRIV(dev)->hotplug_mutex); 438 if (ret) 439 ERROR("Cannot unlock hot-plug mutex(%s)", strerror(ret)); 440 } 441 442 /* 443 * Switch emitting device. 444 * If banned is set, banned must not be considered for 445 * the role of emitting device. 446 */ 447 static inline void 448 fs_switch_dev(struct rte_eth_dev *dev, 449 struct sub_device *banned) 450 { 451 struct sub_device *txd; 452 enum dev_state req_state; 453 454 req_state = PRIV(dev)->state; 455 txd = TX_SUBDEV(dev); 456 if (PREFERRED_SUBDEV(dev)->state >= req_state && 457 PREFERRED_SUBDEV(dev) != banned) { 458 if (txd != PREFERRED_SUBDEV(dev) && 459 (txd == NULL || 460 (req_state == DEV_STARTED) || 461 (txd && txd->state < DEV_STARTED))) { 462 DEBUG("Switching tx_dev to preferred sub_device"); 463 PRIV(dev)->subs_tx = 0; 464 } 465 } else if ((txd && txd->state < req_state) || 466 txd == NULL || 467 txd == banned) { 468 struct sub_device *sdev = NULL; 469 uint8_t i; 470 471 /* Using acceptable device */ 472 FOREACH_SUBDEV_STATE(sdev, i, dev, req_state) { 473 if (sdev == banned) 474 continue; 475 DEBUG("Switching tx_dev to sub_device %d", 476 i); 477 PRIV(dev)->subs_tx = i; 478 break; 479 } 480 if (i >= PRIV(dev)->subs_tail || sdev == NULL) { 481 DEBUG("No device ready, deactivating tx_dev"); 482 PRIV(dev)->subs_tx = PRIV(dev)->subs_tail; 483 } 484 } else { 485 return; 486 } 487 failsafe_set_burst_fn(dev, 0); 488 rte_wmb(); 489 } 490 491 /* 492 * Adjust error value and rte_errno to the fail-safe actual error value. 493 */ 494 static inline int 495 fs_err(struct sub_device *sdev, int err) 496 { 497 /* A device removal shouldn't be reported as an error. */ 498 if (sdev->remove == 1 || err == -EIO) 499 return rte_errno = 0; 500 return err; 501 } 502 #endif /* _ETH_FAILSAFE_PRIVATE_H_ */ 503