1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2017 6WIND S.A. 3 * Copyright 2017 Mellanox Technologies, Ltd 4 */ 5 6 #ifndef _RTE_ETH_FAILSAFE_PRIVATE_H_ 7 #define _RTE_ETH_FAILSAFE_PRIVATE_H_ 8 9 #include <stdint.h> 10 #include <sys/queue.h> 11 #include <pthread.h> 12 13 #include <rte_atomic.h> 14 #include <rte_dev.h> 15 #include <rte_ethdev_driver.h> 16 #include <rte_devargs.h> 17 #include <rte_flow.h> 18 #include <rte_interrupts.h> 19 20 #define FAILSAFE_DRIVER_NAME "Fail-safe PMD" 21 #define FAILSAFE_OWNER_NAME "Fail-safe" 22 23 #define PMD_FAILSAFE_MAC_KVARG "mac" 24 #define PMD_FAILSAFE_HOTPLUG_POLL_KVARG "hotplug_poll" 25 #define PMD_FAILSAFE_PARAM_STRING \ 26 "dev(<ifc>)," \ 27 "exec(<shell command>)," \ 28 "fd(<fd number>)," \ 29 "mac=mac_addr," \ 30 "hotplug_poll=u64" \ 31 "" 32 33 #define FAILSAFE_HOTPLUG_DEFAULT_TIMEOUT_MS 2000 34 35 #define FAILSAFE_MAX_ETHPORTS 2 36 #define FAILSAFE_MAX_ETHADDR 128 37 38 #define DEVARGS_MAXLEN 4096 39 40 enum rxp_service_state { 41 SS_NO_SERVICE = 0, 42 SS_REGISTERED, 43 SS_READY, 44 SS_RUNNING, 45 }; 46 47 /* TYPES */ 48 49 struct rx_proxy { 50 /* epoll file descriptor */ 51 int efd; 52 /* event vector to be used by epoll */ 53 struct rte_epoll_event *evec; 54 /* rte service id */ 55 uint32_t sid; 56 /* service core id */ 57 uint32_t scid; 58 enum rxp_service_state sstate; 59 }; 60 61 struct rxq { 62 struct fs_priv *priv; 63 uint16_t qid; 64 /* next sub_device to poll */ 65 struct sub_device *sdev; 66 unsigned int socket_id; 67 int event_fd; 68 unsigned int enable_events:1; 69 struct rte_eth_rxq_info info; 70 rte_atomic64_t refcnt[]; 71 }; 72 73 struct txq { 74 struct fs_priv *priv; 75 uint16_t qid; 76 unsigned int socket_id; 77 struct rte_eth_txq_info info; 78 rte_atomic64_t refcnt[]; 79 }; 80 81 struct rte_flow { 82 TAILQ_ENTRY(rte_flow) next; 83 /* sub_flows */ 84 struct rte_flow *flows[FAILSAFE_MAX_ETHPORTS]; 85 /* flow description for synchronization */ 86 struct rte_flow_conv_rule rule; 87 uint8_t rule_data[]; 88 }; 89 90 enum dev_state { 91 DEV_UNDEFINED, 92 DEV_PARSED, 93 DEV_PROBED, 94 DEV_ACTIVE, 95 DEV_STARTED, 96 }; 97 98 struct fs_stats { 99 struct rte_eth_stats stats; 100 uint64_t timestamp; 101 }; 102 103 /* 104 * Allocated in shared memory. 105 */ 106 struct sub_device { 107 /* Exhaustive DPDK device description */ 108 struct sub_device *next; 109 struct rte_devargs devargs; 110 struct rte_bus *bus; /* for primary process only. */ 111 struct rte_device *dev; /* for primary process only. */ 112 uint8_t sid; 113 /* Device state machine */ 114 enum dev_state state; 115 /* Last stats snapshot passed to user */ 116 struct fs_stats stats_snapshot; 117 /* Some device are defined as a command line */ 118 char *cmdline; 119 /* Others are retrieved through a file descriptor */ 120 char *fd_str; 121 /* fail-safe device backreference */ 122 uint16_t fs_port_id; /* shared between processes */ 123 /* sub device port id*/ 124 uint16_t sdev_port_id; /* shared between processes */ 125 /* flag calling for recollection */ 126 volatile unsigned int remove:1; 127 /* flow isolation state */ 128 int flow_isolated:1; 129 /* RMV callback registration state */ 130 unsigned int rmv_callback:1; 131 /* LSC callback registration state */ 132 unsigned int lsc_callback:1; 133 }; 134 135 /* 136 * This is referenced by eth_dev->data->dev_private 137 * This is shared between processes. 138 */ 139 struct fs_priv { 140 struct rte_eth_dev_data *data; /* backreference to shared data. */ 141 /* 142 * Set of sub_devices. 143 * subs[0] is the preferred device 144 * any other is just another slave 145 */ 146 struct sub_device *subs; /* shared between processes */ 147 uint8_t subs_head; /* if head == tail, no subs */ 148 uint8_t subs_tail; /* first invalid */ 149 uint8_t subs_tx; /* current emitting device */ 150 uint8_t current_probed; 151 /* flow mapping */ 152 TAILQ_HEAD(sub_flows, rte_flow) flow_list; 153 /* current number of mac_addr slots allocated. */ 154 uint32_t nb_mac_addr; 155 struct ether_addr mac_addrs[FAILSAFE_MAX_ETHADDR]; 156 uint32_t mac_addr_pool[FAILSAFE_MAX_ETHADDR]; 157 uint32_t nb_mcast_addr; 158 struct ether_addr *mcast_addrs; 159 /* current capabilities */ 160 struct rte_eth_dev_info infos; 161 struct rte_eth_dev_owner my_owner; /* Unique owner. */ 162 struct rte_intr_handle intr_handle; /* Port interrupt handle. */ 163 /* 164 * Fail-safe state machine. 165 * This level will be tracking state of the EAL and eth 166 * layer at large as defined by the user application. 167 * It will then steer the sub_devices toward the same 168 * synchronized state. 169 */ 170 enum dev_state state; 171 struct rte_eth_stats stats_accumulator; 172 /* 173 * Rx interrupts/events proxy. 174 * The PMD issues Rx events to the EAL on behalf of its subdevices, 175 * it does that by registering an event-fd for each of its queues with 176 * the EAL. A PMD service thread listens to all the Rx events from the 177 * subdevices, when an Rx event is issued by a subdevice it will be 178 * caught by this service with will trigger an Rx event in the 179 * appropriate failsafe Rx queue. 180 */ 181 struct rx_proxy rxp; 182 pthread_mutex_t hotplug_mutex; 183 /* Hot-plug mutex is locked by the alarm mechanism. */ 184 volatile unsigned int alarm_lock:1; 185 unsigned int pending_alarm:1; /* An alarm is pending */ 186 /* flow isolation state */ 187 int flow_isolated:1; 188 }; 189 190 /* FAILSAFE_INTR */ 191 192 int failsafe_rx_intr_install(struct rte_eth_dev *dev); 193 void failsafe_rx_intr_uninstall(struct rte_eth_dev *dev); 194 int failsafe_rx_intr_install_subdevice(struct sub_device *sdev); 195 void failsafe_rx_intr_uninstall_subdevice(struct sub_device *sdev); 196 197 /* MISC */ 198 199 int failsafe_hotplug_alarm_install(struct rte_eth_dev *dev); 200 int failsafe_hotplug_alarm_cancel(struct rte_eth_dev *dev); 201 202 /* RX / TX */ 203 204 void failsafe_set_burst_fn(struct rte_eth_dev *dev, int force_safe); 205 206 uint16_t failsafe_rx_burst(void *rxq, 207 struct rte_mbuf **rx_pkts, uint16_t nb_pkts); 208 uint16_t failsafe_tx_burst(void *txq, 209 struct rte_mbuf **tx_pkts, uint16_t nb_pkts); 210 211 uint16_t failsafe_rx_burst_fast(void *rxq, 212 struct rte_mbuf **rx_pkts, uint16_t nb_pkts); 213 uint16_t failsafe_tx_burst_fast(void *txq, 214 struct rte_mbuf **tx_pkts, uint16_t nb_pkts); 215 216 /* ARGS */ 217 218 int failsafe_args_parse(struct rte_eth_dev *dev, const char *params); 219 void failsafe_args_free(struct rte_eth_dev *dev); 220 int failsafe_args_count_subdevice(struct rte_eth_dev *dev, const char *params); 221 int failsafe_args_parse_subs(struct rte_eth_dev *dev); 222 223 /* EAL */ 224 225 int failsafe_eal_init(struct rte_eth_dev *dev); 226 int failsafe_eal_uninit(struct rte_eth_dev *dev); 227 228 /* ETH_DEV */ 229 230 int failsafe_eth_dev_state_sync(struct rte_eth_dev *dev); 231 void failsafe_eth_dev_unregister_callbacks(struct sub_device *sdev); 232 void failsafe_dev_remove(struct rte_eth_dev *dev); 233 void failsafe_stats_increment(struct rte_eth_stats *to, 234 struct rte_eth_stats *from); 235 int failsafe_eth_rmv_event_callback(uint16_t port_id, 236 enum rte_eth_event_type type, 237 void *arg, void *out); 238 int failsafe_eth_lsc_event_callback(uint16_t port_id, 239 enum rte_eth_event_type event, 240 void *cb_arg, void *out); 241 int failsafe_eth_new_event_callback(uint16_t port_id, 242 enum rte_eth_event_type event, 243 void *cb_arg, void *out); 244 245 /* GLOBALS */ 246 247 extern const char pmd_failsafe_driver_name[]; 248 extern const struct eth_dev_ops failsafe_ops; 249 extern const struct rte_flow_ops fs_flow_ops; 250 extern uint64_t failsafe_hotplug_poll; 251 extern int failsafe_mac_from_arg; 252 253 /* HELPERS */ 254 255 /* dev: (struct rte_eth_dev *) fail-safe device */ 256 #define PRIV(dev) \ 257 ((struct fs_priv *)(dev)->data->dev_private) 258 259 /* sdev: (struct sub_device *) */ 260 #define ETH(sdev) \ 261 ((sdev)->sdev_port_id == RTE_MAX_ETHPORTS ? \ 262 NULL : &rte_eth_devices[(sdev)->sdev_port_id]) 263 264 /* sdev: (struct sub_device *) */ 265 #define PORT_ID(sdev) \ 266 ((sdev)->sdev_port_id) 267 268 /* sdev: (struct sub_device *) */ 269 #define SUB_ID(sdev) \ 270 ((sdev)->sid) 271 272 /** 273 * Stateful iterator construct over fail-safe sub-devices: 274 * s: (struct sub_device *), iterator 275 * i: (uint8_t), increment 276 * dev: (struct rte_eth_dev *), fail-safe ethdev 277 * state: (enum dev_state), minimum acceptable device state 278 */ 279 #define FOREACH_SUBDEV_STATE(s, i, dev, state) \ 280 for (s = fs_find_next((dev), 0, state, &i); \ 281 s != NULL; \ 282 s = fs_find_next((dev), i + 1, state, &i)) 283 284 /** 285 * Iterator construct over fail-safe sub-devices: 286 * s: (struct sub_device *), iterator 287 * i: (uint8_t), increment 288 * dev: (struct rte_eth_dev *), fail-safe ethdev 289 */ 290 #define FOREACH_SUBDEV(s, i, dev) \ 291 FOREACH_SUBDEV_STATE(s, i, dev, DEV_UNDEFINED) 292 293 /* dev: (struct rte_eth_dev *) fail-safe device */ 294 #define PREFERRED_SUBDEV(dev) \ 295 (&PRIV(dev)->subs[0]) 296 297 /* dev: (struct rte_eth_dev *) fail-safe device */ 298 #define TX_SUBDEV(dev) \ 299 (PRIV(dev)->subs_tx >= PRIV(dev)->subs_tail ? NULL \ 300 : (PRIV(dev)->subs[PRIV(dev)->subs_tx].state < DEV_PROBED ? NULL \ 301 : &PRIV(dev)->subs[PRIV(dev)->subs_tx])) 302 303 /** 304 * s: (struct sub_device *) 305 * ops: (struct eth_dev_ops) member 306 */ 307 #define SUBOPS(s, ops) \ 308 (ETH(s)->dev_ops->ops) 309 310 /** 311 * Atomic guard 312 */ 313 314 /** 315 * a: (rte_atomic64_t) 316 */ 317 #define FS_ATOMIC_P(a) \ 318 rte_atomic64_set(&(a), 1) 319 320 /** 321 * a: (rte_atomic64_t) 322 */ 323 #define FS_ATOMIC_V(a) \ 324 rte_atomic64_set(&(a), 0) 325 326 /** 327 * s: (struct sub_device *) 328 * i: uint16_t qid 329 */ 330 #define FS_ATOMIC_RX(s, i) \ 331 rte_atomic64_read( \ 332 &((struct rxq *) \ 333 (fs_dev(s)->data->rx_queues[i]))->refcnt[(s)->sid]) 334 /** 335 * s: (struct sub_device *) 336 * i: uint16_t qid 337 */ 338 #define FS_ATOMIC_TX(s, i) \ 339 rte_atomic64_read( \ 340 &((struct txq *) \ 341 (fs_dev(s)->data->tx_queues[i]))->refcnt[(s)->sid]) 342 343 #ifdef RTE_EXEC_ENV_FREEBSD 344 #define FS_THREADID_TYPE void* 345 #define FS_THREADID_FMT "p" 346 #else 347 #define FS_THREADID_TYPE unsigned long 348 #define FS_THREADID_FMT "lu" 349 #endif 350 351 extern int failsafe_logtype; 352 353 #define LOG__(l, m, ...) \ 354 rte_log(RTE_LOG_ ## l, failsafe_logtype, \ 355 "net_failsafe: " m "%c", __VA_ARGS__) 356 357 #define LOG_(level, ...) LOG__(level, __VA_ARGS__, '\n') 358 #define DEBUG(...) LOG_(DEBUG, __VA_ARGS__) 359 #define INFO(...) LOG_(INFO, __VA_ARGS__) 360 #define WARN(...) LOG_(WARNING, __VA_ARGS__) 361 #define ERROR(...) LOG_(ERR, __VA_ARGS__) 362 363 /* inlined functions */ 364 365 static inline struct sub_device * 366 fs_find_next(struct rte_eth_dev *dev, 367 uint8_t sid, 368 enum dev_state min_state, 369 uint8_t *sid_out) 370 { 371 struct sub_device *subs; 372 uint8_t tail; 373 374 subs = PRIV(dev)->subs; 375 tail = PRIV(dev)->subs_tail; 376 while (sid < tail) { 377 if (subs[sid].state >= min_state) 378 break; 379 sid++; 380 } 381 *sid_out = sid; 382 if (sid >= tail) 383 return NULL; 384 return &subs[sid]; 385 } 386 387 static inline struct rte_eth_dev * 388 fs_dev(struct sub_device *sdev) { 389 return &rte_eth_devices[sdev->fs_port_id]; 390 } 391 392 /* 393 * Lock hot-plug mutex. 394 * is_alarm means that the caller is, for sure, the hot-plug alarm mechanism. 395 */ 396 static inline int 397 fs_lock(struct rte_eth_dev *dev, unsigned int is_alarm) 398 { 399 int ret; 400 401 if (is_alarm) { 402 ret = pthread_mutex_trylock(&PRIV(dev)->hotplug_mutex); 403 if (ret) { 404 DEBUG("Hot-plug mutex lock trying failed(%s), will try" 405 " again later...", strerror(ret)); 406 return ret; 407 } 408 PRIV(dev)->alarm_lock = 1; 409 } else { 410 ret = pthread_mutex_lock(&PRIV(dev)->hotplug_mutex); 411 if (ret) { 412 ERROR("Cannot lock mutex(%s)", strerror(ret)); 413 return ret; 414 } 415 } 416 return ret; 417 } 418 419 /* 420 * Unlock hot-plug mutex. 421 * is_alarm means that the caller is, for sure, the hot-plug alarm mechanism. 422 */ 423 static inline void 424 fs_unlock(struct rte_eth_dev *dev, unsigned int is_alarm) 425 { 426 int ret; 427 428 if (is_alarm) { 429 RTE_ASSERT(PRIV(dev)->alarm_lock == 1); 430 PRIV(dev)->alarm_lock = 0; 431 } 432 ret = pthread_mutex_unlock(&PRIV(dev)->hotplug_mutex); 433 if (ret) 434 ERROR("Cannot unlock hot-plug mutex(%s)", strerror(ret)); 435 } 436 437 /* 438 * Switch emitting device. 439 * If banned is set, banned must not be considered for 440 * the role of emitting device. 441 */ 442 static inline void 443 fs_switch_dev(struct rte_eth_dev *dev, 444 struct sub_device *banned) 445 { 446 struct sub_device *txd; 447 enum dev_state req_state; 448 449 req_state = PRIV(dev)->state; 450 txd = TX_SUBDEV(dev); 451 if (PREFERRED_SUBDEV(dev)->state >= req_state && 452 PREFERRED_SUBDEV(dev) != banned) { 453 if (txd != PREFERRED_SUBDEV(dev) && 454 (txd == NULL || 455 (req_state == DEV_STARTED) || 456 (txd && txd->state < DEV_STARTED))) { 457 DEBUG("Switching tx_dev to preferred sub_device"); 458 PRIV(dev)->subs_tx = 0; 459 } 460 } else if ((txd && txd->state < req_state) || 461 txd == NULL || 462 txd == banned) { 463 struct sub_device *sdev = NULL; 464 uint8_t i; 465 466 /* Using acceptable device */ 467 FOREACH_SUBDEV_STATE(sdev, i, dev, req_state) { 468 if (sdev == banned) 469 continue; 470 DEBUG("Switching tx_dev to sub_device %d", 471 i); 472 PRIV(dev)->subs_tx = i; 473 break; 474 } 475 if (i >= PRIV(dev)->subs_tail || sdev == NULL) { 476 DEBUG("No device ready, deactivating tx_dev"); 477 PRIV(dev)->subs_tx = PRIV(dev)->subs_tail; 478 } 479 } else { 480 return; 481 } 482 failsafe_set_burst_fn(dev, 0); 483 rte_wmb(); 484 } 485 486 /* 487 * Adjust error value and rte_errno to the fail-safe actual error value. 488 */ 489 static inline int 490 fs_err(struct sub_device *sdev, int err) 491 { 492 /* A device removal shouldn't be reported as an error. */ 493 if (sdev->remove == 1 || err == -EIO) 494 return rte_errno = 0; 495 return err; 496 } 497 #endif /* _RTE_ETH_FAILSAFE_PRIVATE_H_ */ 498