xref: /dpdk/drivers/net/failsafe/failsafe_ops.c (revision 08aa6271c86a561b66c6dd91f9a54fa2f12bc859)
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright 2017 6WIND S.A.
3  * Copyright 2017 Mellanox Technologies, Ltd
4  */
5 
6 #include <stdbool.h>
7 #include <stdint.h>
8 #include <unistd.h>
9 
10 #include <rte_debug.h>
11 #include <rte_atomic.h>
12 #include <rte_ethdev_driver.h>
13 #include <rte_malloc.h>
14 #include <rte_flow.h>
15 #include <rte_cycles.h>
16 #include <rte_ethdev.h>
17 
18 #include "failsafe_private.h"
19 
20 static struct rte_eth_dev_info default_infos = {
21 	/* Max possible number of elements */
22 	.max_rx_pktlen = UINT32_MAX,
23 	.max_rx_queues = RTE_MAX_QUEUES_PER_PORT,
24 	.max_tx_queues = RTE_MAX_QUEUES_PER_PORT,
25 	.max_mac_addrs = FAILSAFE_MAX_ETHADDR,
26 	.max_hash_mac_addrs = UINT32_MAX,
27 	.max_vfs = UINT16_MAX,
28 	.max_vmdq_pools = UINT16_MAX,
29 	.rx_desc_lim = {
30 		.nb_max = UINT16_MAX,
31 		.nb_min = 0,
32 		.nb_align = 1,
33 		.nb_seg_max = UINT16_MAX,
34 		.nb_mtu_seg_max = UINT16_MAX,
35 	},
36 	.tx_desc_lim = {
37 		.nb_max = UINT16_MAX,
38 		.nb_min = 0,
39 		.nb_align = 1,
40 		.nb_seg_max = UINT16_MAX,
41 		.nb_mtu_seg_max = UINT16_MAX,
42 	},
43 	/*
44 	 * Set of capabilities that can be verified upon
45 	 * configuring a sub-device.
46 	 */
47 	.rx_offload_capa =
48 		DEV_RX_OFFLOAD_VLAN_STRIP |
49 		DEV_RX_OFFLOAD_IPV4_CKSUM |
50 		DEV_RX_OFFLOAD_UDP_CKSUM |
51 		DEV_RX_OFFLOAD_TCP_CKSUM |
52 		DEV_RX_OFFLOAD_TCP_LRO |
53 		DEV_RX_OFFLOAD_QINQ_STRIP |
54 		DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
55 		DEV_RX_OFFLOAD_MACSEC_STRIP |
56 		DEV_RX_OFFLOAD_HEADER_SPLIT |
57 		DEV_RX_OFFLOAD_VLAN_FILTER |
58 		DEV_RX_OFFLOAD_VLAN_EXTEND |
59 		DEV_RX_OFFLOAD_JUMBO_FRAME |
60 		DEV_RX_OFFLOAD_CRC_STRIP |
61 		DEV_RX_OFFLOAD_SCATTER |
62 		DEV_RX_OFFLOAD_TIMESTAMP |
63 		DEV_RX_OFFLOAD_SECURITY,
64 	.rx_queue_offload_capa =
65 		DEV_RX_OFFLOAD_VLAN_STRIP |
66 		DEV_RX_OFFLOAD_IPV4_CKSUM |
67 		DEV_RX_OFFLOAD_UDP_CKSUM |
68 		DEV_RX_OFFLOAD_TCP_CKSUM |
69 		DEV_RX_OFFLOAD_TCP_LRO |
70 		DEV_RX_OFFLOAD_QINQ_STRIP |
71 		DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
72 		DEV_RX_OFFLOAD_MACSEC_STRIP |
73 		DEV_RX_OFFLOAD_HEADER_SPLIT |
74 		DEV_RX_OFFLOAD_VLAN_FILTER |
75 		DEV_RX_OFFLOAD_VLAN_EXTEND |
76 		DEV_RX_OFFLOAD_JUMBO_FRAME |
77 		DEV_RX_OFFLOAD_CRC_STRIP |
78 		DEV_RX_OFFLOAD_SCATTER |
79 		DEV_RX_OFFLOAD_TIMESTAMP |
80 		DEV_RX_OFFLOAD_SECURITY,
81 	.tx_offload_capa =
82 		DEV_TX_OFFLOAD_MULTI_SEGS |
83 		DEV_TX_OFFLOAD_IPV4_CKSUM |
84 		DEV_TX_OFFLOAD_UDP_CKSUM |
85 		DEV_TX_OFFLOAD_TCP_CKSUM |
86 		DEV_TX_OFFLOAD_TCP_TSO,
87 	.flow_type_rss_offloads =
88 			ETH_RSS_IP |
89 			ETH_RSS_UDP |
90 			ETH_RSS_TCP,
91 };
92 
93 static int
94 fs_dev_configure(struct rte_eth_dev *dev)
95 {
96 	struct sub_device *sdev;
97 	uint8_t i;
98 	int ret;
99 
100 	fs_lock(dev, 0);
101 	FOREACH_SUBDEV(sdev, i, dev) {
102 		int rmv_interrupt = 0;
103 		int lsc_interrupt = 0;
104 		int lsc_enabled;
105 
106 		if (sdev->state != DEV_PROBED &&
107 		    !(PRIV(dev)->alarm_lock == 0 && sdev->state == DEV_ACTIVE))
108 			continue;
109 
110 		rmv_interrupt = ETH(sdev)->data->dev_flags &
111 				RTE_ETH_DEV_INTR_RMV;
112 		if (rmv_interrupt) {
113 			DEBUG("Enabling RMV interrupts for sub_device %d", i);
114 			dev->data->dev_conf.intr_conf.rmv = 1;
115 		} else {
116 			DEBUG("sub_device %d does not support RMV event", i);
117 		}
118 		lsc_enabled = dev->data->dev_conf.intr_conf.lsc;
119 		lsc_interrupt = lsc_enabled &&
120 				(ETH(sdev)->data->dev_flags &
121 				 RTE_ETH_DEV_INTR_LSC);
122 		if (lsc_interrupt) {
123 			DEBUG("Enabling LSC interrupts for sub_device %d", i);
124 			dev->data->dev_conf.intr_conf.lsc = 1;
125 		} else if (lsc_enabled && !lsc_interrupt) {
126 			DEBUG("Disabling LSC interrupts for sub_device %d", i);
127 			dev->data->dev_conf.intr_conf.lsc = 0;
128 		}
129 		DEBUG("Configuring sub-device %d", i);
130 		ret = rte_eth_dev_configure(PORT_ID(sdev),
131 					dev->data->nb_rx_queues,
132 					dev->data->nb_tx_queues,
133 					&dev->data->dev_conf);
134 		if (ret) {
135 			if (!fs_err(sdev, ret))
136 				continue;
137 			ERROR("Could not configure sub_device %d", i);
138 			fs_unlock(dev, 0);
139 			return ret;
140 		}
141 		if (rmv_interrupt) {
142 			ret = rte_eth_dev_callback_register(PORT_ID(sdev),
143 					RTE_ETH_EVENT_INTR_RMV,
144 					failsafe_eth_rmv_event_callback,
145 					sdev);
146 			if (ret)
147 				WARN("Failed to register RMV callback for sub_device %d",
148 				     SUB_ID(sdev));
149 		}
150 		dev->data->dev_conf.intr_conf.rmv = 0;
151 		if (lsc_interrupt) {
152 			ret = rte_eth_dev_callback_register(PORT_ID(sdev),
153 						RTE_ETH_EVENT_INTR_LSC,
154 						failsafe_eth_lsc_event_callback,
155 						dev);
156 			if (ret)
157 				WARN("Failed to register LSC callback for sub_device %d",
158 				     SUB_ID(sdev));
159 		}
160 		dev->data->dev_conf.intr_conf.lsc = lsc_enabled;
161 		sdev->state = DEV_ACTIVE;
162 	}
163 	if (PRIV(dev)->state < DEV_ACTIVE)
164 		PRIV(dev)->state = DEV_ACTIVE;
165 	fs_unlock(dev, 0);
166 	return 0;
167 }
168 
169 static int
170 fs_dev_start(struct rte_eth_dev *dev)
171 {
172 	struct sub_device *sdev;
173 	uint8_t i;
174 	int ret;
175 
176 	fs_lock(dev, 0);
177 	ret = failsafe_rx_intr_install(dev);
178 	if (ret) {
179 		fs_unlock(dev, 0);
180 		return ret;
181 	}
182 	FOREACH_SUBDEV(sdev, i, dev) {
183 		if (sdev->state != DEV_ACTIVE)
184 			continue;
185 		DEBUG("Starting sub_device %d", i);
186 		ret = rte_eth_dev_start(PORT_ID(sdev));
187 		if (ret) {
188 			if (!fs_err(sdev, ret))
189 				continue;
190 			fs_unlock(dev, 0);
191 			return ret;
192 		}
193 		ret = failsafe_rx_intr_install_subdevice(sdev);
194 		if (ret) {
195 			if (!fs_err(sdev, ret))
196 				continue;
197 			rte_eth_dev_stop(PORT_ID(sdev));
198 			fs_unlock(dev, 0);
199 			return ret;
200 		}
201 		sdev->state = DEV_STARTED;
202 	}
203 	if (PRIV(dev)->state < DEV_STARTED)
204 		PRIV(dev)->state = DEV_STARTED;
205 	fs_switch_dev(dev, NULL);
206 	fs_unlock(dev, 0);
207 	return 0;
208 }
209 
210 static void
211 fs_dev_stop(struct rte_eth_dev *dev)
212 {
213 	struct sub_device *sdev;
214 	uint8_t i;
215 
216 	fs_lock(dev, 0);
217 	PRIV(dev)->state = DEV_STARTED - 1;
218 	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_STARTED) {
219 		rte_eth_dev_stop(PORT_ID(sdev));
220 		failsafe_rx_intr_uninstall_subdevice(sdev);
221 		sdev->state = DEV_STARTED - 1;
222 	}
223 	failsafe_rx_intr_uninstall(dev);
224 	fs_unlock(dev, 0);
225 }
226 
227 static int
228 fs_dev_set_link_up(struct rte_eth_dev *dev)
229 {
230 	struct sub_device *sdev;
231 	uint8_t i;
232 	int ret;
233 
234 	fs_lock(dev, 0);
235 	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
236 		DEBUG("Calling rte_eth_dev_set_link_up on sub_device %d", i);
237 		ret = rte_eth_dev_set_link_up(PORT_ID(sdev));
238 		if ((ret = fs_err(sdev, ret))) {
239 			ERROR("Operation rte_eth_dev_set_link_up failed for sub_device %d"
240 			      " with error %d", i, ret);
241 			fs_unlock(dev, 0);
242 			return ret;
243 		}
244 	}
245 	fs_unlock(dev, 0);
246 	return 0;
247 }
248 
249 static int
250 fs_dev_set_link_down(struct rte_eth_dev *dev)
251 {
252 	struct sub_device *sdev;
253 	uint8_t i;
254 	int ret;
255 
256 	fs_lock(dev, 0);
257 	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
258 		DEBUG("Calling rte_eth_dev_set_link_down on sub_device %d", i);
259 		ret = rte_eth_dev_set_link_down(PORT_ID(sdev));
260 		if ((ret = fs_err(sdev, ret))) {
261 			ERROR("Operation rte_eth_dev_set_link_down failed for sub_device %d"
262 			      " with error %d", i, ret);
263 			fs_unlock(dev, 0);
264 			return ret;
265 		}
266 	}
267 	fs_unlock(dev, 0);
268 	return 0;
269 }
270 
271 static void fs_dev_free_queues(struct rte_eth_dev *dev);
272 static void
273 fs_dev_close(struct rte_eth_dev *dev)
274 {
275 	struct sub_device *sdev;
276 	uint8_t i;
277 
278 	fs_lock(dev, 0);
279 	failsafe_hotplug_alarm_cancel(dev);
280 	if (PRIV(dev)->state == DEV_STARTED)
281 		dev->dev_ops->dev_stop(dev);
282 	PRIV(dev)->state = DEV_ACTIVE - 1;
283 	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
284 		DEBUG("Closing sub_device %d", i);
285 		rte_eth_dev_close(PORT_ID(sdev));
286 		sdev->state = DEV_ACTIVE - 1;
287 	}
288 	fs_dev_free_queues(dev);
289 	fs_unlock(dev, 0);
290 }
291 
292 static void
293 fs_rx_queue_release(void *queue)
294 {
295 	struct rte_eth_dev *dev;
296 	struct sub_device *sdev;
297 	uint8_t i;
298 	struct rxq *rxq;
299 
300 	if (queue == NULL)
301 		return;
302 	rxq = queue;
303 	dev = rxq->priv->dev;
304 	fs_lock(dev, 0);
305 	if (rxq->event_fd > 0)
306 		close(rxq->event_fd);
307 	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
308 		SUBOPS(sdev, rx_queue_release)
309 			(ETH(sdev)->data->rx_queues[rxq->qid]);
310 	dev->data->rx_queues[rxq->qid] = NULL;
311 	rte_free(rxq);
312 	fs_unlock(dev, 0);
313 }
314 
315 static int
316 fs_rx_queue_setup(struct rte_eth_dev *dev,
317 		uint16_t rx_queue_id,
318 		uint16_t nb_rx_desc,
319 		unsigned int socket_id,
320 		const struct rte_eth_rxconf *rx_conf,
321 		struct rte_mempool *mb_pool)
322 {
323 	/*
324 	 * FIXME: Add a proper interface in rte_eal_interrupts for
325 	 * allocating eventfd as an interrupt vector.
326 	 * For the time being, fake as if we are using MSIX interrupts,
327 	 * this will cause rte_intr_efd_enable to allocate an eventfd for us.
328 	 */
329 	struct rte_intr_handle intr_handle = {
330 		.type = RTE_INTR_HANDLE_VFIO_MSIX,
331 		.efds = { -1, },
332 	};
333 	struct sub_device *sdev;
334 	struct rxq *rxq;
335 	uint8_t i;
336 	int ret;
337 
338 	fs_lock(dev, 0);
339 	rxq = dev->data->rx_queues[rx_queue_id];
340 	if (rxq != NULL) {
341 		fs_rx_queue_release(rxq);
342 		dev->data->rx_queues[rx_queue_id] = NULL;
343 	}
344 	rxq = rte_zmalloc(NULL,
345 			  sizeof(*rxq) +
346 			  sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail,
347 			  RTE_CACHE_LINE_SIZE);
348 	if (rxq == NULL) {
349 		fs_unlock(dev, 0);
350 		return -ENOMEM;
351 	}
352 	FOREACH_SUBDEV(sdev, i, dev)
353 		rte_atomic64_init(&rxq->refcnt[i]);
354 	rxq->qid = rx_queue_id;
355 	rxq->socket_id = socket_id;
356 	rxq->info.mp = mb_pool;
357 	rxq->info.conf = *rx_conf;
358 	rxq->info.nb_desc = nb_rx_desc;
359 	rxq->priv = PRIV(dev);
360 	rxq->sdev = PRIV(dev)->subs;
361 	ret = rte_intr_efd_enable(&intr_handle, 1);
362 	if (ret < 0) {
363 		fs_unlock(dev, 0);
364 		return ret;
365 	}
366 	rxq->event_fd = intr_handle.efds[0];
367 	dev->data->rx_queues[rx_queue_id] = rxq;
368 	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
369 		ret = rte_eth_rx_queue_setup(PORT_ID(sdev),
370 				rx_queue_id,
371 				nb_rx_desc, socket_id,
372 				rx_conf, mb_pool);
373 		if ((ret = fs_err(sdev, ret))) {
374 			ERROR("RX queue setup failed for sub_device %d", i);
375 			goto free_rxq;
376 		}
377 	}
378 	fs_unlock(dev, 0);
379 	return 0;
380 free_rxq:
381 	fs_rx_queue_release(rxq);
382 	fs_unlock(dev, 0);
383 	return ret;
384 }
385 
386 static int
387 fs_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx)
388 {
389 	struct rxq *rxq;
390 	struct sub_device *sdev;
391 	uint8_t i;
392 	int ret;
393 	int rc = 0;
394 
395 	fs_lock(dev, 0);
396 	if (idx >= dev->data->nb_rx_queues) {
397 		rc = -EINVAL;
398 		goto unlock;
399 	}
400 	rxq = dev->data->rx_queues[idx];
401 	if (rxq == NULL || rxq->event_fd <= 0) {
402 		rc = -EINVAL;
403 		goto unlock;
404 	}
405 	/* Fail if proxy service is nor running. */
406 	if (PRIV(dev)->rxp.sstate != SS_RUNNING) {
407 		ERROR("failsafe interrupt services are not running");
408 		rc = -EAGAIN;
409 		goto unlock;
410 	}
411 	rxq->enable_events = 1;
412 	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
413 		ret = rte_eth_dev_rx_intr_enable(PORT_ID(sdev), idx);
414 		ret = fs_err(sdev, ret);
415 		if (ret)
416 			rc = ret;
417 	}
418 unlock:
419 	fs_unlock(dev, 0);
420 	if (rc)
421 		rte_errno = -rc;
422 	return rc;
423 }
424 
425 static int
426 fs_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx)
427 {
428 	struct rxq *rxq;
429 	struct sub_device *sdev;
430 	uint64_t u64;
431 	uint8_t i;
432 	int rc = 0;
433 	int ret;
434 
435 	fs_lock(dev, 0);
436 	if (idx >= dev->data->nb_rx_queues) {
437 		rc = -EINVAL;
438 		goto unlock;
439 	}
440 	rxq = dev->data->rx_queues[idx];
441 	if (rxq == NULL || rxq->event_fd <= 0) {
442 		rc = -EINVAL;
443 		goto unlock;
444 	}
445 	rxq->enable_events = 0;
446 	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
447 		ret = rte_eth_dev_rx_intr_disable(PORT_ID(sdev), idx);
448 		ret = fs_err(sdev, ret);
449 		if (ret)
450 			rc = ret;
451 	}
452 	/* Clear pending events */
453 	while (read(rxq->event_fd, &u64, sizeof(uint64_t)) >  0)
454 		;
455 unlock:
456 	fs_unlock(dev, 0);
457 	if (rc)
458 		rte_errno = -rc;
459 	return rc;
460 }
461 
462 static void
463 fs_tx_queue_release(void *queue)
464 {
465 	struct rte_eth_dev *dev;
466 	struct sub_device *sdev;
467 	uint8_t i;
468 	struct txq *txq;
469 
470 	if (queue == NULL)
471 		return;
472 	txq = queue;
473 	dev = txq->priv->dev;
474 	fs_lock(dev, 0);
475 	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
476 		SUBOPS(sdev, tx_queue_release)
477 			(ETH(sdev)->data->tx_queues[txq->qid]);
478 	dev->data->tx_queues[txq->qid] = NULL;
479 	rte_free(txq);
480 	fs_unlock(dev, 0);
481 }
482 
483 static int
484 fs_tx_queue_setup(struct rte_eth_dev *dev,
485 		uint16_t tx_queue_id,
486 		uint16_t nb_tx_desc,
487 		unsigned int socket_id,
488 		const struct rte_eth_txconf *tx_conf)
489 {
490 	struct sub_device *sdev;
491 	struct txq *txq;
492 	uint8_t i;
493 	int ret;
494 
495 	fs_lock(dev, 0);
496 	txq = dev->data->tx_queues[tx_queue_id];
497 	if (txq != NULL) {
498 		fs_tx_queue_release(txq);
499 		dev->data->tx_queues[tx_queue_id] = NULL;
500 	}
501 	txq = rte_zmalloc("ethdev TX queue",
502 			  sizeof(*txq) +
503 			  sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail,
504 			  RTE_CACHE_LINE_SIZE);
505 	if (txq == NULL) {
506 		fs_unlock(dev, 0);
507 		return -ENOMEM;
508 	}
509 	FOREACH_SUBDEV(sdev, i, dev)
510 		rte_atomic64_init(&txq->refcnt[i]);
511 	txq->qid = tx_queue_id;
512 	txq->socket_id = socket_id;
513 	txq->info.conf = *tx_conf;
514 	txq->info.nb_desc = nb_tx_desc;
515 	txq->priv = PRIV(dev);
516 	dev->data->tx_queues[tx_queue_id] = txq;
517 	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
518 		ret = rte_eth_tx_queue_setup(PORT_ID(sdev),
519 				tx_queue_id,
520 				nb_tx_desc, socket_id,
521 				tx_conf);
522 		if ((ret = fs_err(sdev, ret))) {
523 			ERROR("TX queue setup failed for sub_device %d", i);
524 			goto free_txq;
525 		}
526 	}
527 	fs_unlock(dev, 0);
528 	return 0;
529 free_txq:
530 	fs_tx_queue_release(txq);
531 	fs_unlock(dev, 0);
532 	return ret;
533 }
534 
535 static void
536 fs_dev_free_queues(struct rte_eth_dev *dev)
537 {
538 	uint16_t i;
539 
540 	for (i = 0; i < dev->data->nb_rx_queues; i++) {
541 		fs_rx_queue_release(dev->data->rx_queues[i]);
542 		dev->data->rx_queues[i] = NULL;
543 	}
544 	dev->data->nb_rx_queues = 0;
545 	for (i = 0; i < dev->data->nb_tx_queues; i++) {
546 		fs_tx_queue_release(dev->data->tx_queues[i]);
547 		dev->data->tx_queues[i] = NULL;
548 	}
549 	dev->data->nb_tx_queues = 0;
550 }
551 
552 static void
553 fs_promiscuous_enable(struct rte_eth_dev *dev)
554 {
555 	struct sub_device *sdev;
556 	uint8_t i;
557 
558 	fs_lock(dev, 0);
559 	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
560 		rte_eth_promiscuous_enable(PORT_ID(sdev));
561 	fs_unlock(dev, 0);
562 }
563 
564 static void
565 fs_promiscuous_disable(struct rte_eth_dev *dev)
566 {
567 	struct sub_device *sdev;
568 	uint8_t i;
569 
570 	fs_lock(dev, 0);
571 	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
572 		rte_eth_promiscuous_disable(PORT_ID(sdev));
573 	fs_unlock(dev, 0);
574 }
575 
576 static void
577 fs_allmulticast_enable(struct rte_eth_dev *dev)
578 {
579 	struct sub_device *sdev;
580 	uint8_t i;
581 
582 	fs_lock(dev, 0);
583 	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
584 		rte_eth_allmulticast_enable(PORT_ID(sdev));
585 	fs_unlock(dev, 0);
586 }
587 
588 static void
589 fs_allmulticast_disable(struct rte_eth_dev *dev)
590 {
591 	struct sub_device *sdev;
592 	uint8_t i;
593 
594 	fs_lock(dev, 0);
595 	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
596 		rte_eth_allmulticast_disable(PORT_ID(sdev));
597 	fs_unlock(dev, 0);
598 }
599 
600 static int
601 fs_link_update(struct rte_eth_dev *dev,
602 		int wait_to_complete)
603 {
604 	struct sub_device *sdev;
605 	uint8_t i;
606 	int ret;
607 
608 	fs_lock(dev, 0);
609 	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
610 		DEBUG("Calling link_update on sub_device %d", i);
611 		ret = (SUBOPS(sdev, link_update))(ETH(sdev), wait_to_complete);
612 		if (ret && ret != -1 && sdev->remove == 0 &&
613 		    rte_eth_dev_is_removed(PORT_ID(sdev)) == 0) {
614 			ERROR("Link update failed for sub_device %d with error %d",
615 			      i, ret);
616 			fs_unlock(dev, 0);
617 			return ret;
618 		}
619 	}
620 	if (TX_SUBDEV(dev)) {
621 		struct rte_eth_link *l1;
622 		struct rte_eth_link *l2;
623 
624 		l1 = &dev->data->dev_link;
625 		l2 = &ETH(TX_SUBDEV(dev))->data->dev_link;
626 		if (memcmp(l1, l2, sizeof(*l1))) {
627 			*l1 = *l2;
628 			fs_unlock(dev, 0);
629 			return 0;
630 		}
631 	}
632 	fs_unlock(dev, 0);
633 	return -1;
634 }
635 
636 static int
637 fs_stats_get(struct rte_eth_dev *dev,
638 	     struct rte_eth_stats *stats)
639 {
640 	struct rte_eth_stats backup;
641 	struct sub_device *sdev;
642 	uint8_t i;
643 	int ret;
644 
645 	fs_lock(dev, 0);
646 	rte_memcpy(stats, &PRIV(dev)->stats_accumulator, sizeof(*stats));
647 	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
648 		struct rte_eth_stats *snapshot = &sdev->stats_snapshot.stats;
649 		uint64_t *timestamp = &sdev->stats_snapshot.timestamp;
650 
651 		rte_memcpy(&backup, snapshot, sizeof(backup));
652 		ret = rte_eth_stats_get(PORT_ID(sdev), snapshot);
653 		if (ret) {
654 			if (!fs_err(sdev, ret)) {
655 				rte_memcpy(snapshot, &backup, sizeof(backup));
656 				goto inc;
657 			}
658 			ERROR("Operation rte_eth_stats_get failed for sub_device %d with error %d",
659 				  i, ret);
660 			*timestamp = 0;
661 			fs_unlock(dev, 0);
662 			return ret;
663 		}
664 		*timestamp = rte_rdtsc();
665 inc:
666 		failsafe_stats_increment(stats, snapshot);
667 	}
668 	fs_unlock(dev, 0);
669 	return 0;
670 }
671 
672 static void
673 fs_stats_reset(struct rte_eth_dev *dev)
674 {
675 	struct sub_device *sdev;
676 	uint8_t i;
677 
678 	fs_lock(dev, 0);
679 	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
680 		rte_eth_stats_reset(PORT_ID(sdev));
681 		memset(&sdev->stats_snapshot, 0, sizeof(struct rte_eth_stats));
682 	}
683 	memset(&PRIV(dev)->stats_accumulator, 0, sizeof(struct rte_eth_stats));
684 	fs_unlock(dev, 0);
685 }
686 
687 /**
688  * Fail-safe dev_infos_get rules:
689  *
690  * No sub_device:
691  *   Numerables:
692  *      Use the maximum possible values for any field, so as not
693  *      to impede any further configuration effort.
694  *   Capabilities:
695  *      Limits capabilities to those that are understood by the
696  *      fail-safe PMD. This understanding stems from the fail-safe
697  *      being capable of verifying that the related capability is
698  *      expressed within the device configuration (struct rte_eth_conf).
699  *
700  * At least one probed sub_device:
701  *   Numerables:
702  *      Uses values from the active probed sub_device
703  *      The rationale here is that if any sub_device is less capable
704  *      (for example concerning the number of queues) than the active
705  *      sub_device, then its subsequent configuration will fail.
706  *      It is impossible to foresee this failure when the failing sub_device
707  *      is supposed to be plugged-in later on, so the configuration process
708  *      is the single point of failure and error reporting.
709  *   Capabilities:
710  *      Uses a logical AND of RX capabilities among
711  *      all sub_devices and the default capabilities.
712  *      Uses a logical AND of TX capabilities among
713  *      the active probed sub_device and the default capabilities.
714  *
715  */
716 static void
717 fs_dev_infos_get(struct rte_eth_dev *dev,
718 		  struct rte_eth_dev_info *infos)
719 {
720 	struct sub_device *sdev;
721 	uint8_t i;
722 
723 	sdev = TX_SUBDEV(dev);
724 	if (sdev == NULL) {
725 		DEBUG("No probed device, using default infos");
726 		rte_memcpy(&PRIV(dev)->infos, &default_infos,
727 			   sizeof(default_infos));
728 	} else {
729 		uint64_t rx_offload_capa;
730 		uint64_t rxq_offload_capa;
731 		uint64_t rss_hf_offload_capa;
732 
733 		rx_offload_capa = default_infos.rx_offload_capa;
734 		rxq_offload_capa = default_infos.rx_queue_offload_capa;
735 		rss_hf_offload_capa = default_infos.flow_type_rss_offloads;
736 		FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_PROBED) {
737 			rte_eth_dev_info_get(PORT_ID(sdev),
738 					&PRIV(dev)->infos);
739 			rx_offload_capa &= PRIV(dev)->infos.rx_offload_capa;
740 			rxq_offload_capa &=
741 					PRIV(dev)->infos.rx_queue_offload_capa;
742 			rss_hf_offload_capa &=
743 					PRIV(dev)->infos.flow_type_rss_offloads;
744 		}
745 		sdev = TX_SUBDEV(dev);
746 		rte_eth_dev_info_get(PORT_ID(sdev), &PRIV(dev)->infos);
747 		PRIV(dev)->infos.rx_offload_capa = rx_offload_capa;
748 		PRIV(dev)->infos.rx_queue_offload_capa = rxq_offload_capa;
749 		PRIV(dev)->infos.flow_type_rss_offloads = rss_hf_offload_capa;
750 		PRIV(dev)->infos.tx_offload_capa &=
751 					default_infos.tx_offload_capa;
752 		PRIV(dev)->infos.tx_queue_offload_capa &=
753 					default_infos.tx_queue_offload_capa;
754 	}
755 	rte_memcpy(infos, &PRIV(dev)->infos, sizeof(*infos));
756 }
757 
758 static const uint32_t *
759 fs_dev_supported_ptypes_get(struct rte_eth_dev *dev)
760 {
761 	struct sub_device *sdev;
762 	struct rte_eth_dev *edev;
763 	const uint32_t *ret;
764 
765 	fs_lock(dev, 0);
766 	sdev = TX_SUBDEV(dev);
767 	if (sdev == NULL) {
768 		ret = NULL;
769 		goto unlock;
770 	}
771 	edev = ETH(sdev);
772 	/* ENOTSUP: counts as no supported ptypes */
773 	if (SUBOPS(sdev, dev_supported_ptypes_get) == NULL) {
774 		ret = NULL;
775 		goto unlock;
776 	}
777 	/*
778 	 * The API does not permit to do a clean AND of all ptypes,
779 	 * It is also incomplete by design and we do not really care
780 	 * to have a best possible value in this context.
781 	 * We just return the ptypes of the device of highest
782 	 * priority, usually the PREFERRED device.
783 	 */
784 	ret = SUBOPS(sdev, dev_supported_ptypes_get)(edev);
785 unlock:
786 	fs_unlock(dev, 0);
787 	return ret;
788 }
789 
790 static int
791 fs_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
792 {
793 	struct sub_device *sdev;
794 	uint8_t i;
795 	int ret;
796 
797 	fs_lock(dev, 0);
798 	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
799 		DEBUG("Calling rte_eth_dev_set_mtu on sub_device %d", i);
800 		ret = rte_eth_dev_set_mtu(PORT_ID(sdev), mtu);
801 		if ((ret = fs_err(sdev, ret))) {
802 			ERROR("Operation rte_eth_dev_set_mtu failed for sub_device %d with error %d",
803 			      i, ret);
804 			fs_unlock(dev, 0);
805 			return ret;
806 		}
807 	}
808 	fs_unlock(dev, 0);
809 	return 0;
810 }
811 
812 static int
813 fs_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
814 {
815 	struct sub_device *sdev;
816 	uint8_t i;
817 	int ret;
818 
819 	fs_lock(dev, 0);
820 	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
821 		DEBUG("Calling rte_eth_dev_vlan_filter on sub_device %d", i);
822 		ret = rte_eth_dev_vlan_filter(PORT_ID(sdev), vlan_id, on);
823 		if ((ret = fs_err(sdev, ret))) {
824 			ERROR("Operation rte_eth_dev_vlan_filter failed for sub_device %d"
825 			      " with error %d", i, ret);
826 			fs_unlock(dev, 0);
827 			return ret;
828 		}
829 	}
830 	fs_unlock(dev, 0);
831 	return 0;
832 }
833 
834 static int
835 fs_flow_ctrl_get(struct rte_eth_dev *dev,
836 		struct rte_eth_fc_conf *fc_conf)
837 {
838 	struct sub_device *sdev;
839 	int ret;
840 
841 	fs_lock(dev, 0);
842 	sdev = TX_SUBDEV(dev);
843 	if (sdev == NULL) {
844 		ret = 0;
845 		goto unlock;
846 	}
847 	if (SUBOPS(sdev, flow_ctrl_get) == NULL) {
848 		ret = -ENOTSUP;
849 		goto unlock;
850 	}
851 	ret = SUBOPS(sdev, flow_ctrl_get)(ETH(sdev), fc_conf);
852 unlock:
853 	fs_unlock(dev, 0);
854 	return ret;
855 }
856 
857 static int
858 fs_flow_ctrl_set(struct rte_eth_dev *dev,
859 		struct rte_eth_fc_conf *fc_conf)
860 {
861 	struct sub_device *sdev;
862 	uint8_t i;
863 	int ret;
864 
865 	fs_lock(dev, 0);
866 	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
867 		DEBUG("Calling rte_eth_dev_flow_ctrl_set on sub_device %d", i);
868 		ret = rte_eth_dev_flow_ctrl_set(PORT_ID(sdev), fc_conf);
869 		if ((ret = fs_err(sdev, ret))) {
870 			ERROR("Operation rte_eth_dev_flow_ctrl_set failed for sub_device %d"
871 			      " with error %d", i, ret);
872 			fs_unlock(dev, 0);
873 			return ret;
874 		}
875 	}
876 	fs_unlock(dev, 0);
877 	return 0;
878 }
879 
880 static void
881 fs_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
882 {
883 	struct sub_device *sdev;
884 	uint8_t i;
885 
886 	fs_lock(dev, 0);
887 	/* No check: already done within the rte_eth_dev_mac_addr_remove
888 	 * call for the fail-safe device.
889 	 */
890 	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
891 		rte_eth_dev_mac_addr_remove(PORT_ID(sdev),
892 				&dev->data->mac_addrs[index]);
893 	PRIV(dev)->mac_addr_pool[index] = 0;
894 	fs_unlock(dev, 0);
895 }
896 
897 static int
898 fs_mac_addr_add(struct rte_eth_dev *dev,
899 		struct ether_addr *mac_addr,
900 		uint32_t index,
901 		uint32_t vmdq)
902 {
903 	struct sub_device *sdev;
904 	int ret;
905 	uint8_t i;
906 
907 	RTE_ASSERT(index < FAILSAFE_MAX_ETHADDR);
908 	fs_lock(dev, 0);
909 	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
910 		ret = rte_eth_dev_mac_addr_add(PORT_ID(sdev), mac_addr, vmdq);
911 		if ((ret = fs_err(sdev, ret))) {
912 			ERROR("Operation rte_eth_dev_mac_addr_add failed for sub_device %"
913 			      PRIu8 " with error %d", i, ret);
914 			fs_unlock(dev, 0);
915 			return ret;
916 		}
917 	}
918 	if (index >= PRIV(dev)->nb_mac_addr) {
919 		DEBUG("Growing mac_addrs array");
920 		PRIV(dev)->nb_mac_addr = index;
921 	}
922 	PRIV(dev)->mac_addr_pool[index] = vmdq;
923 	fs_unlock(dev, 0);
924 	return 0;
925 }
926 
927 static int
928 fs_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
929 {
930 	struct sub_device *sdev;
931 	uint8_t i;
932 	int ret;
933 
934 	fs_lock(dev, 0);
935 	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
936 		ret = rte_eth_dev_default_mac_addr_set(PORT_ID(sdev), mac_addr);
937 		ret = fs_err(sdev, ret);
938 		if (ret) {
939 			ERROR("Operation rte_eth_dev_mac_addr_set failed for sub_device %d with error %d",
940 				i, ret);
941 			fs_unlock(dev, 0);
942 			return ret;
943 		}
944 	}
945 	fs_unlock(dev, 0);
946 
947 	return 0;
948 }
949 
950 static int
951 fs_rss_hash_update(struct rte_eth_dev *dev,
952 			struct rte_eth_rss_conf *rss_conf)
953 {
954 	struct sub_device *sdev;
955 	uint8_t i;
956 	int ret;
957 
958 	fs_lock(dev, 0);
959 	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
960 		ret = rte_eth_dev_rss_hash_update(PORT_ID(sdev), rss_conf);
961 		ret = fs_err(sdev, ret);
962 		if (ret) {
963 			ERROR("Operation rte_eth_dev_rss_hash_update"
964 				" failed for sub_device %d with error %d",
965 				i, ret);
966 			fs_unlock(dev, 0);
967 			return ret;
968 		}
969 	}
970 	fs_unlock(dev, 0);
971 
972 	return 0;
973 }
974 
975 static int
976 fs_filter_ctrl(struct rte_eth_dev *dev,
977 		enum rte_filter_type type,
978 		enum rte_filter_op op,
979 		void *arg)
980 {
981 	struct sub_device *sdev;
982 	uint8_t i;
983 	int ret;
984 
985 	if (type == RTE_ETH_FILTER_GENERIC &&
986 	    op == RTE_ETH_FILTER_GET) {
987 		*(const void **)arg = &fs_flow_ops;
988 		return 0;
989 	}
990 	fs_lock(dev, 0);
991 	FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
992 		DEBUG("Calling rte_eth_dev_filter_ctrl on sub_device %d", i);
993 		ret = rte_eth_dev_filter_ctrl(PORT_ID(sdev), type, op, arg);
994 		if ((ret = fs_err(sdev, ret))) {
995 			ERROR("Operation rte_eth_dev_filter_ctrl failed for sub_device %d"
996 			      " with error %d", i, ret);
997 			fs_unlock(dev, 0);
998 			return ret;
999 		}
1000 	}
1001 	fs_unlock(dev, 0);
1002 	return 0;
1003 }
1004 
1005 const struct eth_dev_ops failsafe_ops = {
1006 	.dev_configure = fs_dev_configure,
1007 	.dev_start = fs_dev_start,
1008 	.dev_stop = fs_dev_stop,
1009 	.dev_set_link_down = fs_dev_set_link_down,
1010 	.dev_set_link_up = fs_dev_set_link_up,
1011 	.dev_close = fs_dev_close,
1012 	.promiscuous_enable = fs_promiscuous_enable,
1013 	.promiscuous_disable = fs_promiscuous_disable,
1014 	.allmulticast_enable = fs_allmulticast_enable,
1015 	.allmulticast_disable = fs_allmulticast_disable,
1016 	.link_update = fs_link_update,
1017 	.stats_get = fs_stats_get,
1018 	.stats_reset = fs_stats_reset,
1019 	.dev_infos_get = fs_dev_infos_get,
1020 	.dev_supported_ptypes_get = fs_dev_supported_ptypes_get,
1021 	.mtu_set = fs_mtu_set,
1022 	.vlan_filter_set = fs_vlan_filter_set,
1023 	.rx_queue_setup = fs_rx_queue_setup,
1024 	.tx_queue_setup = fs_tx_queue_setup,
1025 	.rx_queue_release = fs_rx_queue_release,
1026 	.tx_queue_release = fs_tx_queue_release,
1027 	.rx_queue_intr_enable = fs_rx_intr_enable,
1028 	.rx_queue_intr_disable = fs_rx_intr_disable,
1029 	.flow_ctrl_get = fs_flow_ctrl_get,
1030 	.flow_ctrl_set = fs_flow_ctrl_set,
1031 	.mac_addr_remove = fs_mac_addr_remove,
1032 	.mac_addr_add = fs_mac_addr_add,
1033 	.mac_addr_set = fs_mac_addr_set,
1034 	.rss_hash_update = fs_rss_hash_update,
1035 	.filter_ctrl = fs_filter_ctrl,
1036 };
1037