1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <ctype.h> 6 #include <sys/queue.h> 7 #include <stdalign.h> 8 #include <stdio.h> 9 #include <errno.h> 10 #include <stdint.h> 11 #include <string.h> 12 #include <unistd.h> 13 #include <stdarg.h> 14 #include <inttypes.h> 15 #include <rte_byteorder.h> 16 #include <rte_common.h> 17 #include <rte_os_shim.h> 18 19 #include <rte_interrupts.h> 20 #include <rte_debug.h> 21 #include <rte_pci.h> 22 #include <rte_alarm.h> 23 #include <rte_atomic.h> 24 #include <rte_eal.h> 25 #include <rte_ether.h> 26 #include <ethdev_driver.h> 27 #include <ethdev_pci.h> 28 #include <rte_malloc.h> 29 #include <rte_memzone.h> 30 #include <dev_driver.h> 31 32 #include "iavf.h" 33 #include "iavf_rxtx.h" 34 #include "iavf_generic_flow.h" 35 #include "rte_pmd_iavf.h" 36 #include "iavf_ipsec_crypto.h" 37 38 /* devargs */ 39 #define IAVF_PROTO_XTR_ARG "proto_xtr" 40 #define IAVF_QUANTA_SIZE_ARG "quanta_size" 41 #define IAVF_RESET_WATCHDOG_ARG "watchdog_period" 42 #define IAVF_ENABLE_AUTO_RESET_ARG "auto_reset" 43 #define IAVF_NO_POLL_ON_LINK_DOWN_ARG "no-poll-on-link-down" 44 #define IAVF_MBUF_CHECK_ARG "mbuf_check" 45 uint64_t iavf_timestamp_dynflag; 46 int iavf_timestamp_dynfield_offset = -1; 47 int rte_pmd_iavf_tx_lldp_dynfield_offset = -1; 48 49 static const char * const iavf_valid_args[] = { 50 IAVF_PROTO_XTR_ARG, 51 IAVF_QUANTA_SIZE_ARG, 52 IAVF_RESET_WATCHDOG_ARG, 53 IAVF_ENABLE_AUTO_RESET_ARG, 54 IAVF_NO_POLL_ON_LINK_DOWN_ARG, 55 IAVF_MBUF_CHECK_ARG, 56 NULL 57 }; 58 59 static const struct rte_mbuf_dynfield iavf_proto_xtr_metadata_param = { 60 .name = "intel_pmd_dynfield_proto_xtr_metadata", 61 .size = sizeof(uint32_t), 62 .align = alignof(uint32_t), 63 .flags = 0, 64 }; 65 66 struct iavf_proto_xtr_ol { 67 const struct rte_mbuf_dynflag param; 68 uint64_t *ol_flag; 69 bool required; 70 }; 71 72 static struct iavf_proto_xtr_ol iavf_proto_xtr_params[] = { 73 [IAVF_PROTO_XTR_VLAN] = { 74 .param = { .name = "intel_pmd_dynflag_proto_xtr_vlan" }, 75 .ol_flag = &rte_pmd_ifd_dynflag_proto_xtr_vlan_mask }, 76 [IAVF_PROTO_XTR_IPV4] = { 77 .param = { .name = "intel_pmd_dynflag_proto_xtr_ipv4" }, 78 .ol_flag = &rte_pmd_ifd_dynflag_proto_xtr_ipv4_mask }, 79 [IAVF_PROTO_XTR_IPV6] = { 80 .param = { .name = "intel_pmd_dynflag_proto_xtr_ipv6" }, 81 .ol_flag = &rte_pmd_ifd_dynflag_proto_xtr_ipv6_mask }, 82 [IAVF_PROTO_XTR_IPV6_FLOW] = { 83 .param = { .name = "intel_pmd_dynflag_proto_xtr_ipv6_flow" }, 84 .ol_flag = &rte_pmd_ifd_dynflag_proto_xtr_ipv6_flow_mask }, 85 [IAVF_PROTO_XTR_TCP] = { 86 .param = { .name = "intel_pmd_dynflag_proto_xtr_tcp" }, 87 .ol_flag = &rte_pmd_ifd_dynflag_proto_xtr_tcp_mask }, 88 [IAVF_PROTO_XTR_IP_OFFSET] = { 89 .param = { .name = "intel_pmd_dynflag_proto_xtr_ip_offset" }, 90 .ol_flag = &rte_pmd_ifd_dynflag_proto_xtr_ip_offset_mask }, 91 [IAVF_PROTO_XTR_IPSEC_CRYPTO_SAID] = { 92 .param = { 93 .name = "intel_pmd_dynflag_proto_xtr_ipsec_crypto_said" }, 94 .ol_flag = 95 &rte_pmd_ifd_dynflag_proto_xtr_ipsec_crypto_said_mask }, 96 }; 97 98 static int iavf_dev_configure(struct rte_eth_dev *dev); 99 static int iavf_dev_start(struct rte_eth_dev *dev); 100 static int iavf_dev_stop(struct rte_eth_dev *dev); 101 static int iavf_dev_close(struct rte_eth_dev *dev); 102 static int iavf_dev_reset(struct rte_eth_dev *dev); 103 static int iavf_dev_info_get(struct rte_eth_dev *dev, 104 struct rte_eth_dev_info *dev_info); 105 static const uint32_t *iavf_dev_supported_ptypes_get(struct rte_eth_dev *dev, 106 size_t *no_of_elements); 107 static int iavf_dev_stats_get(struct rte_eth_dev *dev, 108 struct rte_eth_stats *stats); 109 static int iavf_dev_stats_reset(struct rte_eth_dev *dev); 110 static int iavf_dev_xstats_reset(struct rte_eth_dev *dev); 111 static int iavf_dev_xstats_get(struct rte_eth_dev *dev, 112 struct rte_eth_xstat *xstats, unsigned int n); 113 static int iavf_dev_xstats_get_names(struct rte_eth_dev *dev, 114 struct rte_eth_xstat_name *xstats_names, 115 unsigned int limit); 116 static int iavf_dev_promiscuous_enable(struct rte_eth_dev *dev); 117 static int iavf_dev_promiscuous_disable(struct rte_eth_dev *dev); 118 static int iavf_dev_allmulticast_enable(struct rte_eth_dev *dev); 119 static int iavf_dev_allmulticast_disable(struct rte_eth_dev *dev); 120 static int iavf_dev_add_mac_addr(struct rte_eth_dev *dev, 121 struct rte_ether_addr *addr, 122 uint32_t index, 123 uint32_t pool); 124 static void iavf_dev_del_mac_addr(struct rte_eth_dev *dev, uint32_t index); 125 static int iavf_dev_vlan_filter_set(struct rte_eth_dev *dev, 126 uint16_t vlan_id, int on); 127 static int iavf_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask); 128 static int iavf_dev_rss_reta_update(struct rte_eth_dev *dev, 129 struct rte_eth_rss_reta_entry64 *reta_conf, 130 uint16_t reta_size); 131 static int iavf_dev_rss_reta_query(struct rte_eth_dev *dev, 132 struct rte_eth_rss_reta_entry64 *reta_conf, 133 uint16_t reta_size); 134 static int iavf_dev_rss_hash_update(struct rte_eth_dev *dev, 135 struct rte_eth_rss_conf *rss_conf); 136 static int iavf_dev_rss_hash_conf_get(struct rte_eth_dev *dev, 137 struct rte_eth_rss_conf *rss_conf); 138 static int iavf_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 139 static int iavf_dev_set_default_mac_addr(struct rte_eth_dev *dev, 140 struct rte_ether_addr *mac_addr); 141 static int iavf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, 142 uint16_t queue_id); 143 static int iavf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, 144 uint16_t queue_id); 145 static void iavf_dev_interrupt_handler(void *param); 146 static void iavf_disable_irq0(struct iavf_hw *hw); 147 static int iavf_dev_flow_ops_get(struct rte_eth_dev *dev, 148 const struct rte_flow_ops **ops); 149 static int iavf_set_mc_addr_list(struct rte_eth_dev *dev, 150 struct rte_ether_addr *mc_addrs, 151 uint32_t mc_addrs_num); 152 static int iavf_tm_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg); 153 154 static const struct rte_pci_id pci_id_iavf_map[] = { 155 { RTE_PCI_DEVICE(IAVF_INTEL_VENDOR_ID, IAVF_DEV_ID_ADAPTIVE_VF) }, 156 { RTE_PCI_DEVICE(IAVF_INTEL_VENDOR_ID, IAVF_DEV_ID_VF) }, 157 { RTE_PCI_DEVICE(IAVF_INTEL_VENDOR_ID, IAVF_DEV_ID_VF_HV) }, 158 { RTE_PCI_DEVICE(IAVF_INTEL_VENDOR_ID, IAVF_DEV_ID_X722_VF) }, 159 { RTE_PCI_DEVICE(IAVF_INTEL_VENDOR_ID, IAVF_DEV_ID_X722_A0_VF) }, 160 { .vendor_id = 0, /* sentinel */ }, 161 }; 162 163 struct rte_iavf_xstats_name_off { 164 char name[RTE_ETH_XSTATS_NAME_SIZE]; 165 unsigned int offset; 166 }; 167 168 #define _OFF_OF(a) offsetof(struct iavf_eth_xstats, a) 169 static const struct rte_iavf_xstats_name_off rte_iavf_stats_strings[] = { 170 {"rx_bytes", _OFF_OF(eth_stats.rx_bytes)}, 171 {"rx_unicast_packets", _OFF_OF(eth_stats.rx_unicast)}, 172 {"rx_multicast_packets", _OFF_OF(eth_stats.rx_multicast)}, 173 {"rx_broadcast_packets", _OFF_OF(eth_stats.rx_broadcast)}, 174 {"rx_dropped_packets", _OFF_OF(eth_stats.rx_discards)}, 175 {"rx_unknown_protocol_packets", offsetof(struct iavf_eth_stats, 176 rx_unknown_protocol)}, 177 {"tx_bytes", _OFF_OF(eth_stats.tx_bytes)}, 178 {"tx_unicast_packets", _OFF_OF(eth_stats.tx_unicast)}, 179 {"tx_multicast_packets", _OFF_OF(eth_stats.tx_multicast)}, 180 {"tx_broadcast_packets", _OFF_OF(eth_stats.tx_broadcast)}, 181 {"tx_dropped_packets", _OFF_OF(eth_stats.tx_discards)}, 182 {"tx_error_packets", _OFF_OF(eth_stats.tx_errors)}, 183 {"tx_mbuf_error_packets", _OFF_OF(mbuf_stats.tx_pkt_errors)}, 184 185 {"inline_ipsec_crypto_ipackets", _OFF_OF(ips_stats.icount)}, 186 {"inline_ipsec_crypto_ibytes", _OFF_OF(ips_stats.ibytes)}, 187 {"inline_ipsec_crypto_ierrors", _OFF_OF(ips_stats.ierrors.count)}, 188 {"inline_ipsec_crypto_ierrors_sad_lookup", 189 _OFF_OF(ips_stats.ierrors.sad_miss)}, 190 {"inline_ipsec_crypto_ierrors_not_processed", 191 _OFF_OF(ips_stats.ierrors.not_processed)}, 192 {"inline_ipsec_crypto_ierrors_icv_fail", 193 _OFF_OF(ips_stats.ierrors.icv_check)}, 194 {"inline_ipsec_crypto_ierrors_length", 195 _OFF_OF(ips_stats.ierrors.ipsec_length)}, 196 {"inline_ipsec_crypto_ierrors_misc", 197 _OFF_OF(ips_stats.ierrors.misc)}, 198 }; 199 #undef _OFF_OF 200 201 #define IAVF_NB_XSTATS (sizeof(rte_iavf_stats_strings) / \ 202 sizeof(rte_iavf_stats_strings[0])) 203 204 static const struct eth_dev_ops iavf_eth_dev_ops = { 205 .dev_configure = iavf_dev_configure, 206 .dev_start = iavf_dev_start, 207 .dev_stop = iavf_dev_stop, 208 .dev_close = iavf_dev_close, 209 .dev_reset = iavf_dev_reset, 210 .dev_infos_get = iavf_dev_info_get, 211 .dev_supported_ptypes_get = iavf_dev_supported_ptypes_get, 212 .link_update = iavf_dev_link_update, 213 .stats_get = iavf_dev_stats_get, 214 .stats_reset = iavf_dev_stats_reset, 215 .xstats_get = iavf_dev_xstats_get, 216 .xstats_get_names = iavf_dev_xstats_get_names, 217 .xstats_reset = iavf_dev_xstats_reset, 218 .promiscuous_enable = iavf_dev_promiscuous_enable, 219 .promiscuous_disable = iavf_dev_promiscuous_disable, 220 .allmulticast_enable = iavf_dev_allmulticast_enable, 221 .allmulticast_disable = iavf_dev_allmulticast_disable, 222 .mac_addr_add = iavf_dev_add_mac_addr, 223 .mac_addr_remove = iavf_dev_del_mac_addr, 224 .set_mc_addr_list = iavf_set_mc_addr_list, 225 .vlan_filter_set = iavf_dev_vlan_filter_set, 226 .vlan_offload_set = iavf_dev_vlan_offload_set, 227 .rx_queue_start = iavf_dev_rx_queue_start, 228 .rx_queue_stop = iavf_dev_rx_queue_stop, 229 .tx_queue_start = iavf_dev_tx_queue_start, 230 .tx_queue_stop = iavf_dev_tx_queue_stop, 231 .rx_queue_setup = iavf_dev_rx_queue_setup, 232 .rx_queue_release = iavf_dev_rx_queue_release, 233 .tx_queue_setup = iavf_dev_tx_queue_setup, 234 .tx_queue_release = iavf_dev_tx_queue_release, 235 .mac_addr_set = iavf_dev_set_default_mac_addr, 236 .reta_update = iavf_dev_rss_reta_update, 237 .reta_query = iavf_dev_rss_reta_query, 238 .rss_hash_update = iavf_dev_rss_hash_update, 239 .rss_hash_conf_get = iavf_dev_rss_hash_conf_get, 240 .rxq_info_get = iavf_dev_rxq_info_get, 241 .txq_info_get = iavf_dev_txq_info_get, 242 .mtu_set = iavf_dev_mtu_set, 243 .rx_queue_intr_enable = iavf_dev_rx_queue_intr_enable, 244 .rx_queue_intr_disable = iavf_dev_rx_queue_intr_disable, 245 .flow_ops_get = iavf_dev_flow_ops_get, 246 .tx_done_cleanup = iavf_dev_tx_done_cleanup, 247 .get_monitor_addr = iavf_get_monitor_addr, 248 .tm_ops_get = iavf_tm_ops_get, 249 }; 250 251 static int 252 iavf_tm_ops_get(struct rte_eth_dev *dev, 253 void *arg) 254 { 255 struct iavf_adapter *adapter = 256 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 257 258 if (adapter->closed) 259 return -EIO; 260 261 if (!arg) 262 return -EINVAL; 263 264 *(const void **)arg = &iavf_tm_ops; 265 266 return 0; 267 } 268 269 __rte_unused 270 static int 271 iavf_vfr_inprogress(struct iavf_hw *hw) 272 { 273 int inprogress = 0; 274 275 if ((IAVF_READ_REG(hw, IAVF_VFGEN_RSTAT) & 276 IAVF_VFGEN_RSTAT_VFR_STATE_MASK) == 277 VIRTCHNL_VFR_INPROGRESS) 278 inprogress = 1; 279 280 if (inprogress) 281 PMD_DRV_LOG(INFO, "Watchdog detected VFR in progress"); 282 283 return inprogress; 284 } 285 286 __rte_unused 287 static void 288 iavf_dev_watchdog(void *cb_arg) 289 { 290 struct iavf_adapter *adapter = cb_arg; 291 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter); 292 int vfr_inprogress = 0, rc = 0; 293 294 /* check if watchdog has been disabled since last call */ 295 if (!adapter->vf.watchdog_enabled) 296 return; 297 298 /* If in reset then poll vfr_inprogress register for completion */ 299 if (adapter->vf.vf_reset) { 300 vfr_inprogress = iavf_vfr_inprogress(hw); 301 302 if (!vfr_inprogress) { 303 PMD_DRV_LOG(INFO, "VF \"%s\" reset has completed", 304 adapter->vf.eth_dev->data->name); 305 adapter->vf.vf_reset = false; 306 iavf_set_no_poll(adapter, false); 307 } 308 /* If not in reset then poll vfr_inprogress register for VFLR event */ 309 } else { 310 vfr_inprogress = iavf_vfr_inprogress(hw); 311 312 if (vfr_inprogress) { 313 PMD_DRV_LOG(INFO, 314 "VF \"%s\" reset event detected by watchdog", 315 adapter->vf.eth_dev->data->name); 316 317 /* enter reset state with VFLR event */ 318 adapter->vf.vf_reset = true; 319 iavf_set_no_poll(adapter, false); 320 adapter->vf.link_up = false; 321 322 iavf_dev_event_post(adapter->vf.eth_dev, RTE_ETH_EVENT_INTR_RESET, 323 NULL, 0); 324 } 325 } 326 327 if (adapter->devargs.watchdog_period) { 328 /* re-alarm watchdog */ 329 rc = rte_eal_alarm_set(adapter->devargs.watchdog_period, 330 &iavf_dev_watchdog, cb_arg); 331 332 if (rc) 333 PMD_DRV_LOG(ERR, "Failed \"%s\" to reset device watchdog alarm", 334 adapter->vf.eth_dev->data->name); 335 } 336 } 337 338 void 339 iavf_dev_watchdog_enable(struct iavf_adapter *adapter) 340 { 341 if (!adapter->devargs.watchdog_period) { 342 PMD_DRV_LOG(INFO, "Device watchdog is disabled"); 343 } else { 344 if (!adapter->vf.watchdog_enabled) { 345 PMD_DRV_LOG(INFO, "Enabling device watchdog, period is %dμs", 346 adapter->devargs.watchdog_period); 347 adapter->vf.watchdog_enabled = true; 348 if (rte_eal_alarm_set(adapter->devargs.watchdog_period, 349 &iavf_dev_watchdog, (void *)adapter)) 350 PMD_DRV_LOG(ERR, "Failed to enable device watchdog"); 351 } 352 } 353 } 354 355 void 356 iavf_dev_watchdog_disable(struct iavf_adapter *adapter) 357 { 358 if (!adapter->devargs.watchdog_period) { 359 PMD_DRV_LOG(INFO, "Device watchdog is not enabled"); 360 } else { 361 if (adapter->vf.watchdog_enabled) { 362 PMD_DRV_LOG(INFO, "Disabling device watchdog"); 363 adapter->vf.watchdog_enabled = false; 364 rte_eal_alarm_cancel(&iavf_dev_watchdog, (void *)adapter); 365 } 366 } 367 } 368 369 static int 370 iavf_set_mc_addr_list(struct rte_eth_dev *dev, 371 struct rte_ether_addr *mc_addrs, 372 uint32_t mc_addrs_num) 373 { 374 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 375 struct iavf_adapter *adapter = 376 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 377 int err, ret; 378 379 if (mc_addrs_num > IAVF_NUM_MACADDR_MAX) { 380 PMD_DRV_LOG(ERR, 381 "can't add more than a limited number (%u) of addresses.", 382 (uint32_t)IAVF_NUM_MACADDR_MAX); 383 return -EINVAL; 384 } 385 386 if (adapter->closed) 387 return -EIO; 388 389 /* flush previous addresses */ 390 err = iavf_add_del_mc_addr_list(adapter, vf->mc_addrs, vf->mc_addrs_num, 391 false); 392 if (err) 393 return err; 394 395 /* add new ones */ 396 err = iavf_add_del_mc_addr_list(adapter, mc_addrs, mc_addrs_num, true); 397 398 if (err) { 399 /* if adding mac address list fails, should add the previous 400 * addresses back. 401 */ 402 ret = iavf_add_del_mc_addr_list(adapter, vf->mc_addrs, 403 vf->mc_addrs_num, true); 404 if (ret) 405 return ret; 406 } else { 407 vf->mc_addrs_num = mc_addrs_num; 408 memcpy(vf->mc_addrs, 409 mc_addrs, mc_addrs_num * sizeof(*mc_addrs)); 410 } 411 412 return err; 413 } 414 415 static void 416 iavf_config_rss_hf(struct iavf_adapter *adapter, uint64_t rss_hf) 417 { 418 static const uint64_t map_hena_rss[] = { 419 /* IPv4 */ 420 [IAVF_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP] = 421 RTE_ETH_RSS_NONFRAG_IPV4_UDP, 422 [IAVF_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP] = 423 RTE_ETH_RSS_NONFRAG_IPV4_UDP, 424 [IAVF_FILTER_PCTYPE_NONF_IPV4_UDP] = 425 RTE_ETH_RSS_NONFRAG_IPV4_UDP, 426 [IAVF_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK] = 427 RTE_ETH_RSS_NONFRAG_IPV4_TCP, 428 [IAVF_FILTER_PCTYPE_NONF_IPV4_TCP] = 429 RTE_ETH_RSS_NONFRAG_IPV4_TCP, 430 [IAVF_FILTER_PCTYPE_NONF_IPV4_SCTP] = 431 RTE_ETH_RSS_NONFRAG_IPV4_SCTP, 432 [IAVF_FILTER_PCTYPE_NONF_IPV4_OTHER] = 433 RTE_ETH_RSS_NONFRAG_IPV4_OTHER, 434 [IAVF_FILTER_PCTYPE_FRAG_IPV4] = RTE_ETH_RSS_FRAG_IPV4, 435 436 /* IPv6 */ 437 [IAVF_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP] = 438 RTE_ETH_RSS_NONFRAG_IPV6_UDP, 439 [IAVF_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP] = 440 RTE_ETH_RSS_NONFRAG_IPV6_UDP, 441 [IAVF_FILTER_PCTYPE_NONF_IPV6_UDP] = 442 RTE_ETH_RSS_NONFRAG_IPV6_UDP, 443 [IAVF_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK] = 444 RTE_ETH_RSS_NONFRAG_IPV6_TCP, 445 [IAVF_FILTER_PCTYPE_NONF_IPV6_TCP] = 446 RTE_ETH_RSS_NONFRAG_IPV6_TCP, 447 [IAVF_FILTER_PCTYPE_NONF_IPV6_SCTP] = 448 RTE_ETH_RSS_NONFRAG_IPV6_SCTP, 449 [IAVF_FILTER_PCTYPE_NONF_IPV6_OTHER] = 450 RTE_ETH_RSS_NONFRAG_IPV6_OTHER, 451 [IAVF_FILTER_PCTYPE_FRAG_IPV6] = RTE_ETH_RSS_FRAG_IPV6, 452 453 /* L2 Payload */ 454 [IAVF_FILTER_PCTYPE_L2_PAYLOAD] = RTE_ETH_RSS_L2_PAYLOAD 455 }; 456 457 const uint64_t ipv4_rss = RTE_ETH_RSS_NONFRAG_IPV4_UDP | 458 RTE_ETH_RSS_NONFRAG_IPV4_TCP | 459 RTE_ETH_RSS_NONFRAG_IPV4_SCTP | 460 RTE_ETH_RSS_NONFRAG_IPV4_OTHER | 461 RTE_ETH_RSS_FRAG_IPV4; 462 463 const uint64_t ipv6_rss = RTE_ETH_RSS_NONFRAG_IPV6_UDP | 464 RTE_ETH_RSS_NONFRAG_IPV6_TCP | 465 RTE_ETH_RSS_NONFRAG_IPV6_SCTP | 466 RTE_ETH_RSS_NONFRAG_IPV6_OTHER | 467 RTE_ETH_RSS_FRAG_IPV6; 468 469 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 470 uint64_t caps = 0, hena = 0, valid_rss_hf = 0; 471 uint32_t i; 472 int ret; 473 474 ret = iavf_get_hena_caps(adapter, &caps); 475 if (ret) { 476 /** 477 * RSS offload type configuration is not a necessary feature 478 * for VF, so here just print a warning and return. 479 */ 480 PMD_DRV_LOG(WARNING, 481 "fail to get RSS offload type caps, ret: %d", ret); 482 return; 483 } 484 485 /** 486 * RTE_ETH_RSS_IPV4 and RTE_ETH_RSS_IPV6 can be considered as 2 487 * generalizations of all other IPv4 and IPv6 RSS types. 488 */ 489 if (rss_hf & RTE_ETH_RSS_IPV4) 490 rss_hf |= ipv4_rss; 491 492 if (rss_hf & RTE_ETH_RSS_IPV6) 493 rss_hf |= ipv6_rss; 494 495 RTE_BUILD_BUG_ON(RTE_DIM(map_hena_rss) > sizeof(uint64_t) * CHAR_BIT); 496 497 for (i = 0; i < RTE_DIM(map_hena_rss); i++) { 498 uint64_t bit = BIT_ULL(i); 499 500 if ((caps & bit) && (map_hena_rss[i] & rss_hf)) { 501 valid_rss_hf |= map_hena_rss[i]; 502 hena |= bit; 503 } 504 } 505 506 ret = iavf_set_hena(adapter, hena); 507 if (ret) { 508 /** 509 * RSS offload type configuration is not a necessary feature 510 * for VF, so here just print a warning and return. 511 */ 512 PMD_DRV_LOG(WARNING, 513 "fail to set RSS offload types, ret: %d", ret); 514 return; 515 } 516 517 if (valid_rss_hf & ipv4_rss) 518 valid_rss_hf |= rss_hf & RTE_ETH_RSS_IPV4; 519 520 if (valid_rss_hf & ipv6_rss) 521 valid_rss_hf |= rss_hf & RTE_ETH_RSS_IPV6; 522 523 if (rss_hf & ~valid_rss_hf) 524 PMD_DRV_LOG(WARNING, "Unsupported rss_hf 0x%" PRIx64, 525 rss_hf & ~valid_rss_hf); 526 527 vf->rss_hf = valid_rss_hf; 528 } 529 530 static int 531 iavf_init_rss(struct iavf_adapter *adapter) 532 { 533 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 534 struct rte_eth_rss_conf *rss_conf; 535 uint16_t i, j, nb_q; 536 int ret; 537 538 rss_conf = &adapter->dev_data->dev_conf.rx_adv_conf.rss_conf; 539 nb_q = RTE_MIN(adapter->dev_data->nb_rx_queues, 540 vf->max_rss_qregion); 541 542 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) { 543 PMD_DRV_LOG(DEBUG, "RSS is not supported"); 544 return -ENOTSUP; 545 } 546 547 /* configure RSS key */ 548 if (!rss_conf->rss_key) { 549 /* Calculate the default hash key */ 550 for (i = 0; i < vf->vf_res->rss_key_size; i++) 551 vf->rss_key[i] = (uint8_t)rte_rand(); 552 } else 553 rte_memcpy(vf->rss_key, rss_conf->rss_key, 554 RTE_MIN(rss_conf->rss_key_len, 555 vf->vf_res->rss_key_size)); 556 557 /* init RSS LUT table */ 558 for (i = 0, j = 0; i < vf->vf_res->rss_lut_size; i++, j++) { 559 if (j >= nb_q) 560 j = 0; 561 vf->rss_lut[i] = j; 562 } 563 /* send virtchnl ops to configure RSS */ 564 ret = iavf_configure_rss_lut(adapter); 565 if (ret) 566 return ret; 567 ret = iavf_configure_rss_key(adapter); 568 if (ret) 569 return ret; 570 571 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF) { 572 /* Set RSS hash configuration based on rss_conf->rss_hf. */ 573 ret = iavf_rss_hash_set(adapter, rss_conf->rss_hf, true); 574 if (ret) { 575 PMD_DRV_LOG(ERR, "fail to set default RSS"); 576 return ret; 577 } 578 } else { 579 iavf_config_rss_hf(adapter, rss_conf->rss_hf); 580 } 581 582 return 0; 583 } 584 585 static int 586 iavf_queues_req_reset(struct rte_eth_dev *dev, uint16_t num) 587 { 588 struct iavf_adapter *ad = 589 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 590 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad); 591 int ret; 592 593 ret = iavf_request_queues(dev, num); 594 if (ret) { 595 PMD_DRV_LOG(ERR, "request queues from PF failed"); 596 return ret; 597 } 598 PMD_DRV_LOG(INFO, "change queue pairs from %u to %u", 599 vf->vsi_res->num_queue_pairs, num); 600 601 iavf_dev_watchdog_disable(ad); 602 ret = iavf_dev_reset(dev); 603 if (ret) { 604 PMD_DRV_LOG(ERR, "vf reset failed"); 605 return ret; 606 } 607 608 return 0; 609 } 610 611 static int 612 iavf_dev_vlan_insert_set(struct rte_eth_dev *dev) 613 { 614 struct iavf_adapter *adapter = 615 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 616 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 617 bool enable; 618 619 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN_V2)) 620 return 0; 621 622 enable = !!(dev->data->dev_conf.txmode.offloads & 623 RTE_ETH_TX_OFFLOAD_VLAN_INSERT); 624 iavf_config_vlan_insert_v2(adapter, enable); 625 626 return 0; 627 } 628 629 static int 630 iavf_dev_init_vlan(struct rte_eth_dev *dev) 631 { 632 int err; 633 634 err = iavf_dev_vlan_offload_set(dev, 635 RTE_ETH_VLAN_STRIP_MASK | 636 RTE_ETH_QINQ_STRIP_MASK | 637 RTE_ETH_VLAN_FILTER_MASK | 638 RTE_ETH_VLAN_EXTEND_MASK); 639 if (err) { 640 PMD_DRV_LOG(INFO, 641 "VLAN offloading is not supported, or offloading was refused by the PF"); 642 return err; 643 } 644 645 err = iavf_dev_vlan_insert_set(dev); 646 if (err) 647 PMD_DRV_LOG(ERR, "Failed to update vlan insertion"); 648 649 return err; 650 } 651 652 static int 653 iavf_dev_configure(struct rte_eth_dev *dev) 654 { 655 struct iavf_adapter *ad = 656 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 657 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad); 658 uint16_t num_queue_pairs = RTE_MAX(dev->data->nb_rx_queues, 659 dev->data->nb_tx_queues); 660 int ret; 661 662 if (ad->closed) 663 return -EIO; 664 665 ad->rx_bulk_alloc_allowed = true; 666 /* Initialize to TRUE. If any of Rx queues doesn't meet the 667 * vector Rx/Tx preconditions, it will be reset. 668 */ 669 ad->rx_vec_allowed = true; 670 ad->tx_vec_allowed = true; 671 672 if (dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG) 673 dev->data->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_RSS_HASH; 674 675 /* Large VF setting */ 676 if (num_queue_pairs > IAVF_MAX_NUM_QUEUES_DFLT) { 677 if (!(vf->vf_res->vf_cap_flags & 678 VIRTCHNL_VF_LARGE_NUM_QPAIRS)) { 679 PMD_DRV_LOG(ERR, "large VF is not supported"); 680 return -1; 681 } 682 683 if (num_queue_pairs > IAVF_MAX_NUM_QUEUES_LV) { 684 PMD_DRV_LOG(ERR, "queue pairs number cannot be larger than %u", 685 IAVF_MAX_NUM_QUEUES_LV); 686 return -1; 687 } 688 689 ret = iavf_queues_req_reset(dev, num_queue_pairs); 690 if (ret) 691 return ret; 692 693 ret = iavf_get_max_rss_queue_region(ad); 694 if (ret) { 695 PMD_INIT_LOG(ERR, "get max rss queue region failed"); 696 return ret; 697 } 698 699 vf->lv_enabled = true; 700 } else { 701 /* Check if large VF is already enabled. If so, disable and 702 * release redundant queue resource. 703 * Or check if enough queue pairs. If not, request them from PF. 704 */ 705 if (vf->lv_enabled || 706 num_queue_pairs > vf->vsi_res->num_queue_pairs) { 707 ret = iavf_queues_req_reset(dev, num_queue_pairs); 708 if (ret) 709 return ret; 710 711 vf->lv_enabled = false; 712 } 713 /* if large VF is not required, use default rss queue region */ 714 vf->max_rss_qregion = IAVF_MAX_NUM_QUEUES_DFLT; 715 } 716 717 iavf_dev_init_vlan(dev); 718 719 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { 720 if (iavf_init_rss(ad) != 0) { 721 PMD_DRV_LOG(ERR, "configure rss failed"); 722 return -1; 723 } 724 } 725 return 0; 726 } 727 728 static int 729 iavf_init_rxq(struct rte_eth_dev *dev, struct iavf_rx_queue *rxq) 730 { 731 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 732 struct rte_eth_dev_data *dev_data = dev->data; 733 uint16_t buf_size, max_pkt_len; 734 uint32_t frame_size = dev->data->mtu + IAVF_ETH_OVERHEAD; 735 enum iavf_status err; 736 737 buf_size = rte_pktmbuf_data_room_size(rxq->mp) - RTE_PKTMBUF_HEADROOM; 738 739 /* Calculate the maximum packet length allowed */ 740 max_pkt_len = RTE_MIN((uint32_t) 741 rxq->rx_buf_len * IAVF_MAX_CHAINED_RX_BUFFERS, 742 frame_size); 743 744 /* Check if maximum packet length is set correctly. */ 745 if (max_pkt_len <= RTE_ETHER_MIN_LEN || 746 max_pkt_len > IAVF_FRAME_SIZE_MAX) { 747 PMD_DRV_LOG(ERR, "maximum packet length must be " 748 "larger than %u and smaller than %u", 749 (uint32_t)IAVF_ETH_MAX_LEN, 750 (uint32_t)IAVF_FRAME_SIZE_MAX); 751 return -EINVAL; 752 } 753 754 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) { 755 /* Register mbuf field and flag for Rx timestamp */ 756 err = rte_mbuf_dyn_rx_timestamp_register( 757 &iavf_timestamp_dynfield_offset, 758 &iavf_timestamp_dynflag); 759 if (err) { 760 PMD_DRV_LOG(ERR, 761 "Cannot register mbuf field/flag for timestamp"); 762 return -EINVAL; 763 } 764 } 765 766 rxq->max_pkt_len = max_pkt_len; 767 if ((dev_data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_SCATTER) || 768 rxq->max_pkt_len > buf_size) { 769 dev_data->scattered_rx = 1; 770 } 771 IAVF_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1); 772 IAVF_WRITE_FLUSH(hw); 773 774 return 0; 775 } 776 777 static int 778 iavf_init_queues(struct rte_eth_dev *dev) 779 { 780 struct iavf_rx_queue **rxq = 781 (struct iavf_rx_queue **)dev->data->rx_queues; 782 int i, ret = IAVF_SUCCESS; 783 784 for (i = 0; i < dev->data->nb_rx_queues; i++) { 785 if (!rxq[i] || !rxq[i]->q_set) 786 continue; 787 ret = iavf_init_rxq(dev, rxq[i]); 788 if (ret != IAVF_SUCCESS) 789 break; 790 } 791 /* set rx/tx function to vector/scatter/single-segment 792 * according to parameters 793 */ 794 iavf_set_rx_function(dev); 795 iavf_set_tx_function(dev); 796 797 return ret; 798 } 799 800 static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev, 801 struct rte_intr_handle *intr_handle) 802 { 803 struct iavf_adapter *adapter = 804 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 805 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 806 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter); 807 struct iavf_qv_map *qv_map; 808 uint16_t interval, i; 809 int vec; 810 811 if (rte_intr_cap_multiple(intr_handle) && 812 dev->data->dev_conf.intr_conf.rxq) { 813 if (rte_intr_efd_enable(intr_handle, dev->data->nb_rx_queues)) 814 return -1; 815 } 816 817 if (rte_intr_dp_is_en(intr_handle)) { 818 if (rte_intr_vec_list_alloc(intr_handle, "intr_vec", 819 dev->data->nb_rx_queues)) { 820 PMD_DRV_LOG(ERR, "Failed to allocate %d rx intr_vec", 821 dev->data->nb_rx_queues); 822 return -1; 823 } 824 } 825 826 827 qv_map = rte_zmalloc("qv_map", 828 dev->data->nb_rx_queues * sizeof(struct iavf_qv_map), 0); 829 if (!qv_map) { 830 PMD_DRV_LOG(ERR, "Failed to allocate %d queue-vector map", 831 dev->data->nb_rx_queues); 832 goto qv_map_alloc_err; 833 } 834 835 if (!dev->data->dev_conf.intr_conf.rxq || 836 !rte_intr_dp_is_en(intr_handle)) { 837 /* Rx interrupt disabled, Map interrupt only for writeback */ 838 vf->nb_msix = 1; 839 if (vf->vf_res->vf_cap_flags & 840 VIRTCHNL_VF_OFFLOAD_WB_ON_ITR) { 841 /* If WB_ON_ITR supports, enable it */ 842 vf->msix_base = IAVF_RX_VEC_START; 843 /* Set the ITR for index zero, to 2us to make sure that 844 * we leave time for aggregation to occur, but don't 845 * increase latency dramatically. 846 */ 847 IAVF_WRITE_REG(hw, 848 IAVF_VFINT_DYN_CTLN1(vf->msix_base - 1), 849 (0 << IAVF_VFINT_DYN_CTLN1_ITR_INDX_SHIFT) | 850 IAVF_VFINT_DYN_CTLN1_WB_ON_ITR_MASK | 851 (2UL << IAVF_VFINT_DYN_CTLN1_INTERVAL_SHIFT)); 852 /* debug - check for success! the return value 853 * should be 2, offset is 0x2800 854 */ 855 /* IAVF_READ_REG(hw, IAVF_VFINT_ITRN1(0, 0)); */ 856 } else { 857 /* If no WB_ON_ITR offload flags, need to set 858 * interrupt for descriptor write back. 859 */ 860 vf->msix_base = IAVF_MISC_VEC_ID; 861 862 /* set ITR to default */ 863 interval = iavf_calc_itr_interval( 864 IAVF_QUEUE_ITR_INTERVAL_DEFAULT); 865 IAVF_WRITE_REG(hw, IAVF_VFINT_DYN_CTL01, 866 IAVF_VFINT_DYN_CTL01_INTENA_MASK | 867 (IAVF_ITR_INDEX_DEFAULT << 868 IAVF_VFINT_DYN_CTL01_ITR_INDX_SHIFT) | 869 (interval << 870 IAVF_VFINT_DYN_CTL01_INTERVAL_SHIFT)); 871 } 872 IAVF_WRITE_FLUSH(hw); 873 /* map all queues to the same interrupt */ 874 for (i = 0; i < dev->data->nb_rx_queues; i++) { 875 qv_map[i].queue_id = i; 876 qv_map[i].vector_id = vf->msix_base; 877 } 878 vf->qv_map = qv_map; 879 } else { 880 if (!rte_intr_allow_others(intr_handle)) { 881 vf->nb_msix = 1; 882 vf->msix_base = IAVF_MISC_VEC_ID; 883 for (i = 0; i < dev->data->nb_rx_queues; i++) { 884 qv_map[i].queue_id = i; 885 qv_map[i].vector_id = vf->msix_base; 886 rte_intr_vec_list_index_set(intr_handle, 887 i, IAVF_MISC_VEC_ID); 888 } 889 vf->qv_map = qv_map; 890 PMD_DRV_LOG(DEBUG, 891 "vector %u are mapping to all Rx queues", 892 vf->msix_base); 893 } else { 894 /* If Rx interrupt is required, and we can use 895 * multi interrupts, then the vec is from 1 896 */ 897 vf->nb_msix = 898 RTE_MIN(rte_intr_nb_efd_get(intr_handle), 899 (uint16_t)(vf->vf_res->max_vectors - 1)); 900 vf->msix_base = IAVF_RX_VEC_START; 901 vec = IAVF_RX_VEC_START; 902 for (i = 0; i < dev->data->nb_rx_queues; i++) { 903 qv_map[i].queue_id = i; 904 qv_map[i].vector_id = vec; 905 rte_intr_vec_list_index_set(intr_handle, 906 i, vec++); 907 if (vec >= vf->nb_msix + IAVF_RX_VEC_START) 908 vec = IAVF_RX_VEC_START; 909 } 910 vf->qv_map = qv_map; 911 PMD_DRV_LOG(DEBUG, 912 "%u vectors are mapping to %u Rx queues", 913 vf->nb_msix, dev->data->nb_rx_queues); 914 } 915 } 916 917 if (!vf->lv_enabled) { 918 if (iavf_config_irq_map(adapter)) { 919 PMD_DRV_LOG(ERR, "config interrupt mapping failed"); 920 goto config_irq_map_err; 921 } 922 } else { 923 uint16_t num_qv_maps = dev->data->nb_rx_queues; 924 uint16_t index = 0; 925 926 while (num_qv_maps > IAVF_IRQ_MAP_NUM_PER_BUF) { 927 if (iavf_config_irq_map_lv(adapter, 928 IAVF_IRQ_MAP_NUM_PER_BUF, index)) { 929 PMD_DRV_LOG(ERR, "config interrupt mapping for large VF failed"); 930 goto config_irq_map_err; 931 } 932 num_qv_maps -= IAVF_IRQ_MAP_NUM_PER_BUF; 933 index += IAVF_IRQ_MAP_NUM_PER_BUF; 934 } 935 936 if (iavf_config_irq_map_lv(adapter, num_qv_maps, index)) { 937 PMD_DRV_LOG(ERR, "config interrupt mapping for large VF failed"); 938 goto config_irq_map_err; 939 } 940 } 941 return 0; 942 943 config_irq_map_err: 944 rte_free(vf->qv_map); 945 vf->qv_map = NULL; 946 947 qv_map_alloc_err: 948 rte_intr_vec_list_free(intr_handle); 949 950 return -1; 951 } 952 953 static int 954 iavf_start_queues(struct rte_eth_dev *dev) 955 { 956 struct iavf_rx_queue *rxq; 957 struct ci_tx_queue *txq; 958 int i; 959 uint16_t nb_txq, nb_rxq; 960 961 for (nb_txq = 0; nb_txq < dev->data->nb_tx_queues; nb_txq++) { 962 txq = dev->data->tx_queues[nb_txq]; 963 if (txq->tx_deferred_start) 964 continue; 965 if (iavf_dev_tx_queue_start(dev, nb_txq) != 0) { 966 PMD_DRV_LOG(ERR, "Fail to start tx queue %u", nb_txq); 967 goto tx_err; 968 } 969 } 970 971 for (nb_rxq = 0; nb_rxq < dev->data->nb_rx_queues; nb_rxq++) { 972 rxq = dev->data->rx_queues[nb_rxq]; 973 if (rxq->rx_deferred_start) 974 continue; 975 if (iavf_dev_rx_queue_start(dev, nb_rxq) != 0) { 976 PMD_DRV_LOG(ERR, "Fail to start rx queue %u", nb_rxq); 977 goto rx_err; 978 } 979 } 980 981 return 0; 982 983 rx_err: 984 for (i = 0; i < nb_rxq; i++) 985 iavf_dev_rx_queue_stop(dev, i); 986 tx_err: 987 for (i = 0; i < nb_txq; i++) 988 iavf_dev_tx_queue_stop(dev, i); 989 990 return -1; 991 } 992 993 static int 994 iavf_dev_start(struct rte_eth_dev *dev) 995 { 996 struct iavf_adapter *adapter = 997 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 998 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 999 struct rte_intr_handle *intr_handle = dev->intr_handle; 1000 uint16_t num_queue_pairs; 1001 uint16_t index = 0; 1002 1003 PMD_INIT_FUNC_TRACE(); 1004 1005 if (adapter->closed) 1006 return -1; 1007 1008 adapter->stopped = 0; 1009 1010 vf->max_pkt_len = dev->data->mtu + IAVF_ETH_OVERHEAD; 1011 vf->num_queue_pairs = RTE_MAX(dev->data->nb_rx_queues, 1012 dev->data->nb_tx_queues); 1013 num_queue_pairs = vf->num_queue_pairs; 1014 1015 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_QOS) 1016 if (iavf_get_qos_cap(adapter)) { 1017 PMD_INIT_LOG(ERR, "Failed to get qos capability"); 1018 return -1; 1019 } 1020 1021 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_CAP_PTP) { 1022 if (iavf_get_ptp_cap(adapter)) { 1023 PMD_INIT_LOG(ERR, "Failed to get ptp capability"); 1024 return -1; 1025 } 1026 } 1027 1028 /* Check Tx LLDP dynfield */ 1029 rte_pmd_iavf_tx_lldp_dynfield_offset = 1030 rte_mbuf_dynfield_lookup(IAVF_TX_LLDP_DYNFIELD, NULL); 1031 1032 if (iavf_init_queues(dev) != 0) { 1033 PMD_DRV_LOG(ERR, "failed to do Queue init"); 1034 return -1; 1035 } 1036 1037 if (iavf_set_vf_quanta_size(adapter, index, num_queue_pairs) != 0) 1038 PMD_DRV_LOG(WARNING, "configure quanta size failed"); 1039 1040 /* If needed, send configure queues msg multiple times to make the 1041 * adminq buffer length smaller than the 4K limitation. 1042 */ 1043 while (num_queue_pairs > IAVF_CFG_Q_NUM_PER_BUF) { 1044 if (iavf_configure_queues(adapter, 1045 IAVF_CFG_Q_NUM_PER_BUF, index) != 0) { 1046 PMD_DRV_LOG(ERR, "configure queues failed"); 1047 goto error; 1048 } 1049 num_queue_pairs -= IAVF_CFG_Q_NUM_PER_BUF; 1050 index += IAVF_CFG_Q_NUM_PER_BUF; 1051 } 1052 1053 if (iavf_configure_queues(adapter, num_queue_pairs, index) != 0) { 1054 PMD_DRV_LOG(ERR, "configure queues failed"); 1055 goto error; 1056 } 1057 1058 if (iavf_config_rx_queues_irqs(dev, intr_handle) != 0) { 1059 PMD_DRV_LOG(ERR, "configure irq failed"); 1060 goto error; 1061 } 1062 /* re-enable intr again, because efd assign may change */ 1063 if (dev->data->dev_conf.intr_conf.rxq != 0) { 1064 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_WB_ON_ITR) 1065 rte_intr_disable(intr_handle); 1066 rte_intr_enable(intr_handle); 1067 } 1068 1069 /* Set all mac addrs */ 1070 iavf_add_del_all_mac_addr(adapter, true); 1071 1072 /* Set all multicast addresses */ 1073 iavf_add_del_mc_addr_list(adapter, vf->mc_addrs, vf->mc_addrs_num, 1074 true); 1075 1076 rte_spinlock_init(&vf->phc_time_aq_lock); 1077 1078 if (iavf_start_queues(dev) != 0) { 1079 PMD_DRV_LOG(ERR, "enable queues failed"); 1080 goto error; 1081 } 1082 1083 return 0; 1084 1085 error: 1086 return -1; 1087 } 1088 1089 static int 1090 iavf_dev_stop(struct rte_eth_dev *dev) 1091 { 1092 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1093 struct iavf_adapter *adapter = 1094 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1095 struct rte_intr_handle *intr_handle = dev->intr_handle; 1096 1097 PMD_INIT_FUNC_TRACE(); 1098 1099 if (adapter->closed) 1100 return -1; 1101 1102 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_WB_ON_ITR) && 1103 dev->data->dev_conf.intr_conf.rxq != 0) 1104 rte_intr_disable(intr_handle); 1105 1106 if (adapter->stopped == 1) 1107 return 0; 1108 1109 /* Disable the interrupt for Rx */ 1110 rte_intr_efd_disable(intr_handle); 1111 /* Rx interrupt vector mapping free */ 1112 rte_intr_vec_list_free(intr_handle); 1113 1114 iavf_stop_queues(dev); 1115 1116 adapter->stopped = 1; 1117 dev->data->dev_started = 0; 1118 1119 return 0; 1120 } 1121 1122 static int 1123 iavf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 1124 { 1125 struct iavf_adapter *adapter = 1126 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1127 struct iavf_info *vf = &adapter->vf; 1128 1129 if (adapter->closed) 1130 return -EIO; 1131 1132 dev_info->max_rx_queues = IAVF_MAX_NUM_QUEUES_LV; 1133 dev_info->max_tx_queues = IAVF_MAX_NUM_QUEUES_LV; 1134 dev_info->min_rx_bufsize = IAVF_BUF_SIZE_MIN; 1135 dev_info->max_rx_pktlen = IAVF_FRAME_SIZE_MAX; 1136 dev_info->max_mtu = dev_info->max_rx_pktlen - IAVF_ETH_OVERHEAD; 1137 dev_info->min_mtu = RTE_ETHER_MIN_MTU; 1138 dev_info->hash_key_size = vf->vf_res->rss_key_size; 1139 dev_info->reta_size = vf->vf_res->rss_lut_size; 1140 dev_info->flow_type_rss_offloads = IAVF_RSS_OFFLOAD_ALL; 1141 dev_info->max_mac_addrs = IAVF_NUM_MACADDR_MAX; 1142 dev_info->dev_capa = 1143 RTE_ETH_DEV_CAPA_RUNTIME_RX_QUEUE_SETUP | 1144 RTE_ETH_DEV_CAPA_RUNTIME_TX_QUEUE_SETUP; 1145 dev_info->rx_offload_capa = 1146 RTE_ETH_RX_OFFLOAD_VLAN_STRIP | 1147 RTE_ETH_RX_OFFLOAD_QINQ_STRIP | 1148 RTE_ETH_RX_OFFLOAD_IPV4_CKSUM | 1149 RTE_ETH_RX_OFFLOAD_UDP_CKSUM | 1150 RTE_ETH_RX_OFFLOAD_TCP_CKSUM | 1151 RTE_ETH_RX_OFFLOAD_OUTER_IPV4_CKSUM | 1152 RTE_ETH_RX_OFFLOAD_SCATTER | 1153 RTE_ETH_RX_OFFLOAD_VLAN_FILTER | 1154 RTE_ETH_RX_OFFLOAD_VLAN_EXTEND | 1155 RTE_ETH_RX_OFFLOAD_RSS_HASH; 1156 1157 dev_info->tx_offload_capa = 1158 RTE_ETH_TX_OFFLOAD_VLAN_INSERT | 1159 RTE_ETH_TX_OFFLOAD_QINQ_INSERT | 1160 RTE_ETH_TX_OFFLOAD_IPV4_CKSUM | 1161 RTE_ETH_TX_OFFLOAD_UDP_CKSUM | 1162 RTE_ETH_TX_OFFLOAD_TCP_CKSUM | 1163 RTE_ETH_TX_OFFLOAD_SCTP_CKSUM | 1164 RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM | 1165 RTE_ETH_TX_OFFLOAD_TCP_TSO | 1166 RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | 1167 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | 1168 RTE_ETH_TX_OFFLOAD_IPIP_TNL_TSO | 1169 RTE_ETH_TX_OFFLOAD_GENEVE_TNL_TSO | 1170 RTE_ETH_TX_OFFLOAD_MULTI_SEGS | 1171 RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 1172 1173 /* X710 does not support outer udp checksum */ 1174 if (adapter->hw.mac.type != IAVF_MAC_XL710) 1175 dev_info->tx_offload_capa |= RTE_ETH_TX_OFFLOAD_OUTER_UDP_CKSUM; 1176 1177 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_CRC) 1178 dev_info->rx_offload_capa |= RTE_ETH_RX_OFFLOAD_KEEP_CRC; 1179 1180 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_CAP_PTP) 1181 dev_info->rx_offload_capa |= RTE_ETH_RX_OFFLOAD_TIMESTAMP; 1182 1183 if (iavf_ipsec_crypto_supported(adapter)) { 1184 dev_info->rx_offload_capa |= RTE_ETH_RX_OFFLOAD_SECURITY; 1185 dev_info->tx_offload_capa |= RTE_ETH_TX_OFFLOAD_SECURITY; 1186 } 1187 1188 dev_info->default_rxconf = (struct rte_eth_rxconf) { 1189 .rx_free_thresh = IAVF_DEFAULT_RX_FREE_THRESH, 1190 .rx_drop_en = 0, 1191 .offloads = 0, 1192 }; 1193 1194 dev_info->default_txconf = (struct rte_eth_txconf) { 1195 .tx_free_thresh = IAVF_DEFAULT_TX_FREE_THRESH, 1196 .tx_rs_thresh = IAVF_DEFAULT_TX_RS_THRESH, 1197 .offloads = 0, 1198 }; 1199 1200 dev_info->rx_desc_lim = (struct rte_eth_desc_lim) { 1201 .nb_max = IAVF_MAX_RING_DESC, 1202 .nb_min = IAVF_MIN_RING_DESC, 1203 .nb_align = IAVF_ALIGN_RING_DESC, 1204 }; 1205 1206 dev_info->tx_desc_lim = (struct rte_eth_desc_lim) { 1207 .nb_max = IAVF_MAX_RING_DESC, 1208 .nb_min = IAVF_MIN_RING_DESC, 1209 .nb_align = IAVF_ALIGN_RING_DESC, 1210 .nb_mtu_seg_max = IAVF_TX_MAX_MTU_SEG, 1211 .nb_seg_max = IAVF_MAX_RING_DESC, 1212 }; 1213 1214 dev_info->err_handle_mode = RTE_ETH_ERROR_HANDLE_MODE_PASSIVE; 1215 1216 return 0; 1217 } 1218 1219 static const uint32_t * 1220 iavf_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused, 1221 size_t *no_of_elements) 1222 { 1223 static const uint32_t ptypes[] = { 1224 RTE_PTYPE_L2_ETHER, 1225 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, 1226 RTE_PTYPE_L4_FRAG, 1227 RTE_PTYPE_L4_ICMP, 1228 RTE_PTYPE_L4_NONFRAG, 1229 RTE_PTYPE_L4_SCTP, 1230 RTE_PTYPE_L4_TCP, 1231 RTE_PTYPE_L4_UDP, 1232 }; 1233 *no_of_elements = RTE_DIM(ptypes); 1234 return ptypes; 1235 } 1236 1237 int 1238 iavf_dev_link_update(struct rte_eth_dev *dev, 1239 __rte_unused int wait_to_complete) 1240 { 1241 struct rte_eth_link new_link; 1242 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1243 1244 memset(&new_link, 0, sizeof(new_link)); 1245 1246 /* Only read status info stored in VF, and the info is updated 1247 * when receive LINK_CHANGE evnet from PF by Virtchnnl. 1248 */ 1249 switch (vf->link_speed) { 1250 case 10: 1251 new_link.link_speed = RTE_ETH_SPEED_NUM_10M; 1252 break; 1253 case 100: 1254 new_link.link_speed = RTE_ETH_SPEED_NUM_100M; 1255 break; 1256 case 1000: 1257 new_link.link_speed = RTE_ETH_SPEED_NUM_1G; 1258 break; 1259 case 10000: 1260 new_link.link_speed = RTE_ETH_SPEED_NUM_10G; 1261 break; 1262 case 20000: 1263 new_link.link_speed = RTE_ETH_SPEED_NUM_20G; 1264 break; 1265 case 25000: 1266 new_link.link_speed = RTE_ETH_SPEED_NUM_25G; 1267 break; 1268 case 40000: 1269 new_link.link_speed = RTE_ETH_SPEED_NUM_40G; 1270 break; 1271 case 50000: 1272 new_link.link_speed = RTE_ETH_SPEED_NUM_50G; 1273 break; 1274 case 100000: 1275 new_link.link_speed = RTE_ETH_SPEED_NUM_100G; 1276 break; 1277 default: 1278 new_link.link_speed = RTE_ETH_SPEED_NUM_NONE; 1279 break; 1280 } 1281 1282 new_link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX; 1283 new_link.link_status = vf->link_up ? RTE_ETH_LINK_UP : 1284 RTE_ETH_LINK_DOWN; 1285 new_link.link_autoneg = !(dev->data->dev_conf.link_speeds & 1286 RTE_ETH_LINK_SPEED_FIXED); 1287 1288 return rte_eth_linkstatus_set(dev, &new_link); 1289 } 1290 1291 static int 1292 iavf_dev_promiscuous_enable(struct rte_eth_dev *dev) 1293 { 1294 struct iavf_adapter *adapter = 1295 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1296 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1297 1298 return iavf_config_promisc(adapter, 1299 true, vf->promisc_multicast_enabled); 1300 } 1301 1302 static int 1303 iavf_dev_promiscuous_disable(struct rte_eth_dev *dev) 1304 { 1305 struct iavf_adapter *adapter = 1306 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1307 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1308 1309 return iavf_config_promisc(adapter, 1310 false, vf->promisc_multicast_enabled); 1311 } 1312 1313 static int 1314 iavf_dev_allmulticast_enable(struct rte_eth_dev *dev) 1315 { 1316 struct iavf_adapter *adapter = 1317 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1318 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1319 1320 return iavf_config_promisc(adapter, 1321 vf->promisc_unicast_enabled, true); 1322 } 1323 1324 static int 1325 iavf_dev_allmulticast_disable(struct rte_eth_dev *dev) 1326 { 1327 struct iavf_adapter *adapter = 1328 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1329 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1330 1331 return iavf_config_promisc(adapter, 1332 vf->promisc_unicast_enabled, false); 1333 } 1334 1335 static int 1336 iavf_dev_add_mac_addr(struct rte_eth_dev *dev, struct rte_ether_addr *addr, 1337 __rte_unused uint32_t index, 1338 __rte_unused uint32_t pool) 1339 { 1340 struct iavf_adapter *adapter = 1341 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1342 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1343 int err; 1344 1345 if (rte_is_zero_ether_addr(addr)) { 1346 PMD_DRV_LOG(ERR, "Invalid Ethernet Address"); 1347 return -EINVAL; 1348 } 1349 1350 err = iavf_add_del_eth_addr(adapter, addr, true, VIRTCHNL_ETHER_ADDR_EXTRA); 1351 if (err) { 1352 PMD_DRV_LOG(ERR, "fail to add MAC address"); 1353 return -EIO; 1354 } 1355 1356 vf->mac_num++; 1357 1358 return 0; 1359 } 1360 1361 static void 1362 iavf_dev_del_mac_addr(struct rte_eth_dev *dev, uint32_t index) 1363 { 1364 struct iavf_adapter *adapter = 1365 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1366 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1367 struct rte_ether_addr *addr; 1368 int err; 1369 1370 addr = &dev->data->mac_addrs[index]; 1371 1372 err = iavf_add_del_eth_addr(adapter, addr, false, VIRTCHNL_ETHER_ADDR_EXTRA); 1373 if (err) 1374 PMD_DRV_LOG(ERR, "fail to delete MAC address"); 1375 1376 vf->mac_num--; 1377 } 1378 1379 static int 1380 iavf_dev_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 1381 { 1382 struct iavf_adapter *adapter = 1383 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1384 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1385 struct rte_eth_conf *dev_conf = &dev->data->dev_conf; 1386 int err; 1387 1388 if (adapter->closed) 1389 return -EIO; 1390 1391 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN_V2) { 1392 err = iavf_add_del_vlan_v2(adapter, vlan_id, on); 1393 if (err) 1394 return -EIO; 1395 return 0; 1396 } 1397 1398 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)) 1399 return -ENOTSUP; 1400 1401 err = iavf_add_del_vlan(adapter, vlan_id, on); 1402 if (err) 1403 return -EIO; 1404 1405 /* For i40e kernel driver which only supports vlan(v1) VIRTCHNL OP, 1406 * it will set strip on when setting filter on but dpdk side will not 1407 * change strip flag. To be consistent with dpdk side, disable strip 1408 * again. 1409 * 1410 * For i40e kernel driver which supports vlan v2, dpdk will invoke vlan v2 1411 * related function, so it won't go through here. 1412 */ 1413 if (adapter->hw.mac.type == IAVF_MAC_XL710 || 1414 adapter->hw.mac.type == IAVF_MAC_X722_VF) { 1415 if (on && !(dev_conf->rxmode.offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP)) { 1416 err = iavf_disable_vlan_strip(adapter); 1417 if (err) 1418 return -EIO; 1419 } 1420 } 1421 return 0; 1422 } 1423 1424 static void 1425 iavf_iterate_vlan_filters_v2(struct rte_eth_dev *dev, bool enable) 1426 { 1427 struct rte_vlan_filter_conf *vfc = &dev->data->vlan_filter_conf; 1428 struct iavf_adapter *adapter = 1429 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1430 uint32_t i, j; 1431 uint64_t ids; 1432 1433 for (i = 0; i < RTE_DIM(vfc->ids); i++) { 1434 if (vfc->ids[i] == 0) 1435 continue; 1436 1437 ids = vfc->ids[i]; 1438 for (j = 0; ids != 0 && j < 64; j++, ids >>= 1) { 1439 if (ids & 1) 1440 iavf_add_del_vlan_v2(adapter, 1441 64 * i + j, enable); 1442 } 1443 } 1444 } 1445 1446 static int 1447 iavf_dev_vlan_offload_set_v2(struct rte_eth_dev *dev, int mask) 1448 { 1449 struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode; 1450 struct iavf_adapter *adapter = 1451 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1452 bool enable; 1453 int err; 1454 1455 if (mask & RTE_ETH_VLAN_FILTER_MASK) { 1456 enable = !!(rxmode->offloads & RTE_ETH_RX_OFFLOAD_VLAN_FILTER); 1457 1458 iavf_iterate_vlan_filters_v2(dev, enable); 1459 } 1460 1461 if (mask & RTE_ETH_VLAN_STRIP_MASK) { 1462 enable = !!(rxmode->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP); 1463 1464 err = iavf_config_vlan_strip_v2(adapter, enable); 1465 /* If not support, the stripping is already disabled by PF */ 1466 if (err == -ENOTSUP && !enable) 1467 err = 0; 1468 if (err) 1469 return -EIO; 1470 } 1471 1472 return 0; 1473 } 1474 1475 static int 1476 iavf_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask) 1477 { 1478 struct iavf_adapter *adapter = 1479 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1480 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1481 struct rte_eth_conf *dev_conf = &dev->data->dev_conf; 1482 int err; 1483 1484 if (adapter->closed) 1485 return -EIO; 1486 1487 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN_V2) 1488 return iavf_dev_vlan_offload_set_v2(dev, mask); 1489 1490 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)) 1491 return -ENOTSUP; 1492 1493 /* Vlan stripping setting */ 1494 if (mask & RTE_ETH_VLAN_STRIP_MASK) { 1495 /* Enable or disable VLAN stripping */ 1496 if (dev_conf->rxmode.offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP) 1497 err = iavf_enable_vlan_strip(adapter); 1498 else 1499 err = iavf_disable_vlan_strip(adapter); 1500 1501 if (err) 1502 return -EIO; 1503 } 1504 return 0; 1505 } 1506 1507 static int 1508 iavf_dev_rss_reta_update(struct rte_eth_dev *dev, 1509 struct rte_eth_rss_reta_entry64 *reta_conf, 1510 uint16_t reta_size) 1511 { 1512 struct iavf_adapter *adapter = 1513 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1514 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1515 uint8_t *lut; 1516 uint16_t i, idx, shift; 1517 int ret; 1518 1519 if (adapter->closed) 1520 return -EIO; 1521 1522 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) 1523 return -ENOTSUP; 1524 1525 if (reta_size != vf->vf_res->rss_lut_size) { 1526 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 1527 "(%d) doesn't match the number of hardware can " 1528 "support (%d)", reta_size, vf->vf_res->rss_lut_size); 1529 return -EINVAL; 1530 } 1531 1532 lut = rte_zmalloc("rss_lut", reta_size, 0); 1533 if (!lut) { 1534 PMD_DRV_LOG(ERR, "No memory can be allocated"); 1535 return -ENOMEM; 1536 } 1537 /* store the old lut table temporarily */ 1538 rte_memcpy(lut, vf->rss_lut, reta_size); 1539 1540 for (i = 0; i < reta_size; i++) { 1541 idx = i / RTE_ETH_RETA_GROUP_SIZE; 1542 shift = i % RTE_ETH_RETA_GROUP_SIZE; 1543 if (reta_conf[idx].mask & (1ULL << shift)) 1544 lut[i] = reta_conf[idx].reta[shift]; 1545 } 1546 1547 rte_memcpy(vf->rss_lut, lut, reta_size); 1548 /* send virtchnl ops to configure RSS */ 1549 ret = iavf_configure_rss_lut(adapter); 1550 if (ret) /* revert back */ 1551 rte_memcpy(vf->rss_lut, lut, reta_size); 1552 rte_free(lut); 1553 1554 return ret; 1555 } 1556 1557 static int 1558 iavf_dev_rss_reta_query(struct rte_eth_dev *dev, 1559 struct rte_eth_rss_reta_entry64 *reta_conf, 1560 uint16_t reta_size) 1561 { 1562 struct iavf_adapter *adapter = 1563 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1564 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1565 uint16_t i, idx, shift; 1566 1567 if (adapter->closed) 1568 return -EIO; 1569 1570 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) 1571 return -ENOTSUP; 1572 1573 if (reta_size != vf->vf_res->rss_lut_size) { 1574 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 1575 "(%d) doesn't match the number of hardware can " 1576 "support (%d)", reta_size, vf->vf_res->rss_lut_size); 1577 return -EINVAL; 1578 } 1579 1580 for (i = 0; i < reta_size; i++) { 1581 idx = i / RTE_ETH_RETA_GROUP_SIZE; 1582 shift = i % RTE_ETH_RETA_GROUP_SIZE; 1583 if (reta_conf[idx].mask & (1ULL << shift)) 1584 reta_conf[idx].reta[shift] = vf->rss_lut[i]; 1585 } 1586 1587 return 0; 1588 } 1589 1590 static int 1591 iavf_set_rss_key(struct iavf_adapter *adapter, uint8_t *key, uint8_t key_len) 1592 { 1593 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1594 1595 /* HENA setting, it is enabled by default, no change */ 1596 if (!key || key_len == 0) { 1597 PMD_DRV_LOG(DEBUG, "No key to be configured"); 1598 return 0; 1599 } else if (key_len != vf->vf_res->rss_key_size) { 1600 PMD_DRV_LOG(ERR, "The size of hash key configured " 1601 "(%d) doesn't match the size of hardware can " 1602 "support (%d)", key_len, 1603 vf->vf_res->rss_key_size); 1604 return -EINVAL; 1605 } 1606 1607 rte_memcpy(vf->rss_key, key, key_len); 1608 1609 return iavf_configure_rss_key(adapter); 1610 } 1611 1612 static int 1613 iavf_dev_rss_hash_update(struct rte_eth_dev *dev, 1614 struct rte_eth_rss_conf *rss_conf) 1615 { 1616 struct iavf_adapter *adapter = 1617 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1618 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1619 int ret; 1620 1621 adapter->dev_data->dev_conf.rx_adv_conf.rss_conf = *rss_conf; 1622 1623 if (adapter->closed) 1624 return -EIO; 1625 1626 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) 1627 return -ENOTSUP; 1628 1629 /* Set hash key. */ 1630 ret = iavf_set_rss_key(adapter, rss_conf->rss_key, 1631 rss_conf->rss_key_len); 1632 if (ret) 1633 return ret; 1634 1635 if (rss_conf->rss_hf == 0) { 1636 vf->rss_hf = 0; 1637 ret = iavf_set_hena(adapter, 0); 1638 1639 /* It is a workaround, temporarily allow error to be returned 1640 * due to possible lack of PF handling for hena = 0. 1641 */ 1642 if (ret) 1643 PMD_DRV_LOG(WARNING, "fail to clean existing RSS, lack PF support"); 1644 return 0; 1645 } 1646 1647 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF) { 1648 /* Clear existing RSS. */ 1649 ret = iavf_set_hena(adapter, 0); 1650 1651 /* It is a workaround, temporarily allow error to be returned 1652 * due to possible lack of PF handling for hena = 0. 1653 */ 1654 if (ret) 1655 PMD_DRV_LOG(WARNING, "fail to clean existing RSS," 1656 "lack PF support"); 1657 1658 /* Set new RSS configuration. */ 1659 ret = iavf_rss_hash_set(adapter, rss_conf->rss_hf, true); 1660 if (ret) { 1661 PMD_DRV_LOG(ERR, "fail to set new RSS"); 1662 return ret; 1663 } 1664 } else { 1665 iavf_config_rss_hf(adapter, rss_conf->rss_hf); 1666 } 1667 1668 return 0; 1669 } 1670 1671 static int 1672 iavf_dev_rss_hash_conf_get(struct rte_eth_dev *dev, 1673 struct rte_eth_rss_conf *rss_conf) 1674 { 1675 struct iavf_adapter *adapter = 1676 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1677 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1678 1679 if (adapter->closed) 1680 return -EIO; 1681 1682 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) 1683 return -ENOTSUP; 1684 1685 rss_conf->rss_hf = vf->rss_hf; 1686 1687 if (!rss_conf->rss_key) 1688 return 0; 1689 1690 rss_conf->rss_key_len = vf->vf_res->rss_key_size; 1691 rte_memcpy(rss_conf->rss_key, vf->rss_key, rss_conf->rss_key_len); 1692 1693 return 0; 1694 } 1695 1696 static int 1697 iavf_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu __rte_unused) 1698 { 1699 /* mtu setting is forbidden if port is start */ 1700 if (dev->data->dev_started) { 1701 PMD_DRV_LOG(ERR, "port must be stopped before configuration"); 1702 return -EBUSY; 1703 } 1704 1705 return 0; 1706 } 1707 1708 static int 1709 iavf_dev_set_default_mac_addr(struct rte_eth_dev *dev, 1710 struct rte_ether_addr *mac_addr) 1711 { 1712 struct iavf_adapter *adapter = 1713 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1714 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter); 1715 struct rte_ether_addr *old_addr; 1716 int ret; 1717 1718 old_addr = (struct rte_ether_addr *)hw->mac.addr; 1719 1720 if (rte_is_same_ether_addr(old_addr, mac_addr)) 1721 return 0; 1722 1723 ret = iavf_add_del_eth_addr(adapter, old_addr, false, VIRTCHNL_ETHER_ADDR_PRIMARY); 1724 if (ret) 1725 PMD_DRV_LOG(ERR, "Fail to delete old MAC:" 1726 RTE_ETHER_ADDR_PRT_FMT, 1727 RTE_ETHER_ADDR_BYTES(old_addr)); 1728 1729 ret = iavf_add_del_eth_addr(adapter, mac_addr, true, VIRTCHNL_ETHER_ADDR_PRIMARY); 1730 if (ret) 1731 PMD_DRV_LOG(ERR, "Fail to add new MAC:" 1732 RTE_ETHER_ADDR_PRT_FMT, 1733 RTE_ETHER_ADDR_BYTES(mac_addr)); 1734 1735 if (ret) 1736 return -EIO; 1737 1738 rte_ether_addr_copy(mac_addr, (struct rte_ether_addr *)hw->mac.addr); 1739 return 0; 1740 } 1741 1742 static void 1743 iavf_stat_update_48(uint64_t *offset, uint64_t *stat) 1744 { 1745 if (*stat >= *offset) 1746 *stat = *stat - *offset; 1747 else 1748 *stat = (uint64_t)((*stat + 1749 ((uint64_t)1 << IAVF_48_BIT_WIDTH)) - *offset); 1750 1751 *stat &= IAVF_48_BIT_MASK; 1752 } 1753 1754 static void 1755 iavf_stat_update_32(uint64_t *offset, uint64_t *stat) 1756 { 1757 if (*stat >= *offset) 1758 *stat = (uint64_t)(*stat - *offset); 1759 else 1760 *stat = (uint64_t)((*stat + 1761 ((uint64_t)1 << IAVF_32_BIT_WIDTH)) - *offset); 1762 } 1763 1764 static void 1765 iavf_update_stats(struct iavf_vsi *vsi, struct virtchnl_eth_stats *nes) 1766 { 1767 struct virtchnl_eth_stats *oes = &vsi->eth_stats_offset.eth_stats; 1768 1769 iavf_stat_update_48(&oes->rx_bytes, &nes->rx_bytes); 1770 iavf_stat_update_48(&oes->rx_unicast, &nes->rx_unicast); 1771 iavf_stat_update_48(&oes->rx_multicast, &nes->rx_multicast); 1772 iavf_stat_update_48(&oes->rx_broadcast, &nes->rx_broadcast); 1773 iavf_stat_update_32(&oes->rx_discards, &nes->rx_discards); 1774 iavf_stat_update_48(&oes->tx_bytes, &nes->tx_bytes); 1775 iavf_stat_update_48(&oes->tx_unicast, &nes->tx_unicast); 1776 iavf_stat_update_48(&oes->tx_multicast, &nes->tx_multicast); 1777 iavf_stat_update_48(&oes->tx_broadcast, &nes->tx_broadcast); 1778 iavf_stat_update_32(&oes->tx_errors, &nes->tx_errors); 1779 iavf_stat_update_32(&oes->tx_discards, &nes->tx_discards); 1780 } 1781 1782 static int 1783 iavf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 1784 { 1785 struct iavf_adapter *adapter = 1786 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1787 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1788 struct iavf_vsi *vsi = &vf->vsi; 1789 struct virtchnl_eth_stats *pstats = NULL; 1790 int ret; 1791 1792 ret = iavf_query_stats(adapter, &pstats); 1793 if (ret == 0) { 1794 uint8_t crc_stats_len = (dev->data->dev_conf.rxmode.offloads & 1795 RTE_ETH_RX_OFFLOAD_KEEP_CRC) ? 0 : 1796 RTE_ETHER_CRC_LEN; 1797 iavf_update_stats(vsi, pstats); 1798 stats->ipackets = pstats->rx_unicast + pstats->rx_multicast + 1799 pstats->rx_broadcast - pstats->rx_discards; 1800 stats->opackets = pstats->tx_broadcast + pstats->tx_multicast + 1801 pstats->tx_unicast; 1802 stats->imissed = pstats->rx_discards; 1803 stats->oerrors = pstats->tx_errors + pstats->tx_discards; 1804 stats->ibytes = pstats->rx_bytes; 1805 stats->ibytes -= stats->ipackets * crc_stats_len; 1806 stats->obytes = pstats->tx_bytes; 1807 } else { 1808 PMD_DRV_LOG(ERR, "Get statistics failed"); 1809 } 1810 return ret; 1811 } 1812 1813 static int 1814 iavf_dev_stats_reset(struct rte_eth_dev *dev) 1815 { 1816 int ret; 1817 struct iavf_adapter *adapter = 1818 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1819 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1820 struct iavf_vsi *vsi = &vf->vsi; 1821 struct virtchnl_eth_stats *pstats = NULL; 1822 1823 /* read stat values to clear hardware registers */ 1824 ret = iavf_query_stats(adapter, &pstats); 1825 if (ret != 0) 1826 return ret; 1827 1828 /* set stats offset base on current values */ 1829 vsi->eth_stats_offset.eth_stats = *pstats; 1830 1831 return 0; 1832 } 1833 1834 static int 1835 iavf_dev_xstats_reset(struct rte_eth_dev *dev) 1836 { 1837 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1838 iavf_dev_stats_reset(dev); 1839 memset(&vf->vsi.eth_stats_offset.ips_stats, 0, 1840 sizeof(struct iavf_ipsec_crypto_stats)); 1841 memset(&vf->vsi.eth_stats_offset.mbuf_stats, 0, 1842 sizeof(struct iavf_mbuf_stats)); 1843 1844 return 0; 1845 } 1846 1847 static int iavf_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 1848 struct rte_eth_xstat_name *xstats_names, 1849 __rte_unused unsigned int limit) 1850 { 1851 unsigned int i; 1852 1853 if (xstats_names != NULL) 1854 for (i = 0; i < IAVF_NB_XSTATS; i++) { 1855 snprintf(xstats_names[i].name, 1856 sizeof(xstats_names[i].name), 1857 "%s", rte_iavf_stats_strings[i].name); 1858 } 1859 return IAVF_NB_XSTATS; 1860 } 1861 1862 static void 1863 iavf_dev_update_ipsec_xstats(struct rte_eth_dev *ethdev, 1864 struct iavf_ipsec_crypto_stats *ips) 1865 { 1866 uint16_t idx; 1867 for (idx = 0; idx < ethdev->data->nb_rx_queues; idx++) { 1868 struct iavf_rx_queue *rxq; 1869 struct iavf_ipsec_crypto_stats *stats; 1870 rxq = (struct iavf_rx_queue *)ethdev->data->rx_queues[idx]; 1871 stats = &rxq->stats.ipsec_crypto; 1872 ips->icount += stats->icount; 1873 ips->ibytes += stats->ibytes; 1874 ips->ierrors.count += stats->ierrors.count; 1875 ips->ierrors.sad_miss += stats->ierrors.sad_miss; 1876 ips->ierrors.not_processed += stats->ierrors.not_processed; 1877 ips->ierrors.icv_check += stats->ierrors.icv_check; 1878 ips->ierrors.ipsec_length += stats->ierrors.ipsec_length; 1879 ips->ierrors.misc += stats->ierrors.misc; 1880 } 1881 } 1882 1883 static void 1884 iavf_dev_update_mbuf_stats(struct rte_eth_dev *ethdev, 1885 struct iavf_mbuf_stats *mbuf_stats) 1886 { 1887 uint16_t idx; 1888 struct ci_tx_queue *txq; 1889 1890 for (idx = 0; idx < ethdev->data->nb_tx_queues; idx++) { 1891 txq = ethdev->data->tx_queues[idx]; 1892 mbuf_stats->tx_pkt_errors += txq->mbuf_errors; 1893 } 1894 } 1895 1896 static int iavf_dev_xstats_get(struct rte_eth_dev *dev, 1897 struct rte_eth_xstat *xstats, unsigned int n) 1898 { 1899 int ret; 1900 unsigned int i; 1901 struct iavf_adapter *adapter = 1902 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1903 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1904 struct iavf_vsi *vsi = &vf->vsi; 1905 struct virtchnl_eth_stats *pstats = NULL; 1906 struct iavf_eth_xstats iavf_xtats = {{0}}; 1907 1908 if (n < IAVF_NB_XSTATS) 1909 return IAVF_NB_XSTATS; 1910 1911 ret = iavf_query_stats(adapter, &pstats); 1912 if (ret != 0) 1913 return 0; 1914 1915 if (!xstats) 1916 return 0; 1917 1918 iavf_update_stats(vsi, pstats); 1919 iavf_xtats.eth_stats = *pstats; 1920 1921 if (iavf_ipsec_crypto_supported(adapter)) 1922 iavf_dev_update_ipsec_xstats(dev, &iavf_xtats.ips_stats); 1923 1924 if (adapter->devargs.mbuf_check) 1925 iavf_dev_update_mbuf_stats(dev, &iavf_xtats.mbuf_stats); 1926 1927 /* loop over xstats array and values from pstats */ 1928 for (i = 0; i < IAVF_NB_XSTATS; i++) { 1929 xstats[i].id = i; 1930 xstats[i].value = *(uint64_t *)(((char *)&iavf_xtats) + 1931 rte_iavf_stats_strings[i].offset); 1932 } 1933 1934 return IAVF_NB_XSTATS; 1935 } 1936 1937 1938 static int 1939 iavf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) 1940 { 1941 struct iavf_adapter *adapter = 1942 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1943 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1944 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter); 1945 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1946 uint16_t msix_intr; 1947 1948 if (adapter->closed) 1949 return -EIO; 1950 1951 msix_intr = rte_intr_vec_list_index_get(pci_dev->intr_handle, 1952 queue_id); 1953 if (msix_intr == IAVF_MISC_VEC_ID) { 1954 PMD_DRV_LOG(INFO, "MISC is also enabled for control"); 1955 IAVF_WRITE_REG(hw, IAVF_VFINT_DYN_CTL01, 1956 IAVF_VFINT_DYN_CTL01_INTENA_MASK | 1957 IAVF_VFINT_DYN_CTL01_CLEARPBA_MASK | 1958 IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK); 1959 } else { 1960 IAVF_WRITE_REG(hw, 1961 IAVF_VFINT_DYN_CTLN1 1962 (msix_intr - IAVF_RX_VEC_START), 1963 IAVF_VFINT_DYN_CTLN1_INTENA_MASK | 1964 IAVF_VFINT_DYN_CTL01_CLEARPBA_MASK | 1965 IAVF_VFINT_DYN_CTLN1_ITR_INDX_MASK); 1966 } 1967 1968 IAVF_WRITE_FLUSH(hw); 1969 1970 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_WB_ON_ITR) 1971 rte_intr_ack(pci_dev->intr_handle); 1972 1973 return 0; 1974 } 1975 1976 static int 1977 iavf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) 1978 { 1979 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1980 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1981 uint16_t msix_intr; 1982 1983 msix_intr = rte_intr_vec_list_index_get(pci_dev->intr_handle, 1984 queue_id); 1985 if (msix_intr == IAVF_MISC_VEC_ID) { 1986 PMD_DRV_LOG(ERR, "MISC is used for control, cannot disable it"); 1987 return -EIO; 1988 } 1989 1990 IAVF_WRITE_REG(hw, 1991 IAVF_VFINT_DYN_CTLN1(msix_intr - IAVF_RX_VEC_START), 1992 IAVF_VFINT_DYN_CTLN1_WB_ON_ITR_MASK); 1993 1994 IAVF_WRITE_FLUSH(hw); 1995 return 0; 1996 } 1997 1998 static int 1999 iavf_check_vf_reset_done(struct iavf_hw *hw) 2000 { 2001 int i, reset; 2002 2003 for (i = 0; i < IAVF_RESET_WAIT_CNT; i++) { 2004 reset = IAVF_READ_REG(hw, IAVF_VFGEN_RSTAT) & 2005 IAVF_VFGEN_RSTAT_VFR_STATE_MASK; 2006 reset = reset >> IAVF_VFGEN_RSTAT_VFR_STATE_SHIFT; 2007 if (reset == VIRTCHNL_VFR_VFACTIVE || 2008 reset == VIRTCHNL_VFR_COMPLETED) 2009 break; 2010 rte_delay_ms(20); 2011 } 2012 2013 if (i >= IAVF_RESET_WAIT_CNT) 2014 return -1; 2015 2016 return 0; 2017 } 2018 2019 static int 2020 iavf_lookup_proto_xtr_type(const char *flex_name) 2021 { 2022 static struct { 2023 const char *name; 2024 enum iavf_proto_xtr_type type; 2025 } xtr_type_map[] = { 2026 { "vlan", IAVF_PROTO_XTR_VLAN }, 2027 { "ipv4", IAVF_PROTO_XTR_IPV4 }, 2028 { "ipv6", IAVF_PROTO_XTR_IPV6 }, 2029 { "ipv6_flow", IAVF_PROTO_XTR_IPV6_FLOW }, 2030 { "tcp", IAVF_PROTO_XTR_TCP }, 2031 { "ip_offset", IAVF_PROTO_XTR_IP_OFFSET }, 2032 { "ipsec_crypto_said", IAVF_PROTO_XTR_IPSEC_CRYPTO_SAID }, 2033 }; 2034 uint32_t i; 2035 2036 for (i = 0; i < RTE_DIM(xtr_type_map); i++) { 2037 if (strcmp(flex_name, xtr_type_map[i].name) == 0) 2038 return xtr_type_map[i].type; 2039 } 2040 2041 PMD_DRV_LOG(ERR, "wrong proto_xtr type, it should be: " 2042 "vlan|ipv4|ipv6|ipv6_flow|tcp|ip_offset|ipsec_crypto_said"); 2043 2044 return -1; 2045 } 2046 2047 /** 2048 * Parse elem, the elem could be single number/range or '(' ')' group 2049 * 1) A single number elem, it's just a simple digit. e.g. 9 2050 * 2) A single range elem, two digits with a '-' between. e.g. 2-6 2051 * 3) A group elem, combines multiple 1) or 2) with '( )'. e.g (0,2-4,6) 2052 * Within group elem, '-' used for a range separator; 2053 * ',' used for a single number. 2054 */ 2055 static int 2056 iavf_parse_queue_set(const char *input, int xtr_type, 2057 struct iavf_devargs *devargs) 2058 { 2059 const char *str = input; 2060 char *end = NULL; 2061 uint32_t min, max; 2062 uint32_t idx; 2063 2064 while (isblank(*str)) 2065 str++; 2066 2067 if (!isdigit(*str) && *str != '(') 2068 return -1; 2069 2070 /* process single number or single range of number */ 2071 if (*str != '(') { 2072 errno = 0; 2073 idx = strtoul(str, &end, 10); 2074 if (errno || !end || idx >= IAVF_MAX_QUEUE_NUM) 2075 return -1; 2076 2077 while (isblank(*end)) 2078 end++; 2079 2080 min = idx; 2081 max = idx; 2082 2083 /* process single <number>-<number> */ 2084 if (*end == '-') { 2085 end++; 2086 while (isblank(*end)) 2087 end++; 2088 if (!isdigit(*end)) 2089 return -1; 2090 2091 errno = 0; 2092 idx = strtoul(end, &end, 10); 2093 if (errno || !end || idx >= IAVF_MAX_QUEUE_NUM) 2094 return -1; 2095 2096 max = idx; 2097 while (isblank(*end)) 2098 end++; 2099 } 2100 2101 if (*end != ':') 2102 return -1; 2103 2104 for (idx = RTE_MIN(min, max); 2105 idx <= RTE_MAX(min, max); idx++) 2106 devargs->proto_xtr[idx] = xtr_type; 2107 2108 return 0; 2109 } 2110 2111 /* process set within bracket */ 2112 str++; 2113 while (isblank(*str)) 2114 str++; 2115 if (*str == '\0') 2116 return -1; 2117 2118 min = IAVF_MAX_QUEUE_NUM; 2119 do { 2120 /* go ahead to the first digit */ 2121 while (isblank(*str)) 2122 str++; 2123 if (!isdigit(*str)) 2124 return -1; 2125 2126 /* get the digit value */ 2127 errno = 0; 2128 idx = strtoul(str, &end, 10); 2129 if (errno || !end || idx >= IAVF_MAX_QUEUE_NUM) 2130 return -1; 2131 2132 /* go ahead to separator '-',',' and ')' */ 2133 while (isblank(*end)) 2134 end++; 2135 if (*end == '-') { 2136 if (min == IAVF_MAX_QUEUE_NUM) 2137 min = idx; 2138 else /* avoid continuous '-' */ 2139 return -1; 2140 } else if (*end == ',' || *end == ')') { 2141 max = idx; 2142 if (min == IAVF_MAX_QUEUE_NUM) 2143 min = idx; 2144 2145 for (idx = RTE_MIN(min, max); 2146 idx <= RTE_MAX(min, max); idx++) 2147 devargs->proto_xtr[idx] = xtr_type; 2148 2149 min = IAVF_MAX_QUEUE_NUM; 2150 } else { 2151 return -1; 2152 } 2153 2154 str = end + 1; 2155 } while (*end != ')' && *end != '\0'); 2156 2157 return 0; 2158 } 2159 2160 static int 2161 iavf_parse_queue_proto_xtr(const char *queues, struct iavf_devargs *devargs) 2162 { 2163 const char *queue_start; 2164 uint32_t idx; 2165 int xtr_type; 2166 char flex_name[32]; 2167 2168 while (isblank(*queues)) 2169 queues++; 2170 2171 if (*queues != '[') { 2172 xtr_type = iavf_lookup_proto_xtr_type(queues); 2173 if (xtr_type < 0) 2174 return -1; 2175 2176 devargs->proto_xtr_dflt = xtr_type; 2177 2178 return 0; 2179 } 2180 2181 queues++; 2182 do { 2183 while (isblank(*queues)) 2184 queues++; 2185 if (*queues == '\0') 2186 return -1; 2187 2188 queue_start = queues; 2189 2190 /* go across a complete bracket */ 2191 if (*queue_start == '(') { 2192 queues += strcspn(queues, ")"); 2193 if (*queues != ')') 2194 return -1; 2195 } 2196 2197 /* scan the separator ':' */ 2198 queues += strcspn(queues, ":"); 2199 if (*queues++ != ':') 2200 return -1; 2201 while (isblank(*queues)) 2202 queues++; 2203 2204 for (idx = 0; ; idx++) { 2205 if (isblank(queues[idx]) || 2206 queues[idx] == ',' || 2207 queues[idx] == ']' || 2208 queues[idx] == '\0') 2209 break; 2210 2211 if (idx > sizeof(flex_name) - 2) 2212 return -1; 2213 2214 flex_name[idx] = queues[idx]; 2215 } 2216 flex_name[idx] = '\0'; 2217 xtr_type = iavf_lookup_proto_xtr_type(flex_name); 2218 if (xtr_type < 0) 2219 return -1; 2220 2221 queues += idx; 2222 2223 while (isblank(*queues) || *queues == ',' || *queues == ']') 2224 queues++; 2225 2226 if (iavf_parse_queue_set(queue_start, xtr_type, devargs) < 0) 2227 return -1; 2228 } while (*queues != '\0'); 2229 2230 return 0; 2231 } 2232 2233 static int 2234 iavf_handle_proto_xtr_arg(__rte_unused const char *key, const char *value, 2235 void *extra_args) 2236 { 2237 struct iavf_devargs *devargs = extra_args; 2238 2239 if (!value || !extra_args) 2240 return -EINVAL; 2241 2242 if (iavf_parse_queue_proto_xtr(value, devargs) < 0) { 2243 PMD_DRV_LOG(ERR, "the proto_xtr's parameter is wrong : '%s'", 2244 value); 2245 return -1; 2246 } 2247 2248 return 0; 2249 } 2250 2251 static int 2252 parse_u16(__rte_unused const char *key, const char *value, void *args) 2253 { 2254 u16 *num = (u16 *)args; 2255 u16 tmp; 2256 2257 errno = 0; 2258 tmp = strtoull(value, NULL, 10); 2259 if (errno || !tmp) { 2260 PMD_DRV_LOG(WARNING, "%s: \"%s\" is not a valid u16", 2261 key, value); 2262 return -1; 2263 } 2264 2265 *num = tmp; 2266 2267 return 0; 2268 } 2269 2270 static int 2271 parse_bool(const char *key, const char *value, void *args) 2272 { 2273 int *i = (int *)args; 2274 char *end; 2275 int num; 2276 2277 num = strtoul(value, &end, 10); 2278 2279 if (num != 0 && num != 1) { 2280 PMD_DRV_LOG(WARNING, "invalid value:\"%s\" for key:\"%s\", " 2281 "value must be 0 or 1", 2282 value, key); 2283 return -1; 2284 } 2285 2286 *i = num; 2287 return 0; 2288 } 2289 2290 static int 2291 iavf_parse_watchdog_period(__rte_unused const char *key, const char *value, void *args) 2292 { 2293 int *num = (int *)args; 2294 int tmp; 2295 2296 errno = 0; 2297 tmp = atoi(value); 2298 if (tmp < 0) { 2299 PMD_DRV_LOG(WARNING, "%s: \"%s\" is not greater than or equal to zero", 2300 key, value); 2301 return -1; 2302 } 2303 2304 *num = tmp; 2305 2306 return 0; 2307 } 2308 2309 static int 2310 iavf_parse_mbuf_check(__rte_unused const char *key, const char *value, void *args) 2311 { 2312 char *cur; 2313 char *tmp; 2314 int str_len; 2315 int valid_len; 2316 int ret = 0; 2317 uint64_t *mc_flags = args; 2318 char *str2 = strdup(value); 2319 2320 if (str2 == NULL) 2321 return -1; 2322 2323 str_len = strlen(str2); 2324 if (str_len == 0) { 2325 ret = -1; 2326 goto err_end; 2327 } 2328 2329 /* Try stripping the outer square brackets of the parameter string. */ 2330 if (str2[0] == '[' && str2[str_len - 1] == ']') { 2331 if (str_len < 3) { 2332 ret = -1; 2333 goto err_end; 2334 } 2335 valid_len = str_len - 2; 2336 memmove(str2, str2 + 1, valid_len); 2337 memset(str2 + valid_len, '\0', 2); 2338 } 2339 2340 cur = strtok_r(str2, ",", &tmp); 2341 while (cur != NULL) { 2342 if (!strcmp(cur, "mbuf")) 2343 *mc_flags |= IAVF_MBUF_CHECK_F_TX_MBUF; 2344 else if (!strcmp(cur, "size")) 2345 *mc_flags |= IAVF_MBUF_CHECK_F_TX_SIZE; 2346 else if (!strcmp(cur, "segment")) 2347 *mc_flags |= IAVF_MBUF_CHECK_F_TX_SEGMENT; 2348 else if (!strcmp(cur, "offload")) 2349 *mc_flags |= IAVF_MBUF_CHECK_F_TX_OFFLOAD; 2350 else 2351 PMD_DRV_LOG(ERR, "Unsupported diagnostic type: %s", cur); 2352 cur = strtok_r(NULL, ",", &tmp); 2353 } 2354 2355 err_end: 2356 free(str2); 2357 return ret; 2358 } 2359 2360 static int iavf_parse_devargs(struct rte_eth_dev *dev) 2361 { 2362 struct iavf_adapter *ad = 2363 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 2364 struct rte_devargs *devargs = dev->device->devargs; 2365 struct rte_kvargs *kvlist; 2366 int ret; 2367 int watchdog_period = -1; 2368 2369 if (!devargs) 2370 return 0; 2371 2372 kvlist = rte_kvargs_parse(devargs->args, iavf_valid_args); 2373 if (!kvlist) { 2374 PMD_INIT_LOG(ERR, "invalid kvargs key"); 2375 return -EINVAL; 2376 } 2377 2378 ad->devargs.proto_xtr_dflt = IAVF_PROTO_XTR_NONE; 2379 memset(ad->devargs.proto_xtr, IAVF_PROTO_XTR_NONE, 2380 sizeof(ad->devargs.proto_xtr)); 2381 2382 ret = rte_kvargs_process(kvlist, IAVF_PROTO_XTR_ARG, 2383 &iavf_handle_proto_xtr_arg, &ad->devargs); 2384 if (ret) 2385 goto bail; 2386 2387 ret = rte_kvargs_process(kvlist, IAVF_QUANTA_SIZE_ARG, 2388 &parse_u16, &ad->devargs.quanta_size); 2389 if (ret) 2390 goto bail; 2391 2392 ret = rte_kvargs_process(kvlist, IAVF_RESET_WATCHDOG_ARG, 2393 &iavf_parse_watchdog_period, &watchdog_period); 2394 if (ret) 2395 goto bail; 2396 if (watchdog_period == -1) 2397 ad->devargs.watchdog_period = IAVF_DEV_WATCHDOG_PERIOD; 2398 else 2399 ad->devargs.watchdog_period = watchdog_period; 2400 2401 ret = rte_kvargs_process(kvlist, IAVF_NO_POLL_ON_LINK_DOWN_ARG, 2402 &parse_bool, &ad->devargs.no_poll_on_link_down); 2403 if (ret) 2404 goto bail; 2405 2406 if (ad->devargs.quanta_size != 0 && 2407 (ad->devargs.quanta_size < 256 || ad->devargs.quanta_size > 4096 || 2408 ad->devargs.quanta_size & 0x40)) { 2409 PMD_INIT_LOG(ERR, "invalid quanta size"); 2410 ret = -EINVAL; 2411 goto bail; 2412 } 2413 2414 ret = rte_kvargs_process(kvlist, IAVF_MBUF_CHECK_ARG, 2415 &iavf_parse_mbuf_check, &ad->devargs.mbuf_check); 2416 if (ret) 2417 goto bail; 2418 2419 ret = rte_kvargs_process(kvlist, IAVF_ENABLE_AUTO_RESET_ARG, 2420 &parse_bool, &ad->devargs.auto_reset); 2421 if (ret) 2422 goto bail; 2423 2424 if (ad->devargs.auto_reset != 0) 2425 ad->devargs.no_poll_on_link_down = 1; 2426 2427 bail: 2428 rte_kvargs_free(kvlist); 2429 return ret; 2430 } 2431 2432 static void 2433 iavf_init_proto_xtr(struct rte_eth_dev *dev) 2434 { 2435 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 2436 struct iavf_adapter *ad = 2437 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 2438 const struct iavf_proto_xtr_ol *xtr_ol; 2439 bool proto_xtr_enable = false; 2440 int offset; 2441 uint16_t i; 2442 2443 vf->proto_xtr = rte_zmalloc("vf proto xtr", 2444 vf->vsi_res->num_queue_pairs, 0); 2445 if (unlikely(!(vf->proto_xtr))) { 2446 PMD_DRV_LOG(ERR, "no memory for setting up proto_xtr's table"); 2447 return; 2448 } 2449 2450 for (i = 0; i < vf->vsi_res->num_queue_pairs; i++) { 2451 vf->proto_xtr[i] = ad->devargs.proto_xtr[i] != 2452 IAVF_PROTO_XTR_NONE ? 2453 ad->devargs.proto_xtr[i] : 2454 ad->devargs.proto_xtr_dflt; 2455 2456 if (vf->proto_xtr[i] != IAVF_PROTO_XTR_NONE) { 2457 uint8_t type = vf->proto_xtr[i]; 2458 2459 iavf_proto_xtr_params[type].required = true; 2460 proto_xtr_enable = true; 2461 } 2462 } 2463 2464 if (likely(!proto_xtr_enable)) 2465 return; 2466 2467 offset = rte_mbuf_dynfield_register(&iavf_proto_xtr_metadata_param); 2468 if (unlikely(offset == -1)) { 2469 PMD_DRV_LOG(ERR, 2470 "failed to extract protocol metadata, error %d", 2471 -rte_errno); 2472 return; 2473 } 2474 2475 PMD_DRV_LOG(DEBUG, 2476 "proto_xtr metadata offset in mbuf is : %d", 2477 offset); 2478 rte_pmd_ifd_dynfield_proto_xtr_metadata_offs = offset; 2479 2480 for (i = 0; i < RTE_DIM(iavf_proto_xtr_params); i++) { 2481 xtr_ol = &iavf_proto_xtr_params[i]; 2482 2483 uint8_t rxdid = iavf_proto_xtr_type_to_rxdid((uint8_t)i); 2484 2485 if (!xtr_ol->required) 2486 continue; 2487 2488 if (!(vf->supported_rxdid & BIT(rxdid))) { 2489 PMD_DRV_LOG(ERR, 2490 "rxdid[%u] is not supported in hardware", 2491 rxdid); 2492 rte_pmd_ifd_dynfield_proto_xtr_metadata_offs = -1; 2493 break; 2494 } 2495 2496 offset = rte_mbuf_dynflag_register(&xtr_ol->param); 2497 if (unlikely(offset == -1)) { 2498 PMD_DRV_LOG(ERR, 2499 "failed to register proto_xtr offload '%s', error %d", 2500 xtr_ol->param.name, -rte_errno); 2501 2502 rte_pmd_ifd_dynfield_proto_xtr_metadata_offs = -1; 2503 break; 2504 } 2505 2506 PMD_DRV_LOG(DEBUG, 2507 "proto_xtr offload '%s' offset in mbuf is : %d", 2508 xtr_ol->param.name, offset); 2509 *xtr_ol->ol_flag = 1ULL << offset; 2510 } 2511 } 2512 2513 static int 2514 iavf_init_vf(struct rte_eth_dev *dev) 2515 { 2516 int err, bufsz; 2517 struct iavf_adapter *adapter = 2518 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 2519 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2520 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 2521 2522 vf->eth_dev = dev; 2523 2524 err = iavf_parse_devargs(dev); 2525 if (err) { 2526 PMD_INIT_LOG(ERR, "Failed to parse devargs"); 2527 goto err; 2528 } 2529 2530 err = iavf_set_mac_type(hw); 2531 if (err) { 2532 PMD_INIT_LOG(ERR, "set_mac_type failed: %d", err); 2533 goto err; 2534 } 2535 2536 err = iavf_check_vf_reset_done(hw); 2537 if (err) { 2538 PMD_INIT_LOG(ERR, "VF is still resetting"); 2539 goto err; 2540 } 2541 2542 iavf_init_adminq_parameter(hw); 2543 err = iavf_init_adminq(hw); 2544 if (err) { 2545 PMD_INIT_LOG(ERR, "init_adminq failed: %d", err); 2546 goto err; 2547 } 2548 2549 vf->aq_resp = rte_zmalloc("vf_aq_resp", IAVF_AQ_BUF_SZ, 0); 2550 if (!vf->aq_resp) { 2551 PMD_INIT_LOG(ERR, "unable to allocate vf_aq_resp memory"); 2552 goto err_aq; 2553 } 2554 if (iavf_check_api_version(adapter) != 0) { 2555 PMD_INIT_LOG(ERR, "check_api version failed"); 2556 goto err_api; 2557 } 2558 2559 bufsz = sizeof(struct virtchnl_vf_resource) + 2560 (IAVF_MAX_VF_VSI * sizeof(struct virtchnl_vsi_resource)); 2561 vf->vf_res = rte_zmalloc("vf_res", bufsz, 0); 2562 if (!vf->vf_res) { 2563 PMD_INIT_LOG(ERR, "unable to allocate vf_res memory"); 2564 goto err_api; 2565 } 2566 2567 if (iavf_get_vf_resource(adapter) != 0) { 2568 PMD_INIT_LOG(ERR, "iavf_get_vf_config failed"); 2569 goto err_alloc; 2570 } 2571 /* Allocate memort for RSS info */ 2572 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { 2573 vf->rss_key = rte_zmalloc("rss_key", 2574 vf->vf_res->rss_key_size, 0); 2575 if (!vf->rss_key) { 2576 PMD_INIT_LOG(ERR, "unable to allocate rss_key memory"); 2577 goto err_rss; 2578 } 2579 vf->rss_lut = rte_zmalloc("rss_lut", 2580 vf->vf_res->rss_lut_size, 0); 2581 if (!vf->rss_lut) { 2582 PMD_INIT_LOG(ERR, "unable to allocate rss_lut memory"); 2583 goto err_rss; 2584 } 2585 } 2586 2587 if (vf->vsi_res->num_queue_pairs > IAVF_MAX_NUM_QUEUES_DFLT) 2588 vf->lv_enabled = true; 2589 2590 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) { 2591 if (iavf_get_supported_rxdid(adapter) != 0) { 2592 PMD_INIT_LOG(ERR, "failed to do get supported rxdid"); 2593 goto err_rss; 2594 } 2595 } 2596 2597 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN_V2) { 2598 if (iavf_get_vlan_offload_caps_v2(adapter) != 0) { 2599 PMD_INIT_LOG(ERR, "failed to do get VLAN offload v2 capabilities"); 2600 goto err_rss; 2601 } 2602 } 2603 2604 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_QOS) { 2605 bufsz = sizeof(struct virtchnl_qos_cap_list) + 2606 IAVF_MAX_TRAFFIC_CLASS * 2607 sizeof(struct virtchnl_qos_cap_elem); 2608 vf->qos_cap = rte_zmalloc("qos_cap", bufsz, 0); 2609 if (!vf->qos_cap) { 2610 PMD_INIT_LOG(ERR, "unable to allocate qos_cap memory"); 2611 goto err_rss; 2612 } 2613 iavf_tm_conf_init(dev); 2614 } 2615 2616 iavf_init_proto_xtr(dev); 2617 2618 return 0; 2619 err_rss: 2620 rte_free(vf->rss_key); 2621 rte_free(vf->rss_lut); 2622 err_alloc: 2623 rte_free(vf->qos_cap); 2624 rte_free(vf->vf_res); 2625 vf->vsi_res = NULL; 2626 err_api: 2627 rte_free(vf->aq_resp); 2628 err_aq: 2629 iavf_shutdown_adminq(hw); 2630 err: 2631 return -1; 2632 } 2633 2634 static void 2635 iavf_uninit_vf(struct rte_eth_dev *dev) 2636 { 2637 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2638 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 2639 2640 iavf_shutdown_adminq(hw); 2641 2642 rte_free(vf->vf_res); 2643 vf->vsi_res = NULL; 2644 vf->vf_res = NULL; 2645 2646 rte_free(vf->aq_resp); 2647 vf->aq_resp = NULL; 2648 2649 rte_free(vf->qos_cap); 2650 vf->qos_cap = NULL; 2651 2652 rte_free(vf->rss_lut); 2653 vf->rss_lut = NULL; 2654 rte_free(vf->rss_key); 2655 vf->rss_key = NULL; 2656 } 2657 2658 /* Enable default admin queue interrupt setting */ 2659 static inline void 2660 iavf_enable_irq0(struct iavf_hw *hw) 2661 { 2662 /* Enable admin queue interrupt trigger */ 2663 IAVF_WRITE_REG(hw, IAVF_VFINT_ICR0_ENA1, 2664 IAVF_VFINT_ICR0_ENA1_ADMINQ_MASK); 2665 2666 IAVF_WRITE_REG(hw, IAVF_VFINT_DYN_CTL01, 2667 IAVF_VFINT_DYN_CTL01_INTENA_MASK | 2668 IAVF_VFINT_DYN_CTL01_CLEARPBA_MASK | 2669 IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK); 2670 2671 IAVF_WRITE_FLUSH(hw); 2672 } 2673 2674 static inline void 2675 iavf_disable_irq0(struct iavf_hw *hw) 2676 { 2677 /* Disable all interrupt types */ 2678 IAVF_WRITE_REG(hw, IAVF_VFINT_ICR0_ENA1, 0); 2679 IAVF_WRITE_REG(hw, IAVF_VFINT_DYN_CTL01, 2680 IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK); 2681 IAVF_WRITE_FLUSH(hw); 2682 } 2683 2684 static void 2685 iavf_dev_interrupt_handler(void *param) 2686 { 2687 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 2688 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2689 2690 iavf_disable_irq0(hw); 2691 2692 iavf_handle_virtchnl_msg(dev); 2693 2694 iavf_enable_irq0(hw); 2695 } 2696 2697 void 2698 iavf_dev_alarm_handler(void *param) 2699 { 2700 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 2701 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2702 uint32_t icr0; 2703 2704 iavf_disable_irq0(hw); 2705 2706 /* read out interrupt causes */ 2707 icr0 = IAVF_READ_REG(hw, IAVF_VFINT_ICR01); 2708 2709 if (icr0 & IAVF_VFINT_ICR01_ADMINQ_MASK) { 2710 PMD_DRV_LOG(DEBUG, "ICR01_ADMINQ is reported"); 2711 iavf_handle_virtchnl_msg(dev); 2712 } 2713 2714 iavf_enable_irq0(hw); 2715 2716 rte_eal_alarm_set(IAVF_ALARM_INTERVAL, 2717 iavf_dev_alarm_handler, dev); 2718 } 2719 2720 static int 2721 iavf_dev_flow_ops_get(struct rte_eth_dev *dev, 2722 const struct rte_flow_ops **ops) 2723 { 2724 struct iavf_adapter *adapter = 2725 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 2726 2727 if (adapter->closed) 2728 return -EIO; 2729 2730 *ops = &iavf_flow_ops; 2731 return 0; 2732 } 2733 2734 static void 2735 iavf_default_rss_disable(struct iavf_adapter *adapter) 2736 { 2737 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 2738 int ret = 0; 2739 2740 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { 2741 /* Set hena = 0 to ask PF to cleanup all existing RSS. */ 2742 ret = iavf_set_hena(adapter, 0); 2743 if (ret) 2744 /* It is a workaround, temporarily allow error to be 2745 * returned due to possible lack of PF handling for 2746 * hena = 0. 2747 */ 2748 PMD_INIT_LOG(WARNING, "fail to disable default RSS," 2749 "lack PF support"); 2750 } 2751 } 2752 2753 static int 2754 iavf_dev_init(struct rte_eth_dev *eth_dev) 2755 { 2756 struct iavf_adapter *adapter = 2757 IAVF_DEV_PRIVATE_TO_ADAPTER(eth_dev->data->dev_private); 2758 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter); 2759 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 2760 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 2761 int ret = 0; 2762 2763 PMD_INIT_FUNC_TRACE(); 2764 2765 /* assign ops func pointer */ 2766 eth_dev->dev_ops = &iavf_eth_dev_ops; 2767 eth_dev->rx_queue_count = iavf_dev_rxq_count; 2768 eth_dev->rx_descriptor_status = iavf_dev_rx_desc_status; 2769 eth_dev->tx_descriptor_status = iavf_dev_tx_desc_status; 2770 eth_dev->rx_pkt_burst = &iavf_recv_pkts; 2771 eth_dev->tx_pkt_burst = &iavf_xmit_pkts; 2772 eth_dev->tx_pkt_prepare = &iavf_prep_pkts; 2773 2774 /* For secondary processes, we don't initialise any further as primary 2775 * has already done this work. Only check if we need a different RX 2776 * and TX function. 2777 */ 2778 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 2779 iavf_set_rx_function(eth_dev); 2780 iavf_set_tx_function(eth_dev); 2781 return 0; 2782 } 2783 rte_eth_copy_pci_info(eth_dev, pci_dev); 2784 2785 hw->vendor_id = pci_dev->id.vendor_id; 2786 hw->device_id = pci_dev->id.device_id; 2787 hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id; 2788 hw->subsystem_device_id = pci_dev->id.subsystem_device_id; 2789 hw->bus.bus_id = pci_dev->addr.bus; 2790 hw->bus.device = pci_dev->addr.devid; 2791 hw->bus.func = pci_dev->addr.function; 2792 hw->hw_addr = (void *)pci_dev->mem_resource[0].addr; 2793 hw->back = IAVF_DEV_PRIVATE_TO_ADAPTER(eth_dev->data->dev_private); 2794 adapter->dev_data = eth_dev->data; 2795 adapter->stopped = 1; 2796 2797 if (iavf_dev_event_handler_init()) 2798 goto init_vf_err; 2799 2800 if (iavf_init_vf(eth_dev) != 0) { 2801 PMD_INIT_LOG(ERR, "Init vf failed"); 2802 return -1; 2803 } 2804 2805 /* set default ptype table */ 2806 iavf_set_default_ptype_table(eth_dev); 2807 2808 /* copy mac addr */ 2809 eth_dev->data->mac_addrs = rte_zmalloc( 2810 "iavf_mac", RTE_ETHER_ADDR_LEN * IAVF_NUM_MACADDR_MAX, 0); 2811 if (!eth_dev->data->mac_addrs) { 2812 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to" 2813 " store MAC addresses", 2814 RTE_ETHER_ADDR_LEN * IAVF_NUM_MACADDR_MAX); 2815 ret = -ENOMEM; 2816 goto init_vf_err; 2817 } 2818 /* If the MAC address is not configured by host, 2819 * generate a random one. 2820 */ 2821 if (!rte_is_valid_assigned_ether_addr( 2822 (struct rte_ether_addr *)hw->mac.addr)) 2823 rte_eth_random_addr(hw->mac.addr); 2824 rte_ether_addr_copy((struct rte_ether_addr *)hw->mac.addr, 2825 ð_dev->data->mac_addrs[0]); 2826 2827 2828 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_WB_ON_ITR) { 2829 /* register callback func to eal lib */ 2830 rte_intr_callback_register(pci_dev->intr_handle, 2831 iavf_dev_interrupt_handler, 2832 (void *)eth_dev); 2833 2834 /* enable uio intr after callback register */ 2835 rte_intr_enable(pci_dev->intr_handle); 2836 } else { 2837 rte_eal_alarm_set(IAVF_ALARM_INTERVAL, 2838 iavf_dev_alarm_handler, eth_dev); 2839 } 2840 2841 /* configure and enable device interrupt */ 2842 iavf_enable_irq0(hw); 2843 2844 ret = iavf_flow_init(adapter); 2845 if (ret) { 2846 PMD_INIT_LOG(ERR, "Failed to initialize flow"); 2847 goto flow_init_err; 2848 } 2849 2850 /** Check if the IPsec Crypto offload is supported and create 2851 * security_ctx if it is. 2852 */ 2853 if (iavf_ipsec_crypto_supported(adapter)) { 2854 /* Initialize security_ctx only for primary process*/ 2855 ret = iavf_security_ctx_create(adapter); 2856 if (ret) { 2857 PMD_INIT_LOG(ERR, "failed to create ipsec crypto security instance"); 2858 goto flow_init_err; 2859 } 2860 2861 ret = iavf_security_init(adapter); 2862 if (ret) { 2863 PMD_INIT_LOG(ERR, "failed to initialized ipsec crypto resources"); 2864 goto security_init_err; 2865 } 2866 } 2867 2868 iavf_default_rss_disable(adapter); 2869 2870 iavf_dev_stats_reset(eth_dev); 2871 2872 /* Start device watchdog */ 2873 iavf_dev_watchdog_enable(adapter); 2874 adapter->closed = false; 2875 2876 return 0; 2877 2878 security_init_err: 2879 iavf_security_ctx_destroy(adapter); 2880 2881 flow_init_err: 2882 iavf_disable_irq0(hw); 2883 2884 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_WB_ON_ITR) { 2885 /* disable uio intr before callback unregiser */ 2886 rte_intr_disable(pci_dev->intr_handle); 2887 2888 /* unregister callback func from eal lib */ 2889 rte_intr_callback_unregister(pci_dev->intr_handle, 2890 iavf_dev_interrupt_handler, eth_dev); 2891 } else { 2892 rte_eal_alarm_cancel(iavf_dev_alarm_handler, eth_dev); 2893 } 2894 2895 rte_free(eth_dev->data->mac_addrs); 2896 eth_dev->data->mac_addrs = NULL; 2897 2898 init_vf_err: 2899 iavf_uninit_vf(eth_dev); 2900 2901 return ret; 2902 } 2903 2904 static int 2905 iavf_dev_close(struct rte_eth_dev *dev) 2906 { 2907 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2908 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2909 struct rte_intr_handle *intr_handle = pci_dev->intr_handle; 2910 struct iavf_adapter *adapter = 2911 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 2912 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 2913 int ret; 2914 2915 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 2916 return 0; 2917 2918 if (adapter->closed) { 2919 ret = 0; 2920 goto out; 2921 } 2922 2923 ret = iavf_dev_stop(dev); 2924 2925 /* 2926 * Release redundant queue resource when close the dev 2927 * so that other vfs can re-use the queues. 2928 */ 2929 if (vf->lv_enabled) { 2930 ret = iavf_request_queues(dev, IAVF_MAX_NUM_QUEUES_DFLT); 2931 if (ret) 2932 PMD_DRV_LOG(ERR, "Reset the num of queues failed"); 2933 2934 vf->max_rss_qregion = IAVF_MAX_NUM_QUEUES_DFLT; 2935 } 2936 2937 adapter->closed = true; 2938 2939 /* free iAVF security device context all related resources */ 2940 iavf_security_ctx_destroy(adapter); 2941 2942 iavf_flow_flush(dev, NULL); 2943 iavf_flow_uninit(adapter); 2944 2945 /* 2946 * disable promiscuous mode before reset vf 2947 * it is a workaround solution when work with kernel driver 2948 * and it is not the normal way 2949 */ 2950 if (vf->promisc_unicast_enabled || vf->promisc_multicast_enabled) 2951 iavf_config_promisc(adapter, false, false); 2952 2953 iavf_vf_reset(hw); 2954 iavf_shutdown_adminq(hw); 2955 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_WB_ON_ITR) { 2956 /* disable uio intr before callback unregister */ 2957 rte_intr_disable(intr_handle); 2958 2959 /* unregister callback func from eal lib */ 2960 rte_intr_callback_unregister(intr_handle, 2961 iavf_dev_interrupt_handler, dev); 2962 } else { 2963 rte_eal_alarm_cancel(iavf_dev_alarm_handler, dev); 2964 } 2965 iavf_disable_irq0(hw); 2966 2967 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_QOS) 2968 iavf_tm_conf_uninit(dev); 2969 2970 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { 2971 if (vf->rss_lut) { 2972 rte_free(vf->rss_lut); 2973 vf->rss_lut = NULL; 2974 } 2975 if (vf->rss_key) { 2976 rte_free(vf->rss_key); 2977 vf->rss_key = NULL; 2978 } 2979 } 2980 2981 rte_free(vf->vf_res); 2982 vf->vsi_res = NULL; 2983 vf->vf_res = NULL; 2984 2985 rte_free(vf->aq_resp); 2986 vf->aq_resp = NULL; 2987 2988 /* 2989 * If the VF is reset via VFLR, the device will be knocked out of bus 2990 * master mode, and the driver will fail to recover from the reset. Fix 2991 * this by enabling bus mastering after every reset. In a non-VFLR case, 2992 * the bus master bit will not be disabled, and this call will have no 2993 * effect. 2994 */ 2995 out: 2996 if (vf->vf_reset && !rte_pci_set_bus_master(pci_dev, true)) { 2997 vf->vf_reset = false; 2998 iavf_set_no_poll(adapter, false); 2999 } 3000 3001 /* disable watchdog */ 3002 iavf_dev_watchdog_disable(adapter); 3003 3004 return ret; 3005 } 3006 3007 static int 3008 iavf_dev_uninit(struct rte_eth_dev *dev) 3009 { 3010 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 3011 3012 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 3013 return -EPERM; 3014 3015 iavf_dev_close(dev); 3016 3017 if (!vf->in_reset_recovery) 3018 iavf_dev_event_handler_fini(); 3019 3020 return 0; 3021 } 3022 3023 /* 3024 * Reset VF device only to re-initialize resources in PMD layer 3025 */ 3026 static int 3027 iavf_dev_reset(struct rte_eth_dev *dev) 3028 { 3029 int ret; 3030 struct iavf_adapter *adapter = 3031 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 3032 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3033 /* 3034 * Check whether the VF reset has been done and inform application, 3035 * to avoid calling the virtual channel command, which may cause 3036 * the device to be abnormal. 3037 */ 3038 ret = iavf_check_vf_reset_done(hw); 3039 if (ret) { 3040 PMD_DRV_LOG(ERR, "Wait too long for reset done!"); 3041 return ret; 3042 } 3043 iavf_set_no_poll(adapter, false); 3044 3045 PMD_DRV_LOG(DEBUG, "Start dev_reset ..."); 3046 ret = iavf_dev_uninit(dev); 3047 if (ret) 3048 return ret; 3049 3050 return iavf_dev_init(dev); 3051 } 3052 3053 static inline bool 3054 iavf_is_reset(struct iavf_hw *hw) 3055 { 3056 return !(IAVF_READ_REG(hw, IAVF_VF_ARQLEN1) & 3057 IAVF_VF_ARQLEN1_ARQENABLE_MASK); 3058 } 3059 3060 static bool 3061 iavf_is_reset_detected(struct iavf_adapter *adapter) 3062 { 3063 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter); 3064 int i; 3065 3066 /* poll until we see the reset actually happen */ 3067 for (i = 0; i < IAVF_RESET_DETECTED_CNT; i++) { 3068 if (iavf_is_reset(hw)) 3069 return true; 3070 rte_delay_ms(20); 3071 } 3072 3073 return false; 3074 } 3075 3076 /* 3077 * Handle hardware reset 3078 */ 3079 void 3080 iavf_handle_hw_reset(struct rte_eth_dev *dev) 3081 { 3082 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 3083 struct iavf_adapter *adapter = dev->data->dev_private; 3084 int ret; 3085 3086 if (!dev->data->dev_started) 3087 return; 3088 3089 if (!iavf_is_reset_detected(adapter)) { 3090 PMD_DRV_LOG(DEBUG, "reset not start"); 3091 return; 3092 } 3093 3094 vf->in_reset_recovery = true; 3095 iavf_set_no_poll(adapter, false); 3096 3097 ret = iavf_dev_reset(dev); 3098 if (ret) 3099 goto error; 3100 3101 /* VF states restore */ 3102 ret = iavf_dev_configure(dev); 3103 if (ret) 3104 goto error; 3105 3106 iavf_dev_xstats_reset(dev); 3107 3108 /* start the device */ 3109 ret = iavf_dev_start(dev); 3110 if (ret) 3111 goto error; 3112 3113 dev->data->dev_started = 1; 3114 goto exit; 3115 3116 error: 3117 PMD_DRV_LOG(DEBUG, "RESET recover with error code=%dn", ret); 3118 exit: 3119 vf->in_reset_recovery = false; 3120 iavf_set_no_poll(adapter, false); 3121 3122 return; 3123 } 3124 3125 void 3126 iavf_set_no_poll(struct iavf_adapter *adapter, bool link_change) 3127 { 3128 struct iavf_info *vf = &adapter->vf; 3129 3130 adapter->no_poll = (link_change & !vf->link_up) || 3131 vf->vf_reset || vf->in_reset_recovery; 3132 } 3133 3134 static int 3135 iavf_dcf_cap_check_handler(__rte_unused const char *key, 3136 const char *value, __rte_unused void *opaque) 3137 { 3138 if (strcmp(value, "dcf")) 3139 return -1; 3140 3141 return 0; 3142 } 3143 3144 static int 3145 iavf_dcf_cap_selected(struct rte_devargs *devargs) 3146 { 3147 struct rte_kvargs *kvlist; 3148 const char *key = "cap"; 3149 int ret = 0; 3150 3151 if (devargs == NULL) 3152 return 0; 3153 3154 kvlist = rte_kvargs_parse(devargs->args, NULL); 3155 if (kvlist == NULL) 3156 return 0; 3157 3158 if (!rte_kvargs_count(kvlist, key)) 3159 goto exit; 3160 3161 /* dcf capability selected when there's a key-value pair: cap=dcf */ 3162 if (rte_kvargs_process(kvlist, key, 3163 iavf_dcf_cap_check_handler, NULL) < 0) 3164 goto exit; 3165 3166 ret = 1; 3167 3168 exit: 3169 rte_kvargs_free(kvlist); 3170 return ret; 3171 } 3172 3173 static int eth_iavf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 3174 struct rte_pci_device *pci_dev) 3175 { 3176 if (iavf_dcf_cap_selected(pci_dev->device.devargs)) 3177 return 1; 3178 3179 return rte_eth_dev_pci_generic_probe(pci_dev, 3180 sizeof(struct iavf_adapter), iavf_dev_init); 3181 } 3182 3183 static int eth_iavf_pci_remove(struct rte_pci_device *pci_dev) 3184 { 3185 return rte_eth_dev_pci_generic_remove(pci_dev, iavf_dev_uninit); 3186 } 3187 3188 /* Adaptive virtual function driver struct */ 3189 static struct rte_pci_driver rte_iavf_pmd = { 3190 .id_table = pci_id_iavf_map, 3191 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, 3192 .probe = eth_iavf_pci_probe, 3193 .remove = eth_iavf_pci_remove, 3194 }; 3195 3196 RTE_PMD_REGISTER_PCI(net_iavf, rte_iavf_pmd); 3197 RTE_PMD_REGISTER_PCI_TABLE(net_iavf, pci_id_iavf_map); 3198 RTE_PMD_REGISTER_KMOD_DEP(net_iavf, "* igb_uio | vfio-pci"); 3199 RTE_PMD_REGISTER_PARAM_STRING(net_iavf, "cap=dcf"); 3200 RTE_LOG_REGISTER_SUFFIX(iavf_logtype_init, init, NOTICE); 3201 RTE_LOG_REGISTER_SUFFIX(iavf_logtype_driver, driver, NOTICE); 3202 #ifdef RTE_ETHDEV_DEBUG_RX 3203 RTE_LOG_REGISTER_SUFFIX(iavf_logtype_rx, rx, DEBUG); 3204 #endif 3205 #ifdef RTE_ETHDEV_DEBUG_TX 3206 RTE_LOG_REGISTER_SUFFIX(iavf_logtype_tx, tx, DEBUG); 3207 #endif 3208