1 /*- 2 * BSD LICENSE 3 * 4 * Copyright 2017 6WIND S.A. 5 * Copyright 2017 Mellanox. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of 6WIND S.A. nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <unistd.h> 35 36 #include <rte_flow.h> 37 #include <rte_flow_driver.h> 38 #include <rte_cycles.h> 39 40 #include "failsafe_private.h" 41 42 /** Print a message out of a flow error. */ 43 static int 44 fs_flow_complain(struct rte_flow_error *error) 45 { 46 static const char *const errstrlist[] = { 47 [RTE_FLOW_ERROR_TYPE_NONE] = "no error", 48 [RTE_FLOW_ERROR_TYPE_UNSPECIFIED] = "cause unspecified", 49 [RTE_FLOW_ERROR_TYPE_HANDLE] = "flow rule (handle)", 50 [RTE_FLOW_ERROR_TYPE_ATTR_GROUP] = "group field", 51 [RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY] = "priority field", 52 [RTE_FLOW_ERROR_TYPE_ATTR_INGRESS] = "ingress field", 53 [RTE_FLOW_ERROR_TYPE_ATTR_EGRESS] = "egress field", 54 [RTE_FLOW_ERROR_TYPE_ATTR] = "attributes structure", 55 [RTE_FLOW_ERROR_TYPE_ITEM_NUM] = "pattern length", 56 [RTE_FLOW_ERROR_TYPE_ITEM] = "specific pattern item", 57 [RTE_FLOW_ERROR_TYPE_ACTION_NUM] = "number of actions", 58 [RTE_FLOW_ERROR_TYPE_ACTION] = "specific action", 59 }; 60 const char *errstr; 61 char buf[32]; 62 int err = rte_errno; 63 64 if ((unsigned int)error->type >= RTE_DIM(errstrlist) || 65 !errstrlist[error->type]) 66 errstr = "unknown type"; 67 else 68 errstr = errstrlist[error->type]; 69 ERROR("Caught error type %d (%s): %s%s\n", 70 error->type, errstr, 71 error->cause ? (snprintf(buf, sizeof(buf), "cause: %p, ", 72 error->cause), buf) : "", 73 error->message ? error->message : "(no stated reason)"); 74 return -err; 75 } 76 77 static int 78 eth_dev_flow_isolate_set(struct rte_eth_dev *dev, 79 struct sub_device *sdev) 80 { 81 struct rte_flow_error ferror; 82 int ret; 83 84 if (!PRIV(dev)->flow_isolated) { 85 DEBUG("Flow isolation already disabled"); 86 } else { 87 DEBUG("Enabling flow isolation"); 88 ret = rte_flow_isolate(PORT_ID(sdev), 89 PRIV(dev)->flow_isolated, 90 &ferror); 91 if (ret) { 92 fs_flow_complain(&ferror); 93 return ret; 94 } 95 } 96 return 0; 97 } 98 99 static int 100 fs_eth_dev_conf_apply(struct rte_eth_dev *dev, 101 struct sub_device *sdev) 102 { 103 struct rte_eth_dev *edev; 104 struct rte_vlan_filter_conf *vfc1; 105 struct rte_vlan_filter_conf *vfc2; 106 struct rte_flow *flow; 107 struct rte_flow_error ferror; 108 uint32_t i; 109 int ret; 110 111 edev = ETH(sdev); 112 /* RX queue setup */ 113 for (i = 0; i < dev->data->nb_rx_queues; i++) { 114 struct rxq *rxq; 115 116 rxq = dev->data->rx_queues[i]; 117 ret = rte_eth_rx_queue_setup(PORT_ID(sdev), i, 118 rxq->info.nb_desc, rxq->socket_id, 119 &rxq->info.conf, rxq->info.mp); 120 if (ret) { 121 ERROR("rx_queue_setup failed"); 122 return ret; 123 } 124 } 125 /* TX queue setup */ 126 for (i = 0; i < dev->data->nb_tx_queues; i++) { 127 struct txq *txq; 128 129 txq = dev->data->tx_queues[i]; 130 ret = rte_eth_tx_queue_setup(PORT_ID(sdev), i, 131 txq->info.nb_desc, txq->socket_id, 132 &txq->info.conf); 133 if (ret) { 134 ERROR("tx_queue_setup failed"); 135 return ret; 136 } 137 } 138 /* dev_link.link_status */ 139 if (dev->data->dev_link.link_status != 140 edev->data->dev_link.link_status) { 141 DEBUG("Configuring link_status"); 142 if (dev->data->dev_link.link_status) 143 ret = rte_eth_dev_set_link_up(PORT_ID(sdev)); 144 else 145 ret = rte_eth_dev_set_link_down(PORT_ID(sdev)); 146 if (ret) { 147 ERROR("Failed to apply link_status"); 148 return ret; 149 } 150 } else { 151 DEBUG("link_status already set"); 152 } 153 /* promiscuous */ 154 if (dev->data->promiscuous != edev->data->promiscuous) { 155 DEBUG("Configuring promiscuous"); 156 if (dev->data->promiscuous) 157 rte_eth_promiscuous_enable(PORT_ID(sdev)); 158 else 159 rte_eth_promiscuous_disable(PORT_ID(sdev)); 160 } else { 161 DEBUG("promiscuous already set"); 162 } 163 /* all_multicast */ 164 if (dev->data->all_multicast != edev->data->all_multicast) { 165 DEBUG("Configuring all_multicast"); 166 if (dev->data->all_multicast) 167 rte_eth_allmulticast_enable(PORT_ID(sdev)); 168 else 169 rte_eth_allmulticast_disable(PORT_ID(sdev)); 170 } else { 171 DEBUG("all_multicast already set"); 172 } 173 /* MTU */ 174 if (dev->data->mtu != edev->data->mtu) { 175 DEBUG("Configuring MTU"); 176 ret = rte_eth_dev_set_mtu(PORT_ID(sdev), dev->data->mtu); 177 if (ret) { 178 ERROR("Failed to apply MTU"); 179 return ret; 180 } 181 } else { 182 DEBUG("MTU already set"); 183 } 184 /* default MAC */ 185 DEBUG("Configuring default MAC address"); 186 ret = rte_eth_dev_default_mac_addr_set(PORT_ID(sdev), 187 &dev->data->mac_addrs[0]); 188 if (ret) { 189 ERROR("Setting default MAC address failed"); 190 return ret; 191 } 192 /* additional MAC */ 193 if (PRIV(dev)->nb_mac_addr > 1) 194 DEBUG("Configure additional MAC address%s", 195 (PRIV(dev)->nb_mac_addr > 2 ? "es" : "")); 196 for (i = 1; i < PRIV(dev)->nb_mac_addr; i++) { 197 struct ether_addr *ea; 198 199 ea = &dev->data->mac_addrs[i]; 200 ret = rte_eth_dev_mac_addr_add(PORT_ID(sdev), ea, 201 PRIV(dev)->mac_addr_pool[i]); 202 if (ret) { 203 char ea_fmt[ETHER_ADDR_FMT_SIZE]; 204 205 ether_format_addr(ea_fmt, ETHER_ADDR_FMT_SIZE, ea); 206 ERROR("Adding MAC address %s failed", ea_fmt); 207 return ret; 208 } 209 } 210 /* VLAN filter */ 211 vfc1 = &dev->data->vlan_filter_conf; 212 vfc2 = &edev->data->vlan_filter_conf; 213 if (memcmp(vfc1, vfc2, sizeof(struct rte_vlan_filter_conf))) { 214 uint64_t vbit; 215 uint64_t ids; 216 size_t i; 217 uint16_t vlan_id; 218 219 DEBUG("Configuring VLAN filter"); 220 for (i = 0; i < RTE_DIM(vfc1->ids); i++) { 221 if (vfc1->ids[i] == 0) 222 continue; 223 ids = vfc1->ids[i]; 224 while (ids) { 225 vlan_id = 64 * i; 226 /* count trailing zeroes */ 227 vbit = ~ids & (ids - 1); 228 /* clear least significant bit set */ 229 ids ^= (ids ^ (ids - 1)) ^ vbit; 230 for (; vbit; vlan_id++) 231 vbit >>= 1; 232 ret = rte_eth_dev_vlan_filter( 233 PORT_ID(sdev), vlan_id, 1); 234 if (ret) { 235 ERROR("Failed to apply VLAN filter %hu", 236 vlan_id); 237 return ret; 238 } 239 } 240 } 241 } else { 242 DEBUG("VLAN filter already set"); 243 } 244 /* rte_flow */ 245 if (TAILQ_EMPTY(&PRIV(dev)->flow_list)) { 246 DEBUG("rte_flow already set"); 247 } else { 248 DEBUG("Resetting rte_flow configuration"); 249 ret = rte_flow_flush(PORT_ID(sdev), &ferror); 250 if (ret) { 251 fs_flow_complain(&ferror); 252 return ret; 253 } 254 i = 0; 255 rte_errno = 0; 256 DEBUG("Configuring rte_flow"); 257 TAILQ_FOREACH(flow, &PRIV(dev)->flow_list, next) { 258 DEBUG("Creating flow #%" PRIu32, i++); 259 flow->flows[SUB_ID(sdev)] = 260 rte_flow_create(PORT_ID(sdev), 261 &flow->fd->attr, 262 flow->fd->items, 263 flow->fd->actions, 264 &ferror); 265 ret = rte_errno; 266 if (ret) 267 break; 268 } 269 if (ret) { 270 fs_flow_complain(&ferror); 271 return ret; 272 } 273 } 274 return 0; 275 } 276 277 static void 278 fs_dev_remove(struct sub_device *sdev) 279 { 280 int ret; 281 282 if (sdev == NULL) 283 return; 284 switch (sdev->state) { 285 case DEV_STARTED: 286 failsafe_rx_intr_uninstall_subdevice(sdev); 287 rte_eth_dev_stop(PORT_ID(sdev)); 288 sdev->state = DEV_ACTIVE; 289 /* fallthrough */ 290 case DEV_ACTIVE: 291 rte_eth_dev_close(PORT_ID(sdev)); 292 sdev->state = DEV_PROBED; 293 /* fallthrough */ 294 case DEV_PROBED: 295 ret = rte_eal_hotplug_remove(sdev->bus->name, 296 sdev->dev->name); 297 if (ret) { 298 ERROR("Bus detach failed for sub_device %u", 299 SUB_ID(sdev)); 300 } else { 301 rte_eth_dev_release_port(ETH(sdev)); 302 } 303 sdev->state = DEV_PARSED; 304 /* fallthrough */ 305 case DEV_PARSED: 306 case DEV_UNDEFINED: 307 sdev->state = DEV_UNDEFINED; 308 /* the end */ 309 break; 310 } 311 failsafe_hotplug_alarm_install(sdev->fs_dev); 312 } 313 314 static void 315 fs_dev_stats_save(struct sub_device *sdev) 316 { 317 struct rte_eth_stats stats; 318 int err; 319 320 /* Attempt to read current stats. */ 321 err = rte_eth_stats_get(PORT_ID(sdev), &stats); 322 if (err) { 323 uint64_t timestamp = sdev->stats_snapshot.timestamp; 324 325 WARN("Could not access latest statistics from sub-device %d.\n", 326 SUB_ID(sdev)); 327 if (timestamp != 0) 328 WARN("Using latest snapshot taken before %"PRIu64" seconds.\n", 329 (rte_rdtsc() - timestamp) / rte_get_tsc_hz()); 330 } 331 failsafe_stats_increment(&PRIV(sdev->fs_dev)->stats_accumulator, 332 err ? &sdev->stats_snapshot.stats : &stats); 333 memset(&sdev->stats_snapshot, 0, sizeof(sdev->stats_snapshot)); 334 } 335 336 static inline int 337 fs_rxtx_clean(struct sub_device *sdev) 338 { 339 uint16_t i; 340 341 for (i = 0; i < ETH(sdev)->data->nb_rx_queues; i++) 342 if (FS_ATOMIC_RX(sdev, i)) 343 return 0; 344 for (i = 0; i < ETH(sdev)->data->nb_tx_queues; i++) 345 if (FS_ATOMIC_TX(sdev, i)) 346 return 0; 347 return 1; 348 } 349 350 void 351 failsafe_dev_remove(struct rte_eth_dev *dev) 352 { 353 struct sub_device *sdev; 354 uint8_t i; 355 356 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) 357 if (sdev->remove && fs_rxtx_clean(sdev)) { 358 fs_dev_stats_save(sdev); 359 fs_dev_remove(sdev); 360 } 361 } 362 363 int 364 failsafe_eth_dev_state_sync(struct rte_eth_dev *dev) 365 { 366 struct sub_device *sdev; 367 uint32_t inactive; 368 int ret; 369 uint8_t i; 370 371 if (PRIV(dev)->state < DEV_PARSED) 372 return 0; 373 374 ret = failsafe_args_parse_subs(dev); 375 if (ret) 376 goto err_remove; 377 378 if (PRIV(dev)->state < DEV_PROBED) 379 return 0; 380 ret = failsafe_eal_init(dev); 381 if (ret) 382 goto err_remove; 383 if (PRIV(dev)->state < DEV_ACTIVE) 384 return 0; 385 inactive = 0; 386 FOREACH_SUBDEV(sdev, i, dev) { 387 if (sdev->state == DEV_PROBED) { 388 inactive |= UINT32_C(1) << i; 389 ret = eth_dev_flow_isolate_set(dev, sdev); 390 if (ret) { 391 ERROR("Could not apply configuration to sub_device %d", 392 i); 393 goto err_remove; 394 } 395 } 396 } 397 ret = dev->dev_ops->dev_configure(dev); 398 if (ret) 399 goto err_remove; 400 FOREACH_SUBDEV(sdev, i, dev) { 401 if (inactive & (UINT32_C(1) << i)) { 402 ret = fs_eth_dev_conf_apply(dev, sdev); 403 if (ret) { 404 ERROR("Could not apply configuration to sub_device %d", 405 i); 406 goto err_remove; 407 } 408 } 409 } 410 /* 411 * If new devices have been configured, check if 412 * the link state has changed. 413 */ 414 if (inactive) 415 dev->dev_ops->link_update(dev, 1); 416 if (PRIV(dev)->state < DEV_STARTED) 417 return 0; 418 ret = dev->dev_ops->dev_start(dev); 419 if (ret) 420 goto err_remove; 421 return 0; 422 err_remove: 423 FOREACH_SUBDEV(sdev, i, dev) 424 if (sdev->state != PRIV(dev)->state) 425 sdev->remove = 1; 426 return ret; 427 } 428 429 void 430 failsafe_stats_increment(struct rte_eth_stats *to, struct rte_eth_stats *from) 431 { 432 uint32_t i; 433 434 RTE_ASSERT(to != NULL && from != NULL); 435 to->ipackets += from->ipackets; 436 to->opackets += from->opackets; 437 to->ibytes += from->ibytes; 438 to->obytes += from->obytes; 439 to->imissed += from->imissed; 440 to->ierrors += from->ierrors; 441 to->oerrors += from->oerrors; 442 to->rx_nombuf += from->rx_nombuf; 443 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) { 444 to->q_ipackets[i] += from->q_ipackets[i]; 445 to->q_opackets[i] += from->q_opackets[i]; 446 to->q_ibytes[i] += from->q_ibytes[i]; 447 to->q_obytes[i] += from->q_obytes[i]; 448 to->q_errors[i] += from->q_errors[i]; 449 } 450 } 451 452 int 453 failsafe_eth_rmv_event_callback(uint16_t port_id __rte_unused, 454 enum rte_eth_event_type event __rte_unused, 455 void *cb_arg, void *out __rte_unused) 456 { 457 struct sub_device *sdev = cb_arg; 458 459 /* Switch as soon as possible tx_dev. */ 460 fs_switch_dev(sdev->fs_dev, sdev); 461 /* Use safe bursts in any case. */ 462 set_burst_fn(sdev->fs_dev, 1); 463 /* 464 * Async removal, the sub-PMD will try to unregister 465 * the callback at the source of the current thread context. 466 */ 467 sdev->remove = 1; 468 return 0; 469 } 470 471 int 472 failsafe_eth_lsc_event_callback(uint16_t port_id __rte_unused, 473 enum rte_eth_event_type event __rte_unused, 474 void *cb_arg, void *out __rte_unused) 475 { 476 struct rte_eth_dev *dev = cb_arg; 477 int ret; 478 479 ret = dev->dev_ops->link_update(dev, 0); 480 /* We must pass on the LSC event */ 481 if (ret) 482 return _rte_eth_dev_callback_process(dev, 483 RTE_ETH_EVENT_INTR_LSC, 484 NULL); 485 else 486 return 0; 487 } 488