1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2022 NVIDIA Corporation & Affiliates 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <string.h> 8 #include <stdbool.h> 9 #include <stdint.h> 10 11 #include <sys/queue.h> 12 13 #include "testpmd.h" 14 15 /* Hairpin ports configuration mode. */ 16 uint32_t hairpin_mode; 17 18 bool hairpin_multiport_mode; 19 20 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 21 22 static LIST_HEAD(, hairpin_map) hairpin_map_head = LIST_HEAD_INITIALIZER(); 23 24 struct hairpin_map { 25 LIST_ENTRY(hairpin_map) entry; /**< List entry. */ 26 portid_t rx_port; /**< Hairpin Rx port ID. */ 27 portid_t tx_port; /**< Hairpin Tx port ID. */ 28 uint16_t rxq_head; /**< Hairpin Rx queue head. */ 29 uint16_t txq_head; /**< Hairpin Tx queue head. */ 30 uint16_t qnum; /**< Hairpin queues number. */ 31 }; 32 33 void 34 hairpin_add_multiport_map(struct hairpin_map *map) 35 { 36 LIST_INSERT_HEAD(&hairpin_map_head, map, entry); 37 } 38 39 /* 40 * Get the allowed maximum number of hairpin queues. 41 * *pid return the port id which has minimal value of 42 * max_hairpin_queues in all ports. 43 */ 44 queueid_t 45 get_allowed_max_nb_hairpinq(portid_t *pid) 46 { 47 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 48 portid_t pi; 49 struct rte_eth_hairpin_cap cap; 50 51 RTE_ETH_FOREACH_DEV(pi) { 52 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 53 *pid = pi; 54 return 0; 55 } 56 if (cap.max_nb_queues < allowed_max_hairpinq) { 57 allowed_max_hairpinq = cap.max_nb_queues; 58 *pid = pi; 59 } 60 } 61 return allowed_max_hairpinq; 62 } 63 64 /* 65 * Check input hairpin is valid or not. 66 * If input hairpin is not greater than any of maximum number 67 * of hairpin queues of all ports, it is valid. 68 * if valid, return 0, else return -1 69 */ 70 int 71 check_nb_hairpinq(queueid_t hairpinq) 72 { 73 queueid_t allowed_max_hairpinq; 74 portid_t pid = 0; 75 76 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 77 if (hairpinq > allowed_max_hairpinq) { 78 fprintf(stderr, 79 "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n", 80 hairpinq, allowed_max_hairpinq, pid); 81 return -1; 82 } 83 return 0; 84 } 85 86 #define HAIRPIN_MODE_RX_FORCE_MEMORY RTE_BIT32(8) 87 #define HAIRPIN_MODE_TX_FORCE_MEMORY RTE_BIT32(9) 88 89 #define HAIRPIN_MODE_RX_LOCKED_MEMORY RTE_BIT32(12) 90 #define HAIRPIN_MODE_RX_RTE_MEMORY RTE_BIT32(13) 91 92 #define HAIRPIN_MODE_TX_LOCKED_MEMORY RTE_BIT32(16) 93 #define HAIRPIN_MODE_TX_RTE_MEMORY RTE_BIT32(17) 94 95 static int 96 port_config_hairpin_rxq(portid_t pi, uint16_t peer_tx_port, 97 queueid_t rxq_head, queueid_t txq_head, 98 uint16_t qcount, uint32_t manual_bind) 99 { 100 int diag; 101 queueid_t i, qi; 102 struct rte_port *port = &ports[pi]; 103 struct rte_eth_hairpin_conf hairpin_conf = { 104 .peer_count = 1, 105 .peers[0].port = peer_tx_port, 106 .manual_bind = manual_bind, 107 .tx_explicit = !!(hairpin_mode & 0x10), 108 .force_memory = !!(hairpin_mode & HAIRPIN_MODE_RX_FORCE_MEMORY), 109 .use_locked_device_memory = 110 !!(hairpin_mode & HAIRPIN_MODE_RX_LOCKED_MEMORY), 111 .use_rte_memory = !!(hairpin_mode & HAIRPIN_MODE_RX_RTE_MEMORY), 112 }; 113 114 for (qi = rxq_head, i = 0; qi < rxq_head + qcount; qi++, i++) { 115 hairpin_conf.peers[0].queue = i + txq_head; 116 diag = rte_eth_rx_hairpin_queue_setup(pi, qi, nb_rxd, &hairpin_conf); 117 if (diag == 0) 118 continue; 119 120 /* Fail to setup rx queue, return */ 121 if (port->port_status == RTE_PORT_HANDLING) 122 port->port_status = RTE_PORT_STOPPED; 123 else 124 fprintf(stderr, 125 "Port %d can not be set back to stopped\n", pi); 126 fprintf(stderr, 127 "Port %u failed to configure hairpin on rxq %u.\n" 128 "Peer port: %u peer txq: %u\n", 129 pi, qi, peer_tx_port, i); 130 /* try to reconfigure queues next time */ 131 port->need_reconfig_queues = 1; 132 return -1; 133 } 134 return 0; 135 } 136 137 static int 138 port_config_hairpin_txq(portid_t pi, uint16_t peer_rx_port, 139 queueid_t rxq_head, queueid_t txq_head, 140 uint16_t qcount, uint32_t manual_bind) 141 { 142 int diag; 143 queueid_t i, qi; 144 struct rte_port *port = &ports[pi]; 145 struct rte_eth_hairpin_conf hairpin_conf = { 146 .peer_count = 1, 147 .peers[0].port = peer_rx_port, 148 .manual_bind = manual_bind, 149 .tx_explicit = !!(hairpin_mode & 0x10), 150 .force_memory = !!(hairpin_mode & HAIRPIN_MODE_TX_FORCE_MEMORY), 151 .use_locked_device_memory = 152 !!(hairpin_mode & HAIRPIN_MODE_TX_LOCKED_MEMORY), 153 .use_rte_memory = !!(hairpin_mode & HAIRPIN_MODE_TX_RTE_MEMORY), 154 }; 155 156 for (qi = txq_head, i = 0; qi < txq_head + qcount; qi++, i++) { 157 hairpin_conf.peers[0].queue = i + rxq_head; 158 diag = rte_eth_tx_hairpin_queue_setup(pi, qi, nb_txd, &hairpin_conf); 159 if (diag == 0) 160 continue; 161 162 /* Fail to setup rx queue, return */ 163 if (port->port_status == RTE_PORT_HANDLING) 164 port->port_status = RTE_PORT_STOPPED; 165 else 166 fprintf(stderr, 167 "Port %d can not be set back to stopped\n", pi); 168 fprintf(stderr, 169 "Port %d failed to configure hairpin on txq %u.\n" 170 "Peer port: %u peer rxq: %u\n", 171 pi, qi, peer_rx_port, i); 172 /* try to reconfigure queues next time */ 173 port->need_reconfig_queues = 1; 174 return -1; 175 } 176 return 0; 177 } 178 179 static int 180 setup_legacy_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 181 { 182 int diag; 183 uint16_t peer_rx_port = pi; 184 uint16_t peer_tx_port = pi; 185 uint32_t manual = 1; 186 187 if (!(hairpin_mode & 0xf)) { 188 peer_rx_port = pi; 189 peer_tx_port = pi; 190 manual = 0; 191 } else if (hairpin_mode & 0x1) { 192 peer_tx_port = rte_eth_find_next_owned_by(pi + 1, 193 RTE_ETH_DEV_NO_OWNER); 194 if (peer_tx_port >= RTE_MAX_ETHPORTS) 195 peer_tx_port = rte_eth_find_next_owned_by(0, 196 RTE_ETH_DEV_NO_OWNER); 197 if (p_pi != RTE_MAX_ETHPORTS) { 198 peer_rx_port = p_pi; 199 } else { 200 uint16_t next_pi; 201 202 /* Last port will be the peer RX port of the first. */ 203 RTE_ETH_FOREACH_DEV(next_pi) 204 peer_rx_port = next_pi; 205 } 206 manual = 1; 207 } else if (hairpin_mode & 0x2) { 208 if (cnt_pi & 0x1) { 209 peer_rx_port = p_pi; 210 } else { 211 peer_rx_port = rte_eth_find_next_owned_by(pi + 1, 212 RTE_ETH_DEV_NO_OWNER); 213 if (peer_rx_port >= RTE_MAX_ETHPORTS) 214 peer_rx_port = pi; 215 } 216 peer_tx_port = peer_rx_port; 217 manual = 1; 218 } 219 diag = port_config_hairpin_txq(pi, peer_rx_port, nb_rxq, nb_txq, 220 nb_hairpinq, manual); 221 if (diag) 222 return diag; 223 diag = port_config_hairpin_rxq(pi, peer_tx_port, nb_rxq, nb_txq, 224 nb_hairpinq, manual); 225 if (diag) 226 return diag; 227 return 0; 228 } 229 230 static int 231 setup_mapped_harpin_queues(portid_t pi) 232 { 233 int ret = 0; 234 struct hairpin_map *map; 235 236 LIST_FOREACH(map, &hairpin_map_head, entry) { 237 if (map->rx_port == pi) { 238 ret = port_config_hairpin_rxq(pi, map->tx_port, 239 map->rxq_head, 240 map->txq_head, 241 map->qnum, true); 242 if (ret) 243 return ret; 244 } 245 if (map->tx_port == pi) { 246 ret = port_config_hairpin_txq(pi, map->rx_port, 247 map->rxq_head, 248 map->txq_head, 249 map->qnum, true); 250 if (ret) 251 return ret; 252 } 253 } 254 return 0; 255 } 256 257 /* Configure the Rx and Tx hairpin queues for the selected port. */ 258 int 259 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi) 260 { 261 if (hairpin_multiport_mode) 262 return setup_mapped_harpin_queues(pi); 263 264 return setup_legacy_hairpin_queues(pi, p_pi, cnt_pi); 265 } 266 267 int 268 hairpin_bind(uint16_t cfg_pi, portid_t *pl, portid_t *peer_pl) 269 { 270 uint16_t i; 271 portid_t pi; 272 int peer_pi; 273 int diag; 274 int j; 275 276 /* bind all started hairpin ports */ 277 for (i = 0; i < cfg_pi; i++) { 278 pi = pl[i]; 279 /* bind current Tx to all peer Rx */ 280 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 281 RTE_MAX_ETHPORTS, 1); 282 if (peer_pi < 0) 283 return peer_pi; 284 for (j = 0; j < peer_pi; j++) { 285 if (!port_is_started(peer_pl[j])) 286 continue; 287 diag = rte_eth_hairpin_bind(pi, peer_pl[j]); 288 if (diag < 0) { 289 fprintf(stderr, 290 "Error during binding hairpin Tx port %u to %u: %s\n", 291 pi, peer_pl[j], 292 rte_strerror(-diag)); 293 return -1; 294 } 295 } 296 /* bind all peer Tx to current Rx */ 297 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl, 298 RTE_MAX_ETHPORTS, 0); 299 if (peer_pi < 0) 300 return peer_pi; 301 for (j = 0; j < peer_pi; j++) { 302 if (!port_is_started(peer_pl[j])) 303 continue; 304 diag = rte_eth_hairpin_bind(peer_pl[j], pi); 305 if (diag < 0) { 306 fprintf(stderr, 307 "Error during binding hairpin Tx port %u to %u: %s\n", 308 peer_pl[j], pi, 309 rte_strerror(-diag)); 310 return -1; 311 } 312 } 313 } 314 return 0; 315 } 316 317 void 318 hairpin_map_usage(void) 319 { 320 printf(" --hairpin-map=rxpi:rxq:txpi:txq:n: hairpin map.\n" 321 " rxpi - Rx port index.\n" 322 " rxq - Rx queue.\n" 323 " txpi - Tx port index.\n" 324 " txq - Tx queue.\n" 325 " n - hairpin queues number.\n"); 326 } 327 328 int 329 parse_hairpin_map(const char *hpmap) 330 { 331 /* 332 * Testpmd hairpin map format: 333 * <Rx port id:First Rx queue:Tx port id:First Tx queue:queues number> 334 */ 335 int ret; 336 struct hairpin_map *map = calloc(1, sizeof(*map)); 337 338 if (!map) 339 return -ENOMEM; 340 341 ret = sscanf(hpmap, "%hu:%hu:%hu:%hu:%hu", 342 &map->rx_port, &map->rxq_head, 343 &map->tx_port, &map->txq_head, &map->qnum); 344 if (ret != 5) { 345 free(map); 346 return -EINVAL; 347 } 348 hairpin_add_multiport_map(map); 349 return 0; 350 } 351