1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2016 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <stdint.h> 8 9 #include <rte_log.h> 10 #include <rte_ethdev.h> 11 #include <rte_ether.h> 12 #include <rte_ip.h> 13 #include <rte_byteorder.h> 14 15 #include <rte_port_ring.h> 16 #include <rte_table_hash.h> 17 #include <rte_hash.h> 18 #include <rte_table_hash_cuckoo.h> 19 #include <rte_pipeline.h> 20 21 #include "main.h" 22 23 static void 24 translate_options(uint32_t *special, uint32_t *ext, uint32_t *key_size) 25 { 26 switch (app.pipeline_type) { 27 case e_APP_PIPELINE_HASH_KEY8_EXT: 28 *special = 0; *ext = 1; *key_size = 8; return; 29 case e_APP_PIPELINE_HASH_KEY8_LRU: 30 *special = 0; *ext = 0; *key_size = 8; return; 31 case e_APP_PIPELINE_HASH_KEY16_EXT: 32 *special = 0; *ext = 1; *key_size = 16; return; 33 case e_APP_PIPELINE_HASH_KEY16_LRU: 34 *special = 0; *ext = 0; *key_size = 16; return; 35 case e_APP_PIPELINE_HASH_KEY32_EXT: 36 *special = 0; *ext = 1; *key_size = 32; return; 37 case e_APP_PIPELINE_HASH_KEY32_LRU: 38 *special = 0; *ext = 0; *key_size = 32; return; 39 40 case e_APP_PIPELINE_HASH_SPEC_KEY8_EXT: 41 *special = 1; *ext = 1; *key_size = 8; return; 42 case e_APP_PIPELINE_HASH_SPEC_KEY8_LRU: 43 *special = 1; *ext = 0; *key_size = 8; return; 44 case e_APP_PIPELINE_HASH_SPEC_KEY16_EXT: 45 *special = 1; *ext = 1; *key_size = 16; return; 46 case e_APP_PIPELINE_HASH_SPEC_KEY16_LRU: 47 *special = 1; *ext = 0; *key_size = 16; return; 48 case e_APP_PIPELINE_HASH_SPEC_KEY32_EXT: 49 *special = 1; *ext = 1; *key_size = 32; return; 50 case e_APP_PIPELINE_HASH_SPEC_KEY32_LRU: 51 *special = 1; *ext = 0; *key_size = 32; return; 52 53 case e_APP_PIPELINE_HASH_CUCKOO_KEY8: 54 *special = 0; *ext = 0; *key_size = 8; return; 55 case e_APP_PIPELINE_HASH_CUCKOO_KEY16: 56 *special = 0; *ext = 0; *key_size = 16; return; 57 case e_APP_PIPELINE_HASH_CUCKOO_KEY32: 58 *special = 0; *ext = 0; *key_size = 32; return; 59 case e_APP_PIPELINE_HASH_CUCKOO_KEY48: 60 *special = 0; *ext = 0; *key_size = 48; return; 61 case e_APP_PIPELINE_HASH_CUCKOO_KEY64: 62 *special = 0; *ext = 0; *key_size = 64; return; 63 case e_APP_PIPELINE_HASH_CUCKOO_KEY80: 64 *special = 0; *ext = 0; *key_size = 80; return; 65 case e_APP_PIPELINE_HASH_CUCKOO_KEY96: 66 *special = 0; *ext = 0; *key_size = 96; return; 67 case e_APP_PIPELINE_HASH_CUCKOO_KEY112: 68 *special = 0; *ext = 0; *key_size = 112; return; 69 case e_APP_PIPELINE_HASH_CUCKOO_KEY128: 70 *special = 0; *ext = 0; *key_size = 128; return; 71 72 default: 73 rte_panic("Invalid hash table type or key size\n"); 74 } 75 } 76 void 77 app_main_loop_worker_pipeline_hash(void) { 78 struct rte_pipeline_params pipeline_params = { 79 .name = "pipeline", 80 .socket_id = rte_socket_id(), 81 }; 82 83 struct rte_pipeline *p; 84 uint32_t port_in_id[APP_MAX_PORTS]; 85 uint32_t port_out_id[APP_MAX_PORTS]; 86 uint32_t table_id; 87 uint32_t i; 88 uint32_t special, ext, key_size; 89 90 translate_options(&special, &ext, &key_size); 91 92 RTE_LOG(INFO, USER1, "Core %u is doing work " 93 "(pipeline with hash table, %s, %s, %d-byte key)\n", 94 rte_lcore_id(), 95 special ? "specialized" : "non-specialized", 96 ext ? "extendible bucket" : "LRU", 97 key_size); 98 99 /* Pipeline configuration */ 100 p = rte_pipeline_create(&pipeline_params); 101 if (p == NULL) 102 rte_panic("Unable to configure the pipeline\n"); 103 104 /* Input port configuration */ 105 for (i = 0; i < app.n_ports; i++) { 106 struct rte_port_ring_reader_params port_ring_params = { 107 .ring = app.rings_rx[i], 108 }; 109 110 struct rte_pipeline_port_in_params port_params = { 111 .ops = &rte_port_ring_reader_ops, 112 .arg_create = (void *) &port_ring_params, 113 .f_action = NULL, 114 .arg_ah = NULL, 115 .burst_size = app.burst_size_worker_read, 116 }; 117 118 if (rte_pipeline_port_in_create(p, &port_params, 119 &port_in_id[i])) 120 rte_panic("Unable to configure input port for " 121 "ring %d\n", i); 122 } 123 124 /* Output port configuration */ 125 for (i = 0; i < app.n_ports; i++) { 126 struct rte_port_ring_writer_params port_ring_params = { 127 .ring = app.rings_tx[i], 128 .tx_burst_sz = app.burst_size_worker_write, 129 }; 130 131 struct rte_pipeline_port_out_params port_params = { 132 .ops = &rte_port_ring_writer_ops, 133 .arg_create = (void *) &port_ring_params, 134 .f_action = NULL, 135 .arg_ah = NULL, 136 }; 137 138 if (rte_pipeline_port_out_create(p, &port_params, 139 &port_out_id[i])) 140 rte_panic("Unable to configure output port for " 141 "ring %d\n", i); 142 } 143 144 struct rte_table_hash_params table_hash_params = { 145 .name = "TABLE", 146 .key_size = key_size, 147 .key_offset = APP_METADATA_OFFSET(32), 148 .key_mask = NULL, 149 .n_keys = 1 << 24, 150 .n_buckets = 1 << 22, 151 .f_hash = test_hash, 152 .seed = 0, 153 }; 154 155 struct rte_table_hash_cuckoo_params table_hash_cuckoo_params = { 156 .name = "TABLE", 157 .key_size = key_size, 158 .key_offset = APP_METADATA_OFFSET(32), 159 .key_mask = NULL, 160 .n_keys = 1 << 24, 161 .n_buckets = 1 << 22, 162 .f_hash = test_hash_cuckoo, 163 .seed = 0, 164 }; 165 166 /* Table configuration */ 167 switch (app.pipeline_type) { 168 case e_APP_PIPELINE_HASH_KEY8_EXT: 169 case e_APP_PIPELINE_HASH_KEY16_EXT: 170 case e_APP_PIPELINE_HASH_KEY32_EXT: 171 { 172 struct rte_pipeline_table_params table_params = { 173 .ops = &rte_table_hash_ext_ops, 174 .arg_create = &table_hash_params, 175 .f_action_hit = NULL, 176 .f_action_miss = NULL, 177 .arg_ah = NULL, 178 .action_data_size = 0, 179 }; 180 181 if (rte_pipeline_table_create(p, &table_params, &table_id)) 182 rte_panic("Unable to configure the hash table\n"); 183 } 184 break; 185 186 case e_APP_PIPELINE_HASH_KEY8_LRU: 187 case e_APP_PIPELINE_HASH_KEY16_LRU: 188 case e_APP_PIPELINE_HASH_KEY32_LRU: 189 { 190 struct rte_pipeline_table_params table_params = { 191 .ops = &rte_table_hash_lru_ops, 192 .arg_create = &table_hash_params, 193 .f_action_hit = NULL, 194 .f_action_miss = NULL, 195 .arg_ah = NULL, 196 .action_data_size = 0, 197 }; 198 199 if (rte_pipeline_table_create(p, &table_params, &table_id)) 200 rte_panic("Unable to configure the hash table\n"); 201 } 202 break; 203 204 case e_APP_PIPELINE_HASH_SPEC_KEY8_EXT: 205 { 206 struct rte_pipeline_table_params table_params = { 207 .ops = &rte_table_hash_key8_ext_ops, 208 .arg_create = &table_hash_params, 209 .f_action_hit = NULL, 210 .f_action_miss = NULL, 211 .arg_ah = NULL, 212 .action_data_size = 0, 213 }; 214 215 if (rte_pipeline_table_create(p, &table_params, &table_id)) 216 rte_panic("Unable to configure the hash table\n"); 217 } 218 break; 219 220 case e_APP_PIPELINE_HASH_SPEC_KEY8_LRU: 221 { 222 struct rte_pipeline_table_params table_params = { 223 .ops = &rte_table_hash_key8_lru_ops, 224 .arg_create = &table_hash_params, 225 .f_action_hit = NULL, 226 .f_action_miss = NULL, 227 .arg_ah = NULL, 228 .action_data_size = 0, 229 }; 230 231 if (rte_pipeline_table_create(p, &table_params, &table_id)) 232 rte_panic("Unable to configure the hash table\n"); 233 } 234 break; 235 236 case e_APP_PIPELINE_HASH_SPEC_KEY16_EXT: 237 { 238 struct rte_pipeline_table_params table_params = { 239 .ops = &rte_table_hash_key16_ext_ops, 240 .arg_create = &table_hash_params, 241 .f_action_hit = NULL, 242 .f_action_miss = NULL, 243 .arg_ah = NULL, 244 .action_data_size = 0, 245 }; 246 247 if (rte_pipeline_table_create(p, &table_params, &table_id)) 248 rte_panic("Unable to configure the hash table)\n"); 249 } 250 break; 251 252 case e_APP_PIPELINE_HASH_SPEC_KEY16_LRU: 253 { 254 struct rte_pipeline_table_params table_params = { 255 .ops = &rte_table_hash_key16_lru_ops, 256 .arg_create = &table_hash_params, 257 .f_action_hit = NULL, 258 .f_action_miss = NULL, 259 .arg_ah = NULL, 260 .action_data_size = 0, 261 }; 262 263 if (rte_pipeline_table_create(p, &table_params, &table_id)) 264 rte_panic("Unable to configure the hash table\n"); 265 } 266 break; 267 268 case e_APP_PIPELINE_HASH_SPEC_KEY32_EXT: 269 { 270 struct rte_pipeline_table_params table_params = { 271 .ops = &rte_table_hash_key32_ext_ops, 272 .arg_create = &table_hash_params, 273 .f_action_hit = NULL, 274 .f_action_miss = NULL, 275 .arg_ah = NULL, 276 .action_data_size = 0, 277 }; 278 279 if (rte_pipeline_table_create(p, &table_params, &table_id)) 280 rte_panic("Unable to configure the hash table\n"); 281 } 282 break; 283 284 285 case e_APP_PIPELINE_HASH_SPEC_KEY32_LRU: 286 { 287 struct rte_pipeline_table_params table_params = { 288 .ops = &rte_table_hash_key32_lru_ops, 289 .arg_create = &table_hash_params, 290 .f_action_hit = NULL, 291 .f_action_miss = NULL, 292 .arg_ah = NULL, 293 .action_data_size = 0, 294 }; 295 296 if (rte_pipeline_table_create(p, &table_params, &table_id)) 297 rte_panic("Unable to configure the hash table\n"); 298 } 299 break; 300 301 case e_APP_PIPELINE_HASH_CUCKOO_KEY8: 302 case e_APP_PIPELINE_HASH_CUCKOO_KEY16: 303 case e_APP_PIPELINE_HASH_CUCKOO_KEY32: 304 case e_APP_PIPELINE_HASH_CUCKOO_KEY48: 305 case e_APP_PIPELINE_HASH_CUCKOO_KEY64: 306 case e_APP_PIPELINE_HASH_CUCKOO_KEY80: 307 case e_APP_PIPELINE_HASH_CUCKOO_KEY96: 308 case e_APP_PIPELINE_HASH_CUCKOO_KEY112: 309 case e_APP_PIPELINE_HASH_CUCKOO_KEY128: 310 { 311 struct rte_pipeline_table_params table_params = { 312 .ops = &rte_table_hash_cuckoo_ops, 313 .arg_create = &table_hash_cuckoo_params, 314 .f_action_hit = NULL, 315 .f_action_miss = NULL, 316 .arg_ah = NULL, 317 .action_data_size = 0, 318 }; 319 320 if (rte_pipeline_table_create(p, &table_params, &table_id)) 321 rte_panic("Unable to configure the hash table\n"); 322 } 323 break; 324 325 default: 326 rte_panic("Invalid hash table type or key size\n"); 327 } 328 329 /* Interconnecting ports and tables */ 330 for (i = 0; i < app.n_ports; i++) 331 if (rte_pipeline_port_in_connect_to_table(p, port_in_id[i], 332 table_id)) 333 rte_panic("Unable to connect input port %u to " 334 "table %u\n", port_in_id[i], table_id); 335 336 /* Add entries to tables */ 337 for (i = 0; i < (1 << 24); i++) { 338 struct rte_pipeline_table_entry entry = { 339 .action = RTE_PIPELINE_ACTION_PORT, 340 {.port_id = port_out_id[i & (app.n_ports - 1)]}, 341 }; 342 struct rte_pipeline_table_entry *entry_ptr; 343 uint8_t key[32]; 344 uint32_t *k32 = (uint32_t *) key; 345 int key_found, status; 346 347 memset(key, 0, sizeof(key)); 348 k32[0] = rte_be_to_cpu_32(i); 349 350 status = rte_pipeline_table_entry_add(p, table_id, key, &entry, 351 &key_found, &entry_ptr); 352 if (status < 0) 353 rte_panic("Unable to add entry to table %u (%d)\n", 354 table_id, status); 355 } 356 357 /* Enable input ports */ 358 for (i = 0; i < app.n_ports; i++) 359 if (rte_pipeline_port_in_enable(p, port_in_id[i])) 360 rte_panic("Unable to enable input port %u\n", 361 port_in_id[i]); 362 363 /* Check pipeline consistency */ 364 if (rte_pipeline_check(p) < 0) 365 rte_panic("Pipeline consistency check failed\n"); 366 367 /* Run-time */ 368 #if APP_FLUSH == 0 369 while (!force_quit) 370 rte_pipeline_run(p); 371 #else 372 i = 0; 373 while (!force_quit) { 374 rte_pipeline_run(p); 375 376 if ((i & APP_FLUSH) == 0) 377 rte_pipeline_flush(p); 378 i++; 379 } 380 #endif 381 } 382 383 uint64_t test_hash( 384 void *key, 385 __rte_unused void *key_mask, 386 __rte_unused uint32_t key_size, 387 __rte_unused uint64_t seed) 388 { 389 uint32_t *k32 = key; 390 uint32_t ip_dst = rte_be_to_cpu_32(k32[0]); 391 uint64_t signature = (ip_dst >> 2) | ((ip_dst & 0x3) << 30); 392 393 return signature; 394 } 395 396 uint32_t test_hash_cuckoo( 397 const void *key, 398 __rte_unused uint32_t key_size, 399 __rte_unused uint32_t seed) 400 { 401 const uint32_t *k32 = key; 402 uint32_t ip_dst = rte_be_to_cpu_32(k32[0]); 403 uint32_t signature = (ip_dst >> 2) | ((ip_dst & 0x3) << 30); 404 405 return signature; 406 } 407 408 void 409 app_main_loop_rx_metadata(void) { 410 uint32_t i, j; 411 int ret; 412 413 RTE_LOG(INFO, USER1, "Core %u is doing RX (with meta-data)\n", 414 rte_lcore_id()); 415 416 while (!force_quit) { 417 for (i = 0; i < app.n_ports; i++) { 418 uint16_t n_mbufs; 419 420 n_mbufs = rte_eth_rx_burst( 421 app.ports[i], 422 0, 423 app.mbuf_rx.array, 424 app.burst_size_rx_read); 425 426 if (n_mbufs == 0) 427 continue; 428 429 for (j = 0; j < n_mbufs; j++) { 430 struct rte_mbuf *m; 431 uint8_t *m_data, *key; 432 struct rte_ipv4_hdr *ip_hdr; 433 struct rte_ipv6_hdr *ipv6_hdr; 434 uint32_t ip_dst; 435 uint32_t *signature, *k32; 436 437 m = app.mbuf_rx.array[j]; 438 m_data = rte_pktmbuf_mtod(m, uint8_t *); 439 signature = RTE_MBUF_METADATA_UINT32_PTR(m, 440 APP_METADATA_OFFSET(0)); 441 key = RTE_MBUF_METADATA_UINT8_PTR(m, 442 APP_METADATA_OFFSET(32)); 443 444 if (RTE_ETH_IS_IPV4_HDR(m->packet_type)) { 445 ip_hdr = (struct rte_ipv4_hdr *) 446 &m_data[sizeof(struct rte_ether_hdr)]; 447 ip_dst = ip_hdr->dst_addr; 448 449 k32 = (uint32_t *) key; 450 k32[0] = ip_dst & 0xFFFFFF00; 451 } else if (RTE_ETH_IS_IPV6_HDR(m->packet_type)) { 452 ipv6_hdr = (struct rte_ipv6_hdr *) 453 &m_data[sizeof(struct rte_ether_hdr)]; 454 455 memcpy(key, &ipv6_hdr->dst_addr, 16); 456 } else 457 continue; 458 459 *signature = test_hash(key, NULL, 0, 0); 460 } 461 462 do { 463 ret = rte_ring_sp_enqueue_bulk( 464 app.rings_rx[i], 465 (void **) app.mbuf_rx.array, 466 n_mbufs, 467 NULL); 468 } while (ret == 0 && !force_quit); 469 } 470 } 471 } 472