1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2020 Marvell. 3 */ 4 5 #include <stdalign.h> 6 7 #include <rte_graph.h> 8 #include <rte_graph_worker.h> 9 10 #include "pkt_cls_priv.h" 11 #include "node_private.h" 12 13 /* Next node for each ptype, default is '0' is "pkt_drop" */ 14 static const alignas(RTE_CACHE_LINE_SIZE) uint8_t p_nxt[256] = { 15 [RTE_PTYPE_L3_IPV4] = PKT_CLS_NEXT_IP4_LOOKUP, 16 17 [RTE_PTYPE_L3_IPV4_EXT] = PKT_CLS_NEXT_IP4_LOOKUP, 18 19 [RTE_PTYPE_L3_IPV4_EXT_UNKNOWN] = PKT_CLS_NEXT_IP4_LOOKUP, 20 21 [RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L2_ETHER] = 22 PKT_CLS_NEXT_IP4_LOOKUP, 23 24 [RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L2_ETHER] = 25 PKT_CLS_NEXT_IP4_LOOKUP, 26 27 [RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L2_ETHER] = 28 PKT_CLS_NEXT_IP4_LOOKUP, 29 30 [RTE_PTYPE_L3_IPV6] = PKT_CLS_NEXT_IP6_LOOKUP, 31 32 [RTE_PTYPE_L3_IPV6_EXT] = PKT_CLS_NEXT_IP6_LOOKUP, 33 34 [RTE_PTYPE_L3_IPV6_EXT_UNKNOWN] = PKT_CLS_NEXT_IP6_LOOKUP, 35 36 [RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L2_ETHER] = PKT_CLS_NEXT_IP6_LOOKUP, 37 38 [RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L2_ETHER] = PKT_CLS_NEXT_IP6_LOOKUP, 39 40 [RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L2_ETHER] = 41 PKT_CLS_NEXT_IP6_LOOKUP, 42 }; 43 44 static uint16_t 45 pkt_cls_node_process(struct rte_graph *graph, struct rte_node *node, 46 void **objs, uint16_t nb_objs) 47 { 48 struct rte_mbuf *mbuf0, *mbuf1, *mbuf2, *mbuf3, **pkts; 49 uint8_t l0, l1, l2, l3, last_type; 50 uint16_t next_index, n_left_from; 51 uint16_t held = 0, last_spec = 0; 52 struct pkt_cls_node_ctx *ctx; 53 void **to_next, **from; 54 uint32_t i; 55 56 pkts = (struct rte_mbuf **)objs; 57 from = objs; 58 n_left_from = nb_objs; 59 60 for (i = OBJS_PER_CLINE; i < RTE_GRAPH_BURST_SIZE; i += OBJS_PER_CLINE) 61 rte_prefetch0(&objs[i]); 62 63 #if RTE_GRAPH_BURST_SIZE > 64 64 for (i = 0; i < 4 && i < n_left_from; i++) 65 rte_prefetch0(pkts[i]); 66 #endif 67 68 ctx = (struct pkt_cls_node_ctx *)node->ctx; 69 last_type = ctx->l2l3_type; 70 next_index = p_nxt[last_type]; 71 72 /* Get stream for the speculated next node */ 73 to_next = rte_node_next_stream_get(graph, node, 74 next_index, nb_objs); 75 while (n_left_from >= 4) { 76 #if RTE_GRAPH_BURST_SIZE > 64 77 if (likely(n_left_from > 7)) { 78 rte_prefetch0(pkts[4]); 79 rte_prefetch0(pkts[5]); 80 rte_prefetch0(pkts[6]); 81 rte_prefetch0(pkts[7]); 82 } 83 #endif 84 85 mbuf0 = pkts[0]; 86 mbuf1 = pkts[1]; 87 mbuf2 = pkts[2]; 88 mbuf3 = pkts[3]; 89 pkts += 4; 90 n_left_from -= 4; 91 92 l0 = mbuf0->packet_type & 93 (RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK); 94 l1 = mbuf1->packet_type & 95 (RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK); 96 l2 = mbuf2->packet_type & 97 (RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK); 98 l3 = mbuf3->packet_type & 99 (RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK); 100 101 /* Check if they are destined to same 102 * next node based on l2l3 packet type. 103 */ 104 uint8_t fix_spec = (last_type ^ l0) | (last_type ^ l1) | 105 (last_type ^ l2) | (last_type ^ l3); 106 107 if (unlikely(fix_spec)) { 108 /* Copy things successfully speculated till now */ 109 rte_memcpy(to_next, from, 110 last_spec * sizeof(from[0])); 111 from += last_spec; 112 to_next += last_spec; 113 held += last_spec; 114 last_spec = 0; 115 116 /* l0 */ 117 if (p_nxt[l0] == next_index) { 118 to_next[0] = from[0]; 119 to_next++; 120 held++; 121 } else { 122 rte_node_enqueue_x1(graph, node, 123 p_nxt[l0], from[0]); 124 } 125 126 /* l1 */ 127 if (p_nxt[l1] == next_index) { 128 to_next[0] = from[1]; 129 to_next++; 130 held++; 131 } else { 132 rte_node_enqueue_x1(graph, node, 133 p_nxt[l1], from[1]); 134 } 135 136 /* l2 */ 137 if (p_nxt[l2] == next_index) { 138 to_next[0] = from[2]; 139 to_next++; 140 held++; 141 } else { 142 rte_node_enqueue_x1(graph, node, 143 p_nxt[l2], from[2]); 144 } 145 146 /* l3 */ 147 if (p_nxt[l3] == next_index) { 148 to_next[0] = from[3]; 149 to_next++; 150 held++; 151 } else { 152 rte_node_enqueue_x1(graph, node, 153 p_nxt[l3], from[3]); 154 } 155 156 /* Update speculated ptype */ 157 if ((last_type != l3) && (l2 == l3) && 158 (next_index != p_nxt[l3])) { 159 /* Put the current stream for 160 * speculated ltype. 161 */ 162 rte_node_next_stream_put(graph, node, 163 next_index, held); 164 165 held = 0; 166 167 /* Get next stream for new ltype */ 168 next_index = p_nxt[l3]; 169 last_type = l3; 170 to_next = rte_node_next_stream_get(graph, node, 171 next_index, 172 nb_objs); 173 } else if (next_index == p_nxt[l3]) { 174 last_type = l3; 175 } 176 177 from += 4; 178 } else { 179 last_spec += 4; 180 } 181 } 182 183 while (n_left_from > 0) { 184 mbuf0 = pkts[0]; 185 186 pkts += 1; 187 n_left_from -= 1; 188 189 l0 = mbuf0->packet_type & 190 (RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK); 191 if (unlikely((l0 != last_type) && 192 (p_nxt[l0] != next_index))) { 193 /* Copy things successfully speculated till now */ 194 rte_memcpy(to_next, from, 195 last_spec * sizeof(from[0])); 196 from += last_spec; 197 to_next += last_spec; 198 held += last_spec; 199 last_spec = 0; 200 201 rte_node_enqueue_x1(graph, node, 202 p_nxt[l0], from[0]); 203 from += 1; 204 } else { 205 last_spec += 1; 206 } 207 } 208 209 /* !!! Home run !!! */ 210 if (likely(last_spec == nb_objs)) { 211 rte_node_next_stream_move(graph, node, next_index); 212 return nb_objs; 213 } 214 215 held += last_spec; 216 /* Copy things successfully speculated till now */ 217 rte_memcpy(to_next, from, last_spec * sizeof(from[0])); 218 rte_node_next_stream_put(graph, node, next_index, held); 219 220 ctx->l2l3_type = last_type; 221 return nb_objs; 222 } 223 224 /* Packet Classification Node */ 225 struct rte_node_register pkt_cls_node = { 226 .process = pkt_cls_node_process, 227 .name = "pkt_cls", 228 229 .nb_edges = PKT_CLS_NEXT_MAX, 230 .next_nodes = { 231 /* Pkt drop node starts at '0' */ 232 [PKT_CLS_NEXT_PKT_DROP] = "pkt_drop", 233 [PKT_CLS_NEXT_IP4_LOOKUP] = "ip4_lookup", 234 [PKT_CLS_NEXT_IP6_LOOKUP] = "ip6_lookup", 235 }, 236 }; 237 RTE_NODE_REGISTER(pkt_cls_node); 238