1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 */ 4 #include <string.h> 5 6 #include <rte_mbuf.h> 7 #include <rte_malloc.h> 8 9 #include "rte_port_sched.h" 10 11 /* 12 * Reader 13 */ 14 #ifdef RTE_PORT_STATS_COLLECT 15 16 #define RTE_PORT_SCHED_READER_PKTS_IN_ADD(port, val) \ 17 port->stats.n_pkts_in += val 18 #define RTE_PORT_SCHED_READER_PKTS_DROP_ADD(port, val) \ 19 port->stats.n_pkts_drop += val 20 21 #else 22 23 #define RTE_PORT_SCHED_READER_PKTS_IN_ADD(port, val) 24 #define RTE_PORT_SCHED_READER_PKTS_DROP_ADD(port, val) 25 26 #endif 27 28 struct rte_port_sched_reader { 29 struct rte_port_in_stats stats; 30 31 struct rte_sched_port *sched; 32 }; 33 34 static void * 35 rte_port_sched_reader_create(void *params, int socket_id) 36 { 37 struct rte_port_sched_reader_params *conf = 38 params; 39 struct rte_port_sched_reader *port; 40 41 /* Check input parameters */ 42 if ((conf == NULL) || 43 (conf->sched == NULL)) { 44 RTE_LOG(ERR, PORT, "%s: Invalid params\n", __func__); 45 return NULL; 46 } 47 48 /* Memory allocation */ 49 port = rte_zmalloc_socket("PORT", sizeof(*port), 50 RTE_CACHE_LINE_SIZE, socket_id); 51 if (port == NULL) { 52 RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__); 53 return NULL; 54 } 55 56 /* Initialization */ 57 port->sched = conf->sched; 58 59 return port; 60 } 61 62 static int 63 rte_port_sched_reader_rx(void *port, struct rte_mbuf **pkts, uint32_t n_pkts) 64 { 65 struct rte_port_sched_reader *p = port; 66 uint32_t nb_rx; 67 68 nb_rx = rte_sched_port_dequeue(p->sched, pkts, n_pkts); 69 RTE_PORT_SCHED_READER_PKTS_IN_ADD(p, nb_rx); 70 71 return nb_rx; 72 } 73 74 static int 75 rte_port_sched_reader_free(void *port) 76 { 77 if (port == NULL) { 78 RTE_LOG(ERR, PORT, "%s: port is NULL\n", __func__); 79 return -EINVAL; 80 } 81 82 rte_free(port); 83 84 return 0; 85 } 86 87 static int 88 rte_port_sched_reader_stats_read(void *port, 89 struct rte_port_in_stats *stats, int clear) 90 { 91 struct rte_port_sched_reader *p = 92 port; 93 94 if (stats != NULL) 95 memcpy(stats, &p->stats, sizeof(p->stats)); 96 97 if (clear) 98 memset(&p->stats, 0, sizeof(p->stats)); 99 100 return 0; 101 } 102 103 /* 104 * Writer 105 */ 106 #ifdef RTE_PORT_STATS_COLLECT 107 108 #define RTE_PORT_SCHED_WRITER_STATS_PKTS_IN_ADD(port, val) \ 109 port->stats.n_pkts_in += val 110 #define RTE_PORT_SCHED_WRITER_STATS_PKTS_DROP_ADD(port, val) \ 111 port->stats.n_pkts_drop += val 112 113 #else 114 115 #define RTE_PORT_SCHED_WRITER_STATS_PKTS_IN_ADD(port, val) 116 #define RTE_PORT_SCHED_WRITER_STATS_PKTS_DROP_ADD(port, val) 117 118 #endif 119 120 struct rte_port_sched_writer { 121 struct rte_port_out_stats stats; 122 123 struct rte_mbuf *tx_buf[2 * RTE_PORT_IN_BURST_SIZE_MAX]; 124 struct rte_sched_port *sched; 125 uint32_t tx_burst_sz; 126 uint32_t tx_buf_count; 127 uint64_t bsz_mask; 128 }; 129 130 static void * 131 rte_port_sched_writer_create(void *params, int socket_id) 132 { 133 struct rte_port_sched_writer_params *conf = 134 params; 135 struct rte_port_sched_writer *port; 136 137 /* Check input parameters */ 138 if ((conf == NULL) || 139 (conf->sched == NULL) || 140 (conf->tx_burst_sz == 0) || 141 (conf->tx_burst_sz > RTE_PORT_IN_BURST_SIZE_MAX) || 142 (!rte_is_power_of_2(conf->tx_burst_sz))) { 143 RTE_LOG(ERR, PORT, "%s: Invalid params\n", __func__); 144 return NULL; 145 } 146 147 /* Memory allocation */ 148 port = rte_zmalloc_socket("PORT", sizeof(*port), 149 RTE_CACHE_LINE_SIZE, socket_id); 150 if (port == NULL) { 151 RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__); 152 return NULL; 153 } 154 155 /* Initialization */ 156 port->sched = conf->sched; 157 port->tx_burst_sz = conf->tx_burst_sz; 158 port->tx_buf_count = 0; 159 port->bsz_mask = 1LLU << (conf->tx_burst_sz - 1); 160 161 return port; 162 } 163 164 static int 165 rte_port_sched_writer_tx(void *port, struct rte_mbuf *pkt) 166 { 167 struct rte_port_sched_writer *p = (struct rte_port_sched_writer *) port; 168 169 p->tx_buf[p->tx_buf_count++] = pkt; 170 RTE_PORT_SCHED_WRITER_STATS_PKTS_IN_ADD(p, 1); 171 if (p->tx_buf_count >= p->tx_burst_sz) { 172 __rte_unused uint32_t nb_tx; 173 174 nb_tx = rte_sched_port_enqueue(p->sched, p->tx_buf, p->tx_buf_count); 175 RTE_PORT_SCHED_WRITER_STATS_PKTS_DROP_ADD(p, p->tx_buf_count - nb_tx); 176 p->tx_buf_count = 0; 177 } 178 179 return 0; 180 } 181 182 static int 183 rte_port_sched_writer_tx_bulk(void *port, 184 struct rte_mbuf **pkts, 185 uint64_t pkts_mask) 186 { 187 struct rte_port_sched_writer *p = (struct rte_port_sched_writer *) port; 188 uint64_t bsz_mask = p->bsz_mask; 189 uint32_t tx_buf_count = p->tx_buf_count; 190 uint64_t expr = (pkts_mask & (pkts_mask + 1)) | 191 ((pkts_mask & bsz_mask) ^ bsz_mask); 192 193 if (expr == 0) { 194 __rte_unused uint32_t nb_tx; 195 uint64_t n_pkts = __builtin_popcountll(pkts_mask); 196 197 if (tx_buf_count) { 198 nb_tx = rte_sched_port_enqueue(p->sched, p->tx_buf, 199 tx_buf_count); 200 RTE_PORT_SCHED_WRITER_STATS_PKTS_DROP_ADD(p, tx_buf_count - nb_tx); 201 p->tx_buf_count = 0; 202 } 203 204 nb_tx = rte_sched_port_enqueue(p->sched, pkts, n_pkts); 205 RTE_PORT_SCHED_WRITER_STATS_PKTS_DROP_ADD(p, n_pkts - nb_tx); 206 } else { 207 for ( ; pkts_mask; ) { 208 uint32_t pkt_index = __builtin_ctzll(pkts_mask); 209 uint64_t pkt_mask = 1LLU << pkt_index; 210 struct rte_mbuf *pkt = pkts[pkt_index]; 211 212 p->tx_buf[tx_buf_count++] = pkt; 213 RTE_PORT_SCHED_WRITER_STATS_PKTS_IN_ADD(p, 1); 214 pkts_mask &= ~pkt_mask; 215 } 216 p->tx_buf_count = tx_buf_count; 217 218 if (tx_buf_count >= p->tx_burst_sz) { 219 __rte_unused uint32_t nb_tx; 220 221 nb_tx = rte_sched_port_enqueue(p->sched, p->tx_buf, 222 tx_buf_count); 223 RTE_PORT_SCHED_WRITER_STATS_PKTS_DROP_ADD(p, tx_buf_count - nb_tx); 224 p->tx_buf_count = 0; 225 } 226 } 227 228 return 0; 229 } 230 231 static int 232 rte_port_sched_writer_flush(void *port) 233 { 234 struct rte_port_sched_writer *p = (struct rte_port_sched_writer *) port; 235 236 if (p->tx_buf_count) { 237 __rte_unused uint32_t nb_tx; 238 239 nb_tx = rte_sched_port_enqueue(p->sched, p->tx_buf, p->tx_buf_count); 240 RTE_PORT_SCHED_WRITER_STATS_PKTS_DROP_ADD(p, p->tx_buf_count - nb_tx); 241 p->tx_buf_count = 0; 242 } 243 244 return 0; 245 } 246 247 static int 248 rte_port_sched_writer_free(void *port) 249 { 250 if (port == NULL) { 251 RTE_LOG(ERR, PORT, "%s: port is NULL\n", __func__); 252 return -EINVAL; 253 } 254 255 rte_port_sched_writer_flush(port); 256 rte_free(port); 257 258 return 0; 259 } 260 261 static int 262 rte_port_sched_writer_stats_read(void *port, 263 struct rte_port_out_stats *stats, int clear) 264 { 265 struct rte_port_sched_writer *p = 266 port; 267 268 if (stats != NULL) 269 memcpy(stats, &p->stats, sizeof(p->stats)); 270 271 if (clear) 272 memset(&p->stats, 0, sizeof(p->stats)); 273 274 return 0; 275 } 276 277 /* 278 * Summary of port operations 279 */ 280 struct rte_port_in_ops rte_port_sched_reader_ops = { 281 .f_create = rte_port_sched_reader_create, 282 .f_free = rte_port_sched_reader_free, 283 .f_rx = rte_port_sched_reader_rx, 284 .f_stats = rte_port_sched_reader_stats_read, 285 }; 286 287 struct rte_port_out_ops rte_port_sched_writer_ops = { 288 .f_create = rte_port_sched_writer_create, 289 .f_free = rte_port_sched_writer_free, 290 .f_tx = rte_port_sched_writer_tx, 291 .f_tx_bulk = rte_port_sched_writer_tx_bulk, 292 .f_flush = rte_port_sched_writer_flush, 293 .f_stats = rte_port_sched_writer_stats_read, 294 }; 295