1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <rte_common.h> 35 #include <rte_cycles.h> 36 #include <rte_pipeline.h> 37 38 #include "pipeline_common_be.h" 39 #include "app.h" 40 #include "thread.h" 41 42 #if APP_THREAD_HEADROOM_STATS_COLLECT 43 44 #define PIPELINE_RUN_REGULAR(thread, pipeline) \ 45 do { \ 46 uint64_t t0 = rte_rdtsc_precise(); \ 47 int n_pkts = rte_pipeline_run(pipeline->p); \ 48 \ 49 if (n_pkts == 0) { \ 50 uint64_t t1 = rte_rdtsc_precise(); \ 51 \ 52 thread->headroom_cycles += t1 - t0; \ 53 } \ 54 } while (0) 55 56 57 #define PIPELINE_RUN_CUSTOM(thread, data) \ 58 do { \ 59 uint64_t t0 = rte_rdtsc_precise(); \ 60 int n_pkts = data->f_run(data->be); \ 61 \ 62 if (n_pkts == 0) { \ 63 uint64_t t1 = rte_rdtsc_precise(); \ 64 \ 65 thread->headroom_cycles += t1 - t0; \ 66 } \ 67 } while (0) 68 69 #else 70 71 #define PIPELINE_RUN_REGULAR(thread, pipeline) \ 72 rte_pipeline_run(pipeline->p) 73 74 #define PIPELINE_RUN_CUSTOM(thread, data) \ 75 data->f_run(data->be) 76 77 #endif 78 79 static inline void * 80 thread_msg_recv(struct rte_ring *r) 81 { 82 void *msg; 83 int status = rte_ring_sc_dequeue(r, &msg); 84 85 if (status != 0) 86 return NULL; 87 88 return msg; 89 } 90 91 static inline void 92 thread_msg_send(struct rte_ring *r, 93 void *msg) 94 { 95 int status; 96 97 do { 98 status = rte_ring_sp_enqueue(r, msg); 99 } while (status == -ENOBUFS); 100 } 101 102 static int 103 thread_pipeline_enable(struct app_thread_data *t, 104 struct thread_pipeline_enable_msg_req *req) 105 { 106 struct app_thread_pipeline_data *p; 107 108 if (req->f_run == NULL) { 109 if (t->n_regular >= APP_MAX_THREAD_PIPELINES) 110 return -1; 111 } else { 112 if (t->n_custom >= APP_MAX_THREAD_PIPELINES) 113 return -1; 114 } 115 116 p = (req->f_run == NULL) ? 117 &t->regular[t->n_regular] : 118 &t->custom[t->n_custom]; 119 120 p->pipeline_id = req->pipeline_id; 121 p->be = req->be; 122 p->f_run = req->f_run; 123 p->f_timer = req->f_timer; 124 p->timer_period = req->timer_period; 125 p->deadline = 0; 126 127 if (req->f_run == NULL) 128 t->n_regular++; 129 else 130 t->n_custom++; 131 132 return 0; 133 } 134 135 static int 136 thread_pipeline_disable(struct app_thread_data *t, 137 struct thread_pipeline_disable_msg_req *req) 138 { 139 uint32_t n_regular = RTE_MIN(t->n_regular, RTE_DIM(t->regular)); 140 uint32_t n_custom = RTE_MIN(t->n_custom, RTE_DIM(t->custom)); 141 uint32_t i; 142 143 /* search regular pipelines of current thread */ 144 for (i = 0; i < n_regular; i++) { 145 if (t->regular[i].pipeline_id != req->pipeline_id) 146 continue; 147 148 if (i < n_regular - 1) 149 memcpy(&t->regular[i], 150 &t->regular[i+1], 151 (n_regular - 1 - i) * sizeof(struct app_thread_pipeline_data)); 152 153 n_regular--; 154 t->n_regular = n_regular; 155 156 return 0; 157 } 158 159 /* search custom pipelines of current thread */ 160 for (i = 0; i < n_custom; i++) { 161 if (t->custom[i].pipeline_id != req->pipeline_id) 162 continue; 163 164 if (i < n_custom - 1) 165 memcpy(&t->custom[i], 166 &t->custom[i+1], 167 (n_custom - 1 - i) * sizeof(struct app_thread_pipeline_data)); 168 169 n_custom--; 170 t->n_custom = n_custom; 171 172 return 0; 173 } 174 175 /* return if pipeline not found */ 176 return -1; 177 } 178 179 static int 180 thread_msg_req_handle(struct app_thread_data *t) 181 { 182 void *msg_ptr; 183 struct thread_msg_req *req; 184 struct thread_msg_rsp *rsp; 185 186 msg_ptr = thread_msg_recv(t->msgq_in); 187 req = msg_ptr; 188 rsp = msg_ptr; 189 190 if (req != NULL) 191 switch (req->type) { 192 case THREAD_MSG_REQ_PIPELINE_ENABLE: { 193 rsp->status = thread_pipeline_enable(t, 194 (struct thread_pipeline_enable_msg_req *) req); 195 thread_msg_send(t->msgq_out, rsp); 196 break; 197 } 198 199 case THREAD_MSG_REQ_PIPELINE_DISABLE: { 200 rsp->status = thread_pipeline_disable(t, 201 (struct thread_pipeline_disable_msg_req *) req); 202 thread_msg_send(t->msgq_out, rsp); 203 break; 204 } 205 206 case THREAD_MSG_REQ_HEADROOM_READ: { 207 struct thread_headroom_read_msg_rsp *rsp = 208 (struct thread_headroom_read_msg_rsp *) 209 req; 210 211 rsp->headroom_ratio = t->headroom_ratio; 212 rsp->status = 0; 213 thread_msg_send(t->msgq_out, rsp); 214 break; 215 } 216 default: 217 break; 218 } 219 220 return 0; 221 } 222 223 static void 224 thread_headroom_update(struct app_thread_data *t, uint64_t time) 225 { 226 uint64_t time_diff = time - t->headroom_time; 227 228 t->headroom_ratio = 229 ((double) t->headroom_cycles) / ((double) time_diff); 230 231 t->headroom_cycles = 0; 232 t->headroom_time = rte_rdtsc_precise(); 233 } 234 235 int 236 app_thread(void *arg) 237 { 238 struct app_params *app = (struct app_params *) arg; 239 uint32_t core_id = rte_lcore_id(), i, j; 240 struct app_thread_data *t = &app->thread_data[core_id]; 241 242 for (i = 0; ; i++) { 243 uint32_t n_regular = RTE_MIN(t->n_regular, RTE_DIM(t->regular)); 244 uint32_t n_custom = RTE_MIN(t->n_custom, RTE_DIM(t->custom)); 245 246 /* Run regular pipelines */ 247 for (j = 0; j < n_regular; j++) { 248 struct app_thread_pipeline_data *data = &t->regular[j]; 249 struct pipeline *p = data->be; 250 251 PIPELINE_RUN_REGULAR(t, p); 252 } 253 254 /* Run custom pipelines */ 255 for (j = 0; j < n_custom; j++) { 256 struct app_thread_pipeline_data *data = &t->custom[j]; 257 258 PIPELINE_RUN_CUSTOM(t, data); 259 } 260 261 /* Timer */ 262 if ((i & 0xF) == 0) { 263 uint64_t time = rte_get_tsc_cycles(); 264 uint64_t t_deadline = UINT64_MAX; 265 266 if (time < t->deadline) 267 continue; 268 269 /* Timer for regular pipelines */ 270 for (j = 0; j < n_regular; j++) { 271 struct app_thread_pipeline_data *data = 272 &t->regular[j]; 273 uint64_t p_deadline = data->deadline; 274 275 if (p_deadline <= time) { 276 data->f_timer(data->be); 277 p_deadline = time + data->timer_period; 278 data->deadline = p_deadline; 279 } 280 281 if (p_deadline < t_deadline) 282 t_deadline = p_deadline; 283 } 284 285 /* Timer for custom pipelines */ 286 for (j = 0; j < n_custom; j++) { 287 struct app_thread_pipeline_data *data = 288 &t->custom[j]; 289 uint64_t p_deadline = data->deadline; 290 291 if (p_deadline <= time) { 292 data->f_timer(data->be); 293 p_deadline = time + data->timer_period; 294 data->deadline = p_deadline; 295 } 296 297 if (p_deadline < t_deadline) 298 t_deadline = p_deadline; 299 } 300 301 /* Timer for thread message request */ 302 { 303 uint64_t deadline = t->thread_req_deadline; 304 305 if (deadline <= time) { 306 thread_msg_req_handle(t); 307 thread_headroom_update(t, time); 308 deadline = time + t->timer_period; 309 t->thread_req_deadline = deadline; 310 } 311 312 if (deadline < t_deadline) 313 t_deadline = deadline; 314 } 315 316 317 t->deadline = t_deadline; 318 } 319 } 320 321 return 0; 322 } 323