1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2019 6WIND S.A.
3 */
4
5 #include <stdalign.h>
6 #include <sys/queue.h>
7 #include <stdint.h>
8 #include <limits.h>
9
10 #include <rte_common.h>
11 #include <rte_eal.h>
12 #include <rte_eal_memconfig.h>
13 #include <rte_tailq.h>
14 #include <rte_errno.h>
15 #include <rte_malloc.h>
16 #include <rte_string_fns.h>
17 #include <rte_bitops.h>
18 #include <rte_mbuf.h>
19 #include <rte_mbuf_dyn.h>
20
21 #include "mbuf_log.h"
22
23 #define RTE_MBUF_DYN_MZNAME "rte_mbuf_dyn"
24
25 struct mbuf_dynfield_elt {
26 struct rte_mbuf_dynfield params;
27 size_t offset;
28 };
29 TAILQ_HEAD(mbuf_dynfield_list, rte_tailq_entry);
30
31 static struct rte_tailq_elem mbuf_dynfield_tailq = {
32 .name = "RTE_MBUF_DYNFIELD",
33 };
34 EAL_REGISTER_TAILQ(mbuf_dynfield_tailq);
35
36 struct mbuf_dynflag_elt {
37 struct rte_mbuf_dynflag params;
38 unsigned int bitnum;
39 };
40 TAILQ_HEAD(mbuf_dynflag_list, rte_tailq_entry);
41
42 static struct rte_tailq_elem mbuf_dynflag_tailq = {
43 .name = "RTE_MBUF_DYNFLAG",
44 };
45 EAL_REGISTER_TAILQ(mbuf_dynflag_tailq);
46
47 struct mbuf_dyn_shm {
48 /**
49 * For each mbuf byte, free_space[i] != 0 if space is free.
50 * The value is the size of the biggest aligned element that
51 * can fit in the zone.
52 */
53 uint8_t free_space[sizeof(struct rte_mbuf)];
54 /** Bitfield of available flags. */
55 uint64_t free_flags;
56 };
57 static struct mbuf_dyn_shm *shm;
58
59 /* Set the value of free_space[] according to the size and alignment of
60 * the free areas. This helps to select the best place when reserving a
61 * dynamic field. Assume tailq is locked.
62 */
63 static void
process_score(void)64 process_score(void)
65 {
66 size_t off, align, size, i;
67
68 /* first, erase previous info */
69 for (i = 0; i < sizeof(struct rte_mbuf); i++) {
70 if (shm->free_space[i])
71 shm->free_space[i] = 1;
72 }
73
74 off = 0;
75 while (off < sizeof(struct rte_mbuf)) {
76 /* get the size of the free zone */
77 for (size = 0; (off + size) < sizeof(struct rte_mbuf) &&
78 shm->free_space[off + size]; size++)
79 ;
80 if (size == 0) {
81 off++;
82 continue;
83 }
84
85 /* get the alignment of biggest object that can fit in
86 * the zone at this offset.
87 */
88 for (align = 1;
89 (off % (align << 1)) == 0 && (align << 1) <= size;
90 align <<= 1)
91 ;
92
93 /* save it in free_space[] */
94 for (i = off; i < off + align; i++)
95 shm->free_space[i] = RTE_MAX(align, shm->free_space[i]);
96
97 off += align;
98 }
99 }
100
101 /* Mark the area occupied by a mbuf field as available in the shm. */
102 #define mark_free(field) \
103 memset(&shm->free_space[offsetof(struct rte_mbuf, field)], \
104 1, sizeof(((struct rte_mbuf *)0)->field))
105
106 /* Allocate and initialize the shared memory. Assume tailq is locked */
107 static int
init_shared_mem(void)108 init_shared_mem(void)
109 {
110 const struct rte_memzone *mz;
111 uint64_t mask;
112
113 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
114 mz = rte_memzone_reserve_aligned(RTE_MBUF_DYN_MZNAME,
115 sizeof(struct mbuf_dyn_shm),
116 SOCKET_ID_ANY, 0,
117 RTE_CACHE_LINE_SIZE);
118 } else {
119 mz = rte_memzone_lookup(RTE_MBUF_DYN_MZNAME);
120 }
121 if (mz == NULL) {
122 MBUF_LOG(ERR, "Failed to get mbuf dyn shared memory");
123 return -1;
124 }
125
126 shm = mz->addr;
127
128 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
129 /* init free_space, keep it sync'd with
130 * rte_mbuf_dynfield_copy().
131 */
132 memset(shm, 0, sizeof(*shm));
133 mark_free(dynfield1);
134 #if !RTE_IOVA_IN_MBUF
135 mark_free(dynfield2);
136 #endif
137
138 /* init free_flags */
139 for (mask = RTE_MBUF_F_FIRST_FREE; mask <= RTE_MBUF_F_LAST_FREE; mask <<= 1)
140 shm->free_flags |= mask;
141
142 process_score();
143 }
144
145 return 0;
146 }
147
148 /* check if this offset can be used */
149 static int
check_offset(size_t offset,size_t size,size_t align)150 check_offset(size_t offset, size_t size, size_t align)
151 {
152 size_t i;
153
154 if ((offset & (align - 1)) != 0)
155 return -1;
156 if (offset + size > sizeof(struct rte_mbuf))
157 return -1;
158
159 for (i = 0; i < size; i++) {
160 if (!shm->free_space[i + offset])
161 return -1;
162 }
163
164 return 0;
165 }
166
167 /* assume tailq is locked */
168 static struct mbuf_dynfield_elt *
__mbuf_dynfield_lookup(const char * name)169 __mbuf_dynfield_lookup(const char *name)
170 {
171 struct mbuf_dynfield_list *mbuf_dynfield_list;
172 struct mbuf_dynfield_elt *mbuf_dynfield;
173 struct rte_tailq_entry *te;
174
175 mbuf_dynfield_list = RTE_TAILQ_CAST(
176 mbuf_dynfield_tailq.head, mbuf_dynfield_list);
177
178 TAILQ_FOREACH(te, mbuf_dynfield_list, next) {
179 mbuf_dynfield = (struct mbuf_dynfield_elt *)te->data;
180 if (strcmp(name, mbuf_dynfield->params.name) == 0)
181 break;
182 }
183
184 if (te == NULL || mbuf_dynfield == NULL) {
185 rte_errno = ENOENT;
186 return NULL;
187 }
188
189 return mbuf_dynfield;
190 }
191
192 int
rte_mbuf_dynfield_lookup(const char * name,struct rte_mbuf_dynfield * params)193 rte_mbuf_dynfield_lookup(const char *name, struct rte_mbuf_dynfield *params)
194 {
195 struct mbuf_dynfield_elt *mbuf_dynfield;
196
197 rte_mcfg_tailq_read_lock();
198 if (shm == NULL && init_shared_mem() < 0)
199 mbuf_dynfield = NULL;
200 else
201 mbuf_dynfield = __mbuf_dynfield_lookup(name);
202 rte_mcfg_tailq_read_unlock();
203
204 if (mbuf_dynfield == NULL)
205 return -1;
206
207 if (params != NULL)
208 memcpy(params, &mbuf_dynfield->params, sizeof(*params));
209
210 return mbuf_dynfield->offset;
211 }
212
mbuf_dynfield_cmp(const struct rte_mbuf_dynfield * params1,const struct rte_mbuf_dynfield * params2)213 static int mbuf_dynfield_cmp(const struct rte_mbuf_dynfield *params1,
214 const struct rte_mbuf_dynfield *params2)
215 {
216 if (strcmp(params1->name, params2->name))
217 return -1;
218 if (params1->size != params2->size)
219 return -1;
220 if (params1->align != params2->align)
221 return -1;
222 if (params1->flags != params2->flags)
223 return -1;
224 return 0;
225 }
226
227 /* assume tailq is locked */
228 static int
__rte_mbuf_dynfield_register_offset(const struct rte_mbuf_dynfield * params,size_t req)229 __rte_mbuf_dynfield_register_offset(const struct rte_mbuf_dynfield *params,
230 size_t req)
231 {
232 struct mbuf_dynfield_list *mbuf_dynfield_list;
233 struct mbuf_dynfield_elt *mbuf_dynfield = NULL;
234 struct rte_tailq_entry *te = NULL;
235 unsigned int best_zone = UINT_MAX;
236 size_t i, offset;
237 int ret;
238
239 if (shm == NULL && init_shared_mem() < 0)
240 return -1;
241
242 mbuf_dynfield = __mbuf_dynfield_lookup(params->name);
243 if (mbuf_dynfield != NULL) {
244 if (req != SIZE_MAX && req != mbuf_dynfield->offset) {
245 rte_errno = EEXIST;
246 return -1;
247 }
248 if (mbuf_dynfield_cmp(params, &mbuf_dynfield->params) < 0) {
249 rte_errno = EEXIST;
250 return -1;
251 }
252 return mbuf_dynfield->offset;
253 }
254
255 if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
256 rte_errno = EPERM;
257 return -1;
258 }
259
260 if (req == SIZE_MAX) {
261 /* Find the best place to put this field: we search the
262 * lowest value of shm->free_space[offset]: the zones
263 * containing room for larger fields are kept for later.
264 */
265 for (offset = 0;
266 offset < sizeof(struct rte_mbuf);
267 offset++) {
268 if (check_offset(offset, params->size,
269 params->align) == 0 &&
270 shm->free_space[offset] < best_zone) {
271 best_zone = shm->free_space[offset];
272 req = offset;
273 }
274 }
275 if (req == SIZE_MAX) {
276 rte_errno = ENOENT;
277 return -1;
278 }
279 } else {
280 if (check_offset(req, params->size, params->align) < 0) {
281 rte_errno = EBUSY;
282 return -1;
283 }
284 }
285
286 offset = req;
287 mbuf_dynfield_list = RTE_TAILQ_CAST(
288 mbuf_dynfield_tailq.head, mbuf_dynfield_list);
289
290 te = rte_zmalloc("MBUF_DYNFIELD_TAILQ_ENTRY", sizeof(*te), 0);
291 if (te == NULL) {
292 rte_errno = ENOMEM;
293 return -1;
294 }
295
296 mbuf_dynfield = rte_zmalloc("mbuf_dynfield", sizeof(*mbuf_dynfield), 0);
297 if (mbuf_dynfield == NULL) {
298 rte_free(te);
299 rte_errno = ENOMEM;
300 return -1;
301 }
302
303 ret = strlcpy(mbuf_dynfield->params.name, params->name,
304 sizeof(mbuf_dynfield->params.name));
305 if (ret < 0 || ret >= (int)sizeof(mbuf_dynfield->params.name)) {
306 rte_errno = ENAMETOOLONG;
307 rte_free(mbuf_dynfield);
308 rte_free(te);
309 return -1;
310 }
311 memcpy(&mbuf_dynfield->params, params, sizeof(mbuf_dynfield->params));
312 mbuf_dynfield->offset = offset;
313 te->data = mbuf_dynfield;
314
315 TAILQ_INSERT_TAIL(mbuf_dynfield_list, te, next);
316
317 for (i = offset; i < offset + params->size; i++)
318 shm->free_space[i] = 0;
319 process_score();
320
321 MBUF_LOG(DEBUG, "Registered dynamic field %s (sz=%zu, al=%zu, fl=0x%x) -> %zd",
322 params->name, params->size, params->align, params->flags,
323 offset);
324
325 return offset;
326 }
327
328 int
rte_mbuf_dynfield_register_offset(const struct rte_mbuf_dynfield * params,size_t req)329 rte_mbuf_dynfield_register_offset(const struct rte_mbuf_dynfield *params,
330 size_t req)
331 {
332 int ret;
333
334 if (params->size >= sizeof(struct rte_mbuf)) {
335 rte_errno = EINVAL;
336 return -1;
337 }
338 if (!rte_is_power_of_2(params->align)) {
339 rte_errno = EINVAL;
340 return -1;
341 }
342 if (params->flags != 0) {
343 rte_errno = EINVAL;
344 return -1;
345 }
346
347 rte_mcfg_tailq_write_lock();
348 ret = __rte_mbuf_dynfield_register_offset(params, req);
349 rte_mcfg_tailq_write_unlock();
350
351 return ret;
352 }
353
354 int
rte_mbuf_dynfield_register(const struct rte_mbuf_dynfield * params)355 rte_mbuf_dynfield_register(const struct rte_mbuf_dynfield *params)
356 {
357 return rte_mbuf_dynfield_register_offset(params, SIZE_MAX);
358 }
359
360 /* assume tailq is locked */
361 static struct mbuf_dynflag_elt *
__mbuf_dynflag_lookup(const char * name)362 __mbuf_dynflag_lookup(const char *name)
363 {
364 struct mbuf_dynflag_list *mbuf_dynflag_list;
365 struct mbuf_dynflag_elt *mbuf_dynflag;
366 struct rte_tailq_entry *te;
367
368 mbuf_dynflag_list = RTE_TAILQ_CAST(
369 mbuf_dynflag_tailq.head, mbuf_dynflag_list);
370
371 TAILQ_FOREACH(te, mbuf_dynflag_list, next) {
372 mbuf_dynflag = (struct mbuf_dynflag_elt *)te->data;
373 if (strncmp(name, mbuf_dynflag->params.name,
374 RTE_MBUF_DYN_NAMESIZE) == 0)
375 break;
376 }
377
378 if (te == NULL) {
379 rte_errno = ENOENT;
380 return NULL;
381 }
382
383 return mbuf_dynflag;
384 }
385
386 int
rte_mbuf_dynflag_lookup(const char * name,struct rte_mbuf_dynflag * params)387 rte_mbuf_dynflag_lookup(const char *name,
388 struct rte_mbuf_dynflag *params)
389 {
390 struct mbuf_dynflag_elt *mbuf_dynflag;
391
392 rte_mcfg_tailq_read_lock();
393 if (shm == NULL && init_shared_mem() < 0)
394 mbuf_dynflag = NULL;
395 else
396 mbuf_dynflag = __mbuf_dynflag_lookup(name);
397 rte_mcfg_tailq_read_unlock();
398
399 if (mbuf_dynflag == NULL)
400 return -1;
401
402 if (params != NULL)
403 memcpy(params, &mbuf_dynflag->params, sizeof(*params));
404
405 return mbuf_dynflag->bitnum;
406 }
407
mbuf_dynflag_cmp(const struct rte_mbuf_dynflag * params1,const struct rte_mbuf_dynflag * params2)408 static int mbuf_dynflag_cmp(const struct rte_mbuf_dynflag *params1,
409 const struct rte_mbuf_dynflag *params2)
410 {
411 if (strcmp(params1->name, params2->name))
412 return -1;
413 if (params1->flags != params2->flags)
414 return -1;
415 return 0;
416 }
417
418 /* assume tailq is locked */
419 static int
__rte_mbuf_dynflag_register_bitnum(const struct rte_mbuf_dynflag * params,unsigned int req)420 __rte_mbuf_dynflag_register_bitnum(const struct rte_mbuf_dynflag *params,
421 unsigned int req)
422 {
423 struct mbuf_dynflag_list *mbuf_dynflag_list;
424 struct mbuf_dynflag_elt *mbuf_dynflag = NULL;
425 struct rte_tailq_entry *te = NULL;
426 unsigned int bitnum;
427 int ret;
428
429 if (shm == NULL && init_shared_mem() < 0)
430 return -1;
431
432 mbuf_dynflag = __mbuf_dynflag_lookup(params->name);
433 if (mbuf_dynflag != NULL) {
434 if (req != UINT_MAX && req != mbuf_dynflag->bitnum) {
435 rte_errno = EEXIST;
436 return -1;
437 }
438 if (mbuf_dynflag_cmp(params, &mbuf_dynflag->params) < 0) {
439 rte_errno = EEXIST;
440 return -1;
441 }
442 return mbuf_dynflag->bitnum;
443 }
444
445 if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
446 rte_errno = EPERM;
447 return -1;
448 }
449
450 if (req == UINT_MAX) {
451 if (shm->free_flags == 0) {
452 rte_errno = ENOENT;
453 return -1;
454 }
455 bitnum = rte_bsf64(shm->free_flags);
456 } else {
457 if ((shm->free_flags & (1ULL << req)) == 0) {
458 rte_errno = EBUSY;
459 return -1;
460 }
461 bitnum = req;
462 }
463
464 mbuf_dynflag_list = RTE_TAILQ_CAST(
465 mbuf_dynflag_tailq.head, mbuf_dynflag_list);
466
467 te = rte_zmalloc("MBUF_DYNFLAG_TAILQ_ENTRY", sizeof(*te), 0);
468 if (te == NULL) {
469 rte_errno = ENOMEM;
470 return -1;
471 }
472
473 mbuf_dynflag = rte_zmalloc("mbuf_dynflag", sizeof(*mbuf_dynflag), 0);
474 if (mbuf_dynflag == NULL) {
475 rte_free(te);
476 rte_errno = ENOMEM;
477 return -1;
478 }
479
480 ret = strlcpy(mbuf_dynflag->params.name, params->name,
481 sizeof(mbuf_dynflag->params.name));
482 if (ret < 0 || ret >= (int)sizeof(mbuf_dynflag->params.name)) {
483 rte_free(mbuf_dynflag);
484 rte_free(te);
485 rte_errno = ENAMETOOLONG;
486 return -1;
487 }
488 mbuf_dynflag->bitnum = bitnum;
489 te->data = mbuf_dynflag;
490
491 TAILQ_INSERT_TAIL(mbuf_dynflag_list, te, next);
492
493 shm->free_flags &= ~(1ULL << bitnum);
494
495 MBUF_LOG(DEBUG, "Registered dynamic flag %s (fl=0x%x) -> %u",
496 params->name, params->flags, bitnum);
497
498 return bitnum;
499 }
500
501 int
rte_mbuf_dynflag_register_bitnum(const struct rte_mbuf_dynflag * params,unsigned int req)502 rte_mbuf_dynflag_register_bitnum(const struct rte_mbuf_dynflag *params,
503 unsigned int req)
504 {
505 int ret;
506
507 if (params->flags != 0) {
508 rte_errno = EINVAL;
509 return -1;
510 }
511 if (req >= RTE_SIZEOF_FIELD(struct rte_mbuf, ol_flags) * CHAR_BIT &&
512 req != UINT_MAX) {
513 rte_errno = EINVAL;
514 return -1;
515 }
516
517 rte_mcfg_tailq_write_lock();
518 ret = __rte_mbuf_dynflag_register_bitnum(params, req);
519 rte_mcfg_tailq_write_unlock();
520
521 return ret;
522 }
523
524 int
rte_mbuf_dynflag_register(const struct rte_mbuf_dynflag * params)525 rte_mbuf_dynflag_register(const struct rte_mbuf_dynflag *params)
526 {
527 return rte_mbuf_dynflag_register_bitnum(params, UINT_MAX);
528 }
529
rte_mbuf_dyn_dump(FILE * out)530 void rte_mbuf_dyn_dump(FILE *out)
531 {
532 struct mbuf_dynfield_list *mbuf_dynfield_list;
533 struct mbuf_dynfield_elt *dynfield;
534 struct mbuf_dynflag_list *mbuf_dynflag_list;
535 struct mbuf_dynflag_elt *dynflag;
536 struct rte_tailq_entry *te;
537 size_t i;
538
539 rte_mcfg_tailq_write_lock();
540 if (shm == NULL && init_shared_mem() < 0) {
541 rte_mcfg_tailq_write_unlock();
542 return;
543 }
544
545 fprintf(out, "Reserved fields:\n");
546 mbuf_dynfield_list = RTE_TAILQ_CAST(
547 mbuf_dynfield_tailq.head, mbuf_dynfield_list);
548 TAILQ_FOREACH(te, mbuf_dynfield_list, next) {
549 dynfield = (struct mbuf_dynfield_elt *)te->data;
550 fprintf(out, " name=%s offset=%zd size=%zd align=%zd flags=%x\n",
551 dynfield->params.name, dynfield->offset,
552 dynfield->params.size, dynfield->params.align,
553 dynfield->params.flags);
554 }
555 fprintf(out, "Reserved flags:\n");
556 mbuf_dynflag_list = RTE_TAILQ_CAST(
557 mbuf_dynflag_tailq.head, mbuf_dynflag_list);
558 TAILQ_FOREACH(te, mbuf_dynflag_list, next) {
559 dynflag = (struct mbuf_dynflag_elt *)te->data;
560 fprintf(out, " name=%s bitnum=%u flags=%x\n",
561 dynflag->params.name, dynflag->bitnum,
562 dynflag->params.flags);
563 }
564 fprintf(out, "Free space in mbuf (0 = occupied, value = free zone alignment):\n");
565 for (i = 0; i < sizeof(struct rte_mbuf); i++) {
566 if ((i % 8) == 0)
567 fprintf(out, " %4.4zx: ", i);
568 fprintf(out, "%2.2x%s", shm->free_space[i],
569 (i % 8 != 7) ? " " : "\n");
570 }
571 fprintf(out, "Free bit in mbuf->ol_flags (0 = occupied, 1 = free):\n");
572 for (i = 0; i < sizeof(uint64_t) * CHAR_BIT; i++) {
573 if ((i % 8) == 0)
574 fprintf(out, " %4.4zx: ", i);
575 fprintf(out, "%1.1x%s", (shm->free_flags & (1ULL << i)) ? 1 : 0,
576 (i % 8 != 7) ? " " : "\n");
577 }
578
579 rte_mcfg_tailq_write_unlock();
580 }
581
582 static int
rte_mbuf_dyn_timestamp_register(int * field_offset,uint64_t * flag,const char * direction,const char * flag_name)583 rte_mbuf_dyn_timestamp_register(int *field_offset, uint64_t *flag,
584 const char *direction, const char *flag_name)
585 {
586 static const struct rte_mbuf_dynfield field_desc = {
587 .name = RTE_MBUF_DYNFIELD_TIMESTAMP_NAME,
588 .size = sizeof(rte_mbuf_timestamp_t),
589 .align = alignof(rte_mbuf_timestamp_t),
590 };
591 struct rte_mbuf_dynflag flag_desc = {};
592 int offset;
593
594 offset = rte_mbuf_dynfield_register(&field_desc);
595 if (offset < 0) {
596 MBUF_LOG(ERR,
597 "Failed to register mbuf field for timestamp");
598 return -1;
599 }
600 if (field_offset != NULL)
601 *field_offset = offset;
602
603 strlcpy(flag_desc.name, flag_name, sizeof(flag_desc.name));
604 offset = rte_mbuf_dynflag_register(&flag_desc);
605 if (offset < 0) {
606 MBUF_LOG(ERR,
607 "Failed to register mbuf flag for %s timestamp",
608 direction);
609 return -1;
610 }
611 if (flag != NULL)
612 *flag = RTE_BIT64(offset);
613
614 return 0;
615 }
616
617 int
rte_mbuf_dyn_rx_timestamp_register(int * field_offset,uint64_t * rx_flag)618 rte_mbuf_dyn_rx_timestamp_register(int *field_offset, uint64_t *rx_flag)
619 {
620 return rte_mbuf_dyn_timestamp_register(field_offset, rx_flag,
621 "Rx", RTE_MBUF_DYNFLAG_RX_TIMESTAMP_NAME);
622 }
623
624 int
rte_mbuf_dyn_tx_timestamp_register(int * field_offset,uint64_t * tx_flag)625 rte_mbuf_dyn_tx_timestamp_register(int *field_offset, uint64_t *tx_flag)
626 {
627 return rte_mbuf_dyn_timestamp_register(field_offset, tx_flag,
628 "Tx", RTE_MBUF_DYNFLAG_TX_TIMESTAMP_NAME);
629 }
630