Lines Matching defs:src
48 * @param src
56 rte_memcpy(void *dst, const void *src, size_t n);
64 rte_mov15_or_less(void *dst, const void *src, size_t n)
83 ((const struct rte_uint64_alias *)src)->val;
84 src = (const uint64_t *)src + 1;
89 ((const struct rte_uint32_alias *)src)->val;
90 src = (const uint32_t *)src + 1;
95 ((const struct rte_uint16_alias *)src)->val;
96 src = (const uint16_t *)src + 1;
100 *(uint8_t *)dst = *(const uint8_t *)src;
109 rte_mov16(uint8_t *dst, const uint8_t *src)
113 xmm0 = _mm_loadu_si128((const __m128i *)(const void *)src);
122 rte_mov32(uint8_t *dst, const uint8_t *src)
127 ymm0 = _mm256_loadu_si256((const __m256i *)(const void *)src);
130 rte_mov16((uint8_t *)dst + 0 * 16, (const uint8_t *)src + 0 * 16);
131 rte_mov16((uint8_t *)dst + 1 * 16, (const uint8_t *)src + 1 * 16);
140 rte_mov64(uint8_t *dst, const uint8_t *src)
145 zmm0 = _mm512_loadu_si512((const void *)src);
148 rte_mov32((uint8_t *)dst + 0 * 32, (const uint8_t *)src + 0 * 32);
149 rte_mov32((uint8_t *)dst + 1 * 32, (const uint8_t *)src + 1 * 32);
158 rte_mov128(uint8_t *dst, const uint8_t *src)
160 rte_mov64(dst + 0 * 64, src + 0 * 64);
161 rte_mov64(dst + 1 * 64, src + 1 * 64);
169 rte_mov256(uint8_t *dst, const uint8_t *src)
171 rte_mov128(dst + 0 * 128, src + 0 * 128);
172 rte_mov128(dst + 1 * 128, src + 1 * 128);
188 rte_mov128blocks(uint8_t *dst, const uint8_t *src, size_t n)
193 zmm0 = _mm512_loadu_si512((const void *)(src + 0 * 64));
195 zmm1 = _mm512_loadu_si512((const void *)(src + 1 * 64));
196 src = src + 128;
208 rte_mov512blocks(uint8_t *dst, const uint8_t *src, size_t n)
213 zmm0 = _mm512_loadu_si512((const void *)(src + 0 * 64));
215 zmm1 = _mm512_loadu_si512((const void *)(src + 1 * 64));
216 zmm2 = _mm512_loadu_si512((const void *)(src + 2 * 64));
217 zmm3 = _mm512_loadu_si512((const void *)(src + 3 * 64));
218 zmm4 = _mm512_loadu_si512((const void *)(src + 4 * 64));
219 zmm5 = _mm512_loadu_si512((const void *)(src + 5 * 64));
220 zmm6 = _mm512_loadu_si512((const void *)(src + 6 * 64));
221 zmm7 = _mm512_loadu_si512((const void *)(src + 7 * 64));
222 src = src + 512;
236 rte_memcpy_generic(void *dst, const void *src, size_t n)
246 return rte_mov15_or_less(dst, src, n);
253 rte_mov32((uint8_t *)dst, (const uint8_t *)src);
257 rte_mov16((uint8_t *)dst, (const uint8_t *)src);
261 (const uint8_t *)src - 16 + n);
265 rte_mov64((uint8_t *)dst, (const uint8_t *)src);
269 rte_mov32((uint8_t *)dst, (const uint8_t *)src);
271 (const uint8_t *)src - 32 + n);
277 rte_mov256((uint8_t *)dst, (const uint8_t *)src);
278 src = (const uint8_t *)src + 256;
283 rte_mov128((uint8_t *)dst, (const uint8_t *)src);
284 src = (const uint8_t *)src + 128;
289 rte_mov64((uint8_t *)dst, (const uint8_t *)src);
291 (const uint8_t *)src - 64 + n);
296 (const uint8_t *)src - 64 + n);
307 rte_mov64((uint8_t *)dst, (const uint8_t *)src);
308 src = (const uint8_t *)src + dstofss;
317 rte_mov512blocks((uint8_t *)dst, (const uint8_t *)src, n);
321 src = (const uint8_t *)src + bits;
330 rte_mov128blocks((uint8_t *)dst, (const uint8_t *)src, n);
334 src = (const uint8_t *)src + bits;
357 rte_mov128blocks(uint8_t *dst, const uint8_t *src, size_t n)
363 ((const uint8_t *)src + 0 * 32));
366 ((const uint8_t *)src + 1 * 32));
368 ((const uint8_t *)src + 2 * 32));
370 ((const uint8_t *)src + 3 * 32));
371 src = (const uint8_t *)src + 128;
385 rte_memcpy_generic(void *dst, const void *src, size_t n)
395 return rte_mov15_or_less(dst, src, n);
402 rte_mov32((uint8_t *)dst, (const uint8_t *)src);
406 rte_mov16((uint8_t *)dst, (const uint8_t *)src);
410 (const uint8_t *)src - 16 + n);
414 rte_mov32((uint8_t *)dst, (const uint8_t *)src);
416 (const uint8_t *)src - 32 + n);
422 rte_mov128((uint8_t *)dst, (const uint8_t *)src);
423 src = (const uint8_t *)src + 128;
429 rte_mov64((uint8_t *)dst, (const uint8_t *)src);
430 src = (const uint8_t *)src + 64;
434 rte_mov32((uint8_t *)dst, (const uint8_t *)src);
436 (const uint8_t *)src - 32 + n);
441 (const uint8_t *)src - 32 + n);
453 rte_mov32((uint8_t *)dst, (const uint8_t *)src);
454 src = (const uint8_t *)src + dstofss;
461 rte_mov128blocks((uint8_t *)dst, (const uint8_t *)src, n);
465 src = (const uint8_t *)src + bits;
489 * - For <src>, make sure <offset> bit backwards & <16 - offset> bit forwards are available for loading
490 * - <dst>, <src>, <len> must be variables
493 #define MOVEUNALIGNED_LEFT47_IMM(dst, src, len, offset) \
497 xmm0 = _mm_loadu_si128((const __m128i *)(const void *)((const uint8_t *)src - offset + 0 * 16)); \
499 xmm1 = _mm_loadu_si128((const __m128i *)(const void *)((const uint8_t *)src - offset + 1 * 16)); \
500 xmm2 = _mm_loadu_si128((const __m128i *)(const void *)((const uint8_t *)src - offset + 2 * 16)); \
501 xmm3 = _mm_loadu_si128((const __m128i *)(const void *)((const uint8_t *)src - offset + 3 * 16)); \
502 xmm4 = _mm_loadu_si128((const __m128i *)(const void *)((const uint8_t *)src - offset + 4 * 16)); \
503 xmm5 = _mm_loadu_si128((const __m128i *)(const void *)((const uint8_t *)src - offset + 5 * 16)); \
504 xmm6 = _mm_loadu_si128((const __m128i *)(const void *)((const uint8_t *)src - offset + 6 * 16)); \
505 xmm7 = _mm_loadu_si128((const __m128i *)(const void *)((const uint8_t *)src - offset + 7 * 16)); \
506 xmm8 = _mm_loadu_si128((const __m128i *)(const void *)((const uint8_t *)src - offset + 8 * 16)); \
507 src = (const uint8_t *)src + 128; \
521 src = (const uint8_t *)src + tmp; \
525 xmm0 = _mm_loadu_si128((const __m128i *)(const void *)((const uint8_t *)src - offset + 0 * 16)); \
527 xmm1 = _mm_loadu_si128((const __m128i *)(const void *)((const uint8_t *)src - offset + 1 * 16)); \
528 xmm2 = _mm_loadu_si128((const __m128i *)(const void *)((const uint8_t *)src - offset + 2 * 16)); \
529 src = (const uint8_t *)src + 32; \
537 src = (const uint8_t *)src + tmp; \
550 * - For <src>, make sure <offset> bit backwards & <16 - offset> bit forwards are available for loading
551 * - <dst>, <src>, <len> must be variables
554 #define MOVEUNALIGNED_LEFT47(dst, src, len, offset) \
557 case 0x01: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x01); break; \
558 case 0x02: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x02); break; \
559 case 0x03: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x03); break; \
560 case 0x04: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x04); break; \
561 case 0x05: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x05); break; \
562 case 0x06: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x06); break; \
563 case 0x07: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x07); break; \
564 case 0x08: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x08); break; \
565 case 0x09: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x09); break; \
566 case 0x0A: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x0A); break; \
567 case 0x0B: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x0B); break; \
568 case 0x0C: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x0C); break; \
569 case 0x0D: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x0D); break; \
570 case 0x0E: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x0E); break; \
571 case 0x0F: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x0F); break; \
577 rte_memcpy_generic(void *dst, const void *src, size_t n)
588 return rte_mov15_or_less(dst, src, n);
595 rte_mov16((uint8_t *)dst, (const uint8_t *)src);
598 rte_mov16((uint8_t *)dst - 16 + n, (const uint8_t *)src - 16 + n);
602 rte_mov32((uint8_t *)dst, (const uint8_t *)src);
604 rte_mov16((uint8_t *)dst + 32, (const uint8_t *)src + 32);
605 rte_mov16((uint8_t *)dst - 16 + n, (const uint8_t *)src - 16 + n);
614 rte_mov128((uint8_t *)dst, (const uint8_t *)src);
615 rte_mov128((uint8_t *)dst + 128, (const uint8_t *)src + 128);
616 src = (const uint8_t *)src + 256;
622 rte_mov128((uint8_t *)dst, (const uint8_t *)src);
623 src = (const uint8_t *)src + 128;
629 rte_mov64((uint8_t *)dst, (const uint8_t *)src);
630 src = (const uint8_t *)src + 64;
636 rte_mov32((uint8_t *)dst, (const uint8_t *)src);
637 src = (const uint8_t *)src + 32;
641 rte_mov16((uint8_t *)dst, (const uint8_t *)src);
642 rte_mov16((uint8_t *)dst - 16 + n, (const uint8_t *)src - 16 + n);
646 rte_mov16((uint8_t *)dst - 16 + n, (const uint8_t *)src - 16 + n);
661 rte_mov32((uint8_t *)dst, (const uint8_t *)src);
662 src = (const uint8_t *)src + dstofss;
665 srcofs = ((uintptr_t)src & 0x0F);
675 rte_mov256((uint8_t *)dst, (const uint8_t *)src);
677 src = (const uint8_t *)src + 256;
689 MOVEUNALIGNED_LEFT47(dst, src, n, srcofs);
700 rte_memcpy_aligned(void *dst, const void *src, size_t n)
706 return rte_mov15_or_less(dst, src, n);
711 rte_mov32((uint8_t *)dst, (const uint8_t *)src);
715 rte_mov16((uint8_t *)dst, (const uint8_t *)src);
719 (const uint8_t *)src - 16 + n);
726 rte_mov64((uint8_t *)dst, (const uint8_t *)src);
730 rte_mov32((uint8_t *)dst, (const uint8_t *)src);
732 (const uint8_t *)src - 32 + n);
739 rte_mov64((uint8_t *)dst, (const uint8_t *)src);
741 src = (const uint8_t *)src + 64;
746 (const uint8_t *)src - 64 + n);
752 rte_memcpy(void *dst, const void *src, size_t n)
754 if (!(((uintptr_t)dst | (uintptr_t)src) & ALIGNMENT_MASK))
755 return rte_memcpy_aligned(dst, src, n);
757 return rte_memcpy_generic(dst, src, n);