21 DECLARE_REAL(void *, memcpy, void *dest, const void *src, uptr n)
22 DECLARE_REAL(void *, memmove, void *dest, const void *src, uptr n)
43 void CopyOrigin(const void *dst, const void *src, uptr size,  in CopyOrigin()  argument
45   if (!MEM_IS_APP(dst) || !MEM_IS_APP(src)) return;  in CopyOrigin()
51     u32 o = GetOriginIfPoisoned((uptr)src, beg + 4 - d);  in CopyOrigin()
65     u32 o = GetOriginIfPoisoned((uptr)src + (end - d), (d + size) - end);  in CopyOrigin()
74     uptr s = ((uptr)src + 3) & ~3UL;  in CopyOrigin()
77       u32 *src = (u32 *)MEM_TO_ORIGIN(s);  in CopyOrigin()  local
83       for (; src < src_end; ++src, ++src_s, ++dst) {  in CopyOrigin()
85         if (*src != src_o) {  in CopyOrigin()
86           src_o = *src;  in CopyOrigin()
98 void ReverseCopyOrigin(const void *dst, const void *src, uptr size,  in ReverseCopyOrigin()  argument
100   if (!MEM_IS_APP(dst) || !MEM_IS_APP(src))  in ReverseCopyOrigin()
108     u32 o = GetOriginIfPoisoned((uptr)src + (end - d), (d + size) - end);  in ReverseCopyOrigin()
120     uptr s = ((uptr)src + 3) & ~3UL;  in ReverseCopyOrigin()
122       u32 *src = (u32 *)MEM_TO_ORIGIN(s + end - beg - 4);  in ReverseCopyOrigin()  local
128       for (; src >= src_begin; --src, --src_s, --dst) {  in ReverseCopyOrigin()
131         if (*src != src_o) {  in ReverseCopyOrigin()
132           src_o = *src;  in ReverseCopyOrigin()
145     u32 o = GetOriginIfPoisoned((uptr)src, beg + 4 - d);  in ReverseCopyOrigin()
154 void MoveOrigin(const void *dst, const void *src, uptr size,  in MoveOrigin()  argument
159   uptr src_aligned_beg = reinterpret_cast<uptr>(src) & ~3UL;  in MoveOrigin()
160   uptr src_aligned_end = (reinterpret_cast<uptr>(src) + size) & ~3UL;  in MoveOrigin()
163     return ReverseCopyOrigin(dst, src, size, stack);  in MoveOrigin()
164   return CopyOrigin(dst, src, size, stack);  in MoveOrigin()
167 void MoveShadowAndOrigin(const void *dst, const void *src, uptr size,  in MoveShadowAndOrigin()  argument
170   if (!MEM_IS_APP(src)) return;  in MoveShadowAndOrigin()
171   if (src == dst) return;  in MoveShadowAndOrigin()
175     MoveOrigin(dst, src, size, stack);  in MoveShadowAndOrigin()
177                 (void *)MEM_TO_SHADOW((uptr)src), size);  in MoveShadowAndOrigin()
180 void CopyShadowAndOrigin(const void *dst, const void *src, uptr size,  in CopyShadowAndOrigin()  argument
183   if (!MEM_IS_APP(src)) return;  in CopyShadowAndOrigin()
187                (void *)MEM_TO_SHADOW((uptr)src), size);  in CopyShadowAndOrigin()
189     MoveOrigin(dst, src, size, stack);  in CopyShadowAndOrigin()
192 void CopyMemory(void *dst, const void *src, uptr size, StackTrace *stack) {  in CopyMemory()  argument
193   REAL(memcpy)(dst, src, size);  in CopyMemory()
194   CopyShadowAndOrigin(dst, src, size, stack);  in CopyMemory()