Lines Matching defs:shadow
151 // For the free shadow markers the first element (that contains kFreeSid)
234 # define LOAD_CURRENT_SHADOW(cur, shadow_mem) UNUSED int access = 0, shadow = 0
239 bool ContainsSameAccess(RawShadow* unused0, Shadow unused1, m128 shadow,
248 const m128 same = _mm_cmpeq_epi32(shadow, access);
251 // For reads we need to reset read bit in the shadow,
258 // Shadow::kRodata has epoch 0 which cannot appear in shadow normally
262 const m128 masked_shadow = _mm_or_si128(shadow, read_mask);
268 const m128 ro = _mm_cmpeq_epi32(shadow, read_mask);
275 u32 race_mask, m128 shadow, AccessType typ) {
276 // race_mask points which of the shadow elements raced with the current
283 old = _mm_extract_epi32(shadow, 0);
286 old = _mm_extract_epi32(shadow, 1);
289 old = _mm_extract_epi32(shadow, 2);
292 old = _mm_extract_epi32(shadow, 3);
296 // For the free shadow markers the first element (that contains kFreeSid)
300 prev = Shadow(static_cast<RawShadow>(_mm_extract_epi32(shadow, 1)));
306 m128 shadow, m128 access, AccessType typ) {
312 const m128 access_and = _mm_and_si128(access, shadow);
313 const m128 access_xor = _mm_xor_si128(access, shadow);
340 _mm_cmpeq_epi32(_mm_max_epu32(shadow, access_read_atomic), shadow);
345 const m128 empty = _mm_cmpeq_epi32(shadow, zero);
364 u8 sid = _mm_extract_epi8(shadow, idx * 4 + 1); \
374 const m128 shadow_epochs = _mm_and_si128(shadow, mask_epoch);
380 DoReportRaceV(thr, shadow_mem, cur, concurrent_mask, shadow, typ);
386 const m128 shadow = _mm_load_si128(reinterpret_cast<m128*>(shadow_mem))
436 if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
442 CheckRaces(thr, shadow_mem, cur, shadow, access, typ);
465 if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
470 if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ)))
476 if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
480 CheckRaces(thr, shadow_mem, cur, shadow, access, typ);
503 if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
508 if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ)))
518 if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
522 CheckRaces(thr, shadow_mem, cur, shadow, access, typ);
558 // Don't want to touch lots of shadow memory.
594 // As the result a garbage "freed" shadow can lead to a false positive
601 // Processing more than 1k (2k of shadow) is expensive,
616 const m128 shadow = _mm_load_si128((m128*)shadow_mem);
617 if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ)))
653 if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
655 return CheckRaces(thr, shadow_mem, cur, shadow, access, typ);
683 Printf("Bad shadow start addr: %p (%p)\n", shadow_mem, (void*)addr);
690 Printf("Bad shadow end addr: %p (%p)\n", shadow_mem_end,