10fca6ea1SDimitry Andric /* ===-------- intrin.h ---------------------------------------------------=== 20fca6ea1SDimitry Andric * 30fca6ea1SDimitry Andric * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 40fca6ea1SDimitry Andric * See https://llvm.org/LICENSE.txt for license information. 50fca6ea1SDimitry Andric * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 60fca6ea1SDimitry Andric * 70fca6ea1SDimitry Andric *===-----------------------------------------------------------------------=== 80fca6ea1SDimitry Andric */ 90fca6ea1SDimitry Andric 100fca6ea1SDimitry Andric /* Only include this if we're compiling for the windows platform. */ 110fca6ea1SDimitry Andric #ifndef _MSC_VER 120fca6ea1SDimitry Andric #include_next <intrin0.h> 130fca6ea1SDimitry Andric #else 140fca6ea1SDimitry Andric 150fca6ea1SDimitry Andric #ifndef __INTRIN0_H 160fca6ea1SDimitry Andric #define __INTRIN0_H 170fca6ea1SDimitry Andric 180fca6ea1SDimitry Andric #if defined(__x86_64__) && !defined(__arm64ec__) 190fca6ea1SDimitry Andric #include <adcintrin.h> 200fca6ea1SDimitry Andric #endif 210fca6ea1SDimitry Andric 220fca6ea1SDimitry Andric #ifdef __cplusplus 230fca6ea1SDimitry Andric extern "C" { 240fca6ea1SDimitry Andric #endif 250fca6ea1SDimitry Andric 260fca6ea1SDimitry Andric unsigned char _BitScanForward(unsigned long *_Index, unsigned long _Mask); 270fca6ea1SDimitry Andric unsigned char _BitScanReverse(unsigned long *_Index, unsigned long _Mask); 280fca6ea1SDimitry Andric void _ReadWriteBarrier(void); 290fca6ea1SDimitry Andric 300fca6ea1SDimitry Andric #if defined(__aarch64__) || defined(__arm64ec__) 310fca6ea1SDimitry Andric unsigned int _CountLeadingZeros(unsigned long); 320fca6ea1SDimitry Andric unsigned int _CountLeadingZeros64(unsigned _int64); 330fca6ea1SDimitry Andric unsigned char _InterlockedCompareExchange128_acq(__int64 volatile *_Destination, 340fca6ea1SDimitry Andric __int64 _ExchangeHigh, 350fca6ea1SDimitry Andric __int64 _ExchangeLow, 360fca6ea1SDimitry Andric __int64 *_ComparandResult); 370fca6ea1SDimitry Andric unsigned char _InterlockedCompareExchange128_nf(__int64 volatile *_Destination, 380fca6ea1SDimitry Andric __int64 _ExchangeHigh, 390fca6ea1SDimitry Andric __int64 _ExchangeLow, 400fca6ea1SDimitry Andric __int64 *_ComparandResult); 410fca6ea1SDimitry Andric unsigned char _InterlockedCompareExchange128_rel(__int64 volatile *_Destination, 420fca6ea1SDimitry Andric __int64 _ExchangeHigh, 430fca6ea1SDimitry Andric __int64 _ExchangeLow, 440fca6ea1SDimitry Andric __int64 *_ComparandResult); 450fca6ea1SDimitry Andric #endif 460fca6ea1SDimitry Andric 47*5deeebd8SDimitry Andric #if defined(__x86_64__) && !defined(__arm64ec__) 480fca6ea1SDimitry Andric unsigned __int64 _umul128(unsigned __int64, unsigned __int64, 490fca6ea1SDimitry Andric unsigned __int64 *); 500fca6ea1SDimitry Andric unsigned __int64 __shiftleft128(unsigned __int64 _LowPart, 510fca6ea1SDimitry Andric unsigned __int64 _HighPart, 520fca6ea1SDimitry Andric unsigned char _Shift); 530fca6ea1SDimitry Andric unsigned __int64 __shiftright128(unsigned __int64 _LowPart, 540fca6ea1SDimitry Andric unsigned __int64 _HighPart, 550fca6ea1SDimitry Andric unsigned char _Shift); 560fca6ea1SDimitry Andric #endif 570fca6ea1SDimitry Andric 580fca6ea1SDimitry Andric #if defined(__i386__) || (defined(__x86_64__) && !defined(__arm64ec__)) 590fca6ea1SDimitry Andric void _mm_pause(void); 600fca6ea1SDimitry Andric #endif 610fca6ea1SDimitry Andric 620fca6ea1SDimitry Andric #if defined(__x86_64__) || defined(__aarch64__) 630fca6ea1SDimitry Andric unsigned char _InterlockedCompareExchange128(__int64 volatile *_Destination, 640fca6ea1SDimitry Andric __int64 _ExchangeHigh, 650fca6ea1SDimitry Andric __int64 _ExchangeLow, 660fca6ea1SDimitry Andric __int64 *_ComparandResult); 670fca6ea1SDimitry Andric #endif 680fca6ea1SDimitry Andric 690fca6ea1SDimitry Andric #if defined(__x86_64__) || defined(__arm__) || defined(__aarch64__) 700fca6ea1SDimitry Andric unsigned char _BitScanForward64(unsigned long *_Index, unsigned __int64 _Mask); 710fca6ea1SDimitry Andric unsigned char _BitScanReverse64(unsigned long *_Index, unsigned __int64 _Mask); 720fca6ea1SDimitry Andric #endif 730fca6ea1SDimitry Andric 740fca6ea1SDimitry Andric #if defined(__i386__) || defined(__x86_64__) || defined(__arm__) || \ 750fca6ea1SDimitry Andric defined(__aarch64__) 760fca6ea1SDimitry Andric __int64 _InterlockedDecrement64(__int64 volatile *_Addend); 770fca6ea1SDimitry Andric __int64 _InterlockedExchange64(__int64 volatile *_Target, __int64 _Value); 780fca6ea1SDimitry Andric __int64 _InterlockedExchangeAdd64(__int64 volatile *_Addend, __int64 _Value); 790fca6ea1SDimitry Andric __int64 _InterlockedExchangeSub64(__int64 volatile *_Subend, __int64 _Value); 800fca6ea1SDimitry Andric __int64 _InterlockedIncrement64(__int64 volatile *_Addend); 810fca6ea1SDimitry Andric __int64 _InterlockedOr64(__int64 volatile *_Value, __int64 _Mask); 820fca6ea1SDimitry Andric __int64 _InterlockedXor64(__int64 volatile *_Value, __int64 _Mask); 830fca6ea1SDimitry Andric __int64 _InterlockedAnd64(__int64 volatile *_Value, __int64 _Mask); 840fca6ea1SDimitry Andric #endif 850fca6ea1SDimitry Andric 860fca6ea1SDimitry Andric #if defined(__arm__) || defined(__aarch64__) || defined(__arm64ec__) 870fca6ea1SDimitry Andric /*----------------------------------------------------------------------------*\ 880fca6ea1SDimitry Andric |* Interlocked Exchange Add 890fca6ea1SDimitry Andric \*----------------------------------------------------------------------------*/ 900fca6ea1SDimitry Andric char _InterlockedExchangeAdd8_acq(char volatile *_Addend, char _Value); 910fca6ea1SDimitry Andric char _InterlockedExchangeAdd8_nf(char volatile *_Addend, char _Value); 920fca6ea1SDimitry Andric char _InterlockedExchangeAdd8_rel(char volatile *_Addend, char _Value); 930fca6ea1SDimitry Andric short _InterlockedExchangeAdd16_acq(short volatile *_Addend, short _Value); 940fca6ea1SDimitry Andric short _InterlockedExchangeAdd16_nf(short volatile *_Addend, short _Value); 950fca6ea1SDimitry Andric short _InterlockedExchangeAdd16_rel(short volatile *_Addend, short _Value); 960fca6ea1SDimitry Andric long _InterlockedExchangeAdd_acq(long volatile *_Addend, long _Value); 970fca6ea1SDimitry Andric long _InterlockedExchangeAdd_nf(long volatile *_Addend, long _Value); 980fca6ea1SDimitry Andric long _InterlockedExchangeAdd_rel(long volatile *_Addend, long _Value); 990fca6ea1SDimitry Andric __int64 _InterlockedExchangeAdd64_acq(__int64 volatile *_Addend, 1000fca6ea1SDimitry Andric __int64 _Value); 1010fca6ea1SDimitry Andric __int64 _InterlockedExchangeAdd64_nf(__int64 volatile *_Addend, __int64 _Value); 1020fca6ea1SDimitry Andric __int64 _InterlockedExchangeAdd64_rel(__int64 volatile *_Addend, 1030fca6ea1SDimitry Andric __int64 _Value); 1040fca6ea1SDimitry Andric 1050fca6ea1SDimitry Andric /*----------------------------------------------------------------------------*\ 1060fca6ea1SDimitry Andric |* Interlocked Increment 1070fca6ea1SDimitry Andric \*----------------------------------------------------------------------------*/ 1080fca6ea1SDimitry Andric short _InterlockedIncrement16_acq(short volatile *_Value); 1090fca6ea1SDimitry Andric short _InterlockedIncrement16_nf(short volatile *_Value); 1100fca6ea1SDimitry Andric short _InterlockedIncrement16_rel(short volatile *_Value); 1110fca6ea1SDimitry Andric long _InterlockedIncrement_acq(long volatile *_Value); 1120fca6ea1SDimitry Andric long _InterlockedIncrement_nf(long volatile *_Value); 1130fca6ea1SDimitry Andric long _InterlockedIncrement_rel(long volatile *_Value); 1140fca6ea1SDimitry Andric __int64 _InterlockedIncrement64_acq(__int64 volatile *_Value); 1150fca6ea1SDimitry Andric __int64 _InterlockedIncrement64_nf(__int64 volatile *_Value); 1160fca6ea1SDimitry Andric __int64 _InterlockedIncrement64_rel(__int64 volatile *_Value); 1170fca6ea1SDimitry Andric 1180fca6ea1SDimitry Andric /*----------------------------------------------------------------------------*\ 1190fca6ea1SDimitry Andric |* Interlocked Decrement 1200fca6ea1SDimitry Andric \*----------------------------------------------------------------------------*/ 1210fca6ea1SDimitry Andric short _InterlockedDecrement16_acq(short volatile *_Value); 1220fca6ea1SDimitry Andric short _InterlockedDecrement16_nf(short volatile *_Value); 1230fca6ea1SDimitry Andric short _InterlockedDecrement16_rel(short volatile *_Value); 1240fca6ea1SDimitry Andric long _InterlockedDecrement_acq(long volatile *_Value); 1250fca6ea1SDimitry Andric long _InterlockedDecrement_nf(long volatile *_Value); 1260fca6ea1SDimitry Andric long _InterlockedDecrement_rel(long volatile *_Value); 1270fca6ea1SDimitry Andric __int64 _InterlockedDecrement64_acq(__int64 volatile *_Value); 1280fca6ea1SDimitry Andric __int64 _InterlockedDecrement64_nf(__int64 volatile *_Value); 1290fca6ea1SDimitry Andric __int64 _InterlockedDecrement64_rel(__int64 volatile *_Value); 1300fca6ea1SDimitry Andric 1310fca6ea1SDimitry Andric /*----------------------------------------------------------------------------*\ 1320fca6ea1SDimitry Andric |* Interlocked And 1330fca6ea1SDimitry Andric \*----------------------------------------------------------------------------*/ 1340fca6ea1SDimitry Andric char _InterlockedAnd8_acq(char volatile *_Value, char _Mask); 1350fca6ea1SDimitry Andric char _InterlockedAnd8_nf(char volatile *_Value, char _Mask); 1360fca6ea1SDimitry Andric char _InterlockedAnd8_rel(char volatile *_Value, char _Mask); 1370fca6ea1SDimitry Andric short _InterlockedAnd16_acq(short volatile *_Value, short _Mask); 1380fca6ea1SDimitry Andric short _InterlockedAnd16_nf(short volatile *_Value, short _Mask); 1390fca6ea1SDimitry Andric short _InterlockedAnd16_rel(short volatile *_Value, short _Mask); 1400fca6ea1SDimitry Andric long _InterlockedAnd_acq(long volatile *_Value, long _Mask); 1410fca6ea1SDimitry Andric long _InterlockedAnd_nf(long volatile *_Value, long _Mask); 1420fca6ea1SDimitry Andric long _InterlockedAnd_rel(long volatile *_Value, long _Mask); 1430fca6ea1SDimitry Andric __int64 _InterlockedAnd64_acq(__int64 volatile *_Value, __int64 _Mask); 1440fca6ea1SDimitry Andric __int64 _InterlockedAnd64_nf(__int64 volatile *_Value, __int64 _Mask); 1450fca6ea1SDimitry Andric __int64 _InterlockedAnd64_rel(__int64 volatile *_Value, __int64 _Mask); 1460fca6ea1SDimitry Andric 1470fca6ea1SDimitry Andric /*----------------------------------------------------------------------------*\ 1480fca6ea1SDimitry Andric |* Bit Counting and Testing 1490fca6ea1SDimitry Andric \*----------------------------------------------------------------------------*/ 1500fca6ea1SDimitry Andric unsigned char _interlockedbittestandset_acq(long volatile *_BitBase, 1510fca6ea1SDimitry Andric long _BitPos); 1520fca6ea1SDimitry Andric unsigned char _interlockedbittestandset_nf(long volatile *_BitBase, 1530fca6ea1SDimitry Andric long _BitPos); 1540fca6ea1SDimitry Andric unsigned char _interlockedbittestandset_rel(long volatile *_BitBase, 1550fca6ea1SDimitry Andric long _BitPos); 1560fca6ea1SDimitry Andric unsigned char _interlockedbittestandreset_acq(long volatile *_BitBase, 1570fca6ea1SDimitry Andric long _BitPos); 1580fca6ea1SDimitry Andric unsigned char _interlockedbittestandreset_nf(long volatile *_BitBase, 1590fca6ea1SDimitry Andric long _BitPos); 1600fca6ea1SDimitry Andric unsigned char _interlockedbittestandreset_rel(long volatile *_BitBase, 1610fca6ea1SDimitry Andric long _BitPos); 1620fca6ea1SDimitry Andric 1630fca6ea1SDimitry Andric /*----------------------------------------------------------------------------*\ 1640fca6ea1SDimitry Andric |* Interlocked Or 1650fca6ea1SDimitry Andric \*----------------------------------------------------------------------------*/ 1660fca6ea1SDimitry Andric char _InterlockedOr8_acq(char volatile *_Value, char _Mask); 1670fca6ea1SDimitry Andric char _InterlockedOr8_nf(char volatile *_Value, char _Mask); 1680fca6ea1SDimitry Andric char _InterlockedOr8_rel(char volatile *_Value, char _Mask); 1690fca6ea1SDimitry Andric short _InterlockedOr16_acq(short volatile *_Value, short _Mask); 1700fca6ea1SDimitry Andric short _InterlockedOr16_nf(short volatile *_Value, short _Mask); 1710fca6ea1SDimitry Andric short _InterlockedOr16_rel(short volatile *_Value, short _Mask); 1720fca6ea1SDimitry Andric long _InterlockedOr_acq(long volatile *_Value, long _Mask); 1730fca6ea1SDimitry Andric long _InterlockedOr_nf(long volatile *_Value, long _Mask); 1740fca6ea1SDimitry Andric long _InterlockedOr_rel(long volatile *_Value, long _Mask); 1750fca6ea1SDimitry Andric __int64 _InterlockedOr64_acq(__int64 volatile *_Value, __int64 _Mask); 1760fca6ea1SDimitry Andric __int64 _InterlockedOr64_nf(__int64 volatile *_Value, __int64 _Mask); 1770fca6ea1SDimitry Andric __int64 _InterlockedOr64_rel(__int64 volatile *_Value, __int64 _Mask); 1780fca6ea1SDimitry Andric 1790fca6ea1SDimitry Andric /*----------------------------------------------------------------------------*\ 1800fca6ea1SDimitry Andric |* Interlocked Xor 1810fca6ea1SDimitry Andric \*----------------------------------------------------------------------------*/ 1820fca6ea1SDimitry Andric char _InterlockedXor8_acq(char volatile *_Value, char _Mask); 1830fca6ea1SDimitry Andric char _InterlockedXor8_nf(char volatile *_Value, char _Mask); 1840fca6ea1SDimitry Andric char _InterlockedXor8_rel(char volatile *_Value, char _Mask); 1850fca6ea1SDimitry Andric short _InterlockedXor16_acq(short volatile *_Value, short _Mask); 1860fca6ea1SDimitry Andric short _InterlockedXor16_nf(short volatile *_Value, short _Mask); 1870fca6ea1SDimitry Andric short _InterlockedXor16_rel(short volatile *_Value, short _Mask); 1880fca6ea1SDimitry Andric long _InterlockedXor_acq(long volatile *_Value, long _Mask); 1890fca6ea1SDimitry Andric long _InterlockedXor_nf(long volatile *_Value, long _Mask); 1900fca6ea1SDimitry Andric long _InterlockedXor_rel(long volatile *_Value, long _Mask); 1910fca6ea1SDimitry Andric __int64 _InterlockedXor64_acq(__int64 volatile *_Value, __int64 _Mask); 1920fca6ea1SDimitry Andric __int64 _InterlockedXor64_nf(__int64 volatile *_Value, __int64 _Mask); 1930fca6ea1SDimitry Andric __int64 _InterlockedXor64_rel(__int64 volatile *_Value, __int64 _Mask); 1940fca6ea1SDimitry Andric 1950fca6ea1SDimitry Andric /*----------------------------------------------------------------------------*\ 1960fca6ea1SDimitry Andric |* Interlocked Exchange 1970fca6ea1SDimitry Andric \*----------------------------------------------------------------------------*/ 1980fca6ea1SDimitry Andric char _InterlockedExchange8_acq(char volatile *_Target, char _Value); 1990fca6ea1SDimitry Andric char _InterlockedExchange8_nf(char volatile *_Target, char _Value); 2000fca6ea1SDimitry Andric char _InterlockedExchange8_rel(char volatile *_Target, char _Value); 2010fca6ea1SDimitry Andric short _InterlockedExchange16_acq(short volatile *_Target, short _Value); 2020fca6ea1SDimitry Andric short _InterlockedExchange16_nf(short volatile *_Target, short _Value); 2030fca6ea1SDimitry Andric short _InterlockedExchange16_rel(short volatile *_Target, short _Value); 2040fca6ea1SDimitry Andric long _InterlockedExchange_acq(long volatile *_Target, long _Value); 2050fca6ea1SDimitry Andric long _InterlockedExchange_nf(long volatile *_Target, long _Value); 2060fca6ea1SDimitry Andric long _InterlockedExchange_rel(long volatile *_Target, long _Value); 2070fca6ea1SDimitry Andric __int64 _InterlockedExchange64_acq(__int64 volatile *_Target, __int64 _Value); 2080fca6ea1SDimitry Andric __int64 _InterlockedExchange64_nf(__int64 volatile *_Target, __int64 _Value); 2090fca6ea1SDimitry Andric __int64 _InterlockedExchange64_rel(__int64 volatile *_Target, __int64 _Value); 2100fca6ea1SDimitry Andric 2110fca6ea1SDimitry Andric /*----------------------------------------------------------------------------*\ 2120fca6ea1SDimitry Andric |* Interlocked Compare Exchange 2130fca6ea1SDimitry Andric \*----------------------------------------------------------------------------*/ 2140fca6ea1SDimitry Andric char _InterlockedCompareExchange8_acq(char volatile *_Destination, 2150fca6ea1SDimitry Andric char _Exchange, char _Comparand); 2160fca6ea1SDimitry Andric char _InterlockedCompareExchange8_nf(char volatile *_Destination, 2170fca6ea1SDimitry Andric char _Exchange, char _Comparand); 2180fca6ea1SDimitry Andric char _InterlockedCompareExchange8_rel(char volatile *_Destination, 2190fca6ea1SDimitry Andric char _Exchange, char _Comparand); 2200fca6ea1SDimitry Andric short _InterlockedCompareExchange16_acq(short volatile *_Destination, 2210fca6ea1SDimitry Andric short _Exchange, short _Comparand); 2220fca6ea1SDimitry Andric short _InterlockedCompareExchange16_nf(short volatile *_Destination, 2230fca6ea1SDimitry Andric short _Exchange, short _Comparand); 2240fca6ea1SDimitry Andric short _InterlockedCompareExchange16_rel(short volatile *_Destination, 2250fca6ea1SDimitry Andric short _Exchange, short _Comparand); 2260fca6ea1SDimitry Andric long _InterlockedCompareExchange_acq(long volatile *_Destination, 2270fca6ea1SDimitry Andric long _Exchange, long _Comparand); 2280fca6ea1SDimitry Andric long _InterlockedCompareExchange_nf(long volatile *_Destination, long _Exchange, 2290fca6ea1SDimitry Andric long _Comparand); 2300fca6ea1SDimitry Andric long _InterlockedCompareExchange_rel(long volatile *_Destination, 2310fca6ea1SDimitry Andric long _Exchange, long _Comparand); 2320fca6ea1SDimitry Andric __int64 _InterlockedCompareExchange64_acq(__int64 volatile *_Destination, 2330fca6ea1SDimitry Andric __int64 _Exchange, 2340fca6ea1SDimitry Andric __int64 _Comparand); 2350fca6ea1SDimitry Andric __int64 _InterlockedCompareExchange64_nf(__int64 volatile *_Destination, 2360fca6ea1SDimitry Andric __int64 _Exchange, __int64 _Comparand); 2370fca6ea1SDimitry Andric __int64 _InterlockedCompareExchange64_rel(__int64 volatile *_Destination, 2380fca6ea1SDimitry Andric __int64 _Exchange, 2390fca6ea1SDimitry Andric __int64 _Comparand); 2400fca6ea1SDimitry Andric #endif 2410fca6ea1SDimitry Andric 2420fca6ea1SDimitry Andric #ifdef __cplusplus 2430fca6ea1SDimitry Andric } 2440fca6ea1SDimitry Andric #endif 2450fca6ea1SDimitry Andric 2460fca6ea1SDimitry Andric #endif /* __INTRIN0_H */ 2470fca6ea1SDimitry Andric #endif /* _MSC_VER */ 248