365 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,  in emitAtomicCmpXchg()
415 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,  in emitAtomicCmpXchgFailureSet()
493                                          AtomicExpr::AtomicOp Op,  in EmitPostAtomicMinMax()
501   case AtomicExpr::AO__atomic_max_fetch:  in EmitPostAtomicMinMax()
504   case AtomicExpr::AO__atomic_min_fetch:  in EmitPostAtomicMinMax()
512 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,  in EmitAtomicOp()
522   case AtomicExpr::AO__c11_atomic_init:  in EmitAtomicOp()
523   case AtomicExpr::AO__opencl_atomic_init:  in EmitAtomicOp()
526   case AtomicExpr::AO__c11_atomic_compare_exchange_strong:  in EmitAtomicOp()
527   case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:  in EmitAtomicOp()
531   case AtomicExpr::AO__c11_atomic_compare_exchange_weak:  in EmitAtomicOp()
532   case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:  in EmitAtomicOp()
536   case AtomicExpr::AO__atomic_compare_exchange:  in EmitAtomicOp()
537   case AtomicExpr::AO__atomic_compare_exchange_n: {  in EmitAtomicOp()
566   case AtomicExpr::AO__c11_atomic_load:  in EmitAtomicOp()
567   case AtomicExpr::AO__opencl_atomic_load:  in EmitAtomicOp()
568   case AtomicExpr::AO__atomic_load_n:  in EmitAtomicOp()
569   case AtomicExpr::AO__atomic_load: {  in EmitAtomicOp()
577   case AtomicExpr::AO__c11_atomic_store:  in EmitAtomicOp()
578   case AtomicExpr::AO__opencl_atomic_store:  in EmitAtomicOp()
579   case AtomicExpr::AO__atomic_store:  in EmitAtomicOp()
580   case AtomicExpr::AO__atomic_store_n: {  in EmitAtomicOp()
588   case AtomicExpr::AO__c11_atomic_exchange:  in EmitAtomicOp()
589   case AtomicExpr::AO__opencl_atomic_exchange:  in EmitAtomicOp()
590   case AtomicExpr::AO__atomic_exchange_n:  in EmitAtomicOp()
591   case AtomicExpr::AO__atomic_exchange:  in EmitAtomicOp()
595   case AtomicExpr::AO__atomic_add_fetch:  in EmitAtomicOp()
599   case AtomicExpr::AO__c11_atomic_fetch_add:  in EmitAtomicOp()
600   case AtomicExpr::AO__opencl_atomic_fetch_add:  in EmitAtomicOp()
601   case AtomicExpr::AO__atomic_fetch_add:  in EmitAtomicOp()
606   case AtomicExpr::AO__atomic_sub_fetch:  in EmitAtomicOp()
610   case AtomicExpr::AO__c11_atomic_fetch_sub:  in EmitAtomicOp()
611   case AtomicExpr::AO__opencl_atomic_fetch_sub:  in EmitAtomicOp()
612   case AtomicExpr::AO__atomic_fetch_sub:  in EmitAtomicOp()
617   case AtomicExpr::AO__atomic_min_fetch:  in EmitAtomicOp()
620   case AtomicExpr::AO__c11_atomic_fetch_min:  in EmitAtomicOp()
621   case AtomicExpr::AO__opencl_atomic_fetch_min:  in EmitAtomicOp()
622   case AtomicExpr::AO__atomic_fetch_min:  in EmitAtomicOp()
627   case AtomicExpr::AO__atomic_max_fetch:  in EmitAtomicOp()
630   case AtomicExpr::AO__c11_atomic_fetch_max:  in EmitAtomicOp()
631   case AtomicExpr::AO__opencl_atomic_fetch_max:  in EmitAtomicOp()
632   case AtomicExpr::AO__atomic_fetch_max:  in EmitAtomicOp()
637   case AtomicExpr::AO__atomic_and_fetch:  in EmitAtomicOp()
640   case AtomicExpr::AO__c11_atomic_fetch_and:  in EmitAtomicOp()
641   case AtomicExpr::AO__opencl_atomic_fetch_and:  in EmitAtomicOp()
642   case AtomicExpr::AO__atomic_fetch_and:  in EmitAtomicOp()
646   case AtomicExpr::AO__atomic_or_fetch:  in EmitAtomicOp()
649   case AtomicExpr::AO__c11_atomic_fetch_or:  in EmitAtomicOp()
650   case AtomicExpr::AO__opencl_atomic_fetch_or:  in EmitAtomicOp()
651   case AtomicExpr::AO__atomic_fetch_or:  in EmitAtomicOp()
655   case AtomicExpr::AO__atomic_xor_fetch:  in EmitAtomicOp()
658   case AtomicExpr::AO__c11_atomic_fetch_xor:  in EmitAtomicOp()
659   case AtomicExpr::AO__opencl_atomic_fetch_xor:  in EmitAtomicOp()
660   case AtomicExpr::AO__atomic_fetch_xor:  in EmitAtomicOp()
664   case AtomicExpr::AO__atomic_nand_fetch:  in EmitAtomicOp()
667   case AtomicExpr::AO__atomic_fetch_nand:  in EmitAtomicOp()
687   if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)  in EmitAtomicOp()
702 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest,  in EmitAtomicOp()
785 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {  in EmitAtomicExpr()
797   if (E->getOp() == AtomicExpr::AO__c11_atomic_init ||  in EmitAtomicExpr()
798       E->getOp() == AtomicExpr::AO__opencl_atomic_init) {  in EmitAtomicExpr()
834   case AtomicExpr::AO__c11_atomic_init:  in EmitAtomicExpr()
835   case AtomicExpr::AO__opencl_atomic_init:  in EmitAtomicExpr()
838   case AtomicExpr::AO__c11_atomic_load:  in EmitAtomicExpr()
839   case AtomicExpr::AO__opencl_atomic_load:  in EmitAtomicExpr()
840   case AtomicExpr::AO__atomic_load_n:  in EmitAtomicExpr()
843   case AtomicExpr::AO__atomic_load:  in EmitAtomicExpr()
847   case AtomicExpr::AO__atomic_store:  in EmitAtomicExpr()
851   case AtomicExpr::AO__atomic_exchange:  in EmitAtomicExpr()
856   case AtomicExpr::AO__c11_atomic_compare_exchange_strong:  in EmitAtomicExpr()
857   case AtomicExpr::AO__c11_atomic_compare_exchange_weak:  in EmitAtomicExpr()
858   case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:  in EmitAtomicExpr()
859   case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:  in EmitAtomicExpr()
860   case AtomicExpr::AO__atomic_compare_exchange_n:  in EmitAtomicExpr()
861   case AtomicExpr::AO__atomic_compare_exchange:  in EmitAtomicExpr()
863     if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)  in EmitAtomicExpr()
868     if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||  in EmitAtomicExpr()
869         E->getOp() == AtomicExpr::AO__atomic_compare_exchange)  in EmitAtomicExpr()
873   case AtomicExpr::AO__c11_atomic_fetch_add:  in EmitAtomicExpr()
874   case AtomicExpr::AO__c11_atomic_fetch_sub:  in EmitAtomicExpr()
875   case AtomicExpr::AO__opencl_atomic_fetch_add:  in EmitAtomicExpr()
876   case AtomicExpr::AO__opencl_atomic_fetch_sub:  in EmitAtomicExpr()
893   case AtomicExpr::AO__atomic_fetch_add:  in EmitAtomicExpr()
894   case AtomicExpr::AO__atomic_fetch_sub:  in EmitAtomicExpr()
895   case AtomicExpr::AO__atomic_add_fetch:  in EmitAtomicExpr()
896   case AtomicExpr::AO__atomic_sub_fetch:  in EmitAtomicExpr()
900   case AtomicExpr::AO__c11_atomic_store:  in EmitAtomicExpr()
901   case AtomicExpr::AO__c11_atomic_exchange:  in EmitAtomicExpr()
902   case AtomicExpr::AO__opencl_atomic_store:  in EmitAtomicExpr()
903   case AtomicExpr::AO__opencl_atomic_exchange:  in EmitAtomicExpr()
904   case AtomicExpr::AO__atomic_store_n:  in EmitAtomicExpr()
905   case AtomicExpr::AO__atomic_exchange_n:  in EmitAtomicExpr()
906   case AtomicExpr::AO__c11_atomic_fetch_and:  in EmitAtomicExpr()
907   case AtomicExpr::AO__c11_atomic_fetch_or:  in EmitAtomicExpr()
908   case AtomicExpr::AO__c11_atomic_fetch_xor:  in EmitAtomicExpr()
909   case AtomicExpr::AO__c11_atomic_fetch_max:  in EmitAtomicExpr()
910   case AtomicExpr::AO__c11_atomic_fetch_min:  in EmitAtomicExpr()
911   case AtomicExpr::AO__opencl_atomic_fetch_and:  in EmitAtomicExpr()
912   case AtomicExpr::AO__opencl_atomic_fetch_or:  in EmitAtomicExpr()
913   case AtomicExpr::AO__opencl_atomic_fetch_xor:  in EmitAtomicExpr()
914   case AtomicExpr::AO__opencl_atomic_fetch_min:  in EmitAtomicExpr()
915   case AtomicExpr::AO__opencl_atomic_fetch_max:  in EmitAtomicExpr()
916   case AtomicExpr::AO__atomic_fetch_and:  in EmitAtomicExpr()
917   case AtomicExpr::AO__atomic_fetch_or:  in EmitAtomicExpr()
918   case AtomicExpr::AO__atomic_fetch_xor:  in EmitAtomicExpr()
919   case AtomicExpr::AO__atomic_fetch_nand:  in EmitAtomicExpr()
920   case AtomicExpr::AO__atomic_and_fetch:  in EmitAtomicExpr()
921   case AtomicExpr::AO__atomic_or_fetch:  in EmitAtomicExpr()
922   case AtomicExpr::AO__atomic_xor_fetch:  in EmitAtomicExpr()
923   case AtomicExpr::AO__atomic_nand_fetch:  in EmitAtomicExpr()
924   case AtomicExpr::AO__atomic_max_fetch:  in EmitAtomicExpr()
925   case AtomicExpr::AO__atomic_min_fetch:  in EmitAtomicExpr()
926   case AtomicExpr::AO__atomic_fetch_max:  in EmitAtomicExpr()
927   case AtomicExpr::AO__atomic_fetch_min:  in EmitAtomicExpr()
962     case AtomicExpr::AO__c11_atomic_init:  in EmitAtomicExpr()
963     case AtomicExpr::AO__opencl_atomic_init:  in EmitAtomicExpr()
966     case AtomicExpr::AO__c11_atomic_fetch_add:  in EmitAtomicExpr()
967     case AtomicExpr::AO__opencl_atomic_fetch_add:  in EmitAtomicExpr()
968     case AtomicExpr::AO__atomic_fetch_add:  in EmitAtomicExpr()
969     case AtomicExpr::AO__c11_atomic_fetch_and:  in EmitAtomicExpr()
970     case AtomicExpr::AO__opencl_atomic_fetch_and:  in EmitAtomicExpr()
971     case AtomicExpr::AO__atomic_fetch_and:  in EmitAtomicExpr()
972     case AtomicExpr::AO__c11_atomic_fetch_or:  in EmitAtomicExpr()
973     case AtomicExpr::AO__opencl_atomic_fetch_or:  in EmitAtomicExpr()
974     case AtomicExpr::AO__atomic_fetch_or:  in EmitAtomicExpr()
975     case AtomicExpr::AO__atomic_fetch_nand:  in EmitAtomicExpr()
976     case AtomicExpr::AO__c11_atomic_fetch_sub:  in EmitAtomicExpr()
977     case AtomicExpr::AO__opencl_atomic_fetch_sub:  in EmitAtomicExpr()
978     case AtomicExpr::AO__atomic_fetch_sub:  in EmitAtomicExpr()
979     case AtomicExpr::AO__c11_atomic_fetch_xor:  in EmitAtomicExpr()
980     case AtomicExpr::AO__opencl_atomic_fetch_xor:  in EmitAtomicExpr()
981     case AtomicExpr::AO__opencl_atomic_fetch_min:  in EmitAtomicExpr()
982     case AtomicExpr::AO__opencl_atomic_fetch_max:  in EmitAtomicExpr()
983     case AtomicExpr::AO__atomic_fetch_xor:  in EmitAtomicExpr()
984     case AtomicExpr::AO__c11_atomic_fetch_max:  in EmitAtomicExpr()
985     case AtomicExpr::AO__c11_atomic_fetch_min:  in EmitAtomicExpr()
986     case AtomicExpr::AO__atomic_add_fetch:  in EmitAtomicExpr()
987     case AtomicExpr::AO__atomic_and_fetch:  in EmitAtomicExpr()
988     case AtomicExpr::AO__atomic_nand_fetch:  in EmitAtomicExpr()
989     case AtomicExpr::AO__atomic_or_fetch:  in EmitAtomicExpr()
990     case AtomicExpr::AO__atomic_sub_fetch:  in EmitAtomicExpr()
991     case AtomicExpr::AO__atomic_xor_fetch:  in EmitAtomicExpr()
992     case AtomicExpr::AO__atomic_fetch_max:  in EmitAtomicExpr()
993     case AtomicExpr::AO__atomic_fetch_min:  in EmitAtomicExpr()
994     case AtomicExpr::AO__atomic_max_fetch:  in EmitAtomicExpr()
995     case AtomicExpr::AO__atomic_min_fetch:  in EmitAtomicExpr()
1000     case AtomicExpr::AO__atomic_load:  in EmitAtomicExpr()
1001     case AtomicExpr::AO__atomic_store:  in EmitAtomicExpr()
1002     case AtomicExpr::AO__atomic_exchange:  in EmitAtomicExpr()
1003     case AtomicExpr::AO__atomic_compare_exchange:  in EmitAtomicExpr()
1009     case AtomicExpr::AO__c11_atomic_load:  in EmitAtomicExpr()
1010     case AtomicExpr::AO__c11_atomic_store:  in EmitAtomicExpr()
1011     case AtomicExpr::AO__c11_atomic_exchange:  in EmitAtomicExpr()
1012     case AtomicExpr::AO__c11_atomic_compare_exchange_weak:  in EmitAtomicExpr()
1013     case AtomicExpr::AO__c11_atomic_compare_exchange_strong:  in EmitAtomicExpr()
1014     case AtomicExpr::AO__opencl_atomic_load:  in EmitAtomicExpr()
1015     case AtomicExpr::AO__opencl_atomic_store:  in EmitAtomicExpr()
1016     case AtomicExpr::AO__opencl_atomic_exchange:  in EmitAtomicExpr()
1017     case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:  in EmitAtomicExpr()
1018     case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:  in EmitAtomicExpr()
1019     case AtomicExpr::AO__atomic_load_n:  in EmitAtomicExpr()
1020     case AtomicExpr::AO__atomic_store_n:  in EmitAtomicExpr()
1021     case AtomicExpr::AO__atomic_exchange_n:  in EmitAtomicExpr()
1022     case AtomicExpr::AO__atomic_compare_exchange_n:  in EmitAtomicExpr()
1065     case AtomicExpr::AO__c11_atomic_init:  in EmitAtomicExpr()
1066     case AtomicExpr::AO__opencl_atomic_init:  in EmitAtomicExpr()
1076     case AtomicExpr::AO__c11_atomic_compare_exchange_weak:  in EmitAtomicExpr()
1077     case AtomicExpr::AO__c11_atomic_compare_exchange_strong:  in EmitAtomicExpr()
1078     case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:  in EmitAtomicExpr()
1079     case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:  in EmitAtomicExpr()
1080     case AtomicExpr::AO__atomic_compare_exchange:  in EmitAtomicExpr()
1081     case AtomicExpr::AO__atomic_compare_exchange_n:  in EmitAtomicExpr()
1097     case AtomicExpr::AO__c11_atomic_exchange:  in EmitAtomicExpr()
1098     case AtomicExpr::AO__opencl_atomic_exchange:  in EmitAtomicExpr()
1099     case AtomicExpr::AO__atomic_exchange_n:  in EmitAtomicExpr()
1100     case AtomicExpr::AO__atomic_exchange:  in EmitAtomicExpr()
1107     case AtomicExpr::AO__c11_atomic_store:  in EmitAtomicExpr()
1108     case AtomicExpr::AO__opencl_atomic_store:  in EmitAtomicExpr()
1109     case AtomicExpr::AO__atomic_store:  in EmitAtomicExpr()
1110     case AtomicExpr::AO__atomic_store_n:  in EmitAtomicExpr()
1119     case AtomicExpr::AO__c11_atomic_load:  in EmitAtomicExpr()
1120     case AtomicExpr::AO__opencl_atomic_load:  in EmitAtomicExpr()
1121     case AtomicExpr::AO__atomic_load:  in EmitAtomicExpr()
1122     case AtomicExpr::AO__atomic_load_n:  in EmitAtomicExpr()
1127     case AtomicExpr::AO__atomic_add_fetch:  in EmitAtomicExpr()
1130     case AtomicExpr::AO__c11_atomic_fetch_add:  in EmitAtomicExpr()
1131     case AtomicExpr::AO__opencl_atomic_fetch_add:  in EmitAtomicExpr()
1132     case AtomicExpr::AO__atomic_fetch_add:  in EmitAtomicExpr()
1139     case AtomicExpr::AO__atomic_and_fetch:  in EmitAtomicExpr()
1142     case AtomicExpr::AO__c11_atomic_fetch_and:  in EmitAtomicExpr()
1143     case AtomicExpr::AO__opencl_atomic_fetch_and:  in EmitAtomicExpr()
1144     case AtomicExpr::AO__atomic_fetch_and:  in EmitAtomicExpr()
1151     case AtomicExpr::AO__atomic_or_fetch:  in EmitAtomicExpr()
1154     case AtomicExpr::AO__c11_atomic_fetch_or:  in EmitAtomicExpr()
1155     case AtomicExpr::AO__opencl_atomic_fetch_or:  in EmitAtomicExpr()
1156     case AtomicExpr::AO__atomic_fetch_or:  in EmitAtomicExpr()
1163     case AtomicExpr::AO__atomic_sub_fetch:  in EmitAtomicExpr()
1166     case AtomicExpr::AO__c11_atomic_fetch_sub:  in EmitAtomicExpr()
1167     case AtomicExpr::AO__opencl_atomic_fetch_sub:  in EmitAtomicExpr()
1168     case AtomicExpr::AO__atomic_fetch_sub:  in EmitAtomicExpr()
1175     case AtomicExpr::AO__atomic_xor_fetch:  in EmitAtomicExpr()
1178     case AtomicExpr::AO__c11_atomic_fetch_xor:  in EmitAtomicExpr()
1179     case AtomicExpr::AO__opencl_atomic_fetch_xor:  in EmitAtomicExpr()
1180     case AtomicExpr::AO__atomic_fetch_xor:  in EmitAtomicExpr()
1185     case AtomicExpr::AO__atomic_min_fetch:  in EmitAtomicExpr()
1188     case AtomicExpr::AO__c11_atomic_fetch_min:  in EmitAtomicExpr()
1189     case AtomicExpr::AO__atomic_fetch_min:  in EmitAtomicExpr()
1190     case AtomicExpr::AO__opencl_atomic_fetch_min:  in EmitAtomicExpr()
1197     case AtomicExpr::AO__atomic_max_fetch:  in EmitAtomicExpr()
1200     case AtomicExpr::AO__c11_atomic_fetch_max:  in EmitAtomicExpr()
1201     case AtomicExpr::AO__atomic_fetch_max:  in EmitAtomicExpr()
1202     case AtomicExpr::AO__opencl_atomic_fetch_max:  in EmitAtomicExpr()
1211     case AtomicExpr::AO__atomic_nand_fetch:  in EmitAtomicExpr()
1214     case AtomicExpr::AO__atomic_fetch_nand:  in EmitAtomicExpr()
1272       if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)  in EmitAtomicExpr()
1288   bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||  in EmitAtomicExpr()
1289                  E->getOp() == AtomicExpr::AO__opencl_atomic_store ||  in EmitAtomicExpr()
1290                  E->getOp() == AtomicExpr::AO__atomic_store ||  in EmitAtomicExpr()
1291                  E->getOp() == AtomicExpr::AO__atomic_store_n;  in EmitAtomicExpr()
1292   bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||  in EmitAtomicExpr()
1293                 E->getOp() == AtomicExpr::AO__opencl_atomic_load ||  in EmitAtomicExpr()
1294                 E->getOp() == AtomicExpr::AO__atomic_load ||  in EmitAtomicExpr()
1295                 E->getOp() == AtomicExpr::AO__atomic_load_n;  in EmitAtomicExpr()