1e4b17023SJohn Marino;; GCC machine description for i386 synchronization instructions. 2e4b17023SJohn Marino;; Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011 3e4b17023SJohn Marino;; Free Software Foundation, Inc. 4e4b17023SJohn Marino;; 5e4b17023SJohn Marino;; This file is part of GCC. 6e4b17023SJohn Marino;; 7e4b17023SJohn Marino;; GCC is free software; you can redistribute it and/or modify 8e4b17023SJohn Marino;; it under the terms of the GNU General Public License as published by 9e4b17023SJohn Marino;; the Free Software Foundation; either version 3, or (at your option) 10e4b17023SJohn Marino;; any later version. 11e4b17023SJohn Marino;; 12e4b17023SJohn Marino;; GCC is distributed in the hope that it will be useful, 13e4b17023SJohn Marino;; but WITHOUT ANY WARRANTY; without even the implied warranty of 14e4b17023SJohn Marino;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15e4b17023SJohn Marino;; GNU General Public License for more details. 16e4b17023SJohn Marino;; 17e4b17023SJohn Marino;; You should have received a copy of the GNU General Public License 18e4b17023SJohn Marino;; along with GCC; see the file COPYING3. If not see 19e4b17023SJohn Marino;; <http://www.gnu.org/licenses/>. 20e4b17023SJohn Marino 21e4b17023SJohn Marino(define_c_enum "unspec" [ 22e4b17023SJohn Marino UNSPEC_LFENCE 23e4b17023SJohn Marino UNSPEC_SFENCE 24e4b17023SJohn Marino UNSPEC_MFENCE 25e4b17023SJohn Marino UNSPEC_MOVA ; For __atomic support 26e4b17023SJohn Marino UNSPEC_LDA 27e4b17023SJohn Marino UNSPEC_STA 28e4b17023SJohn Marino]) 29e4b17023SJohn Marino 30e4b17023SJohn Marino(define_c_enum "unspecv" [ 31e4b17023SJohn Marino UNSPECV_CMPXCHG 32e4b17023SJohn Marino UNSPECV_XCHG 33e4b17023SJohn Marino UNSPECV_LOCK 34e4b17023SJohn Marino]) 35e4b17023SJohn Marino 36e4b17023SJohn Marino(define_expand "sse2_lfence" 37e4b17023SJohn Marino [(set (match_dup 0) 38e4b17023SJohn Marino (unspec:BLK [(match_dup 0)] UNSPEC_LFENCE))] 39e4b17023SJohn Marino "TARGET_SSE2" 40e4b17023SJohn Marino{ 41e4b17023SJohn Marino operands[0] = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (Pmode)); 42e4b17023SJohn Marino MEM_VOLATILE_P (operands[0]) = 1; 43e4b17023SJohn Marino}) 44e4b17023SJohn Marino 45e4b17023SJohn Marino(define_insn "*sse2_lfence" 46e4b17023SJohn Marino [(set (match_operand:BLK 0 "" "") 47e4b17023SJohn Marino (unspec:BLK [(match_dup 0)] UNSPEC_LFENCE))] 48e4b17023SJohn Marino "TARGET_SSE2" 49e4b17023SJohn Marino "lfence" 50e4b17023SJohn Marino [(set_attr "type" "sse") 51e4b17023SJohn Marino (set_attr "length_address" "0") 52e4b17023SJohn Marino (set_attr "atom_sse_attr" "lfence") 53e4b17023SJohn Marino (set_attr "memory" "unknown")]) 54e4b17023SJohn Marino 55e4b17023SJohn Marino(define_expand "sse_sfence" 56e4b17023SJohn Marino [(set (match_dup 0) 57e4b17023SJohn Marino (unspec:BLK [(match_dup 0)] UNSPEC_SFENCE))] 58e4b17023SJohn Marino "TARGET_SSE || TARGET_3DNOW_A" 59e4b17023SJohn Marino{ 60e4b17023SJohn Marino operands[0] = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (Pmode)); 61e4b17023SJohn Marino MEM_VOLATILE_P (operands[0]) = 1; 62e4b17023SJohn Marino}) 63e4b17023SJohn Marino 64e4b17023SJohn Marino(define_insn "*sse_sfence" 65e4b17023SJohn Marino [(set (match_operand:BLK 0 "" "") 66e4b17023SJohn Marino (unspec:BLK [(match_dup 0)] UNSPEC_SFENCE))] 67e4b17023SJohn Marino "TARGET_SSE || TARGET_3DNOW_A" 68e4b17023SJohn Marino "sfence" 69e4b17023SJohn Marino [(set_attr "type" "sse") 70e4b17023SJohn Marino (set_attr "length_address" "0") 71e4b17023SJohn Marino (set_attr "atom_sse_attr" "fence") 72e4b17023SJohn Marino (set_attr "memory" "unknown")]) 73e4b17023SJohn Marino 74e4b17023SJohn Marino(define_expand "sse2_mfence" 75e4b17023SJohn Marino [(set (match_dup 0) 76e4b17023SJohn Marino (unspec:BLK [(match_dup 0)] UNSPEC_MFENCE))] 77e4b17023SJohn Marino "TARGET_SSE2" 78e4b17023SJohn Marino{ 79e4b17023SJohn Marino operands[0] = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (Pmode)); 80e4b17023SJohn Marino MEM_VOLATILE_P (operands[0]) = 1; 81e4b17023SJohn Marino}) 82e4b17023SJohn Marino 83e4b17023SJohn Marino(define_insn "mfence_sse2" 84e4b17023SJohn Marino [(set (match_operand:BLK 0 "" "") 85e4b17023SJohn Marino (unspec:BLK [(match_dup 0)] UNSPEC_MFENCE))] 86e4b17023SJohn Marino "TARGET_64BIT || TARGET_SSE2" 87e4b17023SJohn Marino "mfence" 88e4b17023SJohn Marino [(set_attr "type" "sse") 89e4b17023SJohn Marino (set_attr "length_address" "0") 90e4b17023SJohn Marino (set_attr "atom_sse_attr" "fence") 91e4b17023SJohn Marino (set_attr "memory" "unknown")]) 92e4b17023SJohn Marino 93e4b17023SJohn Marino(define_insn "mfence_nosse" 94e4b17023SJohn Marino [(set (match_operand:BLK 0 "" "") 95e4b17023SJohn Marino (unspec:BLK [(match_dup 0)] UNSPEC_MFENCE)) 96e4b17023SJohn Marino (clobber (reg:CC FLAGS_REG))] 97e4b17023SJohn Marino "!(TARGET_64BIT || TARGET_SSE2)" 98e4b17023SJohn Marino "lock{%;} or{l}\t{$0, (%%esp)|DWORD PTR [esp], 0}" 99e4b17023SJohn Marino [(set_attr "memory" "unknown")]) 100e4b17023SJohn Marino 101e4b17023SJohn Marino(define_expand "mem_thread_fence" 102e4b17023SJohn Marino [(match_operand:SI 0 "const_int_operand" "")] ;; model 103e4b17023SJohn Marino "" 104e4b17023SJohn Marino{ 105e4b17023SJohn Marino /* Unless this is a SEQ_CST fence, the i386 memory model is strong 106e4b17023SJohn Marino enough not to require barriers of any kind. */ 107e4b17023SJohn Marino if (INTVAL (operands[0]) == MEMMODEL_SEQ_CST) 108e4b17023SJohn Marino { 109e4b17023SJohn Marino rtx (*mfence_insn)(rtx); 110e4b17023SJohn Marino rtx mem; 111e4b17023SJohn Marino 112e4b17023SJohn Marino if (TARGET_64BIT || TARGET_SSE2) 113e4b17023SJohn Marino mfence_insn = gen_mfence_sse2; 114e4b17023SJohn Marino else 115e4b17023SJohn Marino mfence_insn = gen_mfence_nosse; 116e4b17023SJohn Marino 117e4b17023SJohn Marino mem = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (Pmode)); 118e4b17023SJohn Marino MEM_VOLATILE_P (mem) = 1; 119e4b17023SJohn Marino 120e4b17023SJohn Marino emit_insn (mfence_insn (mem)); 121e4b17023SJohn Marino } 122e4b17023SJohn Marino DONE; 123e4b17023SJohn Marino}) 124e4b17023SJohn Marino 125e4b17023SJohn Marino;; ??? From volume 3 section 8.1.1 Guaranteed Atomic Operations, 126e4b17023SJohn Marino;; Only beginning at Pentium family processors do we get any guarantee of 127e4b17023SJohn Marino;; atomicity in aligned 64-bit quantities. Beginning at P6, we get a 128e4b17023SJohn Marino;; guarantee for 64-bit accesses that do not cross a cacheline boundary. 129e4b17023SJohn Marino;; 130e4b17023SJohn Marino;; Note that the TARGET_CMPXCHG8B test below is a stand-in for "Pentium". 131e4b17023SJohn Marino;; 132e4b17023SJohn Marino;; Importantly, *no* processor makes atomicity guarantees for larger 133e4b17023SJohn Marino;; accesses. In particular, there's no way to perform an atomic TImode 134e4b17023SJohn Marino;; move, despite the apparent applicability of MOVDQA et al. 135e4b17023SJohn Marino 136e4b17023SJohn Marino(define_mode_iterator ATOMIC 137e4b17023SJohn Marino [QI HI SI 138e4b17023SJohn Marino (DI "TARGET_64BIT || (TARGET_CMPXCHG8B && (TARGET_80387 || TARGET_SSE))") 139e4b17023SJohn Marino ]) 140e4b17023SJohn Marino 141e4b17023SJohn Marino(define_expand "atomic_load<mode>" 142e4b17023SJohn Marino [(set (match_operand:ATOMIC 0 "register_operand" "") 143e4b17023SJohn Marino (unspec:ATOMIC [(match_operand:ATOMIC 1 "memory_operand" "") 144e4b17023SJohn Marino (match_operand:SI 2 "const_int_operand" "")] 145e4b17023SJohn Marino UNSPEC_MOVA))] 146e4b17023SJohn Marino "" 147e4b17023SJohn Marino{ 148e4b17023SJohn Marino /* For DImode on 32-bit, we can use the FPU to perform the load. */ 149e4b17023SJohn Marino if (<MODE>mode == DImode && !TARGET_64BIT) 150e4b17023SJohn Marino emit_insn (gen_atomic_loaddi_fpu 151e4b17023SJohn Marino (operands[0], operands[1], 152*5ce9237cSJohn Marino assign_386_stack_local (DImode, SLOT_TEMP))); 153e4b17023SJohn Marino else 154e4b17023SJohn Marino emit_move_insn (operands[0], operands[1]); 155e4b17023SJohn Marino DONE; 156e4b17023SJohn Marino}) 157e4b17023SJohn Marino 158e4b17023SJohn Marino(define_insn_and_split "atomic_loaddi_fpu" 159e4b17023SJohn Marino [(set (match_operand:DI 0 "nonimmediate_operand" "=x,m,?r") 160e4b17023SJohn Marino (unspec:DI [(match_operand:DI 1 "memory_operand" "m,m,m")] 161e4b17023SJohn Marino UNSPEC_MOVA)) 162e4b17023SJohn Marino (clobber (match_operand:DI 2 "memory_operand" "=X,X,m")) 163e4b17023SJohn Marino (clobber (match_scratch:DF 3 "=X,xf,xf"))] 164e4b17023SJohn Marino "!TARGET_64BIT && (TARGET_80387 || TARGET_SSE)" 165e4b17023SJohn Marino "#" 166e4b17023SJohn Marino "&& reload_completed" 167e4b17023SJohn Marino [(const_int 0)] 168e4b17023SJohn Marino{ 169e4b17023SJohn Marino rtx dst = operands[0], src = operands[1]; 170e4b17023SJohn Marino rtx mem = operands[2], tmp = operands[3]; 171e4b17023SJohn Marino 172e4b17023SJohn Marino if (SSE_REG_P (dst)) 173e4b17023SJohn Marino emit_move_insn (dst, src); 174e4b17023SJohn Marino else 175e4b17023SJohn Marino { 176e4b17023SJohn Marino if (MEM_P (dst)) 177e4b17023SJohn Marino mem = dst; 178e4b17023SJohn Marino 179e4b17023SJohn Marino if (FP_REG_P (tmp)) 180e4b17023SJohn Marino { 181e4b17023SJohn Marino emit_insn (gen_loaddi_via_fpu (tmp, src)); 182e4b17023SJohn Marino emit_insn (gen_storedi_via_fpu (mem, tmp)); 183e4b17023SJohn Marino } 184e4b17023SJohn Marino else 185e4b17023SJohn Marino { 186e4b17023SJohn Marino adjust_reg_mode (tmp, DImode); 187e4b17023SJohn Marino emit_move_insn (tmp, src); 188e4b17023SJohn Marino emit_move_insn (mem, tmp); 189e4b17023SJohn Marino } 190e4b17023SJohn Marino 191e4b17023SJohn Marino if (mem != dst) 192e4b17023SJohn Marino emit_move_insn (dst, mem); 193e4b17023SJohn Marino } 194e4b17023SJohn Marino DONE; 195e4b17023SJohn Marino}) 196e4b17023SJohn Marino 197e4b17023SJohn Marino(define_expand "atomic_store<mode>" 198e4b17023SJohn Marino [(set (match_operand:ATOMIC 0 "memory_operand" "") 199e4b17023SJohn Marino (unspec:ATOMIC [(match_operand:ATOMIC 1 "register_operand" "") 200e4b17023SJohn Marino (match_operand:SI 2 "const_int_operand" "")] 201e4b17023SJohn Marino UNSPEC_MOVA))] 202e4b17023SJohn Marino "" 203e4b17023SJohn Marino{ 204e4b17023SJohn Marino enum memmodel model = (enum memmodel) INTVAL (operands[2]); 205e4b17023SJohn Marino 206e4b17023SJohn Marino if (<MODE>mode == DImode && !TARGET_64BIT) 207e4b17023SJohn Marino { 208e4b17023SJohn Marino /* For DImode on 32-bit, we can use the FPU to perform the store. */ 209e4b17023SJohn Marino /* Note that while we could perform a cmpxchg8b loop, that turns 210e4b17023SJohn Marino out to be significantly larger than this plus a barrier. */ 211e4b17023SJohn Marino emit_insn (gen_atomic_storedi_fpu 212e4b17023SJohn Marino (operands[0], operands[1], 213*5ce9237cSJohn Marino assign_386_stack_local (DImode, SLOT_TEMP))); 214e4b17023SJohn Marino } 215e4b17023SJohn Marino else 216e4b17023SJohn Marino { 217e4b17023SJohn Marino /* For seq-cst stores, when we lack MFENCE, use XCHG. */ 218e4b17023SJohn Marino if (model == MEMMODEL_SEQ_CST && !(TARGET_64BIT || TARGET_SSE2)) 219e4b17023SJohn Marino { 220e4b17023SJohn Marino emit_insn (gen_atomic_exchange<mode> (gen_reg_rtx (<MODE>mode), 221e4b17023SJohn Marino operands[0], operands[1], 222e4b17023SJohn Marino operands[2])); 223e4b17023SJohn Marino DONE; 224e4b17023SJohn Marino } 225e4b17023SJohn Marino 226*5ce9237cSJohn Marino /* Otherwise use a store. */ 227*5ce9237cSJohn Marino emit_insn (gen_atomic_store<mode>_1 (operands[0], operands[1], 228*5ce9237cSJohn Marino operands[2])); 229e4b17023SJohn Marino } 230e4b17023SJohn Marino /* ... followed by an MFENCE, if required. */ 231e4b17023SJohn Marino if (model == MEMMODEL_SEQ_CST) 232e4b17023SJohn Marino emit_insn (gen_mem_thread_fence (operands[2])); 233e4b17023SJohn Marino DONE; 234e4b17023SJohn Marino}) 235e4b17023SJohn Marino 236*5ce9237cSJohn Marino(define_insn "atomic_store<mode>_1" 237*5ce9237cSJohn Marino [(set (match_operand:SWI 0 "memory_operand" "=m") 238*5ce9237cSJohn Marino (unspec:SWI [(match_operand:SWI 1 "<nonmemory_operand>" "<r><i>") 239*5ce9237cSJohn Marino (match_operand:SI 2 "const_int_operand")] 240*5ce9237cSJohn Marino UNSPEC_MOVA))] 241*5ce9237cSJohn Marino "" 242*5ce9237cSJohn Marino "mov{<imodesuffix>}\t{%1, %0|%0, %1}") 243*5ce9237cSJohn Marino 244e4b17023SJohn Marino(define_insn_and_split "atomic_storedi_fpu" 245e4b17023SJohn Marino [(set (match_operand:DI 0 "memory_operand" "=m,m,m") 246e4b17023SJohn Marino (unspec:DI [(match_operand:DI 1 "register_operand" "x,m,?r")] 247e4b17023SJohn Marino UNSPEC_MOVA)) 248e4b17023SJohn Marino (clobber (match_operand:DI 2 "memory_operand" "=X,X,m")) 249e4b17023SJohn Marino (clobber (match_scratch:DF 3 "=X,xf,xf"))] 250e4b17023SJohn Marino "!TARGET_64BIT && (TARGET_80387 || TARGET_SSE)" 251e4b17023SJohn Marino "#" 252e4b17023SJohn Marino "&& reload_completed" 253e4b17023SJohn Marino [(const_int 0)] 254e4b17023SJohn Marino{ 255e4b17023SJohn Marino rtx dst = operands[0], src = operands[1]; 256e4b17023SJohn Marino rtx mem = operands[2], tmp = operands[3]; 257e4b17023SJohn Marino 258e4b17023SJohn Marino if (!SSE_REG_P (src)) 259e4b17023SJohn Marino { 260e4b17023SJohn Marino if (REG_P (src)) 261e4b17023SJohn Marino { 262e4b17023SJohn Marino emit_move_insn (mem, src); 263e4b17023SJohn Marino src = mem; 264e4b17023SJohn Marino } 265e4b17023SJohn Marino 266e4b17023SJohn Marino if (FP_REG_P (tmp)) 267e4b17023SJohn Marino { 268e4b17023SJohn Marino emit_insn (gen_loaddi_via_fpu (tmp, src)); 269e4b17023SJohn Marino emit_insn (gen_storedi_via_fpu (dst, tmp)); 270e4b17023SJohn Marino DONE; 271e4b17023SJohn Marino } 272e4b17023SJohn Marino else 273e4b17023SJohn Marino { 274e4b17023SJohn Marino adjust_reg_mode (tmp, DImode); 275e4b17023SJohn Marino emit_move_insn (tmp, mem); 276e4b17023SJohn Marino src = tmp; 277e4b17023SJohn Marino } 278e4b17023SJohn Marino } 279e4b17023SJohn Marino emit_move_insn (dst, src); 280e4b17023SJohn Marino DONE; 281e4b17023SJohn Marino}) 282e4b17023SJohn Marino 283e4b17023SJohn Marino;; ??? You'd think that we'd be able to perform this via FLOAT + FIX_TRUNC 284e4b17023SJohn Marino;; operations. But the fix_trunc patterns want way more setup than we want 285e4b17023SJohn Marino;; to provide. Note that the scratch is DFmode instead of XFmode in order 286e4b17023SJohn Marino;; to make it easy to allocate a scratch in either SSE or FP_REGs above. 287e4b17023SJohn Marino 288e4b17023SJohn Marino(define_insn "loaddi_via_fpu" 289e4b17023SJohn Marino [(set (match_operand:DF 0 "register_operand" "=f") 290e4b17023SJohn Marino (unspec:DF [(match_operand:DI 1 "memory_operand" "m")] UNSPEC_LDA))] 291e4b17023SJohn Marino "TARGET_80387" 292e4b17023SJohn Marino "fild%Z1\t%1" 293e4b17023SJohn Marino [(set_attr "type" "fmov") 294e4b17023SJohn Marino (set_attr "mode" "DF") 295e4b17023SJohn Marino (set_attr "fp_int_src" "true")]) 296e4b17023SJohn Marino 297e4b17023SJohn Marino(define_insn "storedi_via_fpu" 298e4b17023SJohn Marino [(set (match_operand:DI 0 "memory_operand" "=m") 299e4b17023SJohn Marino (unspec:DI [(match_operand:DF 1 "register_operand" "f")] UNSPEC_STA))] 300e4b17023SJohn Marino "TARGET_80387" 301e4b17023SJohn Marino{ 302e4b17023SJohn Marino gcc_assert (find_regno_note (insn, REG_DEAD, FIRST_STACK_REG) != NULL_RTX); 303e4b17023SJohn Marino 304e4b17023SJohn Marino return "fistp%Z0\t%0"; 305e4b17023SJohn Marino} 306e4b17023SJohn Marino [(set_attr "type" "fmov") 307e4b17023SJohn Marino (set_attr "mode" "DI")]) 308e4b17023SJohn Marino 309e4b17023SJohn Marino(define_expand "atomic_compare_and_swap<mode>" 310e4b17023SJohn Marino [(match_operand:QI 0 "register_operand" "") ;; bool success output 311e4b17023SJohn Marino (match_operand:SWI124 1 "register_operand" "") ;; oldval output 312e4b17023SJohn Marino (match_operand:SWI124 2 "memory_operand" "") ;; memory 313e4b17023SJohn Marino (match_operand:SWI124 3 "register_operand" "") ;; expected input 314e4b17023SJohn Marino (match_operand:SWI124 4 "register_operand" "") ;; newval input 315e4b17023SJohn Marino (match_operand:SI 5 "const_int_operand" "") ;; is_weak 316e4b17023SJohn Marino (match_operand:SI 6 "const_int_operand" "") ;; success model 317e4b17023SJohn Marino (match_operand:SI 7 "const_int_operand" "")] ;; failure model 318e4b17023SJohn Marino "TARGET_CMPXCHG" 319e4b17023SJohn Marino{ 320e4b17023SJohn Marino emit_insn 321e4b17023SJohn Marino (gen_atomic_compare_and_swap<mode>_1 322e4b17023SJohn Marino (operands[1], operands[2], operands[3], operands[4])); 323e4b17023SJohn Marino ix86_expand_setcc (operands[0], EQ, gen_rtx_REG (CCZmode, FLAGS_REG), 324e4b17023SJohn Marino const0_rtx); 325e4b17023SJohn Marino DONE; 326e4b17023SJohn Marino}) 327e4b17023SJohn Marino 328e4b17023SJohn Marino(define_mode_iterator CASMODE 329e4b17023SJohn Marino [(DI "TARGET_64BIT || TARGET_CMPXCHG8B") 330e4b17023SJohn Marino (TI "TARGET_64BIT && TARGET_CMPXCHG16B")]) 331e4b17023SJohn Marino(define_mode_attr CASHMODE [(DI "SI") (TI "DI")]) 332e4b17023SJohn Marino 333e4b17023SJohn Marino(define_expand "atomic_compare_and_swap<mode>" 334e4b17023SJohn Marino [(match_operand:QI 0 "register_operand" "") ;; bool success output 335e4b17023SJohn Marino (match_operand:CASMODE 1 "register_operand" "") ;; oldval output 336e4b17023SJohn Marino (match_operand:CASMODE 2 "memory_operand" "") ;; memory 337e4b17023SJohn Marino (match_operand:CASMODE 3 "register_operand" "") ;; expected input 338e4b17023SJohn Marino (match_operand:CASMODE 4 "register_operand" "") ;; newval input 339e4b17023SJohn Marino (match_operand:SI 5 "const_int_operand" "") ;; is_weak 340e4b17023SJohn Marino (match_operand:SI 6 "const_int_operand" "") ;; success model 341e4b17023SJohn Marino (match_operand:SI 7 "const_int_operand" "")] ;; failure model 342e4b17023SJohn Marino "TARGET_CMPXCHG" 343e4b17023SJohn Marino{ 344e4b17023SJohn Marino if (<MODE>mode == DImode && TARGET_64BIT) 345e4b17023SJohn Marino { 346e4b17023SJohn Marino emit_insn 347e4b17023SJohn Marino (gen_atomic_compare_and_swapdi_1 348e4b17023SJohn Marino (operands[1], operands[2], operands[3], operands[4])); 349e4b17023SJohn Marino } 350e4b17023SJohn Marino else 351e4b17023SJohn Marino { 352e4b17023SJohn Marino enum machine_mode hmode = <CASHMODE>mode; 353e4b17023SJohn Marino rtx lo_o, lo_e, lo_n, hi_o, hi_e, hi_n, mem; 354e4b17023SJohn Marino 355e4b17023SJohn Marino lo_o = operands[1]; 356e4b17023SJohn Marino mem = operands[2]; 357e4b17023SJohn Marino lo_e = operands[3]; 358e4b17023SJohn Marino lo_n = operands[4]; 359e4b17023SJohn Marino hi_o = gen_highpart (hmode, lo_o); 360e4b17023SJohn Marino hi_e = gen_highpart (hmode, lo_e); 361e4b17023SJohn Marino hi_n = gen_highpart (hmode, lo_n); 362e4b17023SJohn Marino lo_o = gen_lowpart (hmode, lo_o); 363e4b17023SJohn Marino lo_e = gen_lowpart (hmode, lo_e); 364e4b17023SJohn Marino lo_n = gen_lowpart (hmode, lo_n); 365e4b17023SJohn Marino 366e4b17023SJohn Marino if (!cmpxchg8b_pic_memory_operand (mem, <MODE>mode)) 367e4b17023SJohn Marino mem = replace_equiv_address (mem, force_reg (Pmode, XEXP (mem, 0))); 368e4b17023SJohn Marino 369e4b17023SJohn Marino emit_insn 370e4b17023SJohn Marino (gen_atomic_compare_and_swap<mode>_doubleword 371e4b17023SJohn Marino (lo_o, hi_o, mem, lo_e, hi_e, lo_n, hi_n)); 372e4b17023SJohn Marino } 373e4b17023SJohn Marino 374e4b17023SJohn Marino ix86_expand_setcc (operands[0], EQ, gen_rtx_REG (CCZmode, FLAGS_REG), 375e4b17023SJohn Marino const0_rtx); 376e4b17023SJohn Marino DONE; 377e4b17023SJohn Marino}) 378e4b17023SJohn Marino 379e4b17023SJohn Marino(define_insn "atomic_compare_and_swap<mode>_1" 380e4b17023SJohn Marino [(set (match_operand:SWI 0 "register_operand" "=a") 381e4b17023SJohn Marino (unspec_volatile:SWI 382e4b17023SJohn Marino [(match_operand:SWI 1 "memory_operand" "+m") 383e4b17023SJohn Marino (match_operand:SWI 2 "register_operand" "0") 384e4b17023SJohn Marino (match_operand:SWI 3 "register_operand" "<r>")] 385e4b17023SJohn Marino UNSPECV_CMPXCHG)) 386e4b17023SJohn Marino (set (match_dup 1) 387e4b17023SJohn Marino (unspec_volatile:SWI [(const_int 0)] UNSPECV_CMPXCHG)) 388e4b17023SJohn Marino (set (reg:CCZ FLAGS_REG) 389e4b17023SJohn Marino (unspec_volatile:CCZ [(const_int 0)] UNSPECV_CMPXCHG))] 390e4b17023SJohn Marino "TARGET_CMPXCHG" 391e4b17023SJohn Marino "lock{%;} cmpxchg{<imodesuffix>}\t{%3, %1|%1, %3}") 392e4b17023SJohn Marino 393e4b17023SJohn Marino;; For double-word compare and swap, we are obliged to play tricks with 394e4b17023SJohn Marino;; the input newval (op5:op6) because the Intel register numbering does 395e4b17023SJohn Marino;; not match the gcc register numbering, so the pair must be CX:BX. 396e4b17023SJohn Marino;; That said, in order to take advantage of possible lower-subreg opts, 397e4b17023SJohn Marino;; treat all of the integral operands in the same way. 398e4b17023SJohn Marino 399e4b17023SJohn Marino;; Operands 5 and 6 really need to be different registers, which in 400e4b17023SJohn Marino;; this case means op5 must not be ecx. If op5 and op6 are the same 401e4b17023SJohn Marino;; (like when the input is -1LL) GCC might chose to allocate op5 to ecx, 402e4b17023SJohn Marino;; like op6. This breaks, as the xchg will move the PIC register 403e4b17023SJohn Marino;; contents to %ecx then --> boom. 404e4b17023SJohn Marino 405e4b17023SJohn Marino(define_mode_attr doublemodesuffix [(SI "8") (DI "16")]) 406e4b17023SJohn Marino(define_mode_attr regprefix [(SI "e") (DI "r")]) 407e4b17023SJohn Marino 408e4b17023SJohn Marino(define_insn "atomic_compare_and_swap<dwi>_doubleword" 409e4b17023SJohn Marino [(set (match_operand:DWIH 0 "register_operand" "=a,a") 410e4b17023SJohn Marino (unspec_volatile:DWIH 411e4b17023SJohn Marino [(match_operand:<DWI> 2 "cmpxchg8b_pic_memory_operand" "+m,m") 412e4b17023SJohn Marino (match_operand:DWIH 3 "register_operand" "0,0") 413e4b17023SJohn Marino (match_operand:DWIH 4 "register_operand" "1,1") 414e4b17023SJohn Marino (match_operand:DWIH 5 "register_operand" "b,!*r") 415e4b17023SJohn Marino (match_operand:DWIH 6 "register_operand" "c,c")] 416e4b17023SJohn Marino UNSPECV_CMPXCHG)) 417e4b17023SJohn Marino (set (match_operand:DWIH 1 "register_operand" "=d,d") 418e4b17023SJohn Marino (unspec_volatile:DWIH [(const_int 0)] UNSPECV_CMPXCHG)) 419e4b17023SJohn Marino (set (match_dup 2) 420e4b17023SJohn Marino (unspec_volatile:<DWI> [(const_int 0)] UNSPECV_CMPXCHG)) 421e4b17023SJohn Marino (set (reg:CCZ FLAGS_REG) 422e4b17023SJohn Marino (unspec_volatile:CCZ [(const_int 0)] UNSPECV_CMPXCHG)) 423e4b17023SJohn Marino (clobber (match_scratch:DWIH 7 "=X,&5"))] 424e4b17023SJohn Marino "TARGET_CMPXCHG<doublemodesuffix>B" 425e4b17023SJohn Marino{ 426e4b17023SJohn Marino bool swap = REGNO (operands[5]) != BX_REG; 427e4b17023SJohn Marino 428e4b17023SJohn Marino if (swap) 429e4b17023SJohn Marino output_asm_insn ("xchg{<imodesuffix>}\t%%<regprefix>bx, %5", operands); 430e4b17023SJohn Marino 431e4b17023SJohn Marino output_asm_insn ("lock{%;} cmpxchg<doublemodesuffix>b\t%2", operands); 432e4b17023SJohn Marino 433e4b17023SJohn Marino if (swap) 434e4b17023SJohn Marino output_asm_insn ("xchg{<imodesuffix>}\t%%<regprefix>bx, %5", operands); 435e4b17023SJohn Marino 436e4b17023SJohn Marino return ""; 437e4b17023SJohn Marino}) 438e4b17023SJohn Marino 439e4b17023SJohn Marino;; For operand 2 nonmemory_operand predicate is used instead of 440e4b17023SJohn Marino;; register_operand to allow combiner to better optimize atomic 441e4b17023SJohn Marino;; additions of constants. 442e4b17023SJohn Marino(define_insn "atomic_fetch_add<mode>" 443e4b17023SJohn Marino [(set (match_operand:SWI 0 "register_operand" "=<r>") 444e4b17023SJohn Marino (unspec_volatile:SWI 445e4b17023SJohn Marino [(match_operand:SWI 1 "memory_operand" "+m") 446e4b17023SJohn Marino (match_operand:SI 3 "const_int_operand" "")] ;; model 447e4b17023SJohn Marino UNSPECV_XCHG)) 448e4b17023SJohn Marino (set (match_dup 1) 449e4b17023SJohn Marino (plus:SWI (match_dup 1) 450e4b17023SJohn Marino (match_operand:SWI 2 "nonmemory_operand" "0"))) 451e4b17023SJohn Marino (clobber (reg:CC FLAGS_REG))] 452e4b17023SJohn Marino "TARGET_XADD" 453e4b17023SJohn Marino "lock{%;} xadd{<imodesuffix>}\t{%0, %1|%1, %0}") 454e4b17023SJohn Marino 455e4b17023SJohn Marino;; This peephole2 and following insn optimize 456e4b17023SJohn Marino;; __sync_fetch_and_add (x, -N) == N into just lock {add,sub,inc,dec} 457e4b17023SJohn Marino;; followed by testing of flags instead of lock xadd and comparisons. 458e4b17023SJohn Marino(define_peephole2 459e4b17023SJohn Marino [(set (match_operand:SWI 0 "register_operand" "") 460e4b17023SJohn Marino (match_operand:SWI 2 "const_int_operand" "")) 461e4b17023SJohn Marino (parallel [(set (match_dup 0) 462e4b17023SJohn Marino (unspec_volatile:SWI 463e4b17023SJohn Marino [(match_operand:SWI 1 "memory_operand" "") 464e4b17023SJohn Marino (match_operand:SI 4 "const_int_operand" "")] 465e4b17023SJohn Marino UNSPECV_XCHG)) 466e4b17023SJohn Marino (set (match_dup 1) 467e4b17023SJohn Marino (plus:SWI (match_dup 1) 468e4b17023SJohn Marino (match_dup 0))) 469e4b17023SJohn Marino (clobber (reg:CC FLAGS_REG))]) 470e4b17023SJohn Marino (set (reg:CCZ FLAGS_REG) 471e4b17023SJohn Marino (compare:CCZ (match_dup 0) 472e4b17023SJohn Marino (match_operand:SWI 3 "const_int_operand" "")))] 473e4b17023SJohn Marino "peep2_reg_dead_p (3, operands[0]) 474e4b17023SJohn Marino && (unsigned HOST_WIDE_INT) INTVAL (operands[2]) 475e4b17023SJohn Marino == -(unsigned HOST_WIDE_INT) INTVAL (operands[3]) 476e4b17023SJohn Marino && !reg_overlap_mentioned_p (operands[0], operands[1])" 477e4b17023SJohn Marino [(parallel [(set (reg:CCZ FLAGS_REG) 478e4b17023SJohn Marino (compare:CCZ 479e4b17023SJohn Marino (unspec_volatile:SWI [(match_dup 1) (match_dup 4)] 480e4b17023SJohn Marino UNSPECV_XCHG) 481e4b17023SJohn Marino (match_dup 3))) 482e4b17023SJohn Marino (set (match_dup 1) 483e4b17023SJohn Marino (plus:SWI (match_dup 1) 484e4b17023SJohn Marino (match_dup 2)))])]) 485e4b17023SJohn Marino 486e4b17023SJohn Marino(define_insn "*atomic_fetch_add_cmp<mode>" 487e4b17023SJohn Marino [(set (reg:CCZ FLAGS_REG) 488e4b17023SJohn Marino (compare:CCZ (unspec_volatile:SWI 489e4b17023SJohn Marino [(match_operand:SWI 0 "memory_operand" "+m") 490e4b17023SJohn Marino (match_operand:SI 3 "const_int_operand" "")] 491e4b17023SJohn Marino UNSPECV_XCHG) 492e4b17023SJohn Marino (match_operand:SWI 2 "const_int_operand" "i"))) 493e4b17023SJohn Marino (set (match_dup 0) 494e4b17023SJohn Marino (plus:SWI (match_dup 0) 495e4b17023SJohn Marino (match_operand:SWI 1 "const_int_operand" "i")))] 496e4b17023SJohn Marino "(unsigned HOST_WIDE_INT) INTVAL (operands[1]) 497e4b17023SJohn Marino == -(unsigned HOST_WIDE_INT) INTVAL (operands[2])" 498e4b17023SJohn Marino{ 499e4b17023SJohn Marino if (TARGET_USE_INCDEC) 500e4b17023SJohn Marino { 501e4b17023SJohn Marino if (operands[1] == const1_rtx) 502e4b17023SJohn Marino return "lock{%;} inc{<imodesuffix>}\t%0"; 503e4b17023SJohn Marino if (operands[1] == constm1_rtx) 504e4b17023SJohn Marino return "lock{%;} dec{<imodesuffix>}\t%0"; 505e4b17023SJohn Marino } 506e4b17023SJohn Marino 507e4b17023SJohn Marino if (x86_maybe_negate_const_int (&operands[1], <MODE>mode)) 508e4b17023SJohn Marino return "lock{%;} sub{<imodesuffix>}\t{%1, %0|%0, %1}"; 509e4b17023SJohn Marino 510e4b17023SJohn Marino return "lock{%;} add{<imodesuffix>}\t{%1, %0|%0, %1}"; 511e4b17023SJohn Marino}) 512e4b17023SJohn Marino 513e4b17023SJohn Marino;; Recall that xchg implicitly sets LOCK#, so adding it again wastes space. 514e4b17023SJohn Marino;; In addition, it is always a full barrier, so we can ignore the memory model. 515e4b17023SJohn Marino(define_insn "atomic_exchange<mode>" 516e4b17023SJohn Marino [(set (match_operand:SWI 0 "register_operand" "=<r>") ;; output 517e4b17023SJohn Marino (unspec_volatile:SWI 518e4b17023SJohn Marino [(match_operand:SWI 1 "memory_operand" "+m") ;; memory 519e4b17023SJohn Marino (match_operand:SI 3 "const_int_operand" "")] ;; model 520e4b17023SJohn Marino UNSPECV_XCHG)) 521e4b17023SJohn Marino (set (match_dup 1) 522e4b17023SJohn Marino (match_operand:SWI 2 "register_operand" "0"))] ;; input 523e4b17023SJohn Marino "" 524e4b17023SJohn Marino "xchg{<imodesuffix>}\t{%1, %0|%0, %1}") 525e4b17023SJohn Marino 526e4b17023SJohn Marino(define_insn "atomic_add<mode>" 527e4b17023SJohn Marino [(set (match_operand:SWI 0 "memory_operand" "+m") 528e4b17023SJohn Marino (unspec_volatile:SWI 529e4b17023SJohn Marino [(plus:SWI (match_dup 0) 530e4b17023SJohn Marino (match_operand:SWI 1 "nonmemory_operand" "<r><i>")) 531e4b17023SJohn Marino (match_operand:SI 2 "const_int_operand" "")] ;; model 532e4b17023SJohn Marino UNSPECV_LOCK)) 533e4b17023SJohn Marino (clobber (reg:CC FLAGS_REG))] 534e4b17023SJohn Marino "" 535e4b17023SJohn Marino{ 536e4b17023SJohn Marino if (TARGET_USE_INCDEC) 537e4b17023SJohn Marino { 538e4b17023SJohn Marino if (operands[1] == const1_rtx) 539e4b17023SJohn Marino return "lock{%;} inc{<imodesuffix>}\t%0"; 540e4b17023SJohn Marino if (operands[1] == constm1_rtx) 541e4b17023SJohn Marino return "lock{%;} dec{<imodesuffix>}\t%0"; 542e4b17023SJohn Marino } 543e4b17023SJohn Marino 544e4b17023SJohn Marino if (x86_maybe_negate_const_int (&operands[1], <MODE>mode)) 545e4b17023SJohn Marino return "lock{%;} sub{<imodesuffix>}\t{%1, %0|%0, %1}"; 546e4b17023SJohn Marino 547e4b17023SJohn Marino return "lock{%;} add{<imodesuffix>}\t{%1, %0|%0, %1}"; 548e4b17023SJohn Marino}) 549e4b17023SJohn Marino 550e4b17023SJohn Marino(define_insn "atomic_sub<mode>" 551e4b17023SJohn Marino [(set (match_operand:SWI 0 "memory_operand" "+m") 552e4b17023SJohn Marino (unspec_volatile:SWI 553e4b17023SJohn Marino [(minus:SWI (match_dup 0) 554e4b17023SJohn Marino (match_operand:SWI 1 "nonmemory_operand" "<r><i>")) 555e4b17023SJohn Marino (match_operand:SI 2 "const_int_operand" "")] ;; model 556e4b17023SJohn Marino UNSPECV_LOCK)) 557e4b17023SJohn Marino (clobber (reg:CC FLAGS_REG))] 558e4b17023SJohn Marino "" 559e4b17023SJohn Marino{ 560e4b17023SJohn Marino if (TARGET_USE_INCDEC) 561e4b17023SJohn Marino { 562e4b17023SJohn Marino if (operands[1] == const1_rtx) 563e4b17023SJohn Marino return "lock{%;} dec{<imodesuffix>}\t%0"; 564e4b17023SJohn Marino if (operands[1] == constm1_rtx) 565e4b17023SJohn Marino return "lock{%;} inc{<imodesuffix>}\t%0"; 566e4b17023SJohn Marino } 567e4b17023SJohn Marino 568e4b17023SJohn Marino if (x86_maybe_negate_const_int (&operands[1], <MODE>mode)) 569e4b17023SJohn Marino return "lock{%;} add{<imodesuffix>}\t{%1, %0|%0, %1}"; 570e4b17023SJohn Marino 571e4b17023SJohn Marino return "lock{%;} sub{<imodesuffix>}\t{%1, %0|%0, %1}"; 572e4b17023SJohn Marino}) 573e4b17023SJohn Marino 574e4b17023SJohn Marino(define_insn "atomic_<logic><mode>" 575e4b17023SJohn Marino [(set (match_operand:SWI 0 "memory_operand" "+m") 576e4b17023SJohn Marino (unspec_volatile:SWI 577e4b17023SJohn Marino [(any_logic:SWI (match_dup 0) 578e4b17023SJohn Marino (match_operand:SWI 1 "nonmemory_operand" "<r><i>")) 579e4b17023SJohn Marino (match_operand:SI 2 "const_int_operand" "")] ;; model 580e4b17023SJohn Marino UNSPECV_LOCK)) 581e4b17023SJohn Marino (clobber (reg:CC FLAGS_REG))] 582e4b17023SJohn Marino "" 583e4b17023SJohn Marino "lock{%;} <logic>{<imodesuffix>}\t{%1, %0|%0, %1}") 584