xref: /dflybsd-src/contrib/gcc-4.7/gcc/config/i386/sync.md (revision 81fc95a5293ee307c688a350a3feb4734aaddbb4)
1e4b17023SJohn Marino;; GCC machine description for i386 synchronization instructions.
2e4b17023SJohn Marino;; Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011
3e4b17023SJohn Marino;; Free Software Foundation, Inc.
4e4b17023SJohn Marino;;
5e4b17023SJohn Marino;; This file is part of GCC.
6e4b17023SJohn Marino;;
7e4b17023SJohn Marino;; GCC is free software; you can redistribute it and/or modify
8e4b17023SJohn Marino;; it under the terms of the GNU General Public License as published by
9e4b17023SJohn Marino;; the Free Software Foundation; either version 3, or (at your option)
10e4b17023SJohn Marino;; any later version.
11e4b17023SJohn Marino;;
12e4b17023SJohn Marino;; GCC is distributed in the hope that it will be useful,
13e4b17023SJohn Marino;; but WITHOUT ANY WARRANTY; without even the implied warranty of
14e4b17023SJohn Marino;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15e4b17023SJohn Marino;; GNU General Public License for more details.
16e4b17023SJohn Marino;;
17e4b17023SJohn Marino;; You should have received a copy of the GNU General Public License
18e4b17023SJohn Marino;; along with GCC; see the file COPYING3.  If not see
19e4b17023SJohn Marino;; <http://www.gnu.org/licenses/>.
20e4b17023SJohn Marino
21e4b17023SJohn Marino(define_c_enum "unspec" [
22e4b17023SJohn Marino  UNSPEC_LFENCE
23e4b17023SJohn Marino  UNSPEC_SFENCE
24e4b17023SJohn Marino  UNSPEC_MFENCE
25e4b17023SJohn Marino  UNSPEC_MOVA	; For __atomic support
26e4b17023SJohn Marino  UNSPEC_LDA
27e4b17023SJohn Marino  UNSPEC_STA
28e4b17023SJohn Marino])
29e4b17023SJohn Marino
30e4b17023SJohn Marino(define_c_enum "unspecv" [
31e4b17023SJohn Marino  UNSPECV_CMPXCHG
32e4b17023SJohn Marino  UNSPECV_XCHG
33e4b17023SJohn Marino  UNSPECV_LOCK
34e4b17023SJohn Marino])
35e4b17023SJohn Marino
36e4b17023SJohn Marino(define_expand "sse2_lfence"
37e4b17023SJohn Marino  [(set (match_dup 0)
38e4b17023SJohn Marino	(unspec:BLK [(match_dup 0)] UNSPEC_LFENCE))]
39e4b17023SJohn Marino  "TARGET_SSE2"
40e4b17023SJohn Marino{
41e4b17023SJohn Marino  operands[0] = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (Pmode));
42e4b17023SJohn Marino  MEM_VOLATILE_P (operands[0]) = 1;
43e4b17023SJohn Marino})
44e4b17023SJohn Marino
45e4b17023SJohn Marino(define_insn "*sse2_lfence"
46e4b17023SJohn Marino  [(set (match_operand:BLK 0 "" "")
47e4b17023SJohn Marino	(unspec:BLK [(match_dup 0)] UNSPEC_LFENCE))]
48e4b17023SJohn Marino  "TARGET_SSE2"
49e4b17023SJohn Marino  "lfence"
50e4b17023SJohn Marino  [(set_attr "type" "sse")
51e4b17023SJohn Marino   (set_attr "length_address" "0")
52e4b17023SJohn Marino   (set_attr "atom_sse_attr" "lfence")
53e4b17023SJohn Marino   (set_attr "memory" "unknown")])
54e4b17023SJohn Marino
55e4b17023SJohn Marino(define_expand "sse_sfence"
56e4b17023SJohn Marino  [(set (match_dup 0)
57e4b17023SJohn Marino	(unspec:BLK [(match_dup 0)] UNSPEC_SFENCE))]
58e4b17023SJohn Marino  "TARGET_SSE || TARGET_3DNOW_A"
59e4b17023SJohn Marino{
60e4b17023SJohn Marino  operands[0] = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (Pmode));
61e4b17023SJohn Marino  MEM_VOLATILE_P (operands[0]) = 1;
62e4b17023SJohn Marino})
63e4b17023SJohn Marino
64e4b17023SJohn Marino(define_insn "*sse_sfence"
65e4b17023SJohn Marino  [(set (match_operand:BLK 0 "" "")
66e4b17023SJohn Marino	(unspec:BLK [(match_dup 0)] UNSPEC_SFENCE))]
67e4b17023SJohn Marino  "TARGET_SSE || TARGET_3DNOW_A"
68e4b17023SJohn Marino  "sfence"
69e4b17023SJohn Marino  [(set_attr "type" "sse")
70e4b17023SJohn Marino   (set_attr "length_address" "0")
71e4b17023SJohn Marino   (set_attr "atom_sse_attr" "fence")
72e4b17023SJohn Marino   (set_attr "memory" "unknown")])
73e4b17023SJohn Marino
74e4b17023SJohn Marino(define_expand "sse2_mfence"
75e4b17023SJohn Marino  [(set (match_dup 0)
76e4b17023SJohn Marino	(unspec:BLK [(match_dup 0)] UNSPEC_MFENCE))]
77e4b17023SJohn Marino  "TARGET_SSE2"
78e4b17023SJohn Marino{
79e4b17023SJohn Marino  operands[0] = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (Pmode));
80e4b17023SJohn Marino  MEM_VOLATILE_P (operands[0]) = 1;
81e4b17023SJohn Marino})
82e4b17023SJohn Marino
83e4b17023SJohn Marino(define_insn "mfence_sse2"
84e4b17023SJohn Marino  [(set (match_operand:BLK 0 "" "")
85e4b17023SJohn Marino	(unspec:BLK [(match_dup 0)] UNSPEC_MFENCE))]
86e4b17023SJohn Marino  "TARGET_64BIT || TARGET_SSE2"
87e4b17023SJohn Marino  "mfence"
88e4b17023SJohn Marino  [(set_attr "type" "sse")
89e4b17023SJohn Marino   (set_attr "length_address" "0")
90e4b17023SJohn Marino   (set_attr "atom_sse_attr" "fence")
91e4b17023SJohn Marino   (set_attr "memory" "unknown")])
92e4b17023SJohn Marino
93e4b17023SJohn Marino(define_insn "mfence_nosse"
94e4b17023SJohn Marino  [(set (match_operand:BLK 0 "" "")
95e4b17023SJohn Marino	(unspec:BLK [(match_dup 0)] UNSPEC_MFENCE))
96e4b17023SJohn Marino   (clobber (reg:CC FLAGS_REG))]
97e4b17023SJohn Marino  "!(TARGET_64BIT || TARGET_SSE2)"
98e4b17023SJohn Marino  "lock{%;} or{l}\t{$0, (%%esp)|DWORD PTR [esp], 0}"
99e4b17023SJohn Marino  [(set_attr "memory" "unknown")])
100e4b17023SJohn Marino
101e4b17023SJohn Marino(define_expand "mem_thread_fence"
102e4b17023SJohn Marino  [(match_operand:SI 0 "const_int_operand" "")]		;; model
103e4b17023SJohn Marino  ""
104e4b17023SJohn Marino{
105e4b17023SJohn Marino  /* Unless this is a SEQ_CST fence, the i386 memory model is strong
106e4b17023SJohn Marino     enough not to require barriers of any kind.  */
107e4b17023SJohn Marino  if (INTVAL (operands[0]) == MEMMODEL_SEQ_CST)
108e4b17023SJohn Marino    {
109e4b17023SJohn Marino      rtx (*mfence_insn)(rtx);
110e4b17023SJohn Marino      rtx mem;
111e4b17023SJohn Marino
112e4b17023SJohn Marino      if (TARGET_64BIT || TARGET_SSE2)
113e4b17023SJohn Marino	mfence_insn = gen_mfence_sse2;
114e4b17023SJohn Marino      else
115e4b17023SJohn Marino	mfence_insn = gen_mfence_nosse;
116e4b17023SJohn Marino
117e4b17023SJohn Marino      mem = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (Pmode));
118e4b17023SJohn Marino      MEM_VOLATILE_P (mem) = 1;
119e4b17023SJohn Marino
120e4b17023SJohn Marino      emit_insn (mfence_insn (mem));
121e4b17023SJohn Marino    }
122e4b17023SJohn Marino  DONE;
123e4b17023SJohn Marino})
124e4b17023SJohn Marino
125e4b17023SJohn Marino;; ??? From volume 3 section 8.1.1 Guaranteed Atomic Operations,
126e4b17023SJohn Marino;; Only beginning at Pentium family processors do we get any guarantee of
127e4b17023SJohn Marino;; atomicity in aligned 64-bit quantities.  Beginning at P6, we get a
128e4b17023SJohn Marino;; guarantee for 64-bit accesses that do not cross a cacheline boundary.
129e4b17023SJohn Marino;;
130e4b17023SJohn Marino;; Note that the TARGET_CMPXCHG8B test below is a stand-in for "Pentium".
131e4b17023SJohn Marino;;
132e4b17023SJohn Marino;; Importantly, *no* processor makes atomicity guarantees for larger
133e4b17023SJohn Marino;; accesses.  In particular, there's no way to perform an atomic TImode
134e4b17023SJohn Marino;; move, despite the apparent applicability of MOVDQA et al.
135e4b17023SJohn Marino
136e4b17023SJohn Marino(define_mode_iterator ATOMIC
137e4b17023SJohn Marino   [QI HI SI
138e4b17023SJohn Marino    (DI "TARGET_64BIT || (TARGET_CMPXCHG8B && (TARGET_80387 || TARGET_SSE))")
139e4b17023SJohn Marino   ])
140e4b17023SJohn Marino
141e4b17023SJohn Marino(define_expand "atomic_load<mode>"
142e4b17023SJohn Marino  [(set (match_operand:ATOMIC 0 "register_operand" "")
143e4b17023SJohn Marino	(unspec:ATOMIC [(match_operand:ATOMIC 1 "memory_operand" "")
144e4b17023SJohn Marino			(match_operand:SI 2 "const_int_operand" "")]
145e4b17023SJohn Marino		       UNSPEC_MOVA))]
146e4b17023SJohn Marino  ""
147e4b17023SJohn Marino{
148e4b17023SJohn Marino  /* For DImode on 32-bit, we can use the FPU to perform the load.  */
149e4b17023SJohn Marino  if (<MODE>mode == DImode && !TARGET_64BIT)
150e4b17023SJohn Marino    emit_insn (gen_atomic_loaddi_fpu
151e4b17023SJohn Marino	       (operands[0], operands[1],
152*5ce9237cSJohn Marino	        assign_386_stack_local (DImode, SLOT_TEMP)));
153e4b17023SJohn Marino  else
154e4b17023SJohn Marino    emit_move_insn (operands[0], operands[1]);
155e4b17023SJohn Marino  DONE;
156e4b17023SJohn Marino})
157e4b17023SJohn Marino
158e4b17023SJohn Marino(define_insn_and_split "atomic_loaddi_fpu"
159e4b17023SJohn Marino  [(set (match_operand:DI 0 "nonimmediate_operand" "=x,m,?r")
160e4b17023SJohn Marino	(unspec:DI [(match_operand:DI 1 "memory_operand" "m,m,m")]
161e4b17023SJohn Marino		   UNSPEC_MOVA))
162e4b17023SJohn Marino   (clobber (match_operand:DI 2 "memory_operand" "=X,X,m"))
163e4b17023SJohn Marino   (clobber (match_scratch:DF 3 "=X,xf,xf"))]
164e4b17023SJohn Marino  "!TARGET_64BIT && (TARGET_80387 || TARGET_SSE)"
165e4b17023SJohn Marino  "#"
166e4b17023SJohn Marino  "&& reload_completed"
167e4b17023SJohn Marino  [(const_int 0)]
168e4b17023SJohn Marino{
169e4b17023SJohn Marino  rtx dst = operands[0], src = operands[1];
170e4b17023SJohn Marino  rtx mem = operands[2], tmp = operands[3];
171e4b17023SJohn Marino
172e4b17023SJohn Marino  if (SSE_REG_P (dst))
173e4b17023SJohn Marino    emit_move_insn (dst, src);
174e4b17023SJohn Marino  else
175e4b17023SJohn Marino    {
176e4b17023SJohn Marino      if (MEM_P (dst))
177e4b17023SJohn Marino	mem = dst;
178e4b17023SJohn Marino
179e4b17023SJohn Marino      if (FP_REG_P (tmp))
180e4b17023SJohn Marino        {
181e4b17023SJohn Marino	  emit_insn (gen_loaddi_via_fpu (tmp, src));
182e4b17023SJohn Marino	  emit_insn (gen_storedi_via_fpu (mem, tmp));
183e4b17023SJohn Marino	}
184e4b17023SJohn Marino      else
185e4b17023SJohn Marino	{
186e4b17023SJohn Marino	  adjust_reg_mode (tmp, DImode);
187e4b17023SJohn Marino	  emit_move_insn (tmp, src);
188e4b17023SJohn Marino	  emit_move_insn (mem, tmp);
189e4b17023SJohn Marino	}
190e4b17023SJohn Marino
191e4b17023SJohn Marino      if (mem != dst)
192e4b17023SJohn Marino	emit_move_insn (dst, mem);
193e4b17023SJohn Marino    }
194e4b17023SJohn Marino  DONE;
195e4b17023SJohn Marino})
196e4b17023SJohn Marino
197e4b17023SJohn Marino(define_expand "atomic_store<mode>"
198e4b17023SJohn Marino  [(set (match_operand:ATOMIC 0 "memory_operand" "")
199e4b17023SJohn Marino	(unspec:ATOMIC [(match_operand:ATOMIC 1 "register_operand" "")
200e4b17023SJohn Marino			(match_operand:SI 2 "const_int_operand" "")]
201e4b17023SJohn Marino		       UNSPEC_MOVA))]
202e4b17023SJohn Marino  ""
203e4b17023SJohn Marino{
204e4b17023SJohn Marino  enum memmodel model = (enum memmodel) INTVAL (operands[2]);
205e4b17023SJohn Marino
206e4b17023SJohn Marino  if (<MODE>mode == DImode && !TARGET_64BIT)
207e4b17023SJohn Marino    {
208e4b17023SJohn Marino      /* For DImode on 32-bit, we can use the FPU to perform the store.  */
209e4b17023SJohn Marino      /* Note that while we could perform a cmpxchg8b loop, that turns
210e4b17023SJohn Marino	 out to be significantly larger than this plus a barrier.  */
211e4b17023SJohn Marino      emit_insn (gen_atomic_storedi_fpu
212e4b17023SJohn Marino		 (operands[0], operands[1],
213*5ce9237cSJohn Marino	          assign_386_stack_local (DImode, SLOT_TEMP)));
214e4b17023SJohn Marino    }
215e4b17023SJohn Marino  else
216e4b17023SJohn Marino    {
217e4b17023SJohn Marino      /* For seq-cst stores, when we lack MFENCE, use XCHG.  */
218e4b17023SJohn Marino      if (model == MEMMODEL_SEQ_CST && !(TARGET_64BIT || TARGET_SSE2))
219e4b17023SJohn Marino	{
220e4b17023SJohn Marino	  emit_insn (gen_atomic_exchange<mode> (gen_reg_rtx (<MODE>mode),
221e4b17023SJohn Marino						operands[0], operands[1],
222e4b17023SJohn Marino						operands[2]));
223e4b17023SJohn Marino	  DONE;
224e4b17023SJohn Marino	}
225e4b17023SJohn Marino
226*5ce9237cSJohn Marino      /* Otherwise use a store.  */
227*5ce9237cSJohn Marino      emit_insn (gen_atomic_store<mode>_1 (operands[0], operands[1],
228*5ce9237cSJohn Marino					   operands[2]));
229e4b17023SJohn Marino    }
230e4b17023SJohn Marino  /* ... followed by an MFENCE, if required.  */
231e4b17023SJohn Marino  if (model == MEMMODEL_SEQ_CST)
232e4b17023SJohn Marino    emit_insn (gen_mem_thread_fence (operands[2]));
233e4b17023SJohn Marino  DONE;
234e4b17023SJohn Marino})
235e4b17023SJohn Marino
236*5ce9237cSJohn Marino(define_insn "atomic_store<mode>_1"
237*5ce9237cSJohn Marino  [(set (match_operand:SWI 0 "memory_operand" "=m")
238*5ce9237cSJohn Marino	(unspec:SWI [(match_operand:SWI 1 "<nonmemory_operand>" "<r><i>")
239*5ce9237cSJohn Marino		     (match_operand:SI 2 "const_int_operand")]
240*5ce9237cSJohn Marino		    UNSPEC_MOVA))]
241*5ce9237cSJohn Marino  ""
242*5ce9237cSJohn Marino  "mov{<imodesuffix>}\t{%1, %0|%0, %1}")
243*5ce9237cSJohn Marino
244e4b17023SJohn Marino(define_insn_and_split "atomic_storedi_fpu"
245e4b17023SJohn Marino  [(set (match_operand:DI 0 "memory_operand" "=m,m,m")
246e4b17023SJohn Marino	(unspec:DI [(match_operand:DI 1 "register_operand" "x,m,?r")]
247e4b17023SJohn Marino		   UNSPEC_MOVA))
248e4b17023SJohn Marino   (clobber (match_operand:DI 2 "memory_operand" "=X,X,m"))
249e4b17023SJohn Marino   (clobber (match_scratch:DF 3 "=X,xf,xf"))]
250e4b17023SJohn Marino  "!TARGET_64BIT && (TARGET_80387 || TARGET_SSE)"
251e4b17023SJohn Marino  "#"
252e4b17023SJohn Marino  "&& reload_completed"
253e4b17023SJohn Marino  [(const_int 0)]
254e4b17023SJohn Marino{
255e4b17023SJohn Marino  rtx dst = operands[0], src = operands[1];
256e4b17023SJohn Marino  rtx mem = operands[2], tmp = operands[3];
257e4b17023SJohn Marino
258e4b17023SJohn Marino  if (!SSE_REG_P (src))
259e4b17023SJohn Marino    {
260e4b17023SJohn Marino      if (REG_P (src))
261e4b17023SJohn Marino	{
262e4b17023SJohn Marino	  emit_move_insn (mem, src);
263e4b17023SJohn Marino	  src = mem;
264e4b17023SJohn Marino	}
265e4b17023SJohn Marino
266e4b17023SJohn Marino      if (FP_REG_P (tmp))
267e4b17023SJohn Marino	{
268e4b17023SJohn Marino	  emit_insn (gen_loaddi_via_fpu (tmp, src));
269e4b17023SJohn Marino	  emit_insn (gen_storedi_via_fpu (dst, tmp));
270e4b17023SJohn Marino	  DONE;
271e4b17023SJohn Marino	}
272e4b17023SJohn Marino      else
273e4b17023SJohn Marino	{
274e4b17023SJohn Marino	  adjust_reg_mode (tmp, DImode);
275e4b17023SJohn Marino	  emit_move_insn (tmp, mem);
276e4b17023SJohn Marino	  src = tmp;
277e4b17023SJohn Marino	}
278e4b17023SJohn Marino    }
279e4b17023SJohn Marino  emit_move_insn (dst, src);
280e4b17023SJohn Marino  DONE;
281e4b17023SJohn Marino})
282e4b17023SJohn Marino
283e4b17023SJohn Marino;; ??? You'd think that we'd be able to perform this via FLOAT + FIX_TRUNC
284e4b17023SJohn Marino;; operations.  But the fix_trunc patterns want way more setup than we want
285e4b17023SJohn Marino;; to provide.  Note that the scratch is DFmode instead of XFmode in order
286e4b17023SJohn Marino;; to make it easy to allocate a scratch in either SSE or FP_REGs above.
287e4b17023SJohn Marino
288e4b17023SJohn Marino(define_insn "loaddi_via_fpu"
289e4b17023SJohn Marino  [(set (match_operand:DF 0 "register_operand" "=f")
290e4b17023SJohn Marino	(unspec:DF [(match_operand:DI 1 "memory_operand" "m")] UNSPEC_LDA))]
291e4b17023SJohn Marino  "TARGET_80387"
292e4b17023SJohn Marino  "fild%Z1\t%1"
293e4b17023SJohn Marino  [(set_attr "type" "fmov")
294e4b17023SJohn Marino   (set_attr "mode" "DF")
295e4b17023SJohn Marino   (set_attr "fp_int_src" "true")])
296e4b17023SJohn Marino
297e4b17023SJohn Marino(define_insn "storedi_via_fpu"
298e4b17023SJohn Marino  [(set (match_operand:DI 0 "memory_operand" "=m")
299e4b17023SJohn Marino	(unspec:DI [(match_operand:DF 1 "register_operand" "f")] UNSPEC_STA))]
300e4b17023SJohn Marino  "TARGET_80387"
301e4b17023SJohn Marino{
302e4b17023SJohn Marino  gcc_assert (find_regno_note (insn, REG_DEAD, FIRST_STACK_REG) != NULL_RTX);
303e4b17023SJohn Marino
304e4b17023SJohn Marino  return "fistp%Z0\t%0";
305e4b17023SJohn Marino}
306e4b17023SJohn Marino  [(set_attr "type" "fmov")
307e4b17023SJohn Marino   (set_attr "mode" "DI")])
308e4b17023SJohn Marino
309e4b17023SJohn Marino(define_expand "atomic_compare_and_swap<mode>"
310e4b17023SJohn Marino  [(match_operand:QI 0 "register_operand" "")		;; bool success output
311e4b17023SJohn Marino   (match_operand:SWI124 1 "register_operand" "")	;; oldval output
312e4b17023SJohn Marino   (match_operand:SWI124 2 "memory_operand" "")		;; memory
313e4b17023SJohn Marino   (match_operand:SWI124 3 "register_operand" "")	;; expected input
314e4b17023SJohn Marino   (match_operand:SWI124 4 "register_operand" "")	;; newval input
315e4b17023SJohn Marino   (match_operand:SI 5 "const_int_operand" "")		;; is_weak
316e4b17023SJohn Marino   (match_operand:SI 6 "const_int_operand" "")		;; success model
317e4b17023SJohn Marino   (match_operand:SI 7 "const_int_operand" "")]		;; failure model
318e4b17023SJohn Marino  "TARGET_CMPXCHG"
319e4b17023SJohn Marino{
320e4b17023SJohn Marino  emit_insn
321e4b17023SJohn Marino   (gen_atomic_compare_and_swap<mode>_1
322e4b17023SJohn Marino    (operands[1], operands[2], operands[3], operands[4]));
323e4b17023SJohn Marino  ix86_expand_setcc (operands[0], EQ, gen_rtx_REG (CCZmode, FLAGS_REG),
324e4b17023SJohn Marino		     const0_rtx);
325e4b17023SJohn Marino  DONE;
326e4b17023SJohn Marino})
327e4b17023SJohn Marino
328e4b17023SJohn Marino(define_mode_iterator CASMODE
329e4b17023SJohn Marino  [(DI "TARGET_64BIT || TARGET_CMPXCHG8B")
330e4b17023SJohn Marino   (TI "TARGET_64BIT && TARGET_CMPXCHG16B")])
331e4b17023SJohn Marino(define_mode_attr CASHMODE [(DI "SI") (TI "DI")])
332e4b17023SJohn Marino
333e4b17023SJohn Marino(define_expand "atomic_compare_and_swap<mode>"
334e4b17023SJohn Marino  [(match_operand:QI 0 "register_operand" "")		;; bool success output
335e4b17023SJohn Marino   (match_operand:CASMODE 1 "register_operand" "")	;; oldval output
336e4b17023SJohn Marino   (match_operand:CASMODE 2 "memory_operand" "")	;; memory
337e4b17023SJohn Marino   (match_operand:CASMODE 3 "register_operand" "")	;; expected input
338e4b17023SJohn Marino   (match_operand:CASMODE 4 "register_operand" "")	;; newval input
339e4b17023SJohn Marino   (match_operand:SI 5 "const_int_operand" "")		;; is_weak
340e4b17023SJohn Marino   (match_operand:SI 6 "const_int_operand" "")		;; success model
341e4b17023SJohn Marino   (match_operand:SI 7 "const_int_operand" "")]		;; failure model
342e4b17023SJohn Marino  "TARGET_CMPXCHG"
343e4b17023SJohn Marino{
344e4b17023SJohn Marino  if (<MODE>mode == DImode && TARGET_64BIT)
345e4b17023SJohn Marino    {
346e4b17023SJohn Marino      emit_insn
347e4b17023SJohn Marino       (gen_atomic_compare_and_swapdi_1
348e4b17023SJohn Marino	(operands[1], operands[2], operands[3], operands[4]));
349e4b17023SJohn Marino    }
350e4b17023SJohn Marino  else
351e4b17023SJohn Marino    {
352e4b17023SJohn Marino      enum machine_mode hmode = <CASHMODE>mode;
353e4b17023SJohn Marino      rtx lo_o, lo_e, lo_n, hi_o, hi_e, hi_n, mem;
354e4b17023SJohn Marino
355e4b17023SJohn Marino      lo_o = operands[1];
356e4b17023SJohn Marino      mem  = operands[2];
357e4b17023SJohn Marino      lo_e = operands[3];
358e4b17023SJohn Marino      lo_n = operands[4];
359e4b17023SJohn Marino      hi_o = gen_highpart (hmode, lo_o);
360e4b17023SJohn Marino      hi_e = gen_highpart (hmode, lo_e);
361e4b17023SJohn Marino      hi_n = gen_highpart (hmode, lo_n);
362e4b17023SJohn Marino      lo_o = gen_lowpart (hmode, lo_o);
363e4b17023SJohn Marino      lo_e = gen_lowpart (hmode, lo_e);
364e4b17023SJohn Marino      lo_n = gen_lowpart (hmode, lo_n);
365e4b17023SJohn Marino
366e4b17023SJohn Marino      if (!cmpxchg8b_pic_memory_operand (mem, <MODE>mode))
367e4b17023SJohn Marino 	mem = replace_equiv_address (mem, force_reg (Pmode, XEXP (mem, 0)));
368e4b17023SJohn Marino
369e4b17023SJohn Marino      emit_insn
370e4b17023SJohn Marino       (gen_atomic_compare_and_swap<mode>_doubleword
371e4b17023SJohn Marino        (lo_o, hi_o, mem, lo_e, hi_e, lo_n, hi_n));
372e4b17023SJohn Marino    }
373e4b17023SJohn Marino
374e4b17023SJohn Marino  ix86_expand_setcc (operands[0], EQ, gen_rtx_REG (CCZmode, FLAGS_REG),
375e4b17023SJohn Marino		     const0_rtx);
376e4b17023SJohn Marino  DONE;
377e4b17023SJohn Marino})
378e4b17023SJohn Marino
379e4b17023SJohn Marino(define_insn "atomic_compare_and_swap<mode>_1"
380e4b17023SJohn Marino  [(set (match_operand:SWI 0 "register_operand" "=a")
381e4b17023SJohn Marino	(unspec_volatile:SWI
382e4b17023SJohn Marino	  [(match_operand:SWI 1 "memory_operand" "+m")
383e4b17023SJohn Marino	   (match_operand:SWI 2 "register_operand" "0")
384e4b17023SJohn Marino	   (match_operand:SWI 3 "register_operand" "<r>")]
385e4b17023SJohn Marino	  UNSPECV_CMPXCHG))
386e4b17023SJohn Marino   (set (match_dup 1)
387e4b17023SJohn Marino	(unspec_volatile:SWI [(const_int 0)] UNSPECV_CMPXCHG))
388e4b17023SJohn Marino   (set (reg:CCZ FLAGS_REG)
389e4b17023SJohn Marino        (unspec_volatile:CCZ [(const_int 0)] UNSPECV_CMPXCHG))]
390e4b17023SJohn Marino  "TARGET_CMPXCHG"
391e4b17023SJohn Marino  "lock{%;} cmpxchg{<imodesuffix>}\t{%3, %1|%1, %3}")
392e4b17023SJohn Marino
393e4b17023SJohn Marino;; For double-word compare and swap, we are obliged to play tricks with
394e4b17023SJohn Marino;; the input newval (op5:op6) because the Intel register numbering does
395e4b17023SJohn Marino;; not match the gcc register numbering, so the pair must be CX:BX.
396e4b17023SJohn Marino;; That said, in order to take advantage of possible lower-subreg opts,
397e4b17023SJohn Marino;; treat all of the integral operands in the same way.
398e4b17023SJohn Marino
399e4b17023SJohn Marino;; Operands 5 and 6 really need to be different registers, which in
400e4b17023SJohn Marino;; this case means op5 must not be ecx.  If op5 and op6 are the same
401e4b17023SJohn Marino;; (like when the input is -1LL) GCC might chose to allocate op5 to ecx,
402e4b17023SJohn Marino;; like op6.  This breaks, as the xchg will move the PIC register
403e4b17023SJohn Marino;; contents to %ecx then --> boom.
404e4b17023SJohn Marino
405e4b17023SJohn Marino(define_mode_attr doublemodesuffix [(SI "8") (DI "16")])
406e4b17023SJohn Marino(define_mode_attr regprefix [(SI "e") (DI "r")])
407e4b17023SJohn Marino
408e4b17023SJohn Marino(define_insn "atomic_compare_and_swap<dwi>_doubleword"
409e4b17023SJohn Marino  [(set (match_operand:DWIH 0 "register_operand" "=a,a")
410e4b17023SJohn Marino	(unspec_volatile:DWIH
411e4b17023SJohn Marino	  [(match_operand:<DWI> 2 "cmpxchg8b_pic_memory_operand" "+m,m")
412e4b17023SJohn Marino	   (match_operand:DWIH 3 "register_operand" "0,0")
413e4b17023SJohn Marino	   (match_operand:DWIH 4 "register_operand" "1,1")
414e4b17023SJohn Marino	   (match_operand:DWIH 5 "register_operand" "b,!*r")
415e4b17023SJohn Marino	   (match_operand:DWIH 6 "register_operand" "c,c")]
416e4b17023SJohn Marino	  UNSPECV_CMPXCHG))
417e4b17023SJohn Marino   (set (match_operand:DWIH 1 "register_operand" "=d,d")
418e4b17023SJohn Marino	(unspec_volatile:DWIH [(const_int 0)] UNSPECV_CMPXCHG))
419e4b17023SJohn Marino   (set (match_dup 2)
420e4b17023SJohn Marino	(unspec_volatile:<DWI> [(const_int 0)] UNSPECV_CMPXCHG))
421e4b17023SJohn Marino   (set (reg:CCZ FLAGS_REG)
422e4b17023SJohn Marino        (unspec_volatile:CCZ [(const_int 0)] UNSPECV_CMPXCHG))
423e4b17023SJohn Marino   (clobber (match_scratch:DWIH 7 "=X,&5"))]
424e4b17023SJohn Marino  "TARGET_CMPXCHG<doublemodesuffix>B"
425e4b17023SJohn Marino{
426e4b17023SJohn Marino  bool swap = REGNO (operands[5]) != BX_REG;
427e4b17023SJohn Marino
428e4b17023SJohn Marino  if (swap)
429e4b17023SJohn Marino    output_asm_insn ("xchg{<imodesuffix>}\t%%<regprefix>bx, %5", operands);
430e4b17023SJohn Marino
431e4b17023SJohn Marino  output_asm_insn ("lock{%;} cmpxchg<doublemodesuffix>b\t%2", operands);
432e4b17023SJohn Marino
433e4b17023SJohn Marino  if (swap)
434e4b17023SJohn Marino    output_asm_insn ("xchg{<imodesuffix>}\t%%<regprefix>bx, %5", operands);
435e4b17023SJohn Marino
436e4b17023SJohn Marino  return "";
437e4b17023SJohn Marino})
438e4b17023SJohn Marino
439e4b17023SJohn Marino;; For operand 2 nonmemory_operand predicate is used instead of
440e4b17023SJohn Marino;; register_operand to allow combiner to better optimize atomic
441e4b17023SJohn Marino;; additions of constants.
442e4b17023SJohn Marino(define_insn "atomic_fetch_add<mode>"
443e4b17023SJohn Marino  [(set (match_operand:SWI 0 "register_operand" "=<r>")
444e4b17023SJohn Marino	(unspec_volatile:SWI
445e4b17023SJohn Marino	  [(match_operand:SWI 1 "memory_operand" "+m")
446e4b17023SJohn Marino	   (match_operand:SI 3 "const_int_operand" "")]		;; model
447e4b17023SJohn Marino	  UNSPECV_XCHG))
448e4b17023SJohn Marino   (set (match_dup 1)
449e4b17023SJohn Marino	(plus:SWI (match_dup 1)
450e4b17023SJohn Marino		  (match_operand:SWI 2 "nonmemory_operand" "0")))
451e4b17023SJohn Marino   (clobber (reg:CC FLAGS_REG))]
452e4b17023SJohn Marino  "TARGET_XADD"
453e4b17023SJohn Marino  "lock{%;} xadd{<imodesuffix>}\t{%0, %1|%1, %0}")
454e4b17023SJohn Marino
455e4b17023SJohn Marino;; This peephole2 and following insn optimize
456e4b17023SJohn Marino;; __sync_fetch_and_add (x, -N) == N into just lock {add,sub,inc,dec}
457e4b17023SJohn Marino;; followed by testing of flags instead of lock xadd and comparisons.
458e4b17023SJohn Marino(define_peephole2
459e4b17023SJohn Marino  [(set (match_operand:SWI 0 "register_operand" "")
460e4b17023SJohn Marino	(match_operand:SWI 2 "const_int_operand" ""))
461e4b17023SJohn Marino   (parallel [(set (match_dup 0)
462e4b17023SJohn Marino		   (unspec_volatile:SWI
463e4b17023SJohn Marino		     [(match_operand:SWI 1 "memory_operand" "")
464e4b17023SJohn Marino		      (match_operand:SI 4 "const_int_operand" "")]
465e4b17023SJohn Marino		     UNSPECV_XCHG))
466e4b17023SJohn Marino	      (set (match_dup 1)
467e4b17023SJohn Marino		   (plus:SWI (match_dup 1)
468e4b17023SJohn Marino			     (match_dup 0)))
469e4b17023SJohn Marino	      (clobber (reg:CC FLAGS_REG))])
470e4b17023SJohn Marino   (set (reg:CCZ FLAGS_REG)
471e4b17023SJohn Marino	(compare:CCZ (match_dup 0)
472e4b17023SJohn Marino		     (match_operand:SWI 3 "const_int_operand" "")))]
473e4b17023SJohn Marino  "peep2_reg_dead_p (3, operands[0])
474e4b17023SJohn Marino   && (unsigned HOST_WIDE_INT) INTVAL (operands[2])
475e4b17023SJohn Marino      == -(unsigned HOST_WIDE_INT) INTVAL (operands[3])
476e4b17023SJohn Marino   && !reg_overlap_mentioned_p (operands[0], operands[1])"
477e4b17023SJohn Marino  [(parallel [(set (reg:CCZ FLAGS_REG)
478e4b17023SJohn Marino		   (compare:CCZ
479e4b17023SJohn Marino		     (unspec_volatile:SWI [(match_dup 1) (match_dup 4)]
480e4b17023SJohn Marino					  UNSPECV_XCHG)
481e4b17023SJohn Marino		     (match_dup 3)))
482e4b17023SJohn Marino	      (set (match_dup 1)
483e4b17023SJohn Marino		   (plus:SWI (match_dup 1)
484e4b17023SJohn Marino			     (match_dup 2)))])])
485e4b17023SJohn Marino
486e4b17023SJohn Marino(define_insn "*atomic_fetch_add_cmp<mode>"
487e4b17023SJohn Marino  [(set (reg:CCZ FLAGS_REG)
488e4b17023SJohn Marino	(compare:CCZ (unspec_volatile:SWI
489e4b17023SJohn Marino		       [(match_operand:SWI 0 "memory_operand" "+m")
490e4b17023SJohn Marino		        (match_operand:SI 3 "const_int_operand" "")]
491e4b17023SJohn Marino		       UNSPECV_XCHG)
492e4b17023SJohn Marino		     (match_operand:SWI 2 "const_int_operand" "i")))
493e4b17023SJohn Marino   (set (match_dup 0)
494e4b17023SJohn Marino	(plus:SWI (match_dup 0)
495e4b17023SJohn Marino		  (match_operand:SWI 1 "const_int_operand" "i")))]
496e4b17023SJohn Marino  "(unsigned HOST_WIDE_INT) INTVAL (operands[1])
497e4b17023SJohn Marino   == -(unsigned HOST_WIDE_INT) INTVAL (operands[2])"
498e4b17023SJohn Marino{
499e4b17023SJohn Marino  if (TARGET_USE_INCDEC)
500e4b17023SJohn Marino    {
501e4b17023SJohn Marino      if (operands[1] == const1_rtx)
502e4b17023SJohn Marino	return "lock{%;} inc{<imodesuffix>}\t%0";
503e4b17023SJohn Marino      if (operands[1] == constm1_rtx)
504e4b17023SJohn Marino	return "lock{%;} dec{<imodesuffix>}\t%0";
505e4b17023SJohn Marino    }
506e4b17023SJohn Marino
507e4b17023SJohn Marino  if (x86_maybe_negate_const_int (&operands[1], <MODE>mode))
508e4b17023SJohn Marino    return "lock{%;} sub{<imodesuffix>}\t{%1, %0|%0, %1}";
509e4b17023SJohn Marino
510e4b17023SJohn Marino  return "lock{%;} add{<imodesuffix>}\t{%1, %0|%0, %1}";
511e4b17023SJohn Marino})
512e4b17023SJohn Marino
513e4b17023SJohn Marino;; Recall that xchg implicitly sets LOCK#, so adding it again wastes space.
514e4b17023SJohn Marino;; In addition, it is always a full barrier, so we can ignore the memory model.
515e4b17023SJohn Marino(define_insn "atomic_exchange<mode>"
516e4b17023SJohn Marino  [(set (match_operand:SWI 0 "register_operand" "=<r>")		;; output
517e4b17023SJohn Marino	(unspec_volatile:SWI
518e4b17023SJohn Marino	  [(match_operand:SWI 1 "memory_operand" "+m")		;; memory
519e4b17023SJohn Marino	   (match_operand:SI 3 "const_int_operand" "")]		;; model
520e4b17023SJohn Marino	  UNSPECV_XCHG))
521e4b17023SJohn Marino   (set (match_dup 1)
522e4b17023SJohn Marino	(match_operand:SWI 2 "register_operand" "0"))]		;; input
523e4b17023SJohn Marino  ""
524e4b17023SJohn Marino  "xchg{<imodesuffix>}\t{%1, %0|%0, %1}")
525e4b17023SJohn Marino
526e4b17023SJohn Marino(define_insn "atomic_add<mode>"
527e4b17023SJohn Marino  [(set (match_operand:SWI 0 "memory_operand" "+m")
528e4b17023SJohn Marino	(unspec_volatile:SWI
529e4b17023SJohn Marino	  [(plus:SWI (match_dup 0)
530e4b17023SJohn Marino		     (match_operand:SWI 1 "nonmemory_operand" "<r><i>"))
531e4b17023SJohn Marino	   (match_operand:SI 2 "const_int_operand" "")]		;; model
532e4b17023SJohn Marino	  UNSPECV_LOCK))
533e4b17023SJohn Marino   (clobber (reg:CC FLAGS_REG))]
534e4b17023SJohn Marino  ""
535e4b17023SJohn Marino{
536e4b17023SJohn Marino  if (TARGET_USE_INCDEC)
537e4b17023SJohn Marino    {
538e4b17023SJohn Marino      if (operands[1] == const1_rtx)
539e4b17023SJohn Marino	return "lock{%;} inc{<imodesuffix>}\t%0";
540e4b17023SJohn Marino      if (operands[1] == constm1_rtx)
541e4b17023SJohn Marino	return "lock{%;} dec{<imodesuffix>}\t%0";
542e4b17023SJohn Marino    }
543e4b17023SJohn Marino
544e4b17023SJohn Marino  if (x86_maybe_negate_const_int (&operands[1], <MODE>mode))
545e4b17023SJohn Marino    return "lock{%;} sub{<imodesuffix>}\t{%1, %0|%0, %1}";
546e4b17023SJohn Marino
547e4b17023SJohn Marino  return "lock{%;} add{<imodesuffix>}\t{%1, %0|%0, %1}";
548e4b17023SJohn Marino})
549e4b17023SJohn Marino
550e4b17023SJohn Marino(define_insn "atomic_sub<mode>"
551e4b17023SJohn Marino  [(set (match_operand:SWI 0 "memory_operand" "+m")
552e4b17023SJohn Marino	(unspec_volatile:SWI
553e4b17023SJohn Marino	  [(minus:SWI (match_dup 0)
554e4b17023SJohn Marino		      (match_operand:SWI 1 "nonmemory_operand" "<r><i>"))
555e4b17023SJohn Marino	   (match_operand:SI 2 "const_int_operand" "")]		;; model
556e4b17023SJohn Marino	  UNSPECV_LOCK))
557e4b17023SJohn Marino   (clobber (reg:CC FLAGS_REG))]
558e4b17023SJohn Marino  ""
559e4b17023SJohn Marino{
560e4b17023SJohn Marino  if (TARGET_USE_INCDEC)
561e4b17023SJohn Marino    {
562e4b17023SJohn Marino      if (operands[1] == const1_rtx)
563e4b17023SJohn Marino	return "lock{%;} dec{<imodesuffix>}\t%0";
564e4b17023SJohn Marino      if (operands[1] == constm1_rtx)
565e4b17023SJohn Marino	return "lock{%;} inc{<imodesuffix>}\t%0";
566e4b17023SJohn Marino    }
567e4b17023SJohn Marino
568e4b17023SJohn Marino  if (x86_maybe_negate_const_int (&operands[1], <MODE>mode))
569e4b17023SJohn Marino    return "lock{%;} add{<imodesuffix>}\t{%1, %0|%0, %1}";
570e4b17023SJohn Marino
571e4b17023SJohn Marino  return "lock{%;} sub{<imodesuffix>}\t{%1, %0|%0, %1}";
572e4b17023SJohn Marino})
573e4b17023SJohn Marino
574e4b17023SJohn Marino(define_insn "atomic_<logic><mode>"
575e4b17023SJohn Marino  [(set (match_operand:SWI 0 "memory_operand" "+m")
576e4b17023SJohn Marino	(unspec_volatile:SWI
577e4b17023SJohn Marino	  [(any_logic:SWI (match_dup 0)
578e4b17023SJohn Marino			  (match_operand:SWI 1 "nonmemory_operand" "<r><i>"))
579e4b17023SJohn Marino	   (match_operand:SI 2 "const_int_operand" "")]		;; model
580e4b17023SJohn Marino	  UNSPECV_LOCK))
581e4b17023SJohn Marino   (clobber (reg:CC FLAGS_REG))]
582e4b17023SJohn Marino  ""
583e4b17023SJohn Marino  "lock{%;} <logic>{<imodesuffix>}\t{%1, %0|%0, %1}")
584