1*5d9d9091SRichard Lowe/* 2*5d9d9091SRichard Lowe * CDDL HEADER START 3*5d9d9091SRichard Lowe * 4*5d9d9091SRichard Lowe * The contents of this file are subject to the terms of the 5*5d9d9091SRichard Lowe * Common Development and Distribution License (the "License"). 6*5d9d9091SRichard Lowe * You may not use this file except in compliance with the License. 7*5d9d9091SRichard Lowe * 8*5d9d9091SRichard Lowe * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9*5d9d9091SRichard Lowe * or http://www.opensolaris.org/os/licensing. 10*5d9d9091SRichard Lowe * See the License for the specific language governing permissions 11*5d9d9091SRichard Lowe * and limitations under the License. 12*5d9d9091SRichard Lowe * 13*5d9d9091SRichard Lowe * When distributing Covered Code, include this CDDL HEADER in each 14*5d9d9091SRichard Lowe * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15*5d9d9091SRichard Lowe * If applicable, add the following below this CDDL HEADER, with the 16*5d9d9091SRichard Lowe * fields enclosed by brackets "[]" replaced with your own identifying 17*5d9d9091SRichard Lowe * information: Portions Copyright [yyyy] [name of copyright owner] 18*5d9d9091SRichard Lowe * 19*5d9d9091SRichard Lowe * CDDL HEADER END 20*5d9d9091SRichard Lowe */ 21*5d9d9091SRichard Lowe 22*5d9d9091SRichard Lowe/* 23*5d9d9091SRichard Lowe * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved. 24*5d9d9091SRichard Lowe */ 25*5d9d9091SRichard Lowe 26*5d9d9091SRichard Lowe .file "atomic.s" 27*5d9d9091SRichard Lowe 28*5d9d9091SRichard Lowe#include <sys/asm_linkage.h> 29*5d9d9091SRichard Lowe 30*5d9d9091SRichard Lowe#if defined(_KERNEL) 31*5d9d9091SRichard Lowe /* 32*5d9d9091SRichard Lowe * Legacy kernel interfaces; they will go away the moment our closed 33*5d9d9091SRichard Lowe * bins no longer require them. 34*5d9d9091SRichard Lowe */ 35*5d9d9091SRichard Lowe ANSI_PRAGMA_WEAK2(cas8,atomic_cas_8,function) 36*5d9d9091SRichard Lowe ANSI_PRAGMA_WEAK2(cas32,atomic_cas_32,function) 37*5d9d9091SRichard Lowe ANSI_PRAGMA_WEAK2(cas64,atomic_cas_64,function) 38*5d9d9091SRichard Lowe ANSI_PRAGMA_WEAK2(caslong,atomic_cas_ulong,function) 39*5d9d9091SRichard Lowe ANSI_PRAGMA_WEAK2(casptr,atomic_cas_ptr,function) 40*5d9d9091SRichard Lowe ANSI_PRAGMA_WEAK2(atomic_and_long,atomic_and_ulong,function) 41*5d9d9091SRichard Lowe ANSI_PRAGMA_WEAK2(atomic_or_long,atomic_or_ulong,function) 42*5d9d9091SRichard Lowe#endif 43*5d9d9091SRichard Lowe 44*5d9d9091SRichard Lowe ENTRY(atomic_inc_8) 45*5d9d9091SRichard Lowe ALTENTRY(atomic_inc_uchar) 46*5d9d9091SRichard Lowe lock 47*5d9d9091SRichard Lowe incb (%rdi) 48*5d9d9091SRichard Lowe ret 49*5d9d9091SRichard Lowe SET_SIZE(atomic_inc_uchar) 50*5d9d9091SRichard Lowe SET_SIZE(atomic_inc_8) 51*5d9d9091SRichard Lowe 52*5d9d9091SRichard Lowe ENTRY(atomic_inc_16) 53*5d9d9091SRichard Lowe ALTENTRY(atomic_inc_ushort) 54*5d9d9091SRichard Lowe lock 55*5d9d9091SRichard Lowe incw (%rdi) 56*5d9d9091SRichard Lowe ret 57*5d9d9091SRichard Lowe SET_SIZE(atomic_inc_ushort) 58*5d9d9091SRichard Lowe SET_SIZE(atomic_inc_16) 59*5d9d9091SRichard Lowe 60*5d9d9091SRichard Lowe ENTRY(atomic_inc_32) 61*5d9d9091SRichard Lowe ALTENTRY(atomic_inc_uint) 62*5d9d9091SRichard Lowe lock 63*5d9d9091SRichard Lowe incl (%rdi) 64*5d9d9091SRichard Lowe ret 65*5d9d9091SRichard Lowe SET_SIZE(atomic_inc_uint) 66*5d9d9091SRichard Lowe SET_SIZE(atomic_inc_32) 67*5d9d9091SRichard Lowe 68*5d9d9091SRichard Lowe ENTRY(atomic_inc_64) 69*5d9d9091SRichard Lowe ALTENTRY(atomic_inc_ulong) 70*5d9d9091SRichard Lowe lock 71*5d9d9091SRichard Lowe incq (%rdi) 72*5d9d9091SRichard Lowe ret 73*5d9d9091SRichard Lowe SET_SIZE(atomic_inc_ulong) 74*5d9d9091SRichard Lowe SET_SIZE(atomic_inc_64) 75*5d9d9091SRichard Lowe 76*5d9d9091SRichard Lowe ENTRY(atomic_inc_8_nv) 77*5d9d9091SRichard Lowe ALTENTRY(atomic_inc_uchar_nv) 78*5d9d9091SRichard Lowe xorl %eax, %eax / clear upper bits of %eax return register 79*5d9d9091SRichard Lowe incb %al / %al = 1 80*5d9d9091SRichard Lowe lock 81*5d9d9091SRichard Lowe xaddb %al, (%rdi) / %al = old value, (%rdi) = new value 82*5d9d9091SRichard Lowe incb %al / return new value 83*5d9d9091SRichard Lowe ret 84*5d9d9091SRichard Lowe SET_SIZE(atomic_inc_uchar_nv) 85*5d9d9091SRichard Lowe SET_SIZE(atomic_inc_8_nv) 86*5d9d9091SRichard Lowe 87*5d9d9091SRichard Lowe ENTRY(atomic_inc_16_nv) 88*5d9d9091SRichard Lowe ALTENTRY(atomic_inc_ushort_nv) 89*5d9d9091SRichard Lowe xorl %eax, %eax / clear upper bits of %eax return register 90*5d9d9091SRichard Lowe incw %ax / %ax = 1 91*5d9d9091SRichard Lowe lock 92*5d9d9091SRichard Lowe xaddw %ax, (%rdi) / %ax = old value, (%rdi) = new value 93*5d9d9091SRichard Lowe incw %ax / return new value 94*5d9d9091SRichard Lowe ret 95*5d9d9091SRichard Lowe SET_SIZE(atomic_inc_ushort_nv) 96*5d9d9091SRichard Lowe SET_SIZE(atomic_inc_16_nv) 97*5d9d9091SRichard Lowe 98*5d9d9091SRichard Lowe ENTRY(atomic_inc_32_nv) 99*5d9d9091SRichard Lowe ALTENTRY(atomic_inc_uint_nv) 100*5d9d9091SRichard Lowe xorl %eax, %eax / %eax = 0 101*5d9d9091SRichard Lowe incl %eax / %eax = 1 102*5d9d9091SRichard Lowe lock 103*5d9d9091SRichard Lowe xaddl %eax, (%rdi) / %eax = old value, (%rdi) = new value 104*5d9d9091SRichard Lowe incl %eax / return new value 105*5d9d9091SRichard Lowe ret 106*5d9d9091SRichard Lowe SET_SIZE(atomic_inc_uint_nv) 107*5d9d9091SRichard Lowe SET_SIZE(atomic_inc_32_nv) 108*5d9d9091SRichard Lowe 109*5d9d9091SRichard Lowe ENTRY(atomic_inc_64_nv) 110*5d9d9091SRichard Lowe ALTENTRY(atomic_inc_ulong_nv) 111*5d9d9091SRichard Lowe xorq %rax, %rax / %rax = 0 112*5d9d9091SRichard Lowe incq %rax / %rax = 1 113*5d9d9091SRichard Lowe lock 114*5d9d9091SRichard Lowe xaddq %rax, (%rdi) / %rax = old value, (%rdi) = new value 115*5d9d9091SRichard Lowe incq %rax / return new value 116*5d9d9091SRichard Lowe ret 117*5d9d9091SRichard Lowe SET_SIZE(atomic_inc_ulong_nv) 118*5d9d9091SRichard Lowe SET_SIZE(atomic_inc_64_nv) 119*5d9d9091SRichard Lowe 120*5d9d9091SRichard Lowe ENTRY(atomic_dec_8) 121*5d9d9091SRichard Lowe ALTENTRY(atomic_dec_uchar) 122*5d9d9091SRichard Lowe lock 123*5d9d9091SRichard Lowe decb (%rdi) 124*5d9d9091SRichard Lowe ret 125*5d9d9091SRichard Lowe SET_SIZE(atomic_dec_uchar) 126*5d9d9091SRichard Lowe SET_SIZE(atomic_dec_8) 127*5d9d9091SRichard Lowe 128*5d9d9091SRichard Lowe ENTRY(atomic_dec_16) 129*5d9d9091SRichard Lowe ALTENTRY(atomic_dec_ushort) 130*5d9d9091SRichard Lowe lock 131*5d9d9091SRichard Lowe decw (%rdi) 132*5d9d9091SRichard Lowe ret 133*5d9d9091SRichard Lowe SET_SIZE(atomic_dec_ushort) 134*5d9d9091SRichard Lowe SET_SIZE(atomic_dec_16) 135*5d9d9091SRichard Lowe 136*5d9d9091SRichard Lowe ENTRY(atomic_dec_32) 137*5d9d9091SRichard Lowe ALTENTRY(atomic_dec_uint) 138*5d9d9091SRichard Lowe lock 139*5d9d9091SRichard Lowe decl (%rdi) 140*5d9d9091SRichard Lowe ret 141*5d9d9091SRichard Lowe SET_SIZE(atomic_dec_uint) 142*5d9d9091SRichard Lowe SET_SIZE(atomic_dec_32) 143*5d9d9091SRichard Lowe 144*5d9d9091SRichard Lowe ENTRY(atomic_dec_64) 145*5d9d9091SRichard Lowe ALTENTRY(atomic_dec_ulong) 146*5d9d9091SRichard Lowe lock 147*5d9d9091SRichard Lowe decq (%rdi) 148*5d9d9091SRichard Lowe ret 149*5d9d9091SRichard Lowe SET_SIZE(atomic_dec_ulong) 150*5d9d9091SRichard Lowe SET_SIZE(atomic_dec_64) 151*5d9d9091SRichard Lowe 152*5d9d9091SRichard Lowe ENTRY(atomic_dec_8_nv) 153*5d9d9091SRichard Lowe ALTENTRY(atomic_dec_uchar_nv) 154*5d9d9091SRichard Lowe xorl %eax, %eax / clear upper bits of %eax return register 155*5d9d9091SRichard Lowe decb %al / %al = -1 156*5d9d9091SRichard Lowe lock 157*5d9d9091SRichard Lowe xaddb %al, (%rdi) / %al = old value, (%rdi) = new value 158*5d9d9091SRichard Lowe decb %al / return new value 159*5d9d9091SRichard Lowe ret 160*5d9d9091SRichard Lowe SET_SIZE(atomic_dec_uchar_nv) 161*5d9d9091SRichard Lowe SET_SIZE(atomic_dec_8_nv) 162*5d9d9091SRichard Lowe 163*5d9d9091SRichard Lowe ENTRY(atomic_dec_16_nv) 164*5d9d9091SRichard Lowe ALTENTRY(atomic_dec_ushort_nv) 165*5d9d9091SRichard Lowe xorl %eax, %eax / clear upper bits of %eax return register 166*5d9d9091SRichard Lowe decw %ax / %ax = -1 167*5d9d9091SRichard Lowe lock 168*5d9d9091SRichard Lowe xaddw %ax, (%rdi) / %ax = old value, (%rdi) = new value 169*5d9d9091SRichard Lowe decw %ax / return new value 170*5d9d9091SRichard Lowe ret 171*5d9d9091SRichard Lowe SET_SIZE(atomic_dec_ushort_nv) 172*5d9d9091SRichard Lowe SET_SIZE(atomic_dec_16_nv) 173*5d9d9091SRichard Lowe 174*5d9d9091SRichard Lowe ENTRY(atomic_dec_32_nv) 175*5d9d9091SRichard Lowe ALTENTRY(atomic_dec_uint_nv) 176*5d9d9091SRichard Lowe xorl %eax, %eax / %eax = 0 177*5d9d9091SRichard Lowe decl %eax / %eax = -1 178*5d9d9091SRichard Lowe lock 179*5d9d9091SRichard Lowe xaddl %eax, (%rdi) / %eax = old value, (%rdi) = new value 180*5d9d9091SRichard Lowe decl %eax / return new value 181*5d9d9091SRichard Lowe ret 182*5d9d9091SRichard Lowe SET_SIZE(atomic_dec_uint_nv) 183*5d9d9091SRichard Lowe SET_SIZE(atomic_dec_32_nv) 184*5d9d9091SRichard Lowe 185*5d9d9091SRichard Lowe ENTRY(atomic_dec_64_nv) 186*5d9d9091SRichard Lowe ALTENTRY(atomic_dec_ulong_nv) 187*5d9d9091SRichard Lowe xorq %rax, %rax / %rax = 0 188*5d9d9091SRichard Lowe decq %rax / %rax = -1 189*5d9d9091SRichard Lowe lock 190*5d9d9091SRichard Lowe xaddq %rax, (%rdi) / %rax = old value, (%rdi) = new value 191*5d9d9091SRichard Lowe decq %rax / return new value 192*5d9d9091SRichard Lowe ret 193*5d9d9091SRichard Lowe SET_SIZE(atomic_dec_ulong_nv) 194*5d9d9091SRichard Lowe SET_SIZE(atomic_dec_64_nv) 195*5d9d9091SRichard Lowe 196*5d9d9091SRichard Lowe ENTRY(atomic_add_8) 197*5d9d9091SRichard Lowe ALTENTRY(atomic_add_char) 198*5d9d9091SRichard Lowe lock 199*5d9d9091SRichard Lowe addb %sil, (%rdi) 200*5d9d9091SRichard Lowe ret 201*5d9d9091SRichard Lowe SET_SIZE(atomic_add_char) 202*5d9d9091SRichard Lowe SET_SIZE(atomic_add_8) 203*5d9d9091SRichard Lowe 204*5d9d9091SRichard Lowe ENTRY(atomic_add_16) 205*5d9d9091SRichard Lowe ALTENTRY(atomic_add_short) 206*5d9d9091SRichard Lowe lock 207*5d9d9091SRichard Lowe addw %si, (%rdi) 208*5d9d9091SRichard Lowe ret 209*5d9d9091SRichard Lowe SET_SIZE(atomic_add_short) 210*5d9d9091SRichard Lowe SET_SIZE(atomic_add_16) 211*5d9d9091SRichard Lowe 212*5d9d9091SRichard Lowe ENTRY(atomic_add_32) 213*5d9d9091SRichard Lowe ALTENTRY(atomic_add_int) 214*5d9d9091SRichard Lowe lock 215*5d9d9091SRichard Lowe addl %esi, (%rdi) 216*5d9d9091SRichard Lowe ret 217*5d9d9091SRichard Lowe SET_SIZE(atomic_add_int) 218*5d9d9091SRichard Lowe SET_SIZE(atomic_add_32) 219*5d9d9091SRichard Lowe 220*5d9d9091SRichard Lowe ENTRY(atomic_add_64) 221*5d9d9091SRichard Lowe ALTENTRY(atomic_add_ptr) 222*5d9d9091SRichard Lowe ALTENTRY(atomic_add_long) 223*5d9d9091SRichard Lowe lock 224*5d9d9091SRichard Lowe addq %rsi, (%rdi) 225*5d9d9091SRichard Lowe ret 226*5d9d9091SRichard Lowe SET_SIZE(atomic_add_long) 227*5d9d9091SRichard Lowe SET_SIZE(atomic_add_ptr) 228*5d9d9091SRichard Lowe SET_SIZE(atomic_add_64) 229*5d9d9091SRichard Lowe 230*5d9d9091SRichard Lowe ENTRY(atomic_or_8) 231*5d9d9091SRichard Lowe ALTENTRY(atomic_or_uchar) 232*5d9d9091SRichard Lowe lock 233*5d9d9091SRichard Lowe orb %sil, (%rdi) 234*5d9d9091SRichard Lowe ret 235*5d9d9091SRichard Lowe SET_SIZE(atomic_or_uchar) 236*5d9d9091SRichard Lowe SET_SIZE(atomic_or_8) 237*5d9d9091SRichard Lowe 238*5d9d9091SRichard Lowe ENTRY(atomic_or_16) 239*5d9d9091SRichard Lowe ALTENTRY(atomic_or_ushort) 240*5d9d9091SRichard Lowe lock 241*5d9d9091SRichard Lowe orw %si, (%rdi) 242*5d9d9091SRichard Lowe ret 243*5d9d9091SRichard Lowe SET_SIZE(atomic_or_ushort) 244*5d9d9091SRichard Lowe SET_SIZE(atomic_or_16) 245*5d9d9091SRichard Lowe 246*5d9d9091SRichard Lowe ENTRY(atomic_or_32) 247*5d9d9091SRichard Lowe ALTENTRY(atomic_or_uint) 248*5d9d9091SRichard Lowe lock 249*5d9d9091SRichard Lowe orl %esi, (%rdi) 250*5d9d9091SRichard Lowe ret 251*5d9d9091SRichard Lowe SET_SIZE(atomic_or_uint) 252*5d9d9091SRichard Lowe SET_SIZE(atomic_or_32) 253*5d9d9091SRichard Lowe 254*5d9d9091SRichard Lowe ENTRY(atomic_or_64) 255*5d9d9091SRichard Lowe ALTENTRY(atomic_or_ulong) 256*5d9d9091SRichard Lowe lock 257*5d9d9091SRichard Lowe orq %rsi, (%rdi) 258*5d9d9091SRichard Lowe ret 259*5d9d9091SRichard Lowe SET_SIZE(atomic_or_ulong) 260*5d9d9091SRichard Lowe SET_SIZE(atomic_or_64) 261*5d9d9091SRichard Lowe 262*5d9d9091SRichard Lowe ENTRY(atomic_and_8) 263*5d9d9091SRichard Lowe ALTENTRY(atomic_and_uchar) 264*5d9d9091SRichard Lowe lock 265*5d9d9091SRichard Lowe andb %sil, (%rdi) 266*5d9d9091SRichard Lowe ret 267*5d9d9091SRichard Lowe SET_SIZE(atomic_and_uchar) 268*5d9d9091SRichard Lowe SET_SIZE(atomic_and_8) 269*5d9d9091SRichard Lowe 270*5d9d9091SRichard Lowe ENTRY(atomic_and_16) 271*5d9d9091SRichard Lowe ALTENTRY(atomic_and_ushort) 272*5d9d9091SRichard Lowe lock 273*5d9d9091SRichard Lowe andw %si, (%rdi) 274*5d9d9091SRichard Lowe ret 275*5d9d9091SRichard Lowe SET_SIZE(atomic_and_ushort) 276*5d9d9091SRichard Lowe SET_SIZE(atomic_and_16) 277*5d9d9091SRichard Lowe 278*5d9d9091SRichard Lowe ENTRY(atomic_and_32) 279*5d9d9091SRichard Lowe ALTENTRY(atomic_and_uint) 280*5d9d9091SRichard Lowe lock 281*5d9d9091SRichard Lowe andl %esi, (%rdi) 282*5d9d9091SRichard Lowe ret 283*5d9d9091SRichard Lowe SET_SIZE(atomic_and_uint) 284*5d9d9091SRichard Lowe SET_SIZE(atomic_and_32) 285*5d9d9091SRichard Lowe 286*5d9d9091SRichard Lowe ENTRY(atomic_and_64) 287*5d9d9091SRichard Lowe ALTENTRY(atomic_and_ulong) 288*5d9d9091SRichard Lowe lock 289*5d9d9091SRichard Lowe andq %rsi, (%rdi) 290*5d9d9091SRichard Lowe ret 291*5d9d9091SRichard Lowe SET_SIZE(atomic_and_ulong) 292*5d9d9091SRichard Lowe SET_SIZE(atomic_and_64) 293*5d9d9091SRichard Lowe 294*5d9d9091SRichard Lowe ENTRY(atomic_add_8_nv) 295*5d9d9091SRichard Lowe ALTENTRY(atomic_add_char_nv) 296*5d9d9091SRichard Lowe movzbl %sil, %eax / %al = delta addend, clear upper bits 297*5d9d9091SRichard Lowe lock 298*5d9d9091SRichard Lowe xaddb %sil, (%rdi) / %sil = old value, (%rdi) = sum 299*5d9d9091SRichard Lowe addb %sil, %al / new value = original value + delta 300*5d9d9091SRichard Lowe ret 301*5d9d9091SRichard Lowe SET_SIZE(atomic_add_char_nv) 302*5d9d9091SRichard Lowe SET_SIZE(atomic_add_8_nv) 303*5d9d9091SRichard Lowe 304*5d9d9091SRichard Lowe ENTRY(atomic_add_16_nv) 305*5d9d9091SRichard Lowe ALTENTRY(atomic_add_short_nv) 306*5d9d9091SRichard Lowe movzwl %si, %eax / %ax = delta addend, clean upper bits 307*5d9d9091SRichard Lowe lock 308*5d9d9091SRichard Lowe xaddw %si, (%rdi) / %si = old value, (%rdi) = sum 309*5d9d9091SRichard Lowe addw %si, %ax / new value = original value + delta 310*5d9d9091SRichard Lowe ret 311*5d9d9091SRichard Lowe SET_SIZE(atomic_add_short_nv) 312*5d9d9091SRichard Lowe SET_SIZE(atomic_add_16_nv) 313*5d9d9091SRichard Lowe 314*5d9d9091SRichard Lowe ENTRY(atomic_add_32_nv) 315*5d9d9091SRichard Lowe ALTENTRY(atomic_add_int_nv) 316*5d9d9091SRichard Lowe mov %esi, %eax / %eax = delta addend 317*5d9d9091SRichard Lowe lock 318*5d9d9091SRichard Lowe xaddl %esi, (%rdi) / %esi = old value, (%rdi) = sum 319*5d9d9091SRichard Lowe add %esi, %eax / new value = original value + delta 320*5d9d9091SRichard Lowe ret 321*5d9d9091SRichard Lowe SET_SIZE(atomic_add_int_nv) 322*5d9d9091SRichard Lowe SET_SIZE(atomic_add_32_nv) 323*5d9d9091SRichard Lowe 324*5d9d9091SRichard Lowe ENTRY(atomic_add_64_nv) 325*5d9d9091SRichard Lowe ALTENTRY(atomic_add_ptr_nv) 326*5d9d9091SRichard Lowe ALTENTRY(atomic_add_long_nv) 327*5d9d9091SRichard Lowe mov %rsi, %rax / %rax = delta addend 328*5d9d9091SRichard Lowe lock 329*5d9d9091SRichard Lowe xaddq %rsi, (%rdi) / %rsi = old value, (%rdi) = sum 330*5d9d9091SRichard Lowe addq %rsi, %rax / new value = original value + delta 331*5d9d9091SRichard Lowe ret 332*5d9d9091SRichard Lowe SET_SIZE(atomic_add_long_nv) 333*5d9d9091SRichard Lowe SET_SIZE(atomic_add_ptr_nv) 334*5d9d9091SRichard Lowe SET_SIZE(atomic_add_64_nv) 335*5d9d9091SRichard Lowe 336*5d9d9091SRichard Lowe ENTRY(atomic_and_8_nv) 337*5d9d9091SRichard Lowe ALTENTRY(atomic_and_uchar_nv) 338*5d9d9091SRichard Lowe movb (%rdi), %al / %al = old value 339*5d9d9091SRichard Lowe1: 340*5d9d9091SRichard Lowe movb %sil, %cl 341*5d9d9091SRichard Lowe andb %al, %cl / %cl = new value 342*5d9d9091SRichard Lowe lock 343*5d9d9091SRichard Lowe cmpxchgb %cl, (%rdi) / try to stick it in 344*5d9d9091SRichard Lowe jne 1b 345*5d9d9091SRichard Lowe movzbl %cl, %eax / return new value 346*5d9d9091SRichard Lowe ret 347*5d9d9091SRichard Lowe SET_SIZE(atomic_and_uchar_nv) 348*5d9d9091SRichard Lowe SET_SIZE(atomic_and_8_nv) 349*5d9d9091SRichard Lowe 350*5d9d9091SRichard Lowe ENTRY(atomic_and_16_nv) 351*5d9d9091SRichard Lowe ALTENTRY(atomic_and_ushort_nv) 352*5d9d9091SRichard Lowe movw (%rdi), %ax / %ax = old value 353*5d9d9091SRichard Lowe1: 354*5d9d9091SRichard Lowe movw %si, %cx 355*5d9d9091SRichard Lowe andw %ax, %cx / %cx = new value 356*5d9d9091SRichard Lowe lock 357*5d9d9091SRichard Lowe cmpxchgw %cx, (%rdi) / try to stick it in 358*5d9d9091SRichard Lowe jne 1b 359*5d9d9091SRichard Lowe movzwl %cx, %eax / return new value 360*5d9d9091SRichard Lowe ret 361*5d9d9091SRichard Lowe SET_SIZE(atomic_and_ushort_nv) 362*5d9d9091SRichard Lowe SET_SIZE(atomic_and_16_nv) 363*5d9d9091SRichard Lowe 364*5d9d9091SRichard Lowe ENTRY(atomic_and_32_nv) 365*5d9d9091SRichard Lowe ALTENTRY(atomic_and_uint_nv) 366*5d9d9091SRichard Lowe movl (%rdi), %eax 367*5d9d9091SRichard Lowe1: 368*5d9d9091SRichard Lowe movl %esi, %ecx 369*5d9d9091SRichard Lowe andl %eax, %ecx 370*5d9d9091SRichard Lowe lock 371*5d9d9091SRichard Lowe cmpxchgl %ecx, (%rdi) 372*5d9d9091SRichard Lowe jne 1b 373*5d9d9091SRichard Lowe movl %ecx, %eax 374*5d9d9091SRichard Lowe ret 375*5d9d9091SRichard Lowe SET_SIZE(atomic_and_uint_nv) 376*5d9d9091SRichard Lowe SET_SIZE(atomic_and_32_nv) 377*5d9d9091SRichard Lowe 378*5d9d9091SRichard Lowe ENTRY(atomic_and_64_nv) 379*5d9d9091SRichard Lowe ALTENTRY(atomic_and_ulong_nv) 380*5d9d9091SRichard Lowe movq (%rdi), %rax 381*5d9d9091SRichard Lowe1: 382*5d9d9091SRichard Lowe movq %rsi, %rcx 383*5d9d9091SRichard Lowe andq %rax, %rcx 384*5d9d9091SRichard Lowe lock 385*5d9d9091SRichard Lowe cmpxchgq %rcx, (%rdi) 386*5d9d9091SRichard Lowe jne 1b 387*5d9d9091SRichard Lowe movq %rcx, %rax 388*5d9d9091SRichard Lowe ret 389*5d9d9091SRichard Lowe SET_SIZE(atomic_and_ulong_nv) 390*5d9d9091SRichard Lowe SET_SIZE(atomic_and_64_nv) 391*5d9d9091SRichard Lowe 392*5d9d9091SRichard Lowe ENTRY(atomic_or_8_nv) 393*5d9d9091SRichard Lowe ALTENTRY(atomic_or_uchar_nv) 394*5d9d9091SRichard Lowe movb (%rdi), %al / %al = old value 395*5d9d9091SRichard Lowe1: 396*5d9d9091SRichard Lowe movb %sil, %cl 397*5d9d9091SRichard Lowe orb %al, %cl / %cl = new value 398*5d9d9091SRichard Lowe lock 399*5d9d9091SRichard Lowe cmpxchgb %cl, (%rdi) / try to stick it in 400*5d9d9091SRichard Lowe jne 1b 401*5d9d9091SRichard Lowe movzbl %cl, %eax / return new value 402*5d9d9091SRichard Lowe ret 403*5d9d9091SRichard Lowe SET_SIZE(atomic_or_uchar_nv) 404*5d9d9091SRichard Lowe SET_SIZE(atomic_or_8_nv) 405*5d9d9091SRichard Lowe 406*5d9d9091SRichard Lowe ENTRY(atomic_or_16_nv) 407*5d9d9091SRichard Lowe ALTENTRY(atomic_or_ushort_nv) 408*5d9d9091SRichard Lowe movw (%rdi), %ax / %ax = old value 409*5d9d9091SRichard Lowe1: 410*5d9d9091SRichard Lowe movw %si, %cx 411*5d9d9091SRichard Lowe orw %ax, %cx / %cx = new value 412*5d9d9091SRichard Lowe lock 413*5d9d9091SRichard Lowe cmpxchgw %cx, (%rdi) / try to stick it in 414*5d9d9091SRichard Lowe jne 1b 415*5d9d9091SRichard Lowe movzwl %cx, %eax / return new value 416*5d9d9091SRichard Lowe ret 417*5d9d9091SRichard Lowe SET_SIZE(atomic_or_ushort_nv) 418*5d9d9091SRichard Lowe SET_SIZE(atomic_or_16_nv) 419*5d9d9091SRichard Lowe 420*5d9d9091SRichard Lowe ENTRY(atomic_or_32_nv) 421*5d9d9091SRichard Lowe ALTENTRY(atomic_or_uint_nv) 422*5d9d9091SRichard Lowe movl (%rdi), %eax 423*5d9d9091SRichard Lowe1: 424*5d9d9091SRichard Lowe movl %esi, %ecx 425*5d9d9091SRichard Lowe orl %eax, %ecx 426*5d9d9091SRichard Lowe lock 427*5d9d9091SRichard Lowe cmpxchgl %ecx, (%rdi) 428*5d9d9091SRichard Lowe jne 1b 429*5d9d9091SRichard Lowe movl %ecx, %eax 430*5d9d9091SRichard Lowe ret 431*5d9d9091SRichard Lowe SET_SIZE(atomic_or_uint_nv) 432*5d9d9091SRichard Lowe SET_SIZE(atomic_or_32_nv) 433*5d9d9091SRichard Lowe 434*5d9d9091SRichard Lowe ENTRY(atomic_or_64_nv) 435*5d9d9091SRichard Lowe ALTENTRY(atomic_or_ulong_nv) 436*5d9d9091SRichard Lowe movq (%rdi), %rax 437*5d9d9091SRichard Lowe1: 438*5d9d9091SRichard Lowe movq %rsi, %rcx 439*5d9d9091SRichard Lowe orq %rax, %rcx 440*5d9d9091SRichard Lowe lock 441*5d9d9091SRichard Lowe cmpxchgq %rcx, (%rdi) 442*5d9d9091SRichard Lowe jne 1b 443*5d9d9091SRichard Lowe movq %rcx, %rax 444*5d9d9091SRichard Lowe ret 445*5d9d9091SRichard Lowe SET_SIZE(atomic_or_ulong_nv) 446*5d9d9091SRichard Lowe SET_SIZE(atomic_or_64_nv) 447*5d9d9091SRichard Lowe 448*5d9d9091SRichard Lowe ENTRY(atomic_cas_8) 449*5d9d9091SRichard Lowe ALTENTRY(atomic_cas_uchar) 450*5d9d9091SRichard Lowe movzbl %sil, %eax 451*5d9d9091SRichard Lowe lock 452*5d9d9091SRichard Lowe cmpxchgb %dl, (%rdi) 453*5d9d9091SRichard Lowe ret 454*5d9d9091SRichard Lowe SET_SIZE(atomic_cas_uchar) 455*5d9d9091SRichard Lowe SET_SIZE(atomic_cas_8) 456*5d9d9091SRichard Lowe 457*5d9d9091SRichard Lowe ENTRY(atomic_cas_16) 458*5d9d9091SRichard Lowe ALTENTRY(atomic_cas_ushort) 459*5d9d9091SRichard Lowe movzwl %si, %eax 460*5d9d9091SRichard Lowe lock 461*5d9d9091SRichard Lowe cmpxchgw %dx, (%rdi) 462*5d9d9091SRichard Lowe ret 463*5d9d9091SRichard Lowe SET_SIZE(atomic_cas_ushort) 464*5d9d9091SRichard Lowe SET_SIZE(atomic_cas_16) 465*5d9d9091SRichard Lowe 466*5d9d9091SRichard Lowe ENTRY(atomic_cas_32) 467*5d9d9091SRichard Lowe ALTENTRY(atomic_cas_uint) 468*5d9d9091SRichard Lowe movl %esi, %eax 469*5d9d9091SRichard Lowe lock 470*5d9d9091SRichard Lowe cmpxchgl %edx, (%rdi) 471*5d9d9091SRichard Lowe ret 472*5d9d9091SRichard Lowe SET_SIZE(atomic_cas_uint) 473*5d9d9091SRichard Lowe SET_SIZE(atomic_cas_32) 474*5d9d9091SRichard Lowe 475*5d9d9091SRichard Lowe ENTRY(atomic_cas_64) 476*5d9d9091SRichard Lowe ALTENTRY(atomic_cas_ulong) 477*5d9d9091SRichard Lowe ALTENTRY(atomic_cas_ptr) 478*5d9d9091SRichard Lowe movq %rsi, %rax 479*5d9d9091SRichard Lowe lock 480*5d9d9091SRichard Lowe cmpxchgq %rdx, (%rdi) 481*5d9d9091SRichard Lowe ret 482*5d9d9091SRichard Lowe SET_SIZE(atomic_cas_ptr) 483*5d9d9091SRichard Lowe SET_SIZE(atomic_cas_ulong) 484*5d9d9091SRichard Lowe SET_SIZE(atomic_cas_64) 485*5d9d9091SRichard Lowe 486*5d9d9091SRichard Lowe ENTRY(atomic_swap_8) 487*5d9d9091SRichard Lowe ALTENTRY(atomic_swap_uchar) 488*5d9d9091SRichard Lowe movzbl %sil, %eax 489*5d9d9091SRichard Lowe lock 490*5d9d9091SRichard Lowe xchgb %al, (%rdi) 491*5d9d9091SRichard Lowe ret 492*5d9d9091SRichard Lowe SET_SIZE(atomic_swap_uchar) 493*5d9d9091SRichard Lowe SET_SIZE(atomic_swap_8) 494*5d9d9091SRichard Lowe 495*5d9d9091SRichard Lowe ENTRY(atomic_swap_16) 496*5d9d9091SRichard Lowe ALTENTRY(atomic_swap_ushort) 497*5d9d9091SRichard Lowe movzwl %si, %eax 498*5d9d9091SRichard Lowe lock 499*5d9d9091SRichard Lowe xchgw %ax, (%rdi) 500*5d9d9091SRichard Lowe ret 501*5d9d9091SRichard Lowe SET_SIZE(atomic_swap_ushort) 502*5d9d9091SRichard Lowe SET_SIZE(atomic_swap_16) 503*5d9d9091SRichard Lowe 504*5d9d9091SRichard Lowe ENTRY(atomic_swap_32) 505*5d9d9091SRichard Lowe ALTENTRY(atomic_swap_uint) 506*5d9d9091SRichard Lowe movl %esi, %eax 507*5d9d9091SRichard Lowe lock 508*5d9d9091SRichard Lowe xchgl %eax, (%rdi) 509*5d9d9091SRichard Lowe ret 510*5d9d9091SRichard Lowe SET_SIZE(atomic_swap_uint) 511*5d9d9091SRichard Lowe SET_SIZE(atomic_swap_32) 512*5d9d9091SRichard Lowe 513*5d9d9091SRichard Lowe ENTRY(atomic_swap_64) 514*5d9d9091SRichard Lowe ALTENTRY(atomic_swap_ulong) 515*5d9d9091SRichard Lowe ALTENTRY(atomic_swap_ptr) 516*5d9d9091SRichard Lowe movq %rsi, %rax 517*5d9d9091SRichard Lowe lock 518*5d9d9091SRichard Lowe xchgq %rax, (%rdi) 519*5d9d9091SRichard Lowe ret 520*5d9d9091SRichard Lowe SET_SIZE(atomic_swap_ptr) 521*5d9d9091SRichard Lowe SET_SIZE(atomic_swap_ulong) 522*5d9d9091SRichard Lowe SET_SIZE(atomic_swap_64) 523*5d9d9091SRichard Lowe 524*5d9d9091SRichard Lowe ENTRY(atomic_set_long_excl) 525*5d9d9091SRichard Lowe xorl %eax, %eax 526*5d9d9091SRichard Lowe lock 527*5d9d9091SRichard Lowe btsq %rsi, (%rdi) 528*5d9d9091SRichard Lowe jnc 1f 529*5d9d9091SRichard Lowe decl %eax / return -1 530*5d9d9091SRichard Lowe1: 531*5d9d9091SRichard Lowe ret 532*5d9d9091SRichard Lowe SET_SIZE(atomic_set_long_excl) 533*5d9d9091SRichard Lowe 534*5d9d9091SRichard Lowe ENTRY(atomic_clear_long_excl) 535*5d9d9091SRichard Lowe xorl %eax, %eax 536*5d9d9091SRichard Lowe lock 537*5d9d9091SRichard Lowe btrq %rsi, (%rdi) 538*5d9d9091SRichard Lowe jc 1f 539*5d9d9091SRichard Lowe decl %eax / return -1 540*5d9d9091SRichard Lowe1: 541*5d9d9091SRichard Lowe ret 542*5d9d9091SRichard Lowe SET_SIZE(atomic_clear_long_excl) 543*5d9d9091SRichard Lowe 544*5d9d9091SRichard Lowe#if !defined(_KERNEL) 545*5d9d9091SRichard Lowe 546*5d9d9091SRichard Lowe /* 547*5d9d9091SRichard Lowe * NOTE: membar_enter, and membar_exit are identical routines. 548*5d9d9091SRichard Lowe * We define them separately, instead of using an ALTENTRY 549*5d9d9091SRichard Lowe * definitions to alias them together, so that DTrace and 550*5d9d9091SRichard Lowe * debuggers will see a unique address for them, allowing 551*5d9d9091SRichard Lowe * more accurate tracing. 552*5d9d9091SRichard Lowe */ 553*5d9d9091SRichard Lowe 554*5d9d9091SRichard Lowe ENTRY(membar_enter) 555*5d9d9091SRichard Lowe mfence 556*5d9d9091SRichard Lowe ret 557*5d9d9091SRichard Lowe SET_SIZE(membar_enter) 558*5d9d9091SRichard Lowe 559*5d9d9091SRichard Lowe ENTRY(membar_exit) 560*5d9d9091SRichard Lowe mfence 561*5d9d9091SRichard Lowe ret 562*5d9d9091SRichard Lowe SET_SIZE(membar_exit) 563*5d9d9091SRichard Lowe 564*5d9d9091SRichard Lowe ENTRY(membar_producer) 565*5d9d9091SRichard Lowe sfence 566*5d9d9091SRichard Lowe ret 567*5d9d9091SRichard Lowe SET_SIZE(membar_producer) 568*5d9d9091SRichard Lowe 569*5d9d9091SRichard Lowe ENTRY(membar_consumer) 570*5d9d9091SRichard Lowe lfence 571*5d9d9091SRichard Lowe ret 572*5d9d9091SRichard Lowe SET_SIZE(membar_consumer) 573*5d9d9091SRichard Lowe 574*5d9d9091SRichard Lowe#endif /* !_KERNEL */ 575