1 /* $NetBSD: lock.h,v 1.29 2022/02/12 17:17:54 riastradh Exp $ */
2
3 /*-
4 * Copyright (c) 2000, 2006 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Jason R. Thorpe and Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 /*
33 * Machine-dependent spin lock operations.
34 */
35
36 #ifndef _X86_LOCK_H_
37 #define _X86_LOCK_H_
38
39 #include <sys/param.h>
40
41 static __inline int
__SIMPLELOCK_LOCKED_P(const __cpu_simple_lock_t * __ptr)42 __SIMPLELOCK_LOCKED_P(const __cpu_simple_lock_t *__ptr)
43 {
44 return *__ptr == __SIMPLELOCK_LOCKED;
45 }
46
47 static __inline int
__SIMPLELOCK_UNLOCKED_P(const __cpu_simple_lock_t * __ptr)48 __SIMPLELOCK_UNLOCKED_P(const __cpu_simple_lock_t *__ptr)
49 {
50 return *__ptr == __SIMPLELOCK_UNLOCKED;
51 }
52
53 static __inline void
__cpu_simple_lock_set(__cpu_simple_lock_t * __ptr)54 __cpu_simple_lock_set(__cpu_simple_lock_t *__ptr)
55 {
56
57 *__ptr = __SIMPLELOCK_LOCKED;
58 }
59
60 static __inline void
__cpu_simple_lock_clear(__cpu_simple_lock_t * __ptr)61 __cpu_simple_lock_clear(__cpu_simple_lock_t *__ptr)
62 {
63
64 *__ptr = __SIMPLELOCK_UNLOCKED;
65 }
66
67 #ifdef _HARDKERNEL
68 # include <machine/cpufunc.h>
69 # define SPINLOCK_SPIN_HOOK /* nothing */
70 # ifdef SPINLOCK_BACKOFF_HOOK
71 # undef SPINLOCK_BACKOFF_HOOK
72 # endif
73 # define SPINLOCK_BACKOFF_HOOK x86_pause()
74 # define SPINLOCK_INLINE
75 #else /* !_HARDKERNEL */
76 # define SPINLOCK_BODY
77 # define SPINLOCK_INLINE static __inline __unused
78 #endif /* _HARDKERNEL */
79
80 SPINLOCK_INLINE void __cpu_simple_lock_init(__cpu_simple_lock_t *);
81 SPINLOCK_INLINE void __cpu_simple_lock(__cpu_simple_lock_t *);
82 SPINLOCK_INLINE int __cpu_simple_lock_try(__cpu_simple_lock_t *);
83 SPINLOCK_INLINE void __cpu_simple_unlock(__cpu_simple_lock_t *);
84
85 #ifdef SPINLOCK_BODY
86 SPINLOCK_INLINE void
__cpu_simple_lock_init(__cpu_simple_lock_t * lockp)87 __cpu_simple_lock_init(__cpu_simple_lock_t *lockp)
88 {
89
90 *lockp = __SIMPLELOCK_UNLOCKED;
91 }
92
93 SPINLOCK_INLINE int
__cpu_simple_lock_try(__cpu_simple_lock_t * lockp)94 __cpu_simple_lock_try(__cpu_simple_lock_t *lockp)
95 {
96 uint8_t val;
97
98 val = __SIMPLELOCK_LOCKED;
99 __asm volatile ("xchgb %0,(%2)" :
100 "=qQ" (val)
101 :"0" (val), "r" (lockp));
102 __insn_barrier();
103 return val == __SIMPLELOCK_UNLOCKED;
104 }
105
106 SPINLOCK_INLINE void
__cpu_simple_lock(__cpu_simple_lock_t * lockp)107 __cpu_simple_lock(__cpu_simple_lock_t *lockp)
108 {
109
110 while (!__cpu_simple_lock_try(lockp))
111 /* nothing */;
112 __insn_barrier();
113 }
114
115 /*
116 * Note on x86 memory ordering
117 *
118 * When releasing a lock we must ensure that no stores or loads from within
119 * the critical section are re-ordered by the CPU to occur outside of it:
120 * they must have completed and be visible to other processors once the lock
121 * has been released.
122 *
123 * NetBSD usually runs with the kernel mapped (via MTRR) in a WB (write
124 * back) memory region. In that case, memory ordering on x86 platforms
125 * looks like this:
126 *
127 * i386 All loads/stores occur in instruction sequence.
128 *
129 * i486 All loads/stores occur in instruction sequence. In
130 * Pentium exceptional circumstances, loads can be re-ordered around
131 * stores, but for the purposes of releasing a lock it does
132 * not matter. Stores may not be immediately visible to other
133 * processors as they can be buffered. However, since the
134 * stores are buffered in order the lock release will always be
135 * the last operation in the critical section that becomes
136 * visible to other CPUs.
137 *
138 * Pentium Pro The "Intel 64 and IA-32 Architectures Software Developer's
139 * onwards Manual" volume 3A (order number 248966) says that (1) "Reads
140 * can be carried out speculatively and in any order" and (2)
141 * "Reads can pass buffered stores, but the processor is
142 * self-consistent.". This would be a problem for the below,
143 * and would mandate a locked instruction cycle or load fence
144 * before releasing the simple lock.
145 *
146 * The "Intel Pentium 4 Processor Optimization" guide (order
147 * number 253668-022US) says: "Loads can be moved before stores
148 * that occurred earlier in the program if they are not
149 * predicted to load from the same linear address.". This is
150 * not a problem since the only loads that can be re-ordered
151 * take place once the lock has been released via a store.
152 *
153 * The above two documents seem to contradict each other,
154 * however with the exception of early steppings of the Pentium
155 * Pro, the second document is closer to the truth: a store
156 * will always act as a load fence for all loads that precede
157 * the store in instruction order.
158 *
159 * Again, note that stores can be buffered and will not always
160 * become immediately visible to other CPUs: they are however
161 * buffered in order.
162 *
163 * AMD64 Stores occur in order and are buffered. Loads can be
164 * reordered, however stores act as load fences, meaning that
165 * loads can not be reordered around stores.
166 */
167 SPINLOCK_INLINE void
__cpu_simple_unlock(__cpu_simple_lock_t * lockp)168 __cpu_simple_unlock(__cpu_simple_lock_t *lockp)
169 {
170
171 __insn_barrier();
172 *lockp = __SIMPLELOCK_UNLOCKED;
173 }
174
175 #endif /* SPINLOCK_BODY */
176
177 #endif /* _X86_LOCK_H_ */
178