1 /* $NetBSD: patch.c,v 1.53 2022/08/20 23:48:51 riastradh Exp $ */
2
3 /*-
4 * Copyright (c) 2007, 2008, 2009 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 /*
33 * Patch kernel code at boot time, depending on available CPU features.
34 */
35
36 #include <sys/cdefs.h>
37 __KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.53 2022/08/20 23:48:51 riastradh Exp $");
38
39 #include "opt_lockdebug.h"
40 #ifdef i386
41 #include "opt_spldebug.h"
42 #endif
43
44 #include <sys/types.h>
45 #include <sys/systm.h>
46
47 #include <machine/cpu.h>
48 #include <machine/cpufunc.h>
49 #include <machine/specialreg.h>
50 #include <machine/frameasm.h>
51
52 #include <uvm/uvm.h>
53 #include <machine/pmap.h>
54 #include <machine/pmap_private.h>
55
56 #include <x86/bootspace.h>
57 #include <x86/cpuvar.h>
58 #include <x86/cputypes.h>
59
60 __link_set_decl(x86_hotpatch_descriptors, struct x86_hotpatch_descriptor);
61
62 struct x86_hotpatch_destination {
63 uint8_t name;
64 uint8_t size;
65 void *addr;
66 } __packed;
67
68 /* -------------------------------------------------------------------------- */
69
70 /* CLAC instruction, part of SMAP. */
71 extern uint8_t hp_clac, hp_clac_end;
72 static const struct x86_hotpatch_source hp_clac_source = {
73 .saddr = &hp_clac,
74 .eaddr = &hp_clac_end
75 };
76 static const struct x86_hotpatch_descriptor hp_clac_desc = {
77 .name = HP_NAME_CLAC,
78 .nsrc = 1,
79 .srcs = { &hp_clac_source }
80 };
81 __link_set_add_rodata(x86_hotpatch_descriptors, hp_clac_desc);
82
83 /* STAC instruction, part of SMAP. */
84 extern uint8_t hp_stac, hp_stac_end;
85 static const struct x86_hotpatch_source hp_stac_source = {
86 .saddr = &hp_stac,
87 .eaddr = &hp_stac_end
88 };
89 static const struct x86_hotpatch_descriptor hp_stac_desc = {
90 .name = HP_NAME_STAC,
91 .nsrc = 1,
92 .srcs = { &hp_stac_source }
93 };
94 __link_set_add_rodata(x86_hotpatch_descriptors, hp_stac_desc);
95
96 /* Errata on certain AMD CPUs. */
97 extern uint8_t hp_retfence, hp_retfence_end;
98 static const struct x86_hotpatch_source hp_retfence_source = {
99 .saddr = &hp_retfence,
100 .eaddr = &hp_retfence_end
101 };
102 static const struct x86_hotpatch_descriptor hp_retfence_desc = {
103 .name = HP_NAME_RETFENCE,
104 .nsrc = 1,
105 .srcs = { &hp_retfence_source }
106 };
107 __link_set_add_rodata(x86_hotpatch_descriptors, hp_retfence_desc);
108
109 /* No lock when on a single processor. */
110 extern uint8_t hp_nolock, hp_nolock_end;
111 static const struct x86_hotpatch_source hp_nolock_source = {
112 .saddr = &hp_nolock,
113 .eaddr = &hp_nolock_end
114 };
115 static const struct x86_hotpatch_descriptor hp_nolock_desc = {
116 .name = HP_NAME_NOLOCK,
117 .nsrc = 1,
118 .srcs = { &hp_nolock_source }
119 };
120 __link_set_add_rodata(x86_hotpatch_descriptors, hp_nolock_desc);
121
122 #ifdef i386
123 /* CAS_64. */
124 extern uint8_t _atomic_cas_cx8, _atomic_cas_cx8_end;
125 static const struct x86_hotpatch_source hp_cas_cx8_source = {
126 .saddr = &_atomic_cas_cx8,
127 .eaddr = &_atomic_cas_cx8_end
128 };
129 static const struct x86_hotpatch_descriptor hp_cas_cx8_desc = {
130 .name = HP_NAME_CAS_64,
131 .nsrc = 1,
132 .srcs = { &hp_cas_cx8_source }
133 };
134 __link_set_add_rodata(x86_hotpatch_descriptors, hp_cas_cx8_desc);
135
136 /* SPLLOWER. */
137 extern uint8_t cx8_spllower, cx8_spllower_end;
138 static const struct x86_hotpatch_source hp_cx8_spllower_source = {
139 .saddr = &cx8_spllower,
140 .eaddr = &cx8_spllower_end
141 };
142 static const struct x86_hotpatch_descriptor hp_cx8_spllower_desc = {
143 .name = HP_NAME_SPLLOWER,
144 .nsrc = 1,
145 .srcs = { &hp_cx8_spllower_source }
146 };
147 __link_set_add_rodata(x86_hotpatch_descriptors, hp_cx8_spllower_desc);
148
149 /* MUTEX_EXIT. */
150 #ifndef LOCKDEBUG
151 extern uint8_t i686_mutex_spin_exit, i686_mutex_spin_exit_end;
152 static const struct x86_hotpatch_source hp_i686_mutex_spin_exit_source = {
153 .saddr = &i686_mutex_spin_exit,
154 .eaddr = &i686_mutex_spin_exit_end
155 };
156 static const struct x86_hotpatch_descriptor hp_i686_mutex_spin_exit_desc = {
157 .name = HP_NAME_MUTEX_EXIT,
158 .nsrc = 1,
159 .srcs = { &hp_i686_mutex_spin_exit_source }
160 };
161 __link_set_add_rodata(x86_hotpatch_descriptors, hp_i686_mutex_spin_exit_desc);
162 #endif
163 #endif
164
165 /* -------------------------------------------------------------------------- */
166
167 static inline void __unused
patchbytes(void * addr,const uint8_t * bytes,size_t size)168 patchbytes(void *addr, const uint8_t *bytes, size_t size)
169 {
170 uint8_t *ptr = (uint8_t *)addr;
171 size_t i;
172
173 for (i = 0; i < size; i++) {
174 ptr[i] = bytes[i];
175 }
176 }
177
178 /*
179 * Rules: each pointer accessed in this function MUST be read-only.
180 *
181 * Called from ASM only, prototype not public.
182 */
183 int x86_hotpatch_apply(uint8_t, uint8_t);
184 int
185 __noubsan /* the local variables have unknown alignment to UBSan */
x86_hotpatch_apply(uint8_t name,uint8_t sel)186 x86_hotpatch_apply(uint8_t name, uint8_t sel)
187 {
188 struct x86_hotpatch_descriptor * const *iter;
189 const struct x86_hotpatch_descriptor *desc;
190 const struct x86_hotpatch_source *src;
191 const struct x86_hotpatch_destination *hps, *hpe, *hp;
192 extern char __rodata_hotpatch_start;
193 extern char __rodata_hotpatch_end;
194 const uint8_t *bytes;
195 bool found = false;
196 size_t size;
197
198 /*
199 * Find the descriptor, and perform some sanity checks.
200 */
201 __link_set_foreach(iter, x86_hotpatch_descriptors) {
202 desc = *iter;
203 if (desc->name == name) {
204 found = true;
205 break;
206 }
207 }
208 if (!found)
209 return -1;
210 if (desc->nsrc > 2)
211 return -1;
212 if (sel >= desc->nsrc)
213 return -1;
214
215 /*
216 * Get the hotpatch source.
217 */
218 src = desc->srcs[sel];
219 bytes = src->saddr;
220 size = (size_t)src->eaddr - (size_t)src->saddr;
221
222 /*
223 * Apply the hotpatch on each registered destination.
224 */
225 hps = (struct x86_hotpatch_destination *)&__rodata_hotpatch_start;
226 hpe = (struct x86_hotpatch_destination *)&__rodata_hotpatch_end;
227 for (hp = hps; hp < hpe; hp++) {
228 if (hp->name != name) {
229 continue;
230 }
231 if (hp->size != size) {
232 return -1;
233 }
234 patchbytes(hp->addr, bytes, size);
235 }
236
237 return 0;
238 }
239
240 #ifdef __x86_64__
241 /*
242 * The CPU added the D bit on the text pages while we were writing to them.
243 * Remove that bit. Kinda annoying, but we can't avoid it.
244 */
245 static void
remove_d_bit(void)246 remove_d_bit(void)
247 {
248 extern struct bootspace bootspace;
249 pt_entry_t pte;
250 vaddr_t va;
251 size_t i, n;
252
253 for (i = 0; i < BTSPACE_NSEGS; i++) {
254 if (bootspace.segs[i].type != BTSEG_TEXT)
255 continue;
256 va = bootspace.segs[i].va;
257 n = 0;
258 while (n < bootspace.segs[i].sz) {
259 if (L2_BASE[pl2_i(va)] & PTE_PS) {
260 pte = L2_BASE[pl2_i(va)] & ~PTE_D;
261 pmap_pte_set(&L2_BASE[pl2_i(va)], pte);
262 n += NBPD_L2;
263 va += NBPD_L2;
264 } else {
265 pte = L1_BASE[pl1_i(va)] & ~PTE_D;
266 pmap_pte_set(&L1_BASE[pl1_i(va)], pte);
267 n += NBPD_L1;
268 va += NBPD_L1;
269 }
270 }
271 }
272
273 tlbflushg();
274 }
275 #else
276 #define remove_d_bit() __nothing
277 #endif
278
279 /*
280 * Interrupts disabled here. Called from ASM only, prototype not public.
281 */
282 void x86_hotpatch_cleanup(int);
283 void
x86_hotpatch_cleanup(int retval)284 x86_hotpatch_cleanup(int retval)
285 {
286 if (retval != 0) {
287 panic("x86_hotpatch_apply failed");
288 }
289
290 remove_d_bit();
291 }
292
293 /* -------------------------------------------------------------------------- */
294
295 void
x86_patch(bool early)296 x86_patch(bool early)
297 {
298 static bool first, second;
299
300 if (early) {
301 if (first)
302 return;
303 first = true;
304 } else {
305 if (second)
306 return;
307 second = true;
308 }
309
310 if (!early && ncpu == 1) {
311 #ifndef LOCKDEBUG
312 /*
313 * Uniprocessor: kill LOCK prefixes.
314 */
315 x86_hotpatch(HP_NAME_NOLOCK, 0);
316 #endif
317 }
318
319 #ifdef i386
320 /*
321 * Patch early and late. Second time around the 'lock' prefix
322 * may be gone.
323 */
324 if ((cpu_feature[0] & CPUID_CX8) != 0) {
325 x86_hotpatch(HP_NAME_CAS_64, 0);
326 }
327
328 #if !defined(SPLDEBUG)
329 if (!early && (cpu_feature[0] & CPUID_CX8) != 0) {
330 /* Faster splx(), mutex_spin_exit(). */
331 x86_hotpatch(HP_NAME_SPLLOWER, 0);
332 #if !defined(LOCKDEBUG)
333 x86_hotpatch(HP_NAME_MUTEX_EXIT, 0);
334 #endif
335 }
336 #endif /* !SPLDEBUG */
337 #endif /* i386 */
338
339 /*
340 * On some Opteron revisions, locked operations erroneously
341 * allow memory references to be `bled' outside of critical
342 * sections. Apply workaround.
343 */
344 if (cpu_vendor == CPUVENDOR_AMD &&
345 (CPUID_TO_FAMILY(cpu_info_primary.ci_signature) == 0xe ||
346 (CPUID_TO_FAMILY(cpu_info_primary.ci_signature) == 0xf &&
347 CPUID_TO_EXTMODEL(cpu_info_primary.ci_signature) < 0x4))) {
348 x86_hotpatch(HP_NAME_RETFENCE, 0);
349 }
350
351 /*
352 * If SMAP is present then patch the prepared holes with clac/stac
353 * instructions.
354 */
355 if (!early && cpu_feature[5] & CPUID_SEF_SMAP) {
356 KASSERT(rcr4() & CR4_SMAP);
357
358 x86_hotpatch(HP_NAME_CLAC, 0);
359 x86_hotpatch(HP_NAME_STAC, 0);
360 }
361 }
362