1 /* $NetBSD: patch.c,v 1.50 2022/04/09 12:07:00 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 2007, 2008, 2009 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Patch kernel code at boot time, depending on available CPU features. 34 */ 35 36 #include <sys/cdefs.h> 37 __KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.50 2022/04/09 12:07:00 riastradh Exp $"); 38 39 #include "opt_lockdebug.h" 40 #ifdef i386 41 #include "opt_spldebug.h" 42 #endif 43 44 #include <sys/types.h> 45 #include <sys/systm.h> 46 47 #include <machine/cpu.h> 48 #include <machine/cpufunc.h> 49 #include <machine/specialreg.h> 50 #include <machine/frameasm.h> 51 52 #include <uvm/uvm.h> 53 #include <machine/pmap.h> 54 55 #include <x86/cpuvar.h> 56 #include <x86/cputypes.h> 57 58 __link_set_decl(x86_hotpatch_descriptors, struct x86_hotpatch_descriptor); 59 60 struct x86_hotpatch_destination { 61 uint8_t name; 62 uint8_t size; 63 void *addr; 64 } __packed; 65 66 /* -------------------------------------------------------------------------- */ 67 68 /* CLAC instruction, part of SMAP. */ 69 extern uint8_t hp_clac, hp_clac_end; 70 static const struct x86_hotpatch_source hp_clac_source = { 71 .saddr = &hp_clac, 72 .eaddr = &hp_clac_end 73 }; 74 static const struct x86_hotpatch_descriptor hp_clac_desc = { 75 .name = HP_NAME_CLAC, 76 .nsrc = 1, 77 .srcs = { &hp_clac_source } 78 }; 79 __link_set_add_rodata(x86_hotpatch_descriptors, hp_clac_desc); 80 81 /* STAC instruction, part of SMAP. */ 82 extern uint8_t hp_stac, hp_stac_end; 83 static const struct x86_hotpatch_source hp_stac_source = { 84 .saddr = &hp_stac, 85 .eaddr = &hp_stac_end 86 }; 87 static const struct x86_hotpatch_descriptor hp_stac_desc = { 88 .name = HP_NAME_STAC, 89 .nsrc = 1, 90 .srcs = { &hp_stac_source } 91 }; 92 __link_set_add_rodata(x86_hotpatch_descriptors, hp_stac_desc); 93 94 /* Errata on certain AMD CPUs. */ 95 extern uint8_t hp_retfence, hp_retfence_end; 96 static const struct x86_hotpatch_source hp_retfence_source = { 97 .saddr = &hp_retfence, 98 .eaddr = &hp_retfence_end 99 }; 100 static const struct x86_hotpatch_descriptor hp_retfence_desc = { 101 .name = HP_NAME_RETFENCE, 102 .nsrc = 1, 103 .srcs = { &hp_retfence_source } 104 }; 105 __link_set_add_rodata(x86_hotpatch_descriptors, hp_retfence_desc); 106 107 /* No lock when on a single processor. */ 108 extern uint8_t hp_nolock, hp_nolock_end; 109 static const struct x86_hotpatch_source hp_nolock_source = { 110 .saddr = &hp_nolock, 111 .eaddr = &hp_nolock_end 112 }; 113 static const struct x86_hotpatch_descriptor hp_nolock_desc = { 114 .name = HP_NAME_NOLOCK, 115 .nsrc = 1, 116 .srcs = { &hp_nolock_source } 117 }; 118 __link_set_add_rodata(x86_hotpatch_descriptors, hp_nolock_desc); 119 120 /* Use MFENCE if available, part of SSE2. */ 121 extern uint8_t sse2_mfence, sse2_mfence_end; 122 static const struct x86_hotpatch_source hp_sse2_mfence_source = { 123 .saddr = &sse2_mfence, 124 .eaddr = &sse2_mfence_end 125 }; 126 static const struct x86_hotpatch_descriptor hp_sse2_mfence_desc = { 127 .name = HP_NAME_SSE2_MFENCE, 128 .nsrc = 1, 129 .srcs = { &hp_sse2_mfence_source } 130 }; 131 __link_set_add_rodata(x86_hotpatch_descriptors, hp_sse2_mfence_desc); 132 133 #ifdef i386 134 /* CAS_64. */ 135 extern uint8_t _atomic_cas_cx8, _atomic_cas_cx8_end; 136 static const struct x86_hotpatch_source hp_cas_cx8_source = { 137 .saddr = &_atomic_cas_cx8, 138 .eaddr = &_atomic_cas_cx8_end 139 }; 140 static const struct x86_hotpatch_descriptor hp_cas_cx8_desc = { 141 .name = HP_NAME_CAS_64, 142 .nsrc = 1, 143 .srcs = { &hp_cas_cx8_source } 144 }; 145 __link_set_add_rodata(x86_hotpatch_descriptors, hp_cas_cx8_desc); 146 147 /* SPLLOWER. */ 148 extern uint8_t cx8_spllower, cx8_spllower_end; 149 static const struct x86_hotpatch_source hp_cx8_spllower_source = { 150 .saddr = &cx8_spllower, 151 .eaddr = &cx8_spllower_end 152 }; 153 static const struct x86_hotpatch_descriptor hp_cx8_spllower_desc = { 154 .name = HP_NAME_SPLLOWER, 155 .nsrc = 1, 156 .srcs = { &hp_cx8_spllower_source } 157 }; 158 __link_set_add_rodata(x86_hotpatch_descriptors, hp_cx8_spllower_desc); 159 160 /* MUTEX_EXIT. */ 161 #ifndef LOCKDEBUG 162 extern uint8_t i686_mutex_spin_exit, i686_mutex_spin_exit_end; 163 static const struct x86_hotpatch_source hp_i686_mutex_spin_exit_source = { 164 .saddr = &i686_mutex_spin_exit, 165 .eaddr = &i686_mutex_spin_exit_end 166 }; 167 static const struct x86_hotpatch_descriptor hp_i686_mutex_spin_exit_desc = { 168 .name = HP_NAME_MUTEX_EXIT, 169 .nsrc = 1, 170 .srcs = { &hp_i686_mutex_spin_exit_source } 171 }; 172 __link_set_add_rodata(x86_hotpatch_descriptors, hp_i686_mutex_spin_exit_desc); 173 #endif 174 #endif 175 176 /* -------------------------------------------------------------------------- */ 177 178 static inline void __unused 179 patchbytes(void *addr, const uint8_t *bytes, size_t size) 180 { 181 uint8_t *ptr = (uint8_t *)addr; 182 size_t i; 183 184 for (i = 0; i < size; i++) { 185 ptr[i] = bytes[i]; 186 } 187 } 188 189 /* 190 * Rules: each pointer accessed in this function MUST be read-only. 191 * 192 * Called from ASM only, prototype not public. 193 */ 194 int x86_hotpatch_apply(uint8_t, uint8_t); 195 int 196 __noubsan /* the local variables have unknown alignment to UBSan */ 197 x86_hotpatch_apply(uint8_t name, uint8_t sel) 198 { 199 struct x86_hotpatch_descriptor * const *iter; 200 const struct x86_hotpatch_descriptor *desc; 201 const struct x86_hotpatch_source *src; 202 const struct x86_hotpatch_destination *hps, *hpe, *hp; 203 extern char __rodata_hotpatch_start; 204 extern char __rodata_hotpatch_end; 205 const uint8_t *bytes; 206 bool found = false; 207 size_t size; 208 209 /* 210 * Find the descriptor, and perform some sanity checks. 211 */ 212 __link_set_foreach(iter, x86_hotpatch_descriptors) { 213 desc = *iter; 214 if (desc->name == name) { 215 found = true; 216 break; 217 } 218 } 219 if (!found) 220 return -1; 221 if (desc->nsrc > 2) 222 return -1; 223 if (sel >= desc->nsrc) 224 return -1; 225 226 /* 227 * Get the hotpatch source. 228 */ 229 src = desc->srcs[sel]; 230 bytes = src->saddr; 231 size = (size_t)src->eaddr - (size_t)src->saddr; 232 233 /* 234 * Apply the hotpatch on each registered destination. 235 */ 236 hps = (struct x86_hotpatch_destination *)&__rodata_hotpatch_start; 237 hpe = (struct x86_hotpatch_destination *)&__rodata_hotpatch_end; 238 for (hp = hps; hp < hpe; hp++) { 239 if (hp->name != name) { 240 continue; 241 } 242 if (hp->size != size) { 243 return -1; 244 } 245 patchbytes(hp->addr, bytes, size); 246 } 247 248 return 0; 249 } 250 251 #ifdef __x86_64__ 252 /* 253 * The CPU added the D bit on the text pages while we were writing to them. 254 * Remove that bit. Kinda annoying, but we can't avoid it. 255 */ 256 static void 257 remove_d_bit(void) 258 { 259 extern struct bootspace bootspace; 260 pt_entry_t pte; 261 vaddr_t va; 262 size_t i, n; 263 264 for (i = 0; i < BTSPACE_NSEGS; i++) { 265 if (bootspace.segs[i].type != BTSEG_TEXT) 266 continue; 267 va = bootspace.segs[i].va; 268 n = 0; 269 while (n < bootspace.segs[i].sz) { 270 if (L2_BASE[pl2_i(va)] & PTE_PS) { 271 pte = L2_BASE[pl2_i(va)] & ~PTE_D; 272 pmap_pte_set(&L2_BASE[pl2_i(va)], pte); 273 n += NBPD_L2; 274 va += NBPD_L2; 275 } else { 276 pte = L1_BASE[pl1_i(va)] & ~PTE_D; 277 pmap_pte_set(&L1_BASE[pl1_i(va)], pte); 278 n += NBPD_L1; 279 va += NBPD_L1; 280 } 281 } 282 } 283 284 tlbflushg(); 285 } 286 #else 287 #define remove_d_bit() __nothing 288 #endif 289 290 /* 291 * Interrupts disabled here. Called from ASM only, prototype not public. 292 */ 293 void x86_hotpatch_cleanup(int); 294 void 295 x86_hotpatch_cleanup(int retval) 296 { 297 if (retval != 0) { 298 panic("x86_hotpatch_apply failed"); 299 } 300 301 remove_d_bit(); 302 } 303 304 /* -------------------------------------------------------------------------- */ 305 306 void 307 x86_patch(bool early) 308 { 309 static bool first, second; 310 311 if (early) { 312 if (first) 313 return; 314 first = true; 315 } else { 316 if (second) 317 return; 318 second = true; 319 } 320 321 if (!early && ncpu == 1) { 322 #ifndef LOCKDEBUG 323 /* 324 * Uniprocessor: kill LOCK prefixes. 325 */ 326 x86_hotpatch(HP_NAME_NOLOCK, 0); 327 #endif 328 } 329 330 if (!early && (cpu_feature[0] & CPUID_SSE2) != 0) { 331 /* 332 * Faster memory barriers. The only barrier x86 ever 333 * requires for MI synchronization between CPUs is 334 * MFENCE for store-before-load ordering; all other 335 * ordering is guaranteed already -- every load is a 336 * load-acquire and every store is a store-release. 337 * 338 * LFENCE and SFENCE are relevant only for MD logic 339 * involving I/O devices or non-temporal stores. 340 */ 341 x86_hotpatch(HP_NAME_SSE2_MFENCE, 0); 342 } 343 344 #ifdef i386 345 /* 346 * Patch early and late. Second time around the 'lock' prefix 347 * may be gone. 348 */ 349 if ((cpu_feature[0] & CPUID_CX8) != 0) { 350 x86_hotpatch(HP_NAME_CAS_64, 0); 351 } 352 353 #if !defined(SPLDEBUG) 354 if (!early && (cpu_feature[0] & CPUID_CX8) != 0) { 355 /* Faster splx(), mutex_spin_exit(). */ 356 x86_hotpatch(HP_NAME_SPLLOWER, 0); 357 #if !defined(LOCKDEBUG) 358 x86_hotpatch(HP_NAME_MUTEX_EXIT, 0); 359 #endif 360 } 361 #endif /* !SPLDEBUG */ 362 #endif /* i386 */ 363 364 /* 365 * On some Opteron revisions, locked operations erroneously 366 * allow memory references to be `bled' outside of critical 367 * sections. Apply workaround. 368 */ 369 if (cpu_vendor == CPUVENDOR_AMD && 370 (CPUID_TO_FAMILY(cpu_info_primary.ci_signature) == 0xe || 371 (CPUID_TO_FAMILY(cpu_info_primary.ci_signature) == 0xf && 372 CPUID_TO_EXTMODEL(cpu_info_primary.ci_signature) < 0x4))) { 373 x86_hotpatch(HP_NAME_RETFENCE, 0); 374 } 375 376 /* 377 * If SMAP is present then patch the prepared holes with clac/stac 378 * instructions. 379 */ 380 if (!early && cpu_feature[5] & CPUID_SEF_SMAP) { 381 KASSERT(rcr4() & CR4_SMAP); 382 383 x86_hotpatch(HP_NAME_CLAC, 0); 384 x86_hotpatch(HP_NAME_STAC, 0); 385 } 386 } 387