1 /* $NetBSD: patch.c,v 1.49 2020/05/07 18:13:05 maxv Exp $ */ 2 3 /*- 4 * Copyright (c) 2007, 2008, 2009 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Patch kernel code at boot time, depending on available CPU features. 34 */ 35 36 #include <sys/cdefs.h> 37 __KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.49 2020/05/07 18:13:05 maxv Exp $"); 38 39 #include "opt_lockdebug.h" 40 #ifdef i386 41 #include "opt_spldebug.h" 42 #endif 43 44 #include <sys/types.h> 45 #include <sys/systm.h> 46 47 #include <machine/cpu.h> 48 #include <machine/cpufunc.h> 49 #include <machine/specialreg.h> 50 #include <machine/frameasm.h> 51 52 #include <uvm/uvm.h> 53 #include <machine/pmap.h> 54 55 #include <x86/cpuvar.h> 56 #include <x86/cputypes.h> 57 58 __link_set_decl(x86_hotpatch_descriptors, struct x86_hotpatch_descriptor); 59 60 struct x86_hotpatch_destination { 61 uint8_t name; 62 uint8_t size; 63 void *addr; 64 } __packed; 65 66 /* -------------------------------------------------------------------------- */ 67 68 /* CLAC instruction, part of SMAP. */ 69 extern uint8_t hp_clac, hp_clac_end; 70 static const struct x86_hotpatch_source hp_clac_source = { 71 .saddr = &hp_clac, 72 .eaddr = &hp_clac_end 73 }; 74 static const struct x86_hotpatch_descriptor hp_clac_desc = { 75 .name = HP_NAME_CLAC, 76 .nsrc = 1, 77 .srcs = { &hp_clac_source } 78 }; 79 __link_set_add_rodata(x86_hotpatch_descriptors, hp_clac_desc); 80 81 /* STAC instruction, part of SMAP. */ 82 extern uint8_t hp_stac, hp_stac_end; 83 static const struct x86_hotpatch_source hp_stac_source = { 84 .saddr = &hp_stac, 85 .eaddr = &hp_stac_end 86 }; 87 static const struct x86_hotpatch_descriptor hp_stac_desc = { 88 .name = HP_NAME_STAC, 89 .nsrc = 1, 90 .srcs = { &hp_stac_source } 91 }; 92 __link_set_add_rodata(x86_hotpatch_descriptors, hp_stac_desc); 93 94 /* Errata on certain AMD CPUs. */ 95 extern uint8_t hp_retfence, hp_retfence_end; 96 static const struct x86_hotpatch_source hp_retfence_source = { 97 .saddr = &hp_retfence, 98 .eaddr = &hp_retfence_end 99 }; 100 static const struct x86_hotpatch_descriptor hp_retfence_desc = { 101 .name = HP_NAME_RETFENCE, 102 .nsrc = 1, 103 .srcs = { &hp_retfence_source } 104 }; 105 __link_set_add_rodata(x86_hotpatch_descriptors, hp_retfence_desc); 106 107 /* No lock when on a single processor. */ 108 extern uint8_t hp_nolock, hp_nolock_end; 109 static const struct x86_hotpatch_source hp_nolock_source = { 110 .saddr = &hp_nolock, 111 .eaddr = &hp_nolock_end 112 }; 113 static const struct x86_hotpatch_descriptor hp_nolock_desc = { 114 .name = HP_NAME_NOLOCK, 115 .nsrc = 1, 116 .srcs = { &hp_nolock_source } 117 }; 118 __link_set_add_rodata(x86_hotpatch_descriptors, hp_nolock_desc); 119 120 /* Use LFENCE if available, part of SSE2. */ 121 extern uint8_t sse2_lfence, sse2_lfence_end; 122 static const struct x86_hotpatch_source hp_sse2_lfence_source = { 123 .saddr = &sse2_lfence, 124 .eaddr = &sse2_lfence_end 125 }; 126 static const struct x86_hotpatch_descriptor hp_sse2_lfence_desc = { 127 .name = HP_NAME_SSE2_LFENCE, 128 .nsrc = 1, 129 .srcs = { &hp_sse2_lfence_source } 130 }; 131 __link_set_add_rodata(x86_hotpatch_descriptors, hp_sse2_lfence_desc); 132 133 /* Use MFENCE if available, part of SSE2. */ 134 extern uint8_t sse2_mfence, sse2_mfence_end; 135 static const struct x86_hotpatch_source hp_sse2_mfence_source = { 136 .saddr = &sse2_mfence, 137 .eaddr = &sse2_mfence_end 138 }; 139 static const struct x86_hotpatch_descriptor hp_sse2_mfence_desc = { 140 .name = HP_NAME_SSE2_MFENCE, 141 .nsrc = 1, 142 .srcs = { &hp_sse2_mfence_source } 143 }; 144 __link_set_add_rodata(x86_hotpatch_descriptors, hp_sse2_mfence_desc); 145 146 #ifdef i386 147 /* CAS_64. */ 148 extern uint8_t _atomic_cas_cx8, _atomic_cas_cx8_end; 149 static const struct x86_hotpatch_source hp_cas_cx8_source = { 150 .saddr = &_atomic_cas_cx8, 151 .eaddr = &_atomic_cas_cx8_end 152 }; 153 static const struct x86_hotpatch_descriptor hp_cas_cx8_desc = { 154 .name = HP_NAME_CAS_64, 155 .nsrc = 1, 156 .srcs = { &hp_cas_cx8_source } 157 }; 158 __link_set_add_rodata(x86_hotpatch_descriptors, hp_cas_cx8_desc); 159 160 /* SPLLOWER. */ 161 extern uint8_t cx8_spllower, cx8_spllower_end; 162 static const struct x86_hotpatch_source hp_cx8_spllower_source = { 163 .saddr = &cx8_spllower, 164 .eaddr = &cx8_spllower_end 165 }; 166 static const struct x86_hotpatch_descriptor hp_cx8_spllower_desc = { 167 .name = HP_NAME_SPLLOWER, 168 .nsrc = 1, 169 .srcs = { &hp_cx8_spllower_source } 170 }; 171 __link_set_add_rodata(x86_hotpatch_descriptors, hp_cx8_spllower_desc); 172 173 /* MUTEX_EXIT. */ 174 #ifndef LOCKDEBUG 175 extern uint8_t i686_mutex_spin_exit, i686_mutex_spin_exit_end; 176 static const struct x86_hotpatch_source hp_i686_mutex_spin_exit_source = { 177 .saddr = &i686_mutex_spin_exit, 178 .eaddr = &i686_mutex_spin_exit_end 179 }; 180 static const struct x86_hotpatch_descriptor hp_i686_mutex_spin_exit_desc = { 181 .name = HP_NAME_MUTEX_EXIT, 182 .nsrc = 1, 183 .srcs = { &hp_i686_mutex_spin_exit_source } 184 }; 185 __link_set_add_rodata(x86_hotpatch_descriptors, hp_i686_mutex_spin_exit_desc); 186 #endif 187 #endif 188 189 /* -------------------------------------------------------------------------- */ 190 191 static inline void __unused 192 patchbytes(void *addr, const uint8_t *bytes, size_t size) 193 { 194 uint8_t *ptr = (uint8_t *)addr; 195 size_t i; 196 197 for (i = 0; i < size; i++) { 198 ptr[i] = bytes[i]; 199 } 200 } 201 202 /* 203 * Rules: each pointer accessed in this function MUST be read-only. 204 * 205 * Called from ASM only, prototype not public. 206 */ 207 int x86_hotpatch_apply(uint8_t, uint8_t); 208 int 209 __noubsan /* the local variables have unknown alignment to UBSan */ 210 x86_hotpatch_apply(uint8_t name, uint8_t sel) 211 { 212 struct x86_hotpatch_descriptor * const *iter; 213 const struct x86_hotpatch_descriptor *desc; 214 const struct x86_hotpatch_source *src; 215 const struct x86_hotpatch_destination *hps, *hpe, *hp; 216 extern char __rodata_hotpatch_start; 217 extern char __rodata_hotpatch_end; 218 const uint8_t *bytes; 219 bool found = false; 220 size_t size; 221 222 /* 223 * Find the descriptor, and perform some sanity checks. 224 */ 225 __link_set_foreach(iter, x86_hotpatch_descriptors) { 226 desc = *iter; 227 if (desc->name == name) { 228 found = true; 229 break; 230 } 231 } 232 if (!found) 233 return -1; 234 if (desc->nsrc > 2) 235 return -1; 236 if (sel >= desc->nsrc) 237 return -1; 238 239 /* 240 * Get the hotpatch source. 241 */ 242 src = desc->srcs[sel]; 243 bytes = src->saddr; 244 size = (size_t)src->eaddr - (size_t)src->saddr; 245 246 /* 247 * Apply the hotpatch on each registered destination. 248 */ 249 hps = (struct x86_hotpatch_destination *)&__rodata_hotpatch_start; 250 hpe = (struct x86_hotpatch_destination *)&__rodata_hotpatch_end; 251 for (hp = hps; hp < hpe; hp++) { 252 if (hp->name != name) { 253 continue; 254 } 255 if (hp->size != size) { 256 return -1; 257 } 258 patchbytes(hp->addr, bytes, size); 259 } 260 261 return 0; 262 } 263 264 #ifdef __x86_64__ 265 /* 266 * The CPU added the D bit on the text pages while we were writing to them. 267 * Remove that bit. Kinda annoying, but we can't avoid it. 268 */ 269 static void 270 remove_d_bit(void) 271 { 272 extern struct bootspace bootspace; 273 pt_entry_t pte; 274 vaddr_t va; 275 size_t i, n; 276 277 for (i = 0; i < BTSPACE_NSEGS; i++) { 278 if (bootspace.segs[i].type != BTSEG_TEXT) 279 continue; 280 va = bootspace.segs[i].va; 281 n = 0; 282 while (n < bootspace.segs[i].sz) { 283 if (L2_BASE[pl2_i(va)] & PTE_PS) { 284 pte = L2_BASE[pl2_i(va)] & ~PTE_D; 285 pmap_pte_set(&L2_BASE[pl2_i(va)], pte); 286 n += NBPD_L2; 287 va += NBPD_L2; 288 } else { 289 pte = L1_BASE[pl1_i(va)] & ~PTE_D; 290 pmap_pte_set(&L1_BASE[pl1_i(va)], pte); 291 n += NBPD_L1; 292 va += NBPD_L1; 293 } 294 } 295 } 296 297 tlbflushg(); 298 } 299 #else 300 #define remove_d_bit() __nothing 301 #endif 302 303 /* 304 * Interrupts disabled here. Called from ASM only, prototype not public. 305 */ 306 void x86_hotpatch_cleanup(int); 307 void 308 x86_hotpatch_cleanup(int retval) 309 { 310 if (retval != 0) { 311 panic("x86_hotpatch_apply failed"); 312 } 313 314 remove_d_bit(); 315 } 316 317 /* -------------------------------------------------------------------------- */ 318 319 void 320 x86_patch(bool early) 321 { 322 static bool first, second; 323 324 if (early) { 325 if (first) 326 return; 327 first = true; 328 } else { 329 if (second) 330 return; 331 second = true; 332 } 333 334 if (!early && ncpu == 1) { 335 #ifndef LOCKDEBUG 336 /* 337 * Uniprocessor: kill LOCK prefixes. 338 */ 339 x86_hotpatch(HP_NAME_NOLOCK, 0); 340 #endif 341 } 342 343 if (!early && (cpu_feature[0] & CPUID_SSE2) != 0) { 344 /* 345 * Faster memory barriers. We do not need to patch 346 * membar_producer to use SFENCE because on x86 347 * ordinary non-temporal stores are always issued in 348 * program order to main memory and to other CPUs. 349 */ 350 x86_hotpatch(HP_NAME_SSE2_LFENCE, 0); 351 x86_hotpatch(HP_NAME_SSE2_MFENCE, 0); 352 } 353 354 #ifdef i386 355 /* 356 * Patch early and late. Second time around the 'lock' prefix 357 * may be gone. 358 */ 359 if ((cpu_feature[0] & CPUID_CX8) != 0) { 360 x86_hotpatch(HP_NAME_CAS_64, 0); 361 } 362 363 #if !defined(SPLDEBUG) 364 if (!early && (cpu_feature[0] & CPUID_CX8) != 0) { 365 /* Faster splx(), mutex_spin_exit(). */ 366 x86_hotpatch(HP_NAME_SPLLOWER, 0); 367 #if !defined(LOCKDEBUG) 368 x86_hotpatch(HP_NAME_MUTEX_EXIT, 0); 369 #endif 370 } 371 #endif /* !SPLDEBUG */ 372 #endif /* i386 */ 373 374 /* 375 * On some Opteron revisions, locked operations erroneously 376 * allow memory references to be `bled' outside of critical 377 * sections. Apply workaround. 378 */ 379 if (cpu_vendor == CPUVENDOR_AMD && 380 (CPUID_TO_FAMILY(cpu_info_primary.ci_signature) == 0xe || 381 (CPUID_TO_FAMILY(cpu_info_primary.ci_signature) == 0xf && 382 CPUID_TO_EXTMODEL(cpu_info_primary.ci_signature) < 0x4))) { 383 x86_hotpatch(HP_NAME_RETFENCE, 0); 384 } 385 386 /* 387 * If SMAP is present then patch the prepared holes with clac/stac 388 * instructions. 389 */ 390 if (!early && cpu_feature[5] & CPUID_SEF_SMAP) { 391 KASSERT(rcr4() & CR4_SMAP); 392 393 x86_hotpatch(HP_NAME_CLAC, 0); 394 x86_hotpatch(HP_NAME_STAC, 0); 395 } 396 } 397